From 92279d3c719a3b075e35e112e32e31435e6c77ca Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Mar 2025 22:59:43 -0700 Subject: [PATCH 01/51] Initial work on command-line flags for all configuration (#303). --- borgmatic/commands/arguments.py | 167 ++++++++++++++++++++++++++++++-- borgmatic/commands/borgmatic.py | 30 ++++-- borgmatic/config/arguments.py | 137 ++++++++++++++++++++++++++ borgmatic/config/override.py | 7 ++ borgmatic/config/schema.yaml | 5 + borgmatic/config/validate.py | 15 +-- 6 files changed, 337 insertions(+), 24 deletions(-) create mode 100644 borgmatic/config/arguments.py diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e95ec5f43..b09c56607 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,5 +1,7 @@ import collections import itertools +import json +import re import sys from argparse import ArgumentParser @@ -282,12 +284,155 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -def make_parsers(): +def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' - Build a global arguments parser, individual action parsers, and a combined parser containing - both. Return them as a tuple. The global parser is useful for parsing just global arguments - while ignoring actions, and the combined parser is handy for displaying help that includes - everything: global flags, a list of actions, etc. + Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of + unparsed argument strings, convert the entire schema into corresponding command-line flags and + add them to the arguments group. + + For instance, given a schema of: + + { + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer'} + } + } + } + } + + ... the following flag will be added to the arguments group: + + --foo.bar + + If "foo" is instead an array of objects, it will get added like this + + --foo[0].bar + + And if names are also passed in, they are considered to be the name components of an option + (e.g. "foo" and "bar") and are used to construct a resulting flag. + ''' + if names is None: + names = () + + schema_type = schema.get('type') + + # If this option has multiple types, just use the first one (that isn't "null"). + if isinstance(schema_type, list): + try: + schema_type = next(single_type for single_type in schema_type if single_type != 'null') + except StopIteration: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + # If this is an "object" type, recurse for each child option ("property"). + if schema_type in {'object', 'array'}: + properties = ( + schema.get('items', {}).get('properties') + if schema_type == 'array' + else schema.get('properties') + ) + + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, + child, + unparsed_arguments, + names + ((name + '[0]',) if child.get('type') == 'array' else (name,)), + ) + + return + + flag_name = '.'.join(names) + description = schema.get('description') + metavar = names[-1].upper() + + if schema_type == 'array': + metavar = metavar.rstrip('S') + + if description: + if schema_type == 'array': + items_schema = schema.get('items', {}) + + description += ' Can specify flag multiple times.' + + if '[0]' in flag_name: + description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + + description = description.replace('%', '%%') + + try: + argument_type = {'string': str, 'integer': int, 'boolean': bool, 'array': str}[schema_type] + except KeyError: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + arguments_group.add_argument( + f"--{flag_name.replace('_', '-')}", + type=argument_type, + metavar=metavar, + action='append' if schema_type == 'array' else None, + help=description, + ) + + # We want to support flags that can have arbitrary indices like: + # + # --foo.bar[1].baz + # + # But argparse doesn't support that natively because the index can be an arbitrary number. We + # won't let that stop us though, will we? So, if the current flag name has an array component in + # it (e.g. a name with "[0]"), then make a pattern that would match the flag name regardless of + # the number that's in it. The idea is that we want to look for unparsed arguments that appear + # like the flag name, but instead of "[0]" they have, say, "[1]" or "[123]". + # + # Next, we check each unparsed argument against that pattern. If one of them matches, add an + # argument flag for it to the argument parser group. Example: + # + # Let's say flag_name is: + # + # --foo.bar[0].baz + # + # ... then the regular expression pattern will be: + # + # ^--foo\.bar\[\d+\]\.baz + # + # ... and, if that matches an unparsed argument of: + # + # --foo.bar[1].baz + # + # ... then an argument flag will get added equal to that unparsed argument. And the unparsed + # argument will match it when parsing is performed! In this manner, we're using the actual user + # CLI input to inform what exact flags we support! + if '[0]' not in flag_name or '--help' in unparsed_arguments: + return + + pattern = re.compile(f'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + existing_flags = set( + itertools.chain( + *(group_action.option_strings for group_action in arguments_group._group_actions) + ) + ) + + for unparsed in unparsed_arguments: + unparsed_flag_name = unparsed.split('=', 1)[0] + + if pattern.match(unparsed_flag_name) and unparsed_flag_name not in existing_flags: + arguments_group.add_argument( + unparsed_flag_name, + type=argument_type, + metavar=metavar, + help=description, + ) + + +def make_parsers(schema, unparsed_arguments): + ''' + Given a configuration schema dict, build a global arguments parser, individual action parsers, + and a combined parser containing both. Return them as a tuple. The global parser is useful for + parsing just global arguments while ignoring actions, and the combined parser is handy for + displaying help that includes everything: global flags, a list of actions, etc. ''' config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) @@ -388,6 +533,7 @@ def make_parsers(): action='store_true', help='Display installed version number of borgmatic and exit', ) + add_arguments_from_schema(global_group, schema, unparsed_arguments) global_plus_action_parser = ArgumentParser( description=''' @@ -1523,15 +1669,18 @@ def make_parsers(): return global_parser, action_parsers, global_plus_action_parser -def parse_arguments(*unparsed_arguments): +def parse_arguments(schema, *unparsed_arguments): ''' - Given command-line arguments with which this script was invoked, parse the arguments and return - them as a dict mapping from action name (or "global") to an argparse.Namespace instance. + Given a configuration schema dict and the command-line arguments with which this script was + invoked, parse the arguments and return them as a dict mapping from action name (or "global") to + an argparse.Namespace instance. Raise ValueError if the arguments cannot be parsed. Raise SystemExit with an error code of 0 if "--help" was requested. ''' - global_parser, action_parsers, global_plus_action_parser = make_parsers() + global_parser, action_parsers, global_plus_action_parser = make_parsers( + schema, unparsed_arguments + ) arguments, remaining_action_arguments = parse_arguments_for_actions( unparsed_arguments, action_parsers.choices, global_parser ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5691df71e..9cffbdf85 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -8,6 +8,8 @@ import time from queue import Queue from subprocess import CalledProcessError +import ruamel.yaml + import borgmatic.actions.borg import borgmatic.actions.break_lock import borgmatic.actions.change_passphrase @@ -33,6 +35,7 @@ import borgmatic.actions.restore import borgmatic.actions.transfer import borgmatic.commands.completion.bash import borgmatic.commands.completion.fish +import borgmatic.config.load from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments @@ -570,14 +573,14 @@ def run_actions( ) -def load_configurations(config_filenames, overrides=None, resolve_env=True): +def load_configurations(config_filenames, global_arguments, overrides=None, resolve_env=True): ''' - Given a sequence of configuration filenames, a sequence of configuration file override strings - in the form of "option.suboption=value", and whether to resolve environment variables, load and - validate each configuration file. Return the results as a tuple of: dict of configuration - filename to corresponding parsed configuration, a sequence of paths for all loaded configuration - files (including includes), and a sequence of logging.LogRecord instances containing any parse - errors. + Given a sequence of configuration filenames, global arguments as an argparse.Namespace, a + sequence of configuration file override strings in the form of "option.suboption=value", and + whether to resolve environment variables, load and validate each configuration file. Return the + results as a tuple of: dict of configuration filename to corresponding parsed configuration, a + sequence of paths for all loaded configuration files (including includes), and a sequence of + logging.LogRecord instances containing any parse errors. Log records are returned here instead of being logged directly because logging isn't yet initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this @@ -605,6 +608,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): configs[config_filename], paths, parse_logs = validate.parse_configuration( config_filename, validate.schema_filename(), + global_arguments, overrides, resolve_env, ) @@ -928,9 +932,17 @@ def exit_with_help_link(): # pragma: no cover def main(extra_summary_logs=[]): # pragma: no cover configure_signals() configure_delayed_logging() + schema_filename = validate.schema_filename() try: - arguments = parse_arguments(*sys.argv[1:]) + schema = borgmatic.config.load.load_configuration(schema_filename) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: + configure_logging(logging.CRITICAL) + logger.critical(error) + exit_with_help_link() + + try: + arguments = parse_arguments(schema, *sys.argv[1:]) except ValueError as error: configure_logging(logging.CRITICAL) logger.critical(error) @@ -953,10 +965,10 @@ def main(extra_summary_logs=[]): # pragma: no cover print(borgmatic.commands.completion.fish.fish_completion()) sys.exit(0) - validate = bool('validate' in arguments) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( config_filenames, + global_arguments, global_arguments.overrides, resolve_env=global_arguments.resolve_env and not validate, ) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py new file mode 100644 index 000000000..96a28b4d1 --- /dev/null +++ b/borgmatic/config/arguments.py @@ -0,0 +1,137 @@ +import io +import re + +import ruamel.yaml + + +LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+)\]$') + + +def set_values(config, keys, value): + ''' + Given a configuration dict, a sequence of parsed key strings, and a string value, descend into + the configuration hierarchy based on the keys to set the value into the right place. + ''' + if not keys: + return + + first_key = keys[0] + + # Support "name[0]"-style list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(first_key) + + if match: + list_key = match.group('list_name') + list_index = int(match.group('index')) + + if len(keys) == 1: + config[list_key][list_index] = value + + return + + if list_key not in config: + config[list_key] = [] + + try: + set_values(config[list_key][list_index], keys[1:], value) + except IndexError: + raise ValueError(f'The list index {first_key} is out of range') + + return + + if len(keys) == 1: + config[first_key] = value + return + + if first_key not in config: + config[first_key] = {} + + set_values(config[first_key], keys[1:], value) + + +def type_for_option(schema, option_keys): + ''' + Given a configuration schema dict and a sequence of keys identifying a potentially nested + option, e.g. ('extra_borg_options', 'create'), return the schema type of that option as a + string. + + Return None if the option or its type cannot be found in the schema. + ''' + option_schema = schema + + for key in option_keys: + # Support "name[0]"-style list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(key) + + try: + if match: + option_schema = option_schema['properties'][match.group('list_name')]['items'] + else: + option_schema = option_schema['properties'][key] + except KeyError: + return None + + try: + return option_schema['type'] + except KeyError: + return None + + +def prepare_arguments_for_config(global_arguments, schema): + ''' + Given global arguments as an argparse.Namespace and a configuration schema dict, parse each + argument that corresponds to an option in the schema and return a sequence of tuples (keys, + values) for that option, where keys is a sequence of strings. For instance, given the following + arguments: + + argparse.Namespace(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}) + + ... return this: + + ( + (('my_option', 'sub_option'), 'value1'), + (('other_option'), 'value2'), + ) + + Raise ValueError if an override can't be parsed. + ''' + prepared_values = [] + + for argument_name, value in global_arguments.__dict__.items(): + try: + if value is None: + continue + + keys = tuple(argument_name.split('.')) + option_type = type_for_option(schema, keys) + + # The argument doesn't correspond to any option in the schema, so ignore it. It's + # probably a flag that borgmatic has on the command-line but not in configuration. + if option_type is None: + continue + + prepared_values.append( + ( + keys, + value, + ) + ) + except ruamel.yaml.error.YAMLError as error: + raise ValueError(f"Invalid override '{raw_override}': {error.problem}") + + return tuple(prepared_values) + + +def apply_arguments_to_config(config, schema, global_arguments): + ''' + Given a configuration dict, a corresponding configuration schema dict, and global arguments as + an argparse.Namespace, set those given argument values into their corresponding configuration + options in the configuration dict. + + This supports argument flags of the from "--foo.bar.baz" where each dotted component is a nested + configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list + element in the configuration. + ''' + + for keys, value in prepare_arguments_for_config(global_arguments, schema): + set_values(config, keys, value) diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 8a60cb50e..0893f669c 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -1,8 +1,12 @@ import io +import logging import ruamel.yaml +logger = logging.getLogger(__name__) + + def set_values(config, keys, value): ''' Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value, @@ -134,6 +138,9 @@ def apply_overrides(config, schema, raw_overrides): ''' overrides = parse_overrides(raw_overrides, schema) + if overrides: + logger.warning("The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set.") + for keys, value in overrides: set_values(config, keys, value) set_values(config, strip_section_names(keys), value) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 13ab0686c..3c1bb20f1 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -36,9 +36,14 @@ properties: properties: path: type: string + description: The local path or Borg URL of the repository. example: ssh://user@backupserver/./{fqdn} label: type: string + description: | + An optional label for the repository, used in logging + and to make selecting the repository easier on the + command-line. example: backupserver description: | A required list of local or remote repositories with paths and diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index c80cee4d6..35c8e809d 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -4,7 +4,7 @@ import os import jsonschema import ruamel.yaml -import borgmatic.config +import borgmatic.config.arguments from borgmatic.config import constants, environment, load, normalize, override @@ -84,13 +84,15 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True): +def parse_configuration(config_filename, schema_filename, global_arguments, overrides=None, resolve_env=True): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML - rendition of JSON Schema format, a sequence of configuration file override strings in the form - of "option.suboption=value", and whether to resolve environment variables, return the parsed - configuration as a data structure of nested dicts and lists corresponding to the schema. Example - return value: + rendition of JSON Schema format, global arguments as an argparse.Namespace, a sequence of + configuration file override strings in the form of "option.suboption=value", and whether to + resolve environment variables, return the parsed configuration as a data structure of nested + dicts and lists corresponding to the schema. Example return value. + + Example return value: { 'source_directories': ['/home', '/etc'], @@ -113,6 +115,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) + borgmatic.config.arguments.apply_arguments_to_config(config, schema, global_arguments) override.apply_overrides(config, schema, overrides) constants.apply_constants(config, config.get('constants') if config else {}) From 05900c188f6f0426a0ac78371683a1af17030bc9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Mar 2025 22:58:39 -0700 Subject: [PATCH 02/51] Expand docstrings (#303). --- borgmatic/config/arguments.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 96a28b4d1..ec7fe8eed 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -10,14 +10,25 @@ LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+) def set_values(config, keys, value): ''' Given a configuration dict, a sequence of parsed key strings, and a string value, descend into - the configuration hierarchy based on the keys to set the value into the right place. + the configuration hierarchy based on the given keys and set the value into the right place. + For example, consider these keys: + + ('foo', 'bar', 'baz') + + This looks up "foo" in the given configuration. And within that value, it looks up "bar". And + then within that value, it looks up "baz" and sets it to the given value. Another example: + + ('mylist[0]', 'foo') + + This looks for the zeroth element of "mylist" in the given configuration. And within that value, + it looks up "foo" and sets it to the given value. Finally: ''' if not keys: return first_key = keys[0] - # Support "name[0]"-style list index syntax. + # Support "mylist[0]" list index syntax. match = LIST_INDEX_KEY_PATTERN.match(first_key) if match: @@ -41,6 +52,7 @@ def set_values(config, keys, value): if len(keys) == 1: config[first_key] = value + return if first_key not in config: From eca78fbc2cdeade1c6e4a227ebbc018664468300 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 09:57:25 -0700 Subject: [PATCH 03/51] Support setting whole lists and dicts from the command-line (#303). --- borgmatic/commands/arguments.py | 45 +++++++++++------ borgmatic/config/arguments.py | 22 ++++++-- borgmatic/config/generate.py | 29 ++--------- borgmatic/config/normalize.py | 6 ++- borgmatic/config/schema.py | 22 ++++++++ borgmatic/config/schema.yaml | 36 +++++++++++++- tests/unit/config/test_generate.py | 76 ---------------------------- tests/unit/config/test_schema.py | 80 ++++++++++++++++++++++++++++++ 8 files changed, 194 insertions(+), 122 deletions(-) create mode 100644 borgmatic/config/schema.py create mode 100644 tests/unit/config/test_schema.py diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6504257be..555f41de3 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,11 +1,16 @@ import collections +import decimal import itertools +import io import json import re import sys from argparse import ArgumentParser from borgmatic.config import collect +import borgmatic.config.schema + +import ruamel.yaml ACTION_ALIASES = { 'repo-create': ['rcreate', 'init', '-I'], @@ -308,8 +313,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names --foo.bar - If "foo" is instead an array of objects, it will get added like this + If "foo" is instead an array of objects, both of the following will get added: + --foo --foo[0].bar And if names are also passed in, they are considered to be the name components of an option @@ -328,12 +334,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names raise ValueError(f'Unknown type in configuration schema: {schema_type}') # If this is an "object" type, recurse for each child option ("property"). - if schema_type in {'object', 'array'}: - properties = ( - schema.get('items', {}).get('properties') - if schema_type == 'array' - else schema.get('properties') - ) + if schema_type == 'object': + properties = schema.get('properties') if properties: for name, child in properties.items(): @@ -341,23 +343,37 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names arguments_group, child, unparsed_arguments, - names + ((name + '[0]',) if child.get('type') == 'array' else (name,)), + names + (name,) ) return + # If this is an "array" type, recurse for each child option of its items type. Don't return yet, + # so that a flag also gets added below for the array itself. + if schema_type == 'array': + properties = borgmatic.config.schema.get_properties(schema.get('items', {})) + + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, + child, + unparsed_arguments, + names[:-1] + (f'{names[-1]}[0]',) + (name,) + ) + flag_name = '.'.join(names) description = schema.get('description') metavar = names[-1].upper() - if schema_type == 'array': - metavar = metavar.rstrip('S') - if description: if schema_type == 'array': - items_schema = schema.get('items', {}) + example_buffer = io.StringIO() + yaml = ruamel.yaml.YAML(typ='safe') + yaml.default_flow_style = True + yaml.dump(schema.get('example'), example_buffer) - description += ' Can specify flag multiple times.' + description += f' Example value: "{example_buffer.getvalue().strip()}"' if '[0]' in flag_name: description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' @@ -365,7 +381,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') try: - argument_type = {'string': str, 'integer': int, 'boolean': bool, 'array': str}[schema_type] + argument_type = {'string': str, 'integer': int, 'number': decimal.Decimal, 'boolean': bool, 'array': str}[schema_type] except KeyError: raise ValueError(f'Unknown type in configuration schema: {schema_type}') @@ -373,7 +389,6 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names f"--{flag_name.replace('_', '-')}", type=argument_type, metavar=metavar, - action='append' if schema_type == 'array' else None, help=description, ) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index ec7fe8eed..dab37b5b5 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -21,7 +21,7 @@ def set_values(config, keys, value): ('mylist[0]', 'foo') This looks for the zeroth element of "mylist" in the given configuration. And within that value, - it looks up "foo" and sets it to the given value. Finally: + it looks up "foo" and sets it to the given value. ''' if not keys: return @@ -89,6 +89,22 @@ def type_for_option(schema, option_keys): return None +def convert_value_type(value, option_type): + ''' + Given a string value and its schema type as a string, determine its logical type (string, + boolean, integer, etc.), and return it converted to that type. + + If the option type is a string, leave the value as a string so that special characters in it + don't get interpreted as YAML during conversion. + + Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML. + ''' + if option_type == 'string': + return value + + return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + + def prepare_arguments_for_config(global_arguments, schema): ''' Given global arguments as an argparse.Namespace and a configuration schema dict, parse each @@ -125,11 +141,11 @@ def prepare_arguments_for_config(global_arguments, schema): prepared_values.append( ( keys, - value, + convert_value_type(value, option_type), ) ) except ruamel.yaml.error.YAMLError as error: - raise ValueError(f"Invalid override '{raw_override}': {error.problem}") + raise ValueError(f'Invalid override "{argument_name}": {error.problem}') return tuple(prepared_values) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 58fd03a79..e0a646d85 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -1,12 +1,12 @@ import collections import io -import itertools import os import re import ruamel.yaml from borgmatic.config import load, normalize +import borgmatic.config.schema INDENT = 4 SEQUENCE_INDENT = 2 @@ -22,27 +22,6 @@ def insert_newline_before_comment(config, field_name): ) -def get_properties(schema): - ''' - Given a schema dict, return its properties. But if it's got sub-schemas with multiple different - potential properties, returned their merged properties instead (interleaved so the first - properties of each sub-schema come first). The idea is that the user should see all possible - options even if they're not all possible together. - ''' - if 'oneOf' in schema: - return dict( - item - for item in itertools.chain( - *itertools.zip_longest( - *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] - ) - ) - if item is not None - ) - - return schema['properties'] - - def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False): ''' Given a loaded configuration schema and a source configuration, generate and return sample @@ -78,7 +57,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i sub_schema, (source_config or {}).get(field_name, {}), level + 1 ), ) - for field_name, sub_schema in get_properties(schema).items() + for field_name, sub_schema in borgmatic.config.schema.get_properties(schema).items() ] ) indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) @@ -189,7 +168,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): return for field_name in config[0].keys(): - field_schema = get_properties(schema['items']).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema['items']).get(field_name, {}) description = field_schema.get('description') # No description to use? Skip it. @@ -223,7 +202,7 @@ def add_comments_to_configuration_object( if skip_first and index == 0: continue - field_schema = get_properties(schema).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema).get(field_name, {}) description = field_schema.get('description', '').strip() # If this isn't a default key, add an indicator to the comment flagging it to be commented diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 11f21ce09..f4199e6b4 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -326,7 +326,11 @@ def normalize(config_filename, config): config['repositories'] = [] for repository_dict in repositories: - repository_path = repository_dict['path'] + repository_path = repository_dict.get('path') + + if repository_path is None: + continue + if '~' in repository_path: logs.append( logging.makeLogRecord( diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py new file mode 100644 index 000000000..e46d27cc7 --- /dev/null +++ b/borgmatic/config/schema.py @@ -0,0 +1,22 @@ +import itertools + + +def get_properties(schema): + ''' + Given a schema dict, return its properties. But if it's got sub-schemas with multiple different + potential properties, returned their merged properties instead (interleaved so the first + properties of each sub-schema come first). The idea is that the user should see all possible + options even if they're not all possible together. + ''' + if 'oneOf' in schema: + return dict( + item + for item in itertools.chain( + *itertools.zip_longest( + *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] + ) + ) + if item is not None + ) + + return schema.get('properties', {}) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3c1bb20f1..2f8923371 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -53,8 +53,7 @@ properties: output of "borg help placeholders" for details. See ssh_command for SSH options like identity file or port. If systemd service is used, then add local repository paths in the systemd service file to the - ReadWritePaths list. Prior to borgmatic 1.7.10, repositories was a - list of plain path strings. + ReadWritePaths list. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver @@ -738,6 +737,10 @@ properties: List of one or more consistency checks to run on a periodic basis (if "frequency" is set) or every time borgmatic runs checks (if "frequency" is omitted). + example: + - name: archives + frequency: 2 weeks + - name: repository check_repositories: type: array items: @@ -1115,6 +1118,10 @@ properties: List of one or more command hooks to execute, triggered at particular points during borgmatic's execution. For each command hook, specify one of "before" or "after", not both. + example: + - before: action + when: [create] + run: [echo Backing up.] bootstrap: type: object properties: @@ -1329,6 +1336,9 @@ properties: https://www.postgresql.org/docs/current/app-pgdump.html and https://www.postgresql.org/docs/current/libpq-ssl.html for details. + example: + - name: users + hostname: database.example.org mariadb_databases: type: array items: @@ -1473,6 +1483,9 @@ properties: added to your source directories at runtime and streamed directly to Borg. Requires mariadb-dump/mariadb commands. See https://mariadb.com/kb/en/library/mysqldump/ for details. + example: + - name: users + hostname: database.example.org mysql_databases: type: array items: @@ -1618,6 +1631,9 @@ properties: to Borg. Requires mysqldump/mysql commands. See https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for details. + example: + - name: users + hostname: database.example.org sqlite_databases: type: array items: @@ -1647,6 +1663,15 @@ properties: Path to the SQLite database file to restore to. Defaults to the "path" option. example: /var/lib/sqlite/users.db + description: | + List of one or more SQLite databases to dump before creating a + backup, run once per configuration file. The database dumps are + added to your source directories at runtime and streamed directly to + Borg. Requires the sqlite3 command. See https://sqlite.org/cli.html + for details. + example: + - name: users + path: /var/lib/db.sqlite mongodb_databases: type: array items: @@ -1749,6 +1774,9 @@ properties: to Borg. Requires mongodump/mongorestore commands. See https://docs.mongodb.com/database-tools/mongodump/ and https://docs.mongodb.com/database-tools/mongorestore/ for details. + example: + - name: users + hostname: database.example.org ntfy: type: object required: ['topic'] @@ -2231,9 +2259,13 @@ properties: properties: url: type: string + description: URL of this Apprise service. example: "gotify://hostname/token" label: type: string + description: | + Label used in borgmatic logs for this Apprise + service. example: gotify description: | A list of Apprise services to publish to with URLs and diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 27dcd01f7..abb520fb5 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -4,82 +4,6 @@ from flexmock import flexmock from borgmatic.config import generate as module -def test_get_properties_with_simple_object(): - schema = { - 'type': 'object', - 'properties': dict( - [ - ('field1', {'example': 'Example'}), - ] - ), - } - - assert module.get_properties(schema) == schema['properties'] - - -def test_get_properties_merges_oneof_list_properties(): - schema = { - 'type': 'object', - 'oneOf': [ - { - 'properties': dict( - [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ] - ), - }, - { - 'properties': dict( - [ - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - ], - } - - assert module.get_properties(schema) == dict( - schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] - ) - - -def test_get_properties_interleaves_oneof_list_properties(): - schema = { - 'type': 'object', - 'oneOf': [ - { - 'properties': dict( - [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - { - 'properties': dict( - [ - ('field4', {'example': 'Example 4'}), - ('field5', {'example': 'Example 5'}), - ] - ), - }, - ], - } - - assert module.get_properties(schema) == dict( - [ - ('field1', {'example': 'Example 1'}), - ('field4', {'example': 'Example 4'}), - ('field2', {'example': 'Example 2'}), - ('field5', {'example': 'Example 5'}), - ('field3', {'example': 'Example 3'}), - ] - ) - - def test_schema_to_sample_configuration_generates_config_map_with_examples(): schema = { 'type': 'object', diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py new file mode 100644 index 000000000..e10992b69 --- /dev/null +++ b/tests/unit/config/test_schema.py @@ -0,0 +1,80 @@ +from borgmatic.config import schema as module + + +def test_get_properties_with_simple_object(): + schema = { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'example': 'Example'}), + ] + ), + } + + assert module.get_properties(schema) == schema['properties'] + + +def test_get_properties_merges_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] + ) + + +def test_get_properties_interleaves_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field4', {'example': 'Example 4'}), + ('field5', {'example': 'Example 5'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + [ + ('field1', {'example': 'Example 1'}), + ('field4', {'example': 'Example 4'}), + ('field2', {'example': 'Example 2'}), + ('field5', {'example': 'Example 5'}), + ('field3', {'example': 'Example 3'}), + ] + ) + + + From 87b9ad5aeaebdf02b4d448b97a5ac9ecf221dc6d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 10:02:25 -0700 Subject: [PATCH 04/51] Code formatting (#303). --- borgmatic/commands/arguments.py | 23 +++++++++++++---------- borgmatic/config/arguments.py | 1 - borgmatic/config/generate.py | 2 +- borgmatic/config/override.py | 5 +++-- borgmatic/config/validate.py | 4 +++- tests/unit/config/test_schema.py | 3 --- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 555f41de3..6b3abf9c3 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,17 +1,17 @@ import collections import decimal -import itertools import io +import itertools import json import re import sys from argparse import ArgumentParser -from borgmatic.config import collect -import borgmatic.config.schema - import ruamel.yaml +import borgmatic.config.schema +from borgmatic.config import collect + ACTION_ALIASES = { 'repo-create': ['rcreate', 'init', '-I'], 'prune': ['-p'], @@ -340,10 +340,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names if properties: for name, child in properties.items(): add_arguments_from_schema( - arguments_group, - child, - unparsed_arguments, - names + (name,) + arguments_group, child, unparsed_arguments, names + (name,) ) return @@ -359,7 +356,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names arguments_group, child, unparsed_arguments, - names[:-1] + (f'{names[-1]}[0]',) + (name,) + names[:-1] + (f'{names[-1]}[0]',) + (name,), ) flag_name = '.'.join(names) @@ -381,7 +378,13 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') try: - argument_type = {'string': str, 'integer': int, 'number': decimal.Decimal, 'boolean': bool, 'array': str}[schema_type] + argument_type = { + 'string': str, + 'integer': int, + 'number': decimal.Decimal, + 'boolean': bool, + 'array': str, + }[schema_type] except KeyError: raise ValueError(f'Unknown type in configuration schema: {schema_type}') diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index dab37b5b5..7a71c9264 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -3,7 +3,6 @@ import re import ruamel.yaml - LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+)\]$') diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index e0a646d85..fbc29c815 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -5,8 +5,8 @@ import re import ruamel.yaml -from borgmatic.config import load, normalize import borgmatic.config.schema +from borgmatic.config import load, normalize INDENT = 4 SEQUENCE_INDENT = 2 diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 0893f669c..9067e6f99 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -3,7 +3,6 @@ import logging import ruamel.yaml - logger = logging.getLogger(__name__) @@ -139,7 +138,9 @@ def apply_overrides(config, schema, raw_overrides): overrides = parse_overrides(raw_overrides, schema) if overrides: - logger.warning("The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set.") + logger.warning( + "The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set." + ) for keys, value in overrides: set_values(config, keys, value) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 35c8e809d..063f57676 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -84,7 +84,9 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration(config_filename, schema_filename, global_arguments, overrides=None, resolve_env=True): +def parse_configuration( + config_filename, schema_filename, global_arguments, overrides=None, resolve_env=True +): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML rendition of JSON Schema format, global arguments as an argparse.Namespace, a sequence of diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index e10992b69..a07b19bab 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -75,6 +75,3 @@ def test_get_properties_interleaves_oneof_list_properties(): ('field3', {'example': 'Example 3'}), ] ) - - - From 7b14e8c7f2d1264ae75379181d52a14fb9e9f13f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 10:17:04 -0700 Subject: [PATCH 05/51] Add feature to NEWS (#303). --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index b3d053fb5..60e74e22c 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,7 @@ 2.0.0.dev0 + * #303: Add flags for setting any borgmatic configuration option from the command-line. See the + documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ From 903308864cbf8622c308118ca2c3170504038d4e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 10:46:02 -0700 Subject: [PATCH 06/51] Factor out schema type parsing (#303). --- borgmatic/commands/arguments.py | 12 +----------- borgmatic/config/schema.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6b3abf9c3..dad5584d4 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,5 +1,4 @@ import collections -import decimal import io import itertools import json @@ -377,16 +376,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') - try: - argument_type = { - 'string': str, - 'integer': int, - 'number': decimal.Decimal, - 'boolean': bool, - 'array': str, - }[schema_type] - except KeyError: - raise ValueError(f'Unknown type in configuration schema: {schema_type}') + argument_type = borgmatic.config.schema.parse_type(schema_type) arguments_group.add_argument( f"--{flag_name.replace('_', '-')}", diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index e46d27cc7..118e94370 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -1,3 +1,4 @@ +import decimal import itertools @@ -20,3 +21,16 @@ def get_properties(schema): ) return schema.get('properties', {}) + + +def parse_type(schema_type): + try: + return { + 'string': str, + 'integer': int, + 'number': decimal.Decimal, + 'boolean': bool, + 'array': str, + }[schema_type] + except KeyError: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') From 93e7da823caa9d7fd97d8253af9953547e1d9bf0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 22:24:01 -0700 Subject: [PATCH 07/51] Add an encryption option to repositories (#303). --- NEWS | 4 ++++ borgmatic/actions/repo_create.py | 10 +++++++++- borgmatic/commands/arguments.py | 6 ++++-- borgmatic/config/schema.yaml | 8 ++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 4a1357a22..06f595339 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,10 @@ * #303: Add flags for setting any borgmatic configuration option from the command-line. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides + * #303: Add configuration options that serve as defaults for some (but not all) borgmatic + action flags. For example, each entry in "repositories:" now has an "encryption" option that + applies to the "repo-create" action. See the documentation for more information: + https://torsion.org/borgmatic/docs/reference/configuration/ * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index e6252e4ea..af7b793d9 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -24,13 +24,21 @@ def run_repo_create( return logger.info('Creating repository') + + encryption_mode = repo_create_arguments.encryption_mode or repository.get('encryption') + + if not encryption_mode: + raise ValueError( + 'With the repo-create action, either the --encryption flag or the repository encryption option is required.' + ) + borgmatic.borg.repo_create.create_repository( global_arguments.dry_run, repository['path'], config, local_borg_version, global_arguments, - repo_create_arguments.encryption_mode, + encryption_mode, repo_create_arguments.source_repository, repo_create_arguments.copy_crypt_key, repo_create_arguments.append_only, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index dad5584d4..3f2da08b3 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -445,7 +445,10 @@ def make_parsers(schema, unparsed_arguments): config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) - global_parser = ArgumentParser(add_help=False) + # allow_abbrev=False prevents the global parser from erroring about "ambiguous" options like + # --encryption. Such options are intended for an action parser rather than the global parser, + # and so we don't want to error on them here. + global_parser = ArgumentParser(allow_abbrev=False, add_help=False) global_group = global_parser.add_argument_group('global arguments') global_group.add_argument( @@ -569,7 +572,6 @@ def make_parsers(schema, unparsed_arguments): '--encryption', dest='encryption_mode', help='Borg repository encryption mode', - required=True, ) repo_create_group.add_argument( '--source-repository', diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 7fe30daf7..e4901dc73 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -45,6 +45,14 @@ properties: and to make selecting the repository easier on the command-line. example: backupserver + encryption: + type: string + description: | + The encryption mode with which to create the repository, + only used for the repo-create action. To see the + available encryption modes, run "borg init --help" with + Borg 1 or "borg repo-create --help" with Borg 2. + example: repokey-blake2 description: | A required list of local or remote repositories with paths and optional labels (which can be used with the --repository flag to From 711f5fa6cb9c0d7b8e5626632b2d9d279ab3aa66 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 22:58:25 -0700 Subject: [PATCH 08/51] UX nicety to make default-false boolean options into valueless CLI flags (#303). --- borgmatic/commands/arguments.py | 38 +++++++++++++++++++++++++++------ borgmatic/config/schema.yaml | 12 +++++------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 3f2da08b3..729803277 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,6 +288,24 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) +# As a UX nicety, allow boolean options that have a default of false to have command-line flags +# without values. +DEFAULT_FALSE_FLAG_NAMES = { + 'one_file_system', + 'numeric_ids', + 'read_special', + 'exclude_caches', + 'keep_exclude_tags', + 'exclude_nodump', + 'source_directories_must_exist', + 'relocated_repo_access_is_ok', + 'unknown_unencrypted_repo_access_is_ok', + 'check_i_know_what_i_am_doing', + 'postgresql_databases[0].no_owner', + 'healthchecks.create_slug', +} + + def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of @@ -377,13 +395,21 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') argument_type = borgmatic.config.schema.parse_type(schema_type) + full_flag_name = f"--{flag_name.replace('_', '-')}" - arguments_group.add_argument( - f"--{flag_name.replace('_', '-')}", - type=argument_type, - metavar=metavar, - help=description, - ) + if flag_name in DEFAULT_FALSE_FLAG_NAMES: + arguments_group.add_argument( + full_flag_name, + action='store_true', + help=description, + ) + else: + arguments_group.add_argument( + full_flag_name, + type=argument_type, + metavar=metavar, + help=description, + ) # We want to support flags that can have arbitrary indices like: # diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index e4901dc73..909bc5111 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1983,8 +1983,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -2058,8 +2058,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -2133,8 +2133,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | From 63b0c69794a1fb4f36be0b66811db4d4ef02c15f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 18 Mar 2025 20:54:14 -0700 Subject: [PATCH 09/51] Add additional options under "repositories:" for parity with repo-create #303. --- borgmatic/actions/repo_create.py | 6 +++--- borgmatic/commands/arguments.py | 3 +++ borgmatic/config/arguments.py | 9 +++++++-- borgmatic/config/generate.py | 12 ++++++++---- borgmatic/config/schema.yaml | 20 ++++++++++++++++++++ 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index af7b793d9..49f4e6c05 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -41,9 +41,9 @@ def run_repo_create( encryption_mode, repo_create_arguments.source_repository, repo_create_arguments.copy_crypt_key, - repo_create_arguments.append_only, - repo_create_arguments.storage_quota, - repo_create_arguments.make_parent_dirs, + repo_create_arguments.append_only or repository.get('append_only'), + repo_create_arguments.storage_quota or repository.get('storage_quota'), + repo_create_arguments.make_parent_dirs or repository.get('make_parent_dirs'), local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 729803277..bf65c24b2 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -303,6 +303,8 @@ DEFAULT_FALSE_FLAG_NAMES = { 'check_i_know_what_i_am_doing', 'postgresql_databases[0].no_owner', 'healthchecks.create_slug', + 'repositories[0].append_only', + 'repositories[0].make_parent_dirs', } @@ -401,6 +403,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names arguments_group.add_argument( full_flag_name, action='store_true', + default=None, help=description, ) else: diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 7a71c9264..9aa839982 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -93,11 +93,16 @@ def convert_value_type(value, option_type): Given a string value and its schema type as a string, determine its logical type (string, boolean, integer, etc.), and return it converted to that type. - If the option type is a string, leave the value as a string so that special characters in it - don't get interpreted as YAML during conversion. + If the destination option type is a string, then leave the value as-is so that special + characters in it don't get interpreted as YAML during conversion. + + And if the source value isn't a string, return it as-is. Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML. ''' + if not isinstance(value, str): + return value + if option_type == 'string': return value diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index fbc29c815..c40d9f9b1 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -22,6 +22,9 @@ def insert_newline_before_comment(config, field_name): ) +SCALAR_SCHEMA_TYPES = {'string', 'boolean', 'integer', 'number'} + + def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False): ''' Given a loaded configuration schema and a source configuration, generate and return sample @@ -33,9 +36,6 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i schema_type = schema.get('type') example = schema.get('example') - if example is not None: - return example - if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): config = ruamel.yaml.comments.CommentedSeq( [ @@ -53,7 +53,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i [ ( field_name, - schema_to_sample_configuration( + sub_schema.get('example') if field_name == 'source_directories' else schema_to_sample_configuration( sub_schema, (source_config or {}).get(field_name, {}), level + 1 ), ) @@ -64,6 +64,10 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i add_comments_to_configuration_object( config, schema, source_config, indent=indent, skip_first=parent_is_sequence ) + elif isinstance(schema_type, list) and all(element_schema_type in SCALAR_SCHEMA_TYPES for element_schema_type in schema_type): + return example + elif schema_type in SCALAR_SCHEMA_TYPES: + return example else: raise ValueError(f'Schema at level {level} is unsupported: {schema}') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 909bc5111..04051ece9 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -53,6 +53,26 @@ properties: available encryption modes, run "borg init --help" with Borg 1 or "borg repo-create --help" with Borg 2. example: repokey-blake2 + append_only: + type: boolean + description: | + Whether the repository should be created append-only, + only used for the repo-create action. Defaults to false. + example: false + storage_quota: + type: string + description: | + The storage quota with which to create the repository, + only used for the repo-create action. Defaults to no + quota. + example: 5G + make_parent_dirs: + type: boolean + description: | + Whether any missing parent directories of the repository + path should be created, only used for the repo-create + action. Defaults to false. + example: true description: | A required list of local or remote repositories with paths and optional labels (which can be used with the --repository flag to From 1097a6576f48e7d06482dadc97182c2eed4ff4bd Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 11:06:36 -0700 Subject: [PATCH 10/51] Add "progress" option to configuration (#303). --- borgmatic/actions/compact.py | 2 +- borgmatic/actions/create.py | 2 +- borgmatic/actions/extract.py | 2 +- borgmatic/borg/check.py | 5 +++-- borgmatic/borg/transfer.py | 2 +- borgmatic/commands/arguments.py | 1 + borgmatic/config/schema.yaml | 6 ++++++ 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index a8ab6a6f0..1a0cea3d9 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -37,7 +37,7 @@ def run_compact( global_arguments, local_path=local_path, remote_path=remote_path, - progress=compact_arguments.progress, + progress=compact_arguments.progress or config.get('progress'), cleanup_commits=compact_arguments.cleanup_commits, threshold=compact_arguments.threshold, ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index ecc688b5d..c12bf43ec 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -327,7 +327,7 @@ def run_create( borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - progress=create_arguments.progress, + progress=create_arguments.progress or config.get('progress'), stats=create_arguments.stats, json=create_arguments.json, list_files=create_arguments.list_files, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 6e2e79003..83f811764 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -45,5 +45,5 @@ def run_extract( remote_path=remote_path, destination_path=extract_arguments.destination, strip_components=extract_arguments.strip_components, - progress=extract_arguments.progress, + progress=extract_arguments.progress or config.get('progress'), ) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 00a090a24..98c489463 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -143,6 +143,7 @@ def check_archives( umask = config.get('umask') borg_exit_codes = config.get('borg_exit_codes') working_directory = borgmatic.config.paths.get_working_directory(config) + progress = check_arguments.progress or config.get('progress') if 'data' in checks: checks.add('archives') @@ -170,7 +171,7 @@ def check_archives( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags - + (('--progress',) if check_arguments.progress else ()) + + (('--progress',) if progress else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) @@ -180,7 +181,7 @@ def check_archives( # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. output_file=( - DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None + DO_NOT_CAPTURE if check_arguments.repair or progress else None ), environment=environment.make_environment(config), working_directory=working_directory, diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 3af998a6a..32dc48b38 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -56,7 +56,7 @@ def transfer_archives( return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None, + output_file=DO_NOT_CAPTURE if (transfer_arguments.progress or config.get('progress')) else None, environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index bf65c24b2..19b933d83 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -305,6 +305,7 @@ DEFAULT_FALSE_FLAG_NAMES = { 'healthchecks.create_slug', 'repositories[0].append_only', 'repositories[0].make_parent_dirs', + 'progress', } diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 04051ece9..fbb2f5a40 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -793,6 +793,12 @@ properties: Apply color to console output. Can be overridden with --no-color command-line flag. Defaults to true. example: false + progress: + type: boolean + description: | + Display progress as each file or archive is processed. Defaults to + false. + example: true skip_actions: type: array items: From d02d31f445c2c06fc589abf7a8560cd15e0c050f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 11:37:17 -0700 Subject: [PATCH 11/51] Use schema defaults instead of a flag name whitelist to make valueless boolean flags (#303). --- borgmatic/commands/arguments.py | 25 +++------------------ borgmatic/config/schema.yaml | 39 ++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 19b933d83..5d980a72a 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,27 +288,6 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -# As a UX nicety, allow boolean options that have a default of false to have command-line flags -# without values. -DEFAULT_FALSE_FLAG_NAMES = { - 'one_file_system', - 'numeric_ids', - 'read_special', - 'exclude_caches', - 'keep_exclude_tags', - 'exclude_nodump', - 'source_directories_must_exist', - 'relocated_repo_access_is_ok', - 'unknown_unencrypted_repo_access_is_ok', - 'check_i_know_what_i_am_doing', - 'postgresql_databases[0].no_owner', - 'healthchecks.create_slug', - 'repositories[0].append_only', - 'repositories[0].make_parent_dirs', - 'progress', -} - - def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of @@ -400,7 +379,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names argument_type = borgmatic.config.schema.parse_type(schema_type) full_flag_name = f"--{flag_name.replace('_', '-')}" - if flag_name in DEFAULT_FALSE_FLAG_NAMES: + # As a UX nicety, allow boolean options that have a default of false to have command-line flags + # without values. + if schema_type == 'boolean' and schema.get('default') == False: arguments_group.add_argument( full_flag_name, action='store_true', diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index fbb2f5a40..481b9e78c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -58,7 +58,8 @@ properties: description: | Whether the repository should be created append-only, only used for the repo-create action. Defaults to false. - example: false + default: false + example: true storage_quota: type: string description: | @@ -72,6 +73,7 @@ properties: Whether any missing parent directories of the repository path should be created, only used for the repo-create action. Defaults to false. + default: false example: true description: | A required list of local or remote repositories with paths and @@ -101,12 +103,14 @@ properties: description: | Stay in same file system; do not cross mount points beyond the given source directories. Defaults to false. + default: false example: true numeric_ids: type: boolean description: | Only store/extract numeric user and group identifiers. Defaults to false. + default: false example: true atime: type: boolean @@ -117,11 +121,13 @@ properties: ctime: type: boolean description: Store ctime into archive. Defaults to true. + default: true example: false birthtime: type: boolean description: | Store birthtime (creation date) into archive. Defaults to true. + default: true example: false read_special: type: boolean @@ -131,13 +137,15 @@ properties: used when backing up special devices such as /dev/zero. Defaults to false. But when a database hook is used, the setting here is ignored and read_special is considered true. - example: false + default: false + example: true flags: type: boolean description: | Record filesystem flags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true. - example: true + default: true + example: false files_cache: type: string description: | @@ -210,6 +218,7 @@ properties: Exclude directories that contain a CACHEDIR.TAG file. See http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false. + default: false example: true exclude_if_present: type: array @@ -226,11 +235,13 @@ properties: If true, the exclude_if_present filename is included in backups. Defaults to false, meaning that the exclude_if_present filename is omitted from backups. + default: false example: true exclude_nodump: type: boolean description: | Exclude files with the NODUMP flag. Defaults to false. + default: false example: true borgmatic_source_directory: type: string @@ -262,6 +273,7 @@ properties: description: | If true, then source directories (and root pattern paths) must exist. If they don't, an error is raised. Defaults to false. + default: false example: true encryption_passcommand: type: string @@ -458,19 +470,22 @@ properties: type: boolean description: | Bypass Borg error about a repository that has been moved. Defaults - to not bypassing. + to false. + default: false example: true unknown_unencrypted_repo_access_is_ok: type: boolean description: | Bypass Borg error about a previously unknown unencrypted repository. - Defaults to not bypassing. + Defaults to false. + default: false example: true check_i_know_what_i_am_doing: type: boolean description: | Bypass Borg confirmation about check with repair option. Defaults to - an interactive prompt from Borg. + false and an interactive prompt from Borg. + default: false example: true extra_borg_options: type: object @@ -792,12 +807,14 @@ properties: description: | Apply color to console output. Can be overridden with --no-color command-line flag. Defaults to true. + default: true example: false progress: type: boolean description: | Display progress as each file or archive is processed. Defaults to false. + default: false example: true skip_actions: type: array @@ -1166,6 +1183,7 @@ properties: backup itself. Defaults to true. Changing this to false prevents "borgmatic bootstrap" from extracting configuration files from the backup. + default: true example: false description: | Support for the "borgmatic bootstrap" action, used to extract @@ -1250,6 +1268,7 @@ properties: schema elements. These statements will fail unless the initial connection to the database is made by a superuser. + default: false example: true format: type: string @@ -1488,6 +1507,7 @@ properties: Use the "--add-drop-database" flag with mariadb-dump, causing the database to be dropped right before restore. Defaults to true. + default: true example: false options: type: string @@ -1635,6 +1655,7 @@ properties: Use the "--add-drop-database" flag with mysqldump, causing the database to be dropped right before restore. Defaults to true. + default: true example: false options: type: string @@ -2335,6 +2356,7 @@ properties: description: | Send borgmatic logs to Apprise services as part the "finish", "fail", and "log" states. Defaults to true. + default: true example: false logs_size_limit: type: integer @@ -2440,12 +2462,14 @@ properties: description: | Verify the TLS certificate of the ping URL host. Defaults to true. + default: true example: false send_logs: type: boolean description: | Send borgmatic logs to Healthchecks as part the "finish", "fail", and "log" states. Defaults to true. + default: true example: false ping_body_limit: type: integer @@ -2478,6 +2502,7 @@ properties: the slug URL scheme (https://hc-ping.com// as opposed to https://hc-ping.com/). Defaults to false. + default: false example: true description: | Configuration for a monitoring integration with Healthchecks. Create @@ -2517,6 +2542,7 @@ properties: description: | Verify the TLS certificate of the push URL host. Defaults to true. + default: true example: false description: | Configuration for a monitoring integration with Uptime Kuma using @@ -2553,6 +2579,7 @@ properties: description: | Send borgmatic logs to PagerDuty when a backup errors. Defaults to true. + default: true example: false description: | Configuration for a monitoring integration with PagerDuty. Create an From 3e21cdb5796bfad45a9bb0c3e85ac1b51d6ca094 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 19:43:04 -0700 Subject: [PATCH 12/51] Add "stats" option to configuration (#303). --- borgmatic/actions/create.py | 2 +- borgmatic/borg/prune.py | 5 +++-- borgmatic/config/schema.yaml | 7 +++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index c12bf43ec..47e915e5d 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -328,7 +328,7 @@ def run_create( local_path=local_path, remote_path=remote_path, progress=create_arguments.progress or config.get('progress'), - stats=create_arguments.stats, + stats=create_arguments.stats or config.get('stats'), json=create_arguments.json, list_files=create_arguments.list_files, stream_processes=stream_processes, diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 82a782015..38d0ed67b 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -66,6 +66,7 @@ def prune_archives( borgmatic.logger.add_custom_log_levels() umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) + stats = prune_arguments.stats or config.get('stats') extra_borg_options = config.get('extra_borg_options', {}).get('prune', '') full_command = ( @@ -77,7 +78,7 @@ def prune_archives( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--stats',) - if prune_arguments.stats + if stats and not dry_run and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version) else () @@ -94,7 +95,7 @@ def prune_archives( + flags.make_repository_flags(repository_path, local_borg_version) ) - if prune_arguments.stats or prune_arguments.list_archives: + if stats or prune_arguments.list_archives: output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 481b9e78c..ab0ae1a3c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -816,6 +816,13 @@ properties: false. default: false example: true + stats: + type: boolean + description: | + Display statistics of an archive when running supported actions. + Defaults to false. + default: false + example: true skip_actions: type: array items: From ed6022d4a93b22921d1cb907e2d74e875bda1f18 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 23:05:38 -0700 Subject: [PATCH 13/51] Add "list" option to configuration, corresponding to "--list" (#303). --- borgmatic/actions/create.py | 2 +- borgmatic/actions/export_tar.py | 2 +- borgmatic/borg/delete.py | 2 +- borgmatic/borg/prune.py | 2 +- borgmatic/config/schema.yaml | 32 ++++++++++++++++++++++---------- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 47e915e5d..c771f3d2e 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -330,7 +330,7 @@ def run_create( progress=create_arguments.progress or config.get('progress'), stats=create_arguments.stats or config.get('stats'), json=create_arguments.json, - list_files=create_arguments.list_files, + list_files=create_arguments.list_files or config.get('list'), stream_processes=stream_processes, ) diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index d5c6bacb6..1b41548b5 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -43,6 +43,6 @@ def run_export_tar( local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, - list_files=export_tar_arguments.list_files, + list_files=export_tar_arguments.list_files or config.get('list'), strip_components=export_tar_arguments.strip_components, ) diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index d967582ce..d5b023356 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -34,7 +34,7 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives) + + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives or config.get('list')) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) if delete_arguments.force diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 38d0ed67b..5eef3a291 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -88,7 +88,7 @@ def prune_archives( prune_arguments, excludes=('repository', 'match_archives', 'stats', 'list_archives'), ) - + (('--list',) if prune_arguments.list_archives else ()) + + (('--list',) if prune_arguments.list_archives or config.get('list') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index ab0ae1a3c..71161990a 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -49,30 +49,33 @@ properties: type: string description: | The encryption mode with which to create the repository, - only used for the repo-create action. To see the - available encryption modes, run "borg init --help" with - Borg 1 or "borg repo-create --help" with Borg 2. + only used for the repo-create action. Also set via the + "--encryption" flag. To see the available encryption + modes, run "borg init --help" with Borg 1 or "borg + repo-create --help" with Borg 2. example: repokey-blake2 append_only: type: boolean description: | Whether the repository should be created append-only, - only used for the repo-create action. Defaults to false. + only used for the repo-create action. Also set via the + "--append-only" flag. Defaults to false. default: false example: true storage_quota: type: string description: | The storage quota with which to create the repository, - only used for the repo-create action. Defaults to no - quota. + only used for the repo-create action. Also set via the + "--storage-quota" flag. Defaults to no quota. example: 5G make_parent_dirs: type: boolean description: | Whether any missing parent directories of the repository path should be created, only used for the repo-create - action. Defaults to false. + action. Also set via the "--make-parent-dirs" flag. + Defaults to false. default: false example: true description: | @@ -812,15 +815,24 @@ properties: progress: type: boolean description: | - Display progress as each file or archive is processed. Defaults to + Display progress as each file or archive is processed when running + supported actions. Also set via the "--progress" flag. Defaults to false. default: false example: true stats: type: boolean description: | - Display statistics of an archive when running supported actions. - Defaults to false. + Display statistics for an archive when running supported actions. + Also set via the "--stats" flag. Defaults to false. + default: false + example: true + list: + type: boolean + description: | + Display details for each file or archive as it is processed when + running supported actions. Also set via the "--list" flag. Defaults + to false. default: false example: true skip_actions: From 3119c924b4c97d36948e5dcaa0f2b48eb7c42fb6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 23:08:26 -0700 Subject: [PATCH 14/51] In configuration option descriptions, remove mention of corresponding CLI flags because it looks dumb on the command-line help (#303). --- borgmatic/config/schema.yaml | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 71161990a..db4f4276a 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -49,33 +49,30 @@ properties: type: string description: | The encryption mode with which to create the repository, - only used for the repo-create action. Also set via the - "--encryption" flag. To see the available encryption - modes, run "borg init --help" with Borg 1 or "borg - repo-create --help" with Borg 2. + only used for the repo-create action. To see the + available encryption modes, run "borg init --help" with + Borg 1 or "borg repo-create --help" with Borg 2. example: repokey-blake2 append_only: type: boolean description: | Whether the repository should be created append-only, - only used for the repo-create action. Also set via the - "--append-only" flag. Defaults to false. + only used for the repo-create action. Defaults to false. default: false example: true storage_quota: type: string description: | The storage quota with which to create the repository, - only used for the repo-create action. Also set via the - "--storage-quota" flag. Defaults to no quota. + only used for the repo-create action. Defaults to no + quota. example: 5G make_parent_dirs: type: boolean description: | Whether any missing parent directories of the repository path should be created, only used for the repo-create - action. Also set via the "--make-parent-dirs" flag. - Defaults to false. + action. Defaults to false. default: false example: true description: | @@ -816,23 +813,21 @@ properties: type: boolean description: | Display progress as each file or archive is processed when running - supported actions. Also set via the "--progress" flag. Defaults to - false. + supported actions. Defaults to false. default: false example: true stats: type: boolean description: | Display statistics for an archive when running supported actions. - Also set via the "--stats" flag. Defaults to false. + Defaults to false. default: false example: true list: type: boolean description: | Display details for each file or archive as it is processed when - running supported actions. Also set via the "--list" flag. Defaults - to false. + running supported actions. Defaults to false. default: false example: true skip_actions: From 7d989f727dbb9a218cb548ecce236133f74a814b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Mar 2025 12:23:00 -0700 Subject: [PATCH 15/51] Don't auto-add CLI flags for configuration options that already have per-action CLI flags (#303). --- borgmatic/commands/arguments.py | 8 ++++++++ borgmatic/config/schema.yaml | 9 ++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 5d980a72a..dc72535f0 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,6 +288,9 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) +OMITTED_FLAG_NAMES = {'progress', 'stats', 'list'} + + def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of @@ -376,6 +379,11 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') + # These options already have corresponding flags on individual actions (like "create + # --progress"), so don't bother adding them to the global flags. + if flag_name in OMITTED_FLAG_NAMES: + return + argument_type = borgmatic.config.schema.parse_type(schema_type) full_flag_name = f"--{flag_name.replace('_', '-')}" diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 006b94368..86cc52559 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -813,21 +813,24 @@ properties: type: boolean description: | Display progress as each file or archive is processed when running - supported actions. Defaults to false. + supported actions. Corresponds to the "--progress" flag on those + actions. Defaults to false. default: false example: true stats: type: boolean description: | Display statistics for an archive when running supported actions. - Defaults to false. + Corresponds to the "--stats" flag on those actions. Defaults to + false. default: false example: true list: type: boolean description: | Display details for each file or archive as it is processed when - running supported actions. Defaults to false. + running supported actions. Corresponds to the "--list" flag on those + actions. Defaults to false. default: false example: true skip_actions: From cbfc0bead132c779ec9039ec04f17ef3d235561c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Mar 2025 09:56:42 -0700 Subject: [PATCH 16/51] Exclude --match-archives from global flags since it already exists on several actions (#303). --- borgmatic/commands/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index dc72535f0..cb4bbcef6 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,7 +288,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -OMITTED_FLAG_NAMES = {'progress', 'stats', 'list'} +OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'stats', 'list'} def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): From 976fb8f343429715cf3ec0a353f64277df3e7ae3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Mar 2025 22:44:49 -0700 Subject: [PATCH 17/51] Add "compact_threshold" option, overridden by "compact --threshold" flag (#303). --- borgmatic/actions/compact.py | 2 +- borgmatic/config/schema.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 1a0cea3d9..7386d2309 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -39,7 +39,7 @@ def run_compact( remote_path=remote_path, progress=compact_arguments.progress or config.get('progress'), cleanup_commits=compact_arguments.cleanup_commits, - threshold=compact_arguments.threshold, + threshold=compact_arguments.threshold or config.get('compact_threshold'), ) else: # pragma: nocover logger.info('Skipping compact (only available/needed in Borg 1.2+)') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 86cc52559..a7c9fc4c6 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -565,6 +565,12 @@ properties: not specified, borgmatic defaults to matching archives based on the archive_name_format (see above). example: sourcehostname + compact_threshold: + type: integer + description: | + Minimum saved space percentage threshold for compacting a segment, + defaults to 10. + example: 20 checks: type: array items: From cc1442146098d1b9bc63633dd324454abd9a22bf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Mar 2025 13:58:42 -0700 Subject: [PATCH 18/51] Fix list examples in generated configuration. --- borgmatic/config/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index c40d9f9b1..bf3b3486d 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -38,6 +38,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): config = ruamel.yaml.comments.CommentedSeq( + example if schema['items'].get('type') in SCALAR_SCHEMA_TYPES else [ schema_to_sample_configuration( schema['items'], source_config, level, parent_is_sequence=True @@ -53,7 +54,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i [ ( field_name, - sub_schema.get('example') if field_name == 'source_directories' else schema_to_sample_configuration( + schema_to_sample_configuration( sub_schema, (source_config or {}).get(field_name, {}), level + 1 ), ) From f8eda92379bca390936b233c19e134a828c46eef Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Mar 2025 14:01:39 -0700 Subject: [PATCH 19/51] Code formatting (#303). --- borgmatic/borg/check.py | 4 +--- borgmatic/borg/delete.py | 4 +++- borgmatic/borg/transfer.py | 4 +++- borgmatic/config/generate.py | 9 ++++++--- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 98c489463..13f4ffca6 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -180,9 +180,7 @@ def check_archives( full_command, # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. - output_file=( - DO_NOT_CAPTURE if check_arguments.repair or progress else None - ), + output_file=(DO_NOT_CAPTURE if check_arguments.repair or progress else None), environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index d5b023356..a9c3478cd 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -34,7 +34,9 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives or config.get('list')) + + borgmatic.borg.flags.make_flags( + 'list', delete_arguments.list_archives or config.get('list') + ) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) if delete_arguments.force diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 32dc48b38..ca90063ea 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -56,7 +56,9 @@ def transfer_archives( return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=DO_NOT_CAPTURE if (transfer_arguments.progress or config.get('progress')) else None, + output_file=( + DO_NOT_CAPTURE if (transfer_arguments.progress or config.get('progress')) else None + ), environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index bf3b3486d..53f20133c 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -38,8 +38,9 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): config = ruamel.yaml.comments.CommentedSeq( - example if schema['items'].get('type') in SCALAR_SCHEMA_TYPES else - [ + example + if schema['items'].get('type') in SCALAR_SCHEMA_TYPES + else [ schema_to_sample_configuration( schema['items'], source_config, level, parent_is_sequence=True ) @@ -65,7 +66,9 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i add_comments_to_configuration_object( config, schema, source_config, indent=indent, skip_first=parent_is_sequence ) - elif isinstance(schema_type, list) and all(element_schema_type in SCALAR_SCHEMA_TYPES for element_schema_type in schema_type): + elif isinstance(schema_type, list) and all( + element_schema_type in SCALAR_SCHEMA_TYPES for element_schema_type in schema_type + ): return example elif schema_type in SCALAR_SCHEMA_TYPES: return example From dc9da3832d1c661f06407e9c8dce0c5cf20030ab Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Mar 2025 14:03:44 -0700 Subject: [PATCH 20/51] Bold "not yet released" in docs to prevent confusion (#303). --- docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 7637fd725..3aecf478e 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -17,8 +17,8 @@ points as it runs. feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) instead.) -New in version 2.0.0 (not yet -released) Command hooks are now configured via a list of `commands:` in +New in version 2.0.0 (**not yet +released**) Command hooks are now configured via a list of `commands:` in your borgmatic configuration file. For example: ```yaml From f222bf2c1a5fb6d8a54d73672503259599d29b32 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Mar 2025 22:52:23 -0700 Subject: [PATCH 21/51] Organizational refactoring (#303). --- borgmatic/commands/arguments.py | 152 ++++++++++++++++++-------------- borgmatic/config/arguments.py | 8 +- borgmatic/config/schema.py | 7 +- 3 files changed, 98 insertions(+), 69 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index f00f49bb9..32249f8f1 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -291,6 +291,91 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'stats', 'list'} +def make_argument_description(schema, flag_name): + ''' + Given a configuration schema dict and a flag name for it, extend the schema's description with + an example or additional information as appropriate based on its type. Return the updated + description for use in a command-line argument. + ''' + description = schema.get('description') + schema_type = schema.get('type') + + if not description: + return None + + if schema_type == 'array': + example_buffer = io.StringIO() + yaml = ruamel.yaml.YAML(typ='safe') + yaml.default_flow_style = True + yaml.dump(schema.get('example'), example_buffer) + + description += f' Example value: "{example_buffer.getvalue().strip()}"' + + if '[0]' in flag_name: + description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + + description = description.replace('%', '%%') + + +def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name): + ''' + Given an argparse._ArgumentGroup instance, a configuration schema dict, a sequence of unparsed + argument strings, and a dotted flag name, convert the schema into corresponding command-line + array element flags that correspond to the given unparsed arguments. + + Here's the background. We want to support flags that can have arbitrary indices like: + + --foo.bar[1].baz + + But argparse doesn't support that natively because the index can be an arbitrary number. We + won't let that stop us though, will we? + + If the current flag name has an array component in it (e.g. a name with "[0]"), then make a + pattern that would match the flag name regardless of the number that's in it. The idea is that + we want to look for unparsed arguments that appear like the flag name, but instead of "[0]" they + have, say, "[1]" or "[123]". + + Next, we check each unparsed argument against that pattern. If one of them matches, add an + argument flag for it to the argument parser group. Example: + + Let's say flag_name is: + + --foo.bar[0].baz + + ... then the regular expression pattern will be: + + ^--foo\.bar\[\d+\]\.baz + + ... and, if that matches an unparsed argument of: + + --foo.bar[1].baz + + ... then an argument flag will get added equal to that unparsed argument. And the unparsed + argument will match it when parsing is performed! In this manner, we're using the actual user + CLI input to inform what exact flags we support! + ''' + if '[0]' not in flag_name or '--help' in unparsed_arguments: + return + + pattern = re.compile(f'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + existing_flags = set( + itertools.chain( + *(group_action.option_strings for group_action in arguments_group._group_actions) + ) + ) + + for unparsed in unparsed_arguments: + unparsed_flag_name = unparsed.split('=', 1)[0] + + if pattern.match(unparsed_flag_name) and unparsed_flag_name not in existing_flags: + arguments_group.add_argument( + unparsed_flag_name, + type=argument_type, + metavar=metavar, + help=description, + ) + + def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of @@ -362,28 +447,14 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names ) flag_name = '.'.join(names) - description = schema.get('description') metavar = names[-1].upper() - if description: - if schema_type == 'array': - example_buffer = io.StringIO() - yaml = ruamel.yaml.YAML(typ='safe') - yaml.default_flow_style = True - yaml.dump(schema.get('example'), example_buffer) - - description += f' Example value: "{example_buffer.getvalue().strip()}"' - - if '[0]' in flag_name: - description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' - - description = description.replace('%', '%%') - - # These options already have corresponding flags on individual actions (like "create + # Certain options already have corresponding flags on individual actions (like "create # --progress"), so don't bother adding them to the global flags. if flag_name in OMITTED_FLAG_NAMES: return + description = make_argument_description(schema, flag_name) argument_type = borgmatic.config.schema.parse_type(schema_type) full_flag_name = f"--{flag_name.replace('_', '-')}" @@ -404,54 +475,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names help=description, ) - # We want to support flags that can have arbitrary indices like: - # - # --foo.bar[1].baz - # - # But argparse doesn't support that natively because the index can be an arbitrary number. We - # won't let that stop us though, will we? So, if the current flag name has an array component in - # it (e.g. a name with "[0]"), then make a pattern that would match the flag name regardless of - # the number that's in it. The idea is that we want to look for unparsed arguments that appear - # like the flag name, but instead of "[0]" they have, say, "[1]" or "[123]". - # - # Next, we check each unparsed argument against that pattern. If one of them matches, add an - # argument flag for it to the argument parser group. Example: - # - # Let's say flag_name is: - # - # --foo.bar[0].baz - # - # ... then the regular expression pattern will be: - # - # ^--foo\.bar\[\d+\]\.baz - # - # ... and, if that matches an unparsed argument of: - # - # --foo.bar[1].baz - # - # ... then an argument flag will get added equal to that unparsed argument. And the unparsed - # argument will match it when parsing is performed! In this manner, we're using the actual user - # CLI input to inform what exact flags we support! - if '[0]' not in flag_name or '--help' in unparsed_arguments: - return - - pattern = re.compile(f'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') - existing_flags = set( - itertools.chain( - *(group_action.option_strings for group_action in arguments_group._group_actions) - ) - ) - - for unparsed in unparsed_arguments: - unparsed_flag_name = unparsed.split('=', 1)[0] - - if pattern.match(unparsed_flag_name) and unparsed_flag_name not in existing_flags: - arguments_group.add_argument( - unparsed_flag_name, - type=argument_type, - metavar=metavar, - help=description, - ) + add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name) def make_parsers(schema, unparsed_arguments): diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 9aa839982..8f7023cdb 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -14,13 +14,13 @@ def set_values(config, keys, value): ('foo', 'bar', 'baz') - This looks up "foo" in the given configuration. And within that value, it looks up "bar". And - then within that value, it looks up "baz" and sets it to the given value. Another example: + This looks up "foo" in the given configuration dict. And within that, it looks up "bar". And + then within that, it looks up "baz" and sets it to the given value. Another example: ('mylist[0]', 'foo') - This looks for the zeroth element of "mylist" in the given configuration. And within that value, - it looks up "foo" and sets it to the given value. + This looks for the zeroth element of "mylist" in the given configuration. And within that, it + looks up "foo" and sets it to the given value. ''' if not keys: return diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 118e94370..4442010fb 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -5,7 +5,7 @@ import itertools def get_properties(schema): ''' Given a schema dict, return its properties. But if it's got sub-schemas with multiple different - potential properties, returned their merged properties instead (interleaved so the first + potential properties, return their merged properties instead (interleaved so the first properties of each sub-schema come first). The idea is that the user should see all possible options even if they're not all possible together. ''' @@ -24,6 +24,11 @@ def get_properties(schema): def parse_type(schema_type): + ''' + Given a schema type as a string, return the corresponding Python type. + + Raise ValueError if the schema type is unknown. + ''' try: return { 'string': str, From 57721937a3729ce8e1b6cbb1edcbbf85eb2b8a7a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 11:24:36 -0700 Subject: [PATCH 22/51] Factor out schema type comparion in config generation and get several tests passing (#303). --- borgmatic/commands/arguments.py | 1 + borgmatic/config/generate.py | 12 ++++-------- borgmatic/config/schema.py | 22 ++++++++++++++++++++++ tests/unit/config/test_generate.py | 28 ++++++++++++++++++++-------- 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 32249f8f1..2d285f509 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -297,6 +297,7 @@ def make_argument_description(schema, flag_name): an example or additional information as appropriate based on its type. Return the updated description for use in a command-line argument. ''' + # FIXME: Argument descriptions are apparently broken right now. description = schema.get('description') schema_type = schema.get('type') diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 53f20133c..58360ad0f 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -36,10 +36,10 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i schema_type = schema.get('type') example = schema.get('example') - if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): + if borgmatic.config.schema.compare_types(schema_type, {'array'}): config = ruamel.yaml.comments.CommentedSeq( example - if schema['items'].get('type') in SCALAR_SCHEMA_TYPES + if borgmatic.config.schema.compare_types(schema['items'].get('type'), SCALAR_SCHEMA_TYPES) else [ schema_to_sample_configuration( schema['items'], source_config, level, parent_is_sequence=True @@ -47,7 +47,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i ] ) add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT)) - elif schema_type == 'object' or (isinstance(schema_type, list) and 'object' in schema_type): + elif borgmatic.config.schema.compare_types(schema_type, {'object'}): if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict): source_config = dict(collections.ChainMap(*source_config)) @@ -66,11 +66,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i add_comments_to_configuration_object( config, schema, source_config, indent=indent, skip_first=parent_is_sequence ) - elif isinstance(schema_type, list) and all( - element_schema_type in SCALAR_SCHEMA_TYPES for element_schema_type in schema_type - ): - return example - elif schema_type in SCALAR_SCHEMA_TYPES: + elif borgmatic.config.schema.compare_types(schema_type, SCALAR_SCHEMA_TYPES, match=all): return example else: raise ValueError(f'Schema at level {level} is unsupported: {schema}') diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 4442010fb..d4698c3ad 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -39,3 +39,25 @@ def parse_type(schema_type): }[schema_type] except KeyError: raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + +def compare_types(schema_type, target_types, match=any): + ''' + Given a schema type as a string or a list of strings (representing multiple types) and a set of + target type strings, return whether every schema type is in the set of target types. + + If the schema type is a list of strings, use the given match function (such as any or all) to + compare elements. + ''' + if isinstance(schema_type, list): + if match( + element_schema_type in target_types for element_schema_type in schema_type + ): + return True + + return False + + if schema_type in target_types: + return True + + return False diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index abb520fb5..9c07001db 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -2,6 +2,7 @@ import pytest from flexmock import flexmock from borgmatic.config import generate as module +import borgmatic.config.schema def test_schema_to_sample_configuration_generates_config_map_with_examples(): @@ -9,13 +10,16 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), } - flexmock(module).should_receive('get_properties').and_return(schema['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('object', {'object'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['properties']) flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) flexmock(module).should_receive('add_comments_to_configuration_object') @@ -46,11 +50,15 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e 'items': { 'type': 'object', 'properties': dict( - [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + [('field1', {'type': 'string', 'example': 'Example 1'}), ('field2', {'type': 'string', 'example': 'Example 2'})] ), }, } - flexmock(module).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('array', {'array'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('object', {'object'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['items']['properties']) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') @@ -66,11 +74,15 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_m 'items': { 'type': ['object', 'null'], 'properties': dict( - [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + [('field1', {'type': 'string', 'example': 'Example 1'}), ('field2', {'type': 'string', 'example': 'Example 2'})] ), }, } - flexmock(module).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('array', {'array'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args(['object', 'null'], {'object'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['items']['properties']) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') From ba75958a2f043f491ffc57673951bf71bc8b197a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 11:26:49 -0700 Subject: [PATCH 23/51] Fix missing argument descriptions (#303). --- borgmatic/commands/arguments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 2d285f509..289d163f9 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -297,7 +297,6 @@ def make_argument_description(schema, flag_name): an example or additional information as appropriate based on its type. Return the updated description for use in a command-line argument. ''' - # FIXME: Argument descriptions are apparently broken right now. description = schema.get('description') schema_type = schema.get('type') @@ -317,6 +316,8 @@ def make_argument_description(schema, flag_name): description = description.replace('%', '%%') + return description + def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name): ''' From 9f7c71265e82ec1d470f9a6e8748868e422095f5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 16:32:31 -0700 Subject: [PATCH 24/51] Add Bash completion for completing flags like "--foo[3].bar". --- borgmatic/commands/completion/bash.py | 12 ++++++++++-- borgmatic/commands/completion/fish.py | 23 +++++++++++++++-------- borgmatic/commands/completion/flag.py | 16 ++++++++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) create mode 100644 borgmatic/commands/completion/flag.py diff --git a/borgmatic/commands/completion/bash.py b/borgmatic/commands/completion/bash.py index 7bf28a429..6eb5e92d7 100644 --- a/borgmatic/commands/completion/bash.py +++ b/borgmatic/commands/completion/bash.py @@ -1,5 +1,6 @@ import borgmatic.commands.arguments import borgmatic.commands.completion.actions +import borgmatic.commands.completion.flag def parser_flags(parser): @@ -7,7 +8,11 @@ def parser_flags(parser): Given an argparse.ArgumentParser instance, return its argument flags in a space-separated string. ''' - return ' '.join(option for action in parser._actions for option in action.option_strings) + return ' '.join( + flag_variant + for action in parser._actions for flag_name in action.option_strings + for flag_variant in borgmatic.commands.completion.flag.variants(flag_name) + ) def bash_completion(): @@ -19,7 +24,10 @@ def bash_completion(): unused_global_parser, action_parsers, global_plus_action_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) global_flags = parser_flags(global_plus_action_parser) # Avert your eyes. diff --git a/borgmatic/commands/completion/fish.py b/borgmatic/commands/completion/fish.py index edca0226b..31b83e9cb 100644 --- a/borgmatic/commands/completion/fish.py +++ b/borgmatic/commands/completion/fish.py @@ -4,6 +4,7 @@ from textwrap import dedent import borgmatic.commands.arguments import borgmatic.commands.completion.actions +import borgmatic.config.validate def has_file_options(action: Action): @@ -26,9 +27,11 @@ def has_choice_options(action: Action): def has_unknown_required_param_options(action: Action): ''' A catch-all for options that take a required parameter, but we don't know what the parameter is. - This should be used last. These are actions that take something like a glob, a list of numbers, or a string. + This should be used last. These are actions that take something like a glob, a list of numbers, + or a string. - Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid. + Actions that match this pattern should not show the normal arguments, because those are unlikely + to be valid. ''' return ( action.required is True @@ -52,9 +55,9 @@ def has_exact_options(action: Action): def exact_options_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation that forces file completions, options completion, - or just that some value follow the action, if the action takes such an argument and was the last action on the - command line prior to the cursor. + Given an argparse.Action instance, return a completion invocation that forces file completions, + options completion, or just that some value follow the action, if the action takes such an + argument and was the last action on the command line prior to the cursor. Otherwise, return an empty string. ''' @@ -80,8 +83,9 @@ def exact_options_completion(action: Action): def dedent_strip_as_tuple(string: str): ''' - Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string. - Makes it easier to write multiline strings for completions when you join them with a tuple. + Dedent a string, then strip it to avoid requiring your first line to have content, then return a + tuple of the string. Makes it easier to write multiline strings for completions when you join + them with a tuple. ''' return (dedent(string).strip('\n'),) @@ -95,7 +99,10 @@ def fish_completion(): unused_global_parser, action_parsers, global_plus_action_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) all_action_parsers = ' '.join(action for action in action_parsers.choices.keys()) diff --git a/borgmatic/commands/completion/flag.py b/borgmatic/commands/completion/flag.py new file mode 100644 index 000000000..67cf0712c --- /dev/null +++ b/borgmatic/commands/completion/flag.py @@ -0,0 +1,16 @@ +def variants(flag_name): + ''' + Given an flag name as a string, yield it and any variations that should be complete-able as + well. For instance, for a string like "--foo[0].bar", yield "--foo[0].bar", "--foo[1].bar", ..., + "--foo[9].bar". + ''' + if '[0]' in flag_name: + for index in range(0, 10): + yield flag_name.replace('[0]', f'[{index}]') + + return + + yield flag_name + + + From 423627e67bf81873d65fdc9c1388ed1e54dcb1c6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 17:00:04 -0700 Subject: [PATCH 25/51] Get existing unit/integration tests passing (#303). --- borgmatic/commands/arguments.py | 52 ++--- borgmatic/commands/completion/bash.py | 4 +- borgmatic/commands/completion/flag.py | 3 - borgmatic/config/generate.py | 4 +- borgmatic/config/schema.py | 4 +- borgmatic/config/validate.py | 12 ++ tests/integration/borg/test_commands.py | 12 +- .../commands/completion/test_actions.py | 11 +- tests/integration/commands/test_arguments.py | 180 ++++++++++-------- tests/integration/config/test_generate.py | 24 +-- tests/integration/config/test_validate.py | 43 +++-- tests/unit/commands/test_borgmatic.py | 9 +- tests/unit/config/test_generate.py | 54 ++++-- 13 files changed, 252 insertions(+), 160 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 289d163f9..97dc33618 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -69,9 +69,9 @@ def get_subactions_for_actions(action_parsers): def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments): ''' - Given a sequence of string arguments and a dict from action name to parsed argparse.Namespace - arguments, return the string arguments with any values omitted that happen to be the same as - the name of a borgmatic action. + Given unparsed arguments as a sequence of strings and a dict from action name to parsed + argparse.Namespace arguments, return the string arguments with any values omitted that happen to + be the same as the name of a borgmatic action. This prevents, for instance, "check --only extract" from triggering the "extract" action. ''' @@ -320,15 +320,15 @@ def make_argument_description(schema, flag_name): def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name): - ''' + r''' Given an argparse._ArgumentGroup instance, a configuration schema dict, a sequence of unparsed argument strings, and a dotted flag name, convert the schema into corresponding command-line array element flags that correspond to the given unparsed arguments. Here's the background. We want to support flags that can have arbitrary indices like: - + --foo.bar[1].baz - + But argparse doesn't support that natively because the index can be an arbitrary number. We won't let that stop us though, will we? @@ -336,22 +336,22 @@ def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_ar pattern that would match the flag name regardless of the number that's in it. The idea is that we want to look for unparsed arguments that appear like the flag name, but instead of "[0]" they have, say, "[1]" or "[123]". - + Next, we check each unparsed argument against that pattern. If one of them matches, add an argument flag for it to the argument parser group. Example: - + Let's say flag_name is: - + --foo.bar[0].baz - + ... then the regular expression pattern will be: - + ^--foo\.bar\[\d+\]\.baz - + ... and, if that matches an unparsed argument of: - + --foo.bar[1].baz - + ... then an argument flag will get added equal to that unparsed argument. And the unparsed argument will match it when parsing is performed! In this manner, we're using the actual user CLI input to inform what exact flags we support! @@ -359,7 +359,7 @@ def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_ar if '[0]' not in flag_name or '--help' in unparsed_arguments: return - pattern = re.compile(f'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') existing_flags = set( itertools.chain( *(group_action.option_strings for group_action in arguments_group._group_actions) @@ -409,10 +409,15 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names And if names are also passed in, they are considered to be the name components of an option (e.g. "foo" and "bar") and are used to construct a resulting flag. + + Bail if the schema is not a dict. ''' if names is None: names = () + if not isinstance(schema, dict): + return + schema_type = schema.get('type') # If this option has multiple types, just use the first one (that isn't "null"). @@ -449,13 +454,13 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names ) flag_name = '.'.join(names) - metavar = names[-1].upper() # Certain options already have corresponding flags on individual actions (like "create # --progress"), so don't bother adding them to the global flags. - if flag_name in OMITTED_FLAG_NAMES: + if not flag_name or flag_name in OMITTED_FLAG_NAMES: return + metavar = names[-1].upper() description = make_argument_description(schema, flag_name) argument_type = borgmatic.config.schema.parse_type(schema_type) full_flag_name = f"--{flag_name.replace('_', '-')}" @@ -482,10 +487,11 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names def make_parsers(schema, unparsed_arguments): ''' - Given a configuration schema dict, build a global arguments parser, individual action parsers, - and a combined parser containing both. Return them as a tuple. The global parser is useful for - parsing just global arguments while ignoring actions, and the combined parser is handy for - displaying help that includes everything: global flags, a list of actions, etc. + Given a configuration schema dict and unparsed arguments as a sequence of strings, build a + global arguments parser, individual action parsers, and a combined parser containing both. + Return them as a tuple. The global parser is useful for parsing just global arguments while + ignoring actions, and the combined parser is handy for displaying help that includes everything: + global flags, a list of actions, etc. ''' config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) @@ -1752,8 +1758,8 @@ def make_parsers(schema, unparsed_arguments): def parse_arguments(schema, *unparsed_arguments): ''' Given a configuration schema dict and the command-line arguments with which this script was - invoked, parse the arguments and return them as a dict mapping from action name (or "global") to - an argparse.Namespace instance. + invoked and unparsed arguments as a sequence of strings, parse the arguments and return them as + a dict mapping from action name (or "global") to an argparse.Namespace instance. Raise ValueError if the arguments cannot be parsed. Raise SystemExit with an error code of 0 if "--help" was requested. diff --git a/borgmatic/commands/completion/bash.py b/borgmatic/commands/completion/bash.py index 6eb5e92d7..c72fbbecc 100644 --- a/borgmatic/commands/completion/bash.py +++ b/borgmatic/commands/completion/bash.py @@ -1,6 +1,7 @@ import borgmatic.commands.arguments import borgmatic.commands.completion.actions import borgmatic.commands.completion.flag +import borgmatic.config.validate def parser_flags(parser): @@ -10,7 +11,8 @@ def parser_flags(parser): ''' return ' '.join( flag_variant - for action in parser._actions for flag_name in action.option_strings + for action in parser._actions + for flag_name in action.option_strings for flag_variant in borgmatic.commands.completion.flag.variants(flag_name) ) diff --git a/borgmatic/commands/completion/flag.py b/borgmatic/commands/completion/flag.py index 67cf0712c..6de6517fc 100644 --- a/borgmatic/commands/completion/flag.py +++ b/borgmatic/commands/completion/flag.py @@ -11,6 +11,3 @@ def variants(flag_name): return yield flag_name - - - diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 58360ad0f..5ddc26c04 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -39,7 +39,9 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i if borgmatic.config.schema.compare_types(schema_type, {'array'}): config = ruamel.yaml.comments.CommentedSeq( example - if borgmatic.config.schema.compare_types(schema['items'].get('type'), SCALAR_SCHEMA_TYPES) + if borgmatic.config.schema.compare_types( + schema['items'].get('type'), SCALAR_SCHEMA_TYPES + ) else [ schema_to_sample_configuration( schema['items'], source_config, level, parent_is_sequence=True diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index d4698c3ad..0ce774d9f 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -50,9 +50,7 @@ def compare_types(schema_type, target_types, match=any): compare elements. ''' if isinstance(schema_type, list): - if match( - element_schema_type in target_types for element_schema_type in schema_type - ): + if match(element_schema_type in target_types for element_schema_type in schema_type): return True return False diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 5899de7d7..29215f911 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -21,6 +21,18 @@ def schema_filename(): return schema_path +def load_schema(schema_path): + ''' + Given a schema filename path, load the schema and return it as a dict. + + Raise Validation_error if the schema could not be parsed. + ''' + try: + return load.load_configuration(schema_path) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: + raise Validation_error(schema_path, (str(error),)) + + def format_json_error_path_element(path_element): ''' Given a path element into a JSON data structure, format it for display as a string. diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 4e9261e8d..80de7dcd7 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -53,7 +53,7 @@ def fuzz_argument(arguments, argument_name): def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments( - 'transfer', '--source-repository', 'foo' + {}, 'transfer', '--source-repository', 'foo' )['transfer'] flexmock(borgmatic.borg.transfer).should_receive('execute_command').replace_with( assert_command_does_not_duplicate_flags @@ -74,7 +74,7 @@ def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): def test_prune_archives_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('prune')['prune'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'prune')['prune'] flexmock(borgmatic.borg.prune).should_receive('execute_command').replace_with( assert_command_does_not_duplicate_flags ) @@ -94,7 +94,7 @@ def test_prune_archives_command_does_not_duplicate_flags_or_raise(): def test_mount_archive_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('mount', '--mount-point', 'tmp')[ + arguments = borgmatic.commands.arguments.parse_arguments({}, 'mount', '--mount-point', 'tmp')[ 'mount' ] flexmock(borgmatic.borg.mount).should_receive('execute_command').replace_with( @@ -116,7 +116,7 @@ def test_mount_archive_command_does_not_duplicate_flags_or_raise(): def test_make_list_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('list')['list'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'list')['list'] for argument_name in dir(arguments): if argument_name.startswith('_'): @@ -134,7 +134,7 @@ def test_make_list_command_does_not_duplicate_flags_or_raise(): def test_make_repo_list_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('repo-list')['repo-list'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'repo-list')['repo-list'] for argument_name in dir(arguments): if argument_name.startswith('_'): @@ -152,7 +152,7 @@ def test_make_repo_list_command_does_not_duplicate_flags_or_raise(): def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('info')['info'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'info')['info'] flexmock(borgmatic.borg.info).should_receive('execute_command_and_capture_output').replace_with( assert_command_does_not_duplicate_flags ) diff --git a/tests/integration/commands/completion/test_actions.py b/tests/integration/commands/completion/test_actions.py index 2e6fde9b4..52dfd268f 100644 --- a/tests/integration/commands/completion/test_actions.py +++ b/tests/integration/commands/completion/test_actions.py @@ -1,4 +1,5 @@ import borgmatic.commands.arguments +import borgmatic.config.validate from borgmatic.commands.completion import actions as module @@ -7,7 +8,10 @@ def test_available_actions_uses_only_subactions_for_action_with_subactions(): unused_global_parser, action_parsers, unused_combined_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) actions = module.available_actions(action_parsers, 'config') @@ -20,7 +24,10 @@ def test_available_actions_omits_subactions_for_action_without_subactions(): unused_global_parser, action_parsers, unused_combined_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) actions = module.available_actions(action_parsers, 'list') diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 4399c8678..3469858c3 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -8,7 +8,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments() + arguments = module.parse_arguments({}) global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -21,7 +21,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): def test_parse_arguments_with_multiple_config_flags_parses_as_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', '--config', 'otherconfig') + arguments = module.parse_arguments({}, '--config', 'myconfig', '--config', 'otherconfig') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig', 'otherconfig'] @@ -34,7 +34,7 @@ def test_parse_arguments_with_multiple_config_flags_parses_as_list(): def test_parse_arguments_with_action_after_config_path_omits_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'list', '--json') + arguments = module.parse_arguments({}, '--config', 'myconfig', 'list', '--json') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -45,7 +45,9 @@ def test_parse_arguments_with_action_after_config_path_omits_action(): def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey') + arguments = module.parse_arguments( + {}, '--config', 'myconfig', 'init', '--encryption', 'repokey' + ) global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -56,7 +58,7 @@ def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): def test_parse_arguments_with_action_and_positional_arguments_after_config_path_omits_action_and_arguments(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'borg', 'key', 'export') + arguments = module.parse_arguments({}, '--config', 'myconfig', 'borg', 'key', 'export') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -68,7 +70,7 @@ def test_parse_arguments_with_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--verbosity', '1') + arguments = module.parse_arguments({}, '--verbosity', '1') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -82,7 +84,7 @@ def test_parse_arguments_with_syslog_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--syslog-verbosity', '2') + arguments = module.parse_arguments({}, '--syslog-verbosity', '2') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -96,7 +98,7 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--log-file-verbosity', '-1') + arguments = module.parse_arguments({}, '--log-file-verbosity', '-1') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -109,7 +111,7 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default(): def test_parse_arguments_with_single_override_parses(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--override', 'foo.bar=baz') + arguments = module.parse_arguments({}, '--override', 'foo.bar=baz') global_arguments = arguments['global'] assert global_arguments.overrides == ['foo.bar=baz'] @@ -119,7 +121,7 @@ def test_parse_arguments_with_multiple_overrides_flags_parses(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) arguments = module.parse_arguments( - '--override', 'foo.bar=baz', '--override', 'foo.quux=7', '--override', 'this.that=8' + {}, '--override', 'foo.bar=baz', '--override', 'foo.quux=7', '--override', 'this.that=8' ) global_arguments = arguments['global'] @@ -127,7 +129,7 @@ def test_parse_arguments_with_multiple_overrides_flags_parses(): def test_parse_arguments_with_list_json_overrides_default(): - arguments = module.parse_arguments('list', '--json') + arguments = module.parse_arguments({}, 'list', '--json') assert 'list' in arguments assert arguments['list'].json is True @@ -136,7 +138,7 @@ def test_parse_arguments_with_list_json_overrides_default(): def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments() + arguments = module.parse_arguments({}) assert 'prune' in arguments assert 'create' in arguments @@ -146,7 +148,7 @@ def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): def test_parse_arguments_with_no_actions_passes_argument_to_relevant_actions(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--stats', '--list') + arguments = module.parse_arguments({}, '--stats', '--list') assert 'prune' in arguments assert arguments['prune'].stats @@ -161,7 +163,7 @@ def test_parse_arguments_with_help_and_no_actions_shows_global_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('--help') + module.parse_arguments({}, '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -173,7 +175,7 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('create', '--help') + module.parse_arguments({}, 'create', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -185,7 +187,7 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys): def test_parse_arguments_with_action_before_global_options_parses_options(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('prune', '--verbosity', '2') + arguments = module.parse_arguments({}, 'prune', '--verbosity', '2') assert 'prune' in arguments assert arguments['global'].verbosity == 2 @@ -194,7 +196,7 @@ def test_parse_arguments_with_action_before_global_options_parses_options(): def test_parse_arguments_with_global_options_before_action_parses_options(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--verbosity', '2', 'prune') + arguments = module.parse_arguments({}, '--verbosity', '2', 'prune') assert 'prune' in arguments assert arguments['global'].verbosity == 2 @@ -203,7 +205,7 @@ def test_parse_arguments_with_global_options_before_action_parses_options(): def test_parse_arguments_with_prune_action_leaves_other_actions_disabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('prune') + arguments = module.parse_arguments({}, 'prune') assert 'prune' in arguments assert 'create' not in arguments @@ -213,7 +215,7 @@ def test_parse_arguments_with_prune_action_leaves_other_actions_disabled(): def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('create', 'check') + arguments = module.parse_arguments({}, 'create', 'check') assert 'prune' not in arguments assert 'create' in arguments @@ -224,60 +226,53 @@ def test_parse_arguments_disallows_invalid_argument(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--posix-me-harder') + module.parse_arguments({}, '--posix-me-harder') def test_parse_arguments_disallows_encryption_mode_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey') + module.parse_arguments({}, '--config', 'myconfig', '--encryption', 'repokey') def test_parse_arguments_allows_encryption_mode_with_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey') - - -def test_parse_arguments_requires_encryption_mode_with_init(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'init') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey') def test_parse_arguments_disallows_append_only_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--append-only') + module.parse_arguments({}, '--config', 'myconfig', '--append-only') def test_parse_arguments_disallows_storage_quota_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G') + module.parse_arguments({}, '--config', 'myconfig', '--storage-quota', '5G') def test_parse_arguments_allows_init_and_prune(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune') def test_parse_arguments_allows_init_and_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') def test_parse_arguments_allows_repository_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test' + {}, '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test' ) @@ -285,6 +280,7 @@ def test_parse_arguments_allows_repository_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( + {}, '--config', 'myconfig', 'mount', @@ -300,187 +296,187 @@ def test_parse_arguments_allows_repository_with_mount(): def test_parse_arguments_allows_repository_with_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'list', '--repository', 'test.borg') + module.parse_arguments({}, '--config', 'myconfig', 'list', '--repository', 'test.borg') def test_parse_arguments_disallows_archive_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', '--archive', 'test') def test_parse_arguments_disallows_paths_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--path', 'test') + module.parse_arguments({}, '--config', 'myconfig', '--path', 'test') def test_parse_arguments_disallows_other_actions_with_config_bootstrap(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('config', 'bootstrap', '--repository', 'test.borg', 'list') + module.parse_arguments({}, 'config', 'bootstrap', '--repository', 'test.borg', 'list') def test_parse_arguments_allows_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'extract', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'extract', '--archive', 'test') def test_parse_arguments_allows_archive_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - '--config', 'myconfig', 'mount', '--archive', 'test', '--mount-point', '/mnt' + {}, '--config', 'myconfig', 'mount', '--archive', 'test', '--mount-point', '/mnt' ) def test_parse_arguments_allows_archive_with_restore(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'restore', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'restore', '--archive', 'test') def test_parse_arguments_allows_archive_with_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'list', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'list', '--archive', 'test') def test_parse_arguments_requires_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'extract') + module.parse_arguments({}, '--config', 'myconfig', 'extract') def test_parse_arguments_requires_archive_with_restore(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'restore') + module.parse_arguments({}, '--config', 'myconfig', 'restore') def test_parse_arguments_requires_mount_point_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'mount', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'mount', '--archive', 'test') def test_parse_arguments_requires_mount_point_with_umount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'umount') + module.parse_arguments({}, '--config', 'myconfig', 'umount') def test_parse_arguments_allows_progress_before_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--progress', 'create', 'list') + module.parse_arguments({}, '--progress', 'create', 'list') def test_parse_arguments_allows_progress_after_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('create', '--progress', 'list') + module.parse_arguments({}, 'create', '--progress', 'list') def test_parse_arguments_allows_progress_and_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--progress', 'extract', '--archive', 'test', 'list') + module.parse_arguments({}, '--progress', 'extract', '--archive', 'test', 'list') def test_parse_arguments_disallows_progress_without_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--progress', 'list') + module.parse_arguments({}, '--progress', 'list') def test_parse_arguments_with_stats_and_create_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--stats', 'create', 'list') + module.parse_arguments({}, '--stats', 'create', 'list') def test_parse_arguments_with_stats_and_prune_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--stats', 'prune', 'list') + module.parse_arguments({}, '--stats', 'prune', 'list') def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--stats', 'list') + module.parse_arguments({}, '--stats', 'list') def test_parse_arguments_with_list_and_create_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--list', 'create') + module.parse_arguments({}, '--list', 'create') def test_parse_arguments_with_list_and_prune_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--list', 'prune') + module.parse_arguments({}, '--list', 'prune') def test_parse_arguments_with_list_flag_but_no_relevant_action_raises_value_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): - module.parse_arguments('--list', 'repo-create') + with pytest.raises(ValueError): + module.parse_arguments({}, '--list', 'repo-create') def test_parse_arguments_disallows_list_with_progress_for_create_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('create', '--list', '--progress') + module.parse_arguments({}, 'create', '--list', '--progress') def test_parse_arguments_disallows_list_with_json_for_create_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('create', '--list', '--json') + module.parse_arguments({}, 'create', '--list', '--json') def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('list', '--json') - module.parse_arguments('info', '--json') + module.parse_arguments({}, 'list', '--json') + module.parse_arguments({}, 'info', '--json') def test_parse_arguments_disallows_json_with_both_list_and_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', 'info', '--json') + module.parse_arguments({}, 'list', 'info', '--json') def test_parse_arguments_disallows_json_with_both_list_and_repo_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', 'repo-info', '--json') + module.parse_arguments({}, 'list', 'repo-info', '--json') def test_parse_arguments_disallows_json_with_both_repo_info_and_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('repo-info', 'info', '--json') + module.parse_arguments({}, 'repo-info', 'info', '--json') def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives(): @@ -488,6 +484,7 @@ def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives with pytest.raises(ValueError): module.parse_arguments( + {}, 'transfer', '--source-repository', 'source.borg', @@ -502,74 +499,74 @@ def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'list', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_repo_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('repo-list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'repo-list', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_info_with_both_archive_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--archive', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'info', '--archive', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_info_with_both_archive_and_prefix(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--archive', 'foo', '--prefix', 'bar') + module.parse_arguments({}, 'info', '--archive', 'foo', '--prefix', 'bar') def test_parse_arguments_disallows_info_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'info', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_check_only_extract_does_not_raise_extract_subparser_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('check', '--only', 'extract') + module.parse_arguments({}, 'check', '--only', 'extract') def test_parse_arguments_extract_archive_check_does_not_raise_check_subparser_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('extract', '--archive', 'check') + module.parse_arguments({}, 'extract', '--archive', 'check') def test_parse_arguments_extract_with_check_only_extract_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract') + module.parse_arguments({}, 'extract', '--archive', 'name', 'check', '--only', 'extract') def test_parse_arguments_bootstrap_without_config_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('bootstrap') + module.parse_arguments({}, 'bootstrap') def test_parse_arguments_config_with_no_subaction_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('config') + module.parse_arguments({}, 'config') def test_parse_arguments_config_with_help_shows_config_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', '--help') + module.parse_arguments({}, 'config', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -582,7 +579,7 @@ def test_parse_arguments_config_with_subaction_but_missing_flags_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', 'bootstrap') + module.parse_arguments({}, 'config', 'bootstrap') assert exit.value.code == 2 @@ -591,7 +588,7 @@ def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(cap flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', 'bootstrap', '--help') + module.parse_arguments({}, 'config', 'bootstrap', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -601,26 +598,30 @@ def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(cap def test_parse_arguments_config_with_subaction_and_required_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg') + module.parse_arguments({}, 'config', 'bootstrap', '--repository', 'repo.borg') def test_parse_arguments_config_with_subaction_and_global_flags_at_start_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg') + module.parse_arguments( + {}, '--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg' + ) def test_parse_arguments_config_with_subaction_and_global_flags_at_end_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1') + module.parse_arguments( + {}, 'config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1' + ) def test_parse_arguments_config_with_subaction_and_explicit_config_file_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - 'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml' + {}, 'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml' ) @@ -628,10 +629,23 @@ def test_parse_arguments_with_borg_action_and_dry_run_raises(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--dry-run', 'borg', 'list') + module.parse_arguments({}, '--dry-run', 'borg', 'list') def test_parse_arguments_with_borg_action_and_no_dry_run_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('borg', 'list') + module.parse_arguments({}, 'borg', 'list') + + +def test_parse_arguments_with_argument_from_schema_does_not_raise(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + module.parse_arguments( + { + 'type': 'object', + 'properties': {'foo': {'type': 'object', 'properties': {'bar': {'type': 'integer'}}}}, + }, + '--foo.bar', + '3', + ) diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 426224038..71c0abc56 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -21,9 +21,9 @@ def test_schema_to_sample_configuration_comments_out_non_default_options(): 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('source_directories', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('source_directories', {'type': 'string', 'example': 'Example 3'}), ] ), } @@ -47,9 +47,9 @@ def test_schema_to_sample_configuration_comments_out_non_source_config_options() 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), } @@ -76,9 +76,9 @@ def test_schema_to_sample_configuration_comments_out_non_default_options_in_sequ 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('source_directories', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('source_directories', {'type': 'string', 'example': 'Example 3'}), ] ), }, @@ -105,9 +105,9 @@ def test_schema_to_sample_configuration_comments_out_non_source_config_options_i 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), }, diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 9cd5c9802..61d12d7fb 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -58,7 +58,9 @@ def test_parse_configuration_transforms_file_into_mapping(): ''' ) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': ['/home', '/etc'], @@ -86,7 +88,9 @@ def test_parse_configuration_passes_through_quoted_punctuation(): ''' ) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': [f'/home/{string.punctuation}'], @@ -119,7 +123,7 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): ''', ) - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock()) def test_parse_configuration_inlines_include_inside_deprecated_section(): @@ -145,7 +149,9 @@ def test_parse_configuration_inlines_include_inside_deprecated_section(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': ['/home'], @@ -181,7 +187,9 @@ def test_parse_configuration_merges_include(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': ['/home'], @@ -196,7 +204,9 @@ def test_parse_configuration_merges_include(): def test_parse_configuration_raises_for_missing_config_file(): with pytest.raises(FileNotFoundError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) def test_parse_configuration_raises_for_missing_schema_file(): @@ -208,14 +218,18 @@ def test_parse_configuration_raises_for_missing_schema_file(): builtins.should_receive('open').with_args('/tmp/schema.yaml').and_raise(FileNotFoundError) with pytest.raises(FileNotFoundError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) def test_parse_configuration_raises_for_syntax_error(): mock_config_and_schema('foo:\nbar') with pytest.raises(ValueError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) def test_parse_configuration_raises_for_validation_error(): @@ -228,7 +242,9 @@ def test_parse_configuration_raises_for_validation_error(): ) with pytest.raises(module.Validation_error): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) def test_parse_configuration_applies_overrides(): @@ -245,7 +261,10 @@ def test_parse_configuration_applies_overrides(): ) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', overrides=['local_path=borg2'] + '/tmp/config.yaml', + '/tmp/schema.yaml', + global_arguments=flexmock(), + overrides=['local_path=borg2'], ) assert config == { @@ -273,7 +292,9 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in ) flexmock(os).should_receive('getenv').replace_with(lambda variable_name, default: default) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': ['/home'], diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 160f44543..9faa66321 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1537,6 +1537,7 @@ def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env configs, config_paths, logs = tuple( module.load_configurations( ('test.yaml', 'other.yaml'), + global_arguments=flexmock(), resolve_env=resolve_env, ) ) @@ -1549,7 +1550,9 @@ def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env def test_load_configurations_logs_warning_for_permission_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(PermissionError) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple( + module.load_configurations(('test.yaml',), global_arguments=flexmock()) + ) assert configs == {} assert config_paths == [] @@ -1559,7 +1562,9 @@ def test_load_configurations_logs_warning_for_permission_error(): def test_load_configurations_logs_critical_for_parse_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple( + module.load_configurations(('test.yaml',), global_arguments=flexmock()) + ) assert configs == {} assert config_paths == [] diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 9c07001db..885bdcca0 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -17,9 +17,15 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): ), } flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('object', {'object'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) flexmock(module).should_receive('add_comments_to_configuration_object') @@ -50,15 +56,26 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e 'items': { 'type': 'object', 'properties': dict( - [('field1', {'type': 'string', 'example': 'Example 1'}), ('field2', {'type': 'string', 'example': 'Example 2'})] + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ] ), }, } flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('array', {'array'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('object', {'object'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'array', {'array'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['items']['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') @@ -74,15 +91,26 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_m 'items': { 'type': ['object', 'null'], 'properties': dict( - [('field1', {'type': 'string', 'example': 'Example 1'}), ('field2', {'type': 'string', 'example': 'Example 2'})] + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ] ), }, } flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('array', {'array'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args(['object', 'null'], {'object'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'array', {'array'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + ['object', 'null'], {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['items']['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') From 16a1121649042ea9735dca5bfb0d8f21d5f634d5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 18:45:49 -0700 Subject: [PATCH 26/51] Get existing end-to-end tests passing (#303). --- borgmatic/config/schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index a7c9fc4c6..078c72377 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -37,7 +37,7 @@ properties: path: type: string description: The local path or Borg URL of the repository. - example: ssh://user@backupserver/./{fqdn} + example: ssh://user@backupserver/./sourcehostname.borg label: type: string description: | From 65b1d8e8b22807af005d4c94e6dac35fad8d4ede Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 19:13:07 -0700 Subject: [PATCH 27/51] Clarify NEWS items (#303). --- NEWS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 61b95f34c..8de5ff53c 100644 --- a/NEWS +++ b/NEWS @@ -1,11 +1,11 @@ 2.0.0.dev0 - * #303: Add flags for setting any borgmatic configuration option from the command-line. See the + * #303: Add command-line flags for every borgmatic configuration option. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides - * #303: Add configuration options that serve as defaults for some (but not all) borgmatic + * #303: Add configuration options that serve as defaults for some (but not all) command-line action flags. For example, each entry in "repositories:" now has an "encryption" option that - applies to the "repo-create" action. See the documentation for more information: - https://torsion.org/borgmatic/docs/reference/configuration/ + applies to the "repo-create" action, serving as a default for the "--encryption" flag. See the + documentation for more information: https://torsion.org/borgmatic/docs/reference/configuration/ * #345: Add a "key import" action to import a repository key from backup. * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: From 50beb334dcd561f9bb7c446ebde80136ef21426f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Mar 2025 11:07:25 -0700 Subject: [PATCH 28/51] Add tests for adding array element arguments and fix the code under test (#303). --- borgmatic/commands/arguments.py | 58 ++++-- tests/integration/commands/test_arguments.py | 36 ++++ tests/unit/actions/test_repo_create.py | 54 ++++- tests/unit/commands/test_arguments.py | 206 +++++++++++++++++++ 4 files changed, 331 insertions(+), 23 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 97dc33618..8a92efb96 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -299,31 +299,32 @@ def make_argument_description(schema, flag_name): ''' description = schema.get('description') schema_type = schema.get('type') + example = schema.get('example') if not description: return None - if schema_type == 'array': + if '[0]' in flag_name: + description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + + if example and schema_type == 'array': example_buffer = io.StringIO() yaml = ruamel.yaml.YAML(typ='safe') yaml.default_flow_style = True - yaml.dump(schema.get('example'), example_buffer) + yaml.dump(example, example_buffer) description += f' Example value: "{example_buffer.getvalue().strip()}"' - if '[0]' in flag_name: - description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' - description = description.replace('%', '%%') return description -def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name): +def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): r''' - Given an argparse._ArgumentGroup instance, a configuration schema dict, a sequence of unparsed - argument strings, and a dotted flag name, convert the schema into corresponding command-line - array element flags that correspond to the given unparsed arguments. + Given an argparse._ArgumentGroup instance, a sequence of unparsed argument strings, and a dotted + flag name, convert the schema into corresponding command-line array element flags that + correspond to the given unparsed arguments. Here's the background. We want to support flags that can have arbitrary indices like: @@ -356,26 +357,39 @@ def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_ar argument will match it when parsing is performed! In this manner, we're using the actual user CLI input to inform what exact flags we support! ''' - if '[0]' not in flag_name or '--help' in unparsed_arguments: + if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: return pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') - existing_flags = set( - itertools.chain( - *(group_action.option_strings for group_action in arguments_group._group_actions) + + # Find an existing list index flag (and its action) corresponding to the given flag name. If one + # isn't found, bail. + try: + (argument_action, existing_flag_name) = next( + (action, action_flag_name) + for action in arguments_group._group_actions + for action_flag_name in action.option_strings + if pattern.match(action_flag_name) + if f'--{flag_name}'.startswith(action_flag_name) ) - ) + except StopIteration: + return for unparsed in unparsed_arguments: unparsed_flag_name = unparsed.split('=', 1)[0] - if pattern.match(unparsed_flag_name) and unparsed_flag_name not in existing_flags: - arguments_group.add_argument( - unparsed_flag_name, - type=argument_type, - metavar=metavar, - help=description, - ) + if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name: + continue + + arguments_group.add_argument( + unparsed_flag_name, + choices=argument_action.choices, + default=argument_action.default, + dest=unparsed_flag_name.lstrip('-'), + nargs=argument_action.nargs, + required=argument_action.nargs, + type=argument_action.type, + ) def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): @@ -482,7 +496,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names help=description, ) - add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name) + add_array_element_arguments(arguments_group, unparsed_arguments, flag_name) def make_parsers(schema, unparsed_arguments): diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 3469858c3..5f5396602 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -4,6 +4,42 @@ from flexmock import flexmock from borgmatic.commands import arguments as module +def test_make_argument_description_with_array_adds_example(): + assert module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': [1, '- foo', {'bar': 'baz'}], + }, + flag_name='flag', + ) == 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + arguments_group.add_argument( + '--foo[0].val', + dest='--foo[0].val', + ) + + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo[25].val', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + def test_parse_arguments_with_no_arguments_uses_defaults(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) diff --git a/tests/unit/actions/test_repo_create.py b/tests/unit/actions/test_repo_create.py index 8bb350b17..035dd6232 100644 --- a/tests/unit/actions/test_repo_create.py +++ b/tests/unit/actions/test_repo_create.py @@ -1,9 +1,10 @@ +import pytest from flexmock import flexmock from borgmatic.actions import repo_create as module -def test_run_repo_create_does_not_raise(): +def test_run_repo_create_with_encryption_mode_argument_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') @@ -28,6 +29,57 @@ def test_run_repo_create_does_not_raise(): ) +def test_run_repo_create_with_encryption_mode_option_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') + arguments = flexmock( + encryption_mode=None, + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_dirs=flexmock(), + ) + + module.run_repo_create( + repository={'path': 'repo', 'encryption': flexmock()}, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + +def test_run_repo_create_without_encryption_mode_raises(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') + arguments = flexmock( + encryption_mode=None, + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_dirs=flexmock(), + ) + + with pytest.raises(ValueError): + module.run_repo_create( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + def test_run_repo_create_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index d10cf887d..379dba3b1 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -575,3 +575,209 @@ def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): with pytest.raises(ValueError): module.parse_arguments_for_actions(('config',), action_parsers, global_parser) + + +def test_make_argument_description_without_description_bails(): + assert module.make_argument_description( + schema={ + 'description': None, + 'type': 'not yours', + }, + flag_name='flag', + ) is None + + +def test_make_argument_description_with_array_adds_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('[example]') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) == 'Thing. Example value: "[example]"' + + +def test_make_argument_description_with_array_skips_missing_example(): + yaml = flexmock() + yaml.should_receive('dump').and_return('[example]') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + }, + flag_name='flag', + ) == 'Thing.' + + +def test_make_argument_description_with_array_index_in_flag_name_adds_to_description(): + assert 'list element' in module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'something', + }, + flag_name='flag[0]', + ) + + +def test_make_argument_description_escapes_percent_character(): + assert module.make_argument_description( + schema={ + 'description': '% Thing.', + 'type': 'something', + }, + flag_name='flag', + ) == '%% Thing.' + + +def test_add_array_element_arguments_without_array_index_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo', + ) + + +def test_add_array_element_arguments_with_help_flag_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--help', '--bar'), + flag_name='foo[0]', + ) + + +def test_add_array_element_arguments_without_any_flags_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo[0]', + ) + + +def test_add_array_element_arguments_without_array_index_flags_bails(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + ), + ) + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--bar'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_with_non_matching_array_index_flags_bails(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + ), + ) + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--bar[25].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_with_identical_array_index_flag_bails(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + ), + ) + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[0].val', 'fooval', '--bar'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ) + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_equals_sign(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ) + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val=fooval', '--bar[1].val=barval'), + flag_name='foo[0].val', + ) From 79bf641668bcf4ea438d670c8e7cebc28ab75086 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Mar 2025 12:42:49 -0700 Subject: [PATCH 29/51] Set the action type when cloning an argument for a list index flag (#303). --- borgmatic/commands/arguments.py | 13 +++- tests/integration/commands/test_arguments.py | 2 + tests/unit/commands/test_arguments.py | 69 ++++++++++++++++---- 3 files changed, 71 insertions(+), 13 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8a92efb96..8e0a7d85d 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -362,9 +362,8 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') - # Find an existing list index flag (and its action) corresponding to the given flag name. If one - # isn't found, bail. try: + # Find an existing list index flag (and its action) corresponding to the given flag name. (argument_action, existing_flag_name) = next( (action, action_flag_name) for action in arguments_group._group_actions @@ -372,6 +371,15 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): if pattern.match(action_flag_name) if f'--{flag_name}'.startswith(action_flag_name) ) + + # Based on the type of the action (e.g. argparse._StoreTrueAction), look up the corresponding + # action registry name (e.g., "store_true") to pass to add_argument(action=...) below. + action_registry_name = next( + registry_name + for registry_name, action_type in arguments_group._registries['action'].items() + # Not using isinstance() here because we only want an exact match—no parent classes. + if type(argument_action) == action_type + ) except StopIteration: return @@ -383,6 +391,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): arguments_group.add_argument( unparsed_flag_name, + action=action_registry_name, choices=argument_action.choices, default=argument_action.default, dest=unparsed_flag_name.lstrip('-'), diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 5f5396602..d6841a2c5 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -20,11 +20,13 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): arguments_group = parser.add_argument_group('arguments') arguments_group.add_argument( '--foo[0].val', + action='store_true', dest='--foo[0].val', ) flexmock(arguments_group).should_receive('add_argument').with_args( '--foo[25].val', + action='store_true', choices=object, default=object, dest='foo[25].val', diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 379dba3b1..a9479db58 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -672,13 +672,31 @@ def test_add_array_element_arguments_without_any_flags_bails(): ) +# Use this instead of a flexmock because it's not easy to check the type() of a flexmock instance. +Group_action = collections.namedtuple( + 'Group_action', + ( + 'option_strings', + 'choices', + 'default', + 'nargs', + 'required', + 'type', + ), + defaults=( + flexmock(), flexmock(), flexmock(), flexmock(), flexmock(), + ) +) + + def test_add_array_element_arguments_without_array_index_flags_bails(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), ), - ) + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').never() @@ -692,10 +710,11 @@ def test_add_array_element_arguments_without_array_index_flags_bails(): def test_add_array_element_arguments_with_non_matching_array_index_flags_bails(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), ), - ) + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').never() @@ -709,10 +728,11 @@ def test_add_array_element_arguments_with_non_matching_array_index_flags_bails() def test_add_array_element_arguments_with_identical_array_index_flag_bails(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), ), - ) + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').never() @@ -723,10 +743,10 @@ def test_add_array_element_arguments_with_identical_array_index_flag_bails(): ) -def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): +def test_add_array_element_arguments_without_action_type_in_registry_bails(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), choices=flexmock(), default=flexmock(), @@ -734,10 +754,35 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): required=flexmock(), type=flexmock(), ), - ) + ), + _registries={'action': {'store_stuff': bool}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').with_args( '--foo[25].val', + action='store_stuff', choices=object, default=object, dest='foo[25].val', @@ -756,7 +801,7 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_equals_sign(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), choices=flexmock(), default=flexmock(), @@ -764,10 +809,12 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_e required=flexmock(), type=flexmock(), ), - ) + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').with_args( '--foo[25].val', + action='store_stuff', choices=object, default=object, dest='foo[25].val', From b4c558d013b4c322dcc6ef8d447b7763cc1f1735 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Mar 2025 16:49:14 -0700 Subject: [PATCH 30/51] Add tests for CLI arguments from schema logic (#303). --- borgmatic/commands/arguments.py | 11 +- borgmatic/config/schema.py | 2 + tests/integration/commands/test_arguments.py | 93 ++++- tests/unit/commands/test_arguments.py | 415 +++++++++++++++++-- 4 files changed, 476 insertions(+), 45 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8e0a7d85d..3bd220822 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -323,8 +323,7 @@ def make_argument_description(schema, flag_name): def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): r''' Given an argparse._ArgumentGroup instance, a sequence of unparsed argument strings, and a dotted - flag name, convert the schema into corresponding command-line array element flags that - correspond to the given unparsed arguments. + flag name, add command-line array element flags that correspond to the given unparsed arguments. Here's the background. We want to support flags that can have arbitrary indices like: @@ -353,9 +352,9 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): --foo.bar[1].baz - ... then an argument flag will get added equal to that unparsed argument. And the unparsed + ... then an argument flag will get added equal to that unparsed argument. And so the unparsed argument will match it when parsing is performed! In this manner, we're using the actual user - CLI input to inform what exact flags we support! + CLI input to inform what exact flags we support. ''' if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: return @@ -462,8 +461,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names return - # If this is an "array" type, recurse for each child option of its items type. Don't return yet, - # so that a flag also gets added below for the array itself. + # If this is an "array" type, recurse for each items type child option. Don't return yet so that + # a flag also gets added below for the array itself. if schema_type == 'array': properties = borgmatic.config.schema.get_properties(schema.get('items', {})) diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 0ce774d9f..52358a3e9 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -35,6 +35,8 @@ def parse_type(schema_type): 'integer': int, 'number': decimal.Decimal, 'boolean': bool, + # This is str instead of list to support specifying a list as a YAML string on the + # command-line. 'array': str, }[schema_type] except KeyError: diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index d6841a2c5..200368aa1 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -5,14 +5,17 @@ from borgmatic.commands import arguments as module def test_make_argument_description_with_array_adds_example(): - assert module.make_argument_description( - schema={ - 'description': 'Thing.', - 'type': 'array', - 'example': [1, '- foo', {'bar': 'baz'}], - }, - flag_name='flag', - ) == 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': [1, '- foo', {'bar': 'baz'}], + }, + flag_name='flag', + ) + == 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + ) def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): @@ -42,6 +45,80 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): ) +def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo.baz', + type=str, + metavar='BAZ', + help='help 2', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer', 'description': 'help 1'}, + 'baz': {'type': 'string', 'description': 'help 2'}, + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo[0].bar', + type=int, + metavar='BAR', + help=object, + ).once() + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help 2', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'bar': { + 'type': 'integer', + 'description': 'help 1', + } + } + }, + 'description': 'help 2', + } + } + }, + unparsed_arguments=(), + ) + + def test_parse_arguments_with_no_arguments_uses_defaults(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index a9479db58..fcfa9589c 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -578,13 +578,16 @@ def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): def test_make_argument_description_without_description_bails(): - assert module.make_argument_description( - schema={ - 'description': None, - 'type': 'not yours', - }, - flag_name='flag', - ) is None + assert ( + module.make_argument_description( + schema={ + 'description': None, + 'type': 'not yours', + }, + flag_name='flag', + ) + is None + ) def test_make_argument_description_with_array_adds_example(): @@ -595,14 +598,17 @@ def test_make_argument_description_with_array_adds_example(): yaml.should_receive('dump') flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) - assert module.make_argument_description( - schema={ - 'description': 'Thing.', - 'type': 'array', - 'example': ['example'], - }, - flag_name='flag', - ) == 'Thing. Example value: "[example]"' + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) + == 'Thing. Example value: "[example]"' + ) def test_make_argument_description_with_array_skips_missing_example(): @@ -610,13 +616,16 @@ def test_make_argument_description_with_array_skips_missing_example(): yaml.should_receive('dump').and_return('[example]') flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) - assert module.make_argument_description( - schema={ - 'description': 'Thing.', - 'type': 'array', - }, - flag_name='flag', - ) == 'Thing.' + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + }, + flag_name='flag', + ) + == 'Thing.' + ) def test_make_argument_description_with_array_index_in_flag_name_adds_to_description(): @@ -630,13 +639,16 @@ def test_make_argument_description_with_array_index_in_flag_name_adds_to_descrip def test_make_argument_description_escapes_percent_character(): - assert module.make_argument_description( - schema={ - 'description': '% Thing.', - 'type': 'something', - }, - flag_name='flag', - ) == '%% Thing.' + assert ( + module.make_argument_description( + schema={ + 'description': '% Thing.', + 'type': 'something', + }, + flag_name='flag', + ) + == '%% Thing.' + ) def test_add_array_element_arguments_without_array_index_bails(): @@ -684,8 +696,12 @@ Group_action = collections.namedtuple( 'type', ), defaults=( - flexmock(), flexmock(), flexmock(), flexmock(), flexmock(), - ) + flexmock(), + flexmock(), + flexmock(), + flexmock(), + flexmock(), + ), ) @@ -828,3 +844,340 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_e unparsed_arguments=('--foo[25].val=fooval', '--bar[1].val=barval'), flag_name='foo[0].val', ) + + +def test_add_arguments_from_schema_with_non_dict_schema_bails(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').never() + flexmock(module.borgmatic.config.schema).should_receive('parse_type').never() + arguments_group.should_receive('add_argument').never() + + module.add_arguments_from_schema( + arguments_group=arguments_group, schema='foo', unparsed_arguments=() + ) + + +def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return('help 2') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo.baz', + type=str, + metavar='BAZ', + help='help 2', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer'}, + 'baz': {'type': 'str'}, + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_uses_first_non_null_type_from_multi_type_object(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': ['null', 'object', 'boolean'], + 'properties': { + 'bar': {'type': 'integer'}, + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_empty_multi_type_raises(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int) + arguments_group.should_receive('add_argument').never() + flexmock(module).should_receive('add_array_element_arguments').never() + + with pytest.raises(ValueError): + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': [], + 'properties': { + 'bar': {'type': 'integer'}, + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_propertyless_option_does_not_add_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').never() + flexmock(module.borgmatic.config.schema).should_receive('parse_type').never() + arguments_group.should_receive('add_argument').never() + flexmock(module).should_receive('add_array_element_arguments').never() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_adds_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'integer', + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return('help 2') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo[0].bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help 2', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + flexmock(module).should_receive('add_array_element_arguments').with_args( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo[0].bar', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'bar': { + 'type': 'integer', + } + } + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_default_false_boolean_adds_valueless_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo', + action='store_true', + default=None, + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'boolean', + 'default': False, + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_default_true_boolean_adds_value_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=bool, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'boolean', + 'default': True, + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_defaultless_boolean_adds_value_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=bool, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'boolean', + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_skips_omitted_flag_name(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--match-archives', + type=object, + metavar=object, + help=object, + ).never() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'match_archives': { + 'type': 'string', + }, + 'foo': { + 'type': 'string', + }, + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_rewrites_option_name_to_flag_name(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo-and-stuff', + type=str, + metavar='FOO_AND_STUFF', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo_and_stuff': { + 'type': 'string', + }, + } + }, + unparsed_arguments=(), + ) From 5bf2f546b9db76ed2863fcb9e982adb15f5983f0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Mar 2025 21:01:56 -0700 Subject: [PATCH 31/51] More automated tests (#303). --- borgmatic/commands/arguments.py | 5 +- borgmatic/config/arguments.py | 51 ++--- borgmatic/config/schema.py | 25 ++- tests/integration/commands/test_arguments.py | 8 +- tests/integration/config/test_arguments.py | 35 ++++ tests/unit/commands/test_arguments.py | 50 +++-- tests/unit/config/test_arguments.py | 195 +++++++++++++++++++ 7 files changed, 310 insertions(+), 59 deletions(-) create mode 100644 tests/integration/config/test_arguments.py create mode 100644 tests/unit/config/test_arguments.py diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 3bd220822..140dc1ae9 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -484,7 +484,10 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names metavar = names[-1].upper() description = make_argument_description(schema, flag_name) - argument_type = borgmatic.config.schema.parse_type(schema_type) + + # array=str instead of list here to support specifying a list as a YAML string on the + # command-line. + argument_type = borgmatic.config.schema.parse_type(schema_type, array=str) full_flag_name = f"--{flag_name.replace('_', '-')}" # As a UX nicety, allow boolean options that have a default of false to have command-line flags diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 8f7023cdb..85c8eef97 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -3,6 +3,8 @@ import re import ruamel.yaml +import borgmatic.config.schema + LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+)\]$') @@ -45,7 +47,7 @@ def set_values(config, keys, value): try: set_values(config[list_key][list_index], keys[1:], value) except IndexError: - raise ValueError(f'The list index {first_key} is out of range') + raise ValueError(f'Argument list index {first_key} is out of range') return @@ -99,6 +101,7 @@ def convert_value_type(value, option_type): And if the source value isn't a string, return it as-is. Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML. + Raise ValueError if the parsed value doesn't match the option type. ''' if not isinstance(value, str): return value @@ -106,7 +109,15 @@ def convert_value_type(value, option_type): if option_type == 'string': return value - return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + try: + parsed_value = ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + except ruamel.yaml.error.YAMLError as error: + raise ValueError(f'Argument value "{value}" is invalid: {error.problem}') + + if not isinstance(parsed_value, borgmatic.config.schema.parse_type(option_type)): + raise ValueError(f'Argument value "{value}" is not of the expected type: {option_type}') + + return parsed_value def prepare_arguments_for_config(global_arguments, schema): @@ -122,39 +133,34 @@ def prepare_arguments_for_config(global_arguments, schema): ( (('my_option', 'sub_option'), 'value1'), - (('other_option'), 'value2'), + (('other_option',), 'value2'), ) - - Raise ValueError if an override can't be parsed. ''' prepared_values = [] for argument_name, value in global_arguments.__dict__.items(): - try: - if value is None: - continue + if value is None: + continue - keys = tuple(argument_name.split('.')) - option_type = type_for_option(schema, keys) + keys = tuple(argument_name.split('.')) + option_type = type_for_option(schema, keys) - # The argument doesn't correspond to any option in the schema, so ignore it. It's - # probably a flag that borgmatic has on the command-line but not in configuration. - if option_type is None: - continue + # The argument doesn't correspond to any option in the schema, so ignore it. It's + # probably a flag that borgmatic has on the command-line but not in configuration. + if option_type is None: + continue - prepared_values.append( - ( - keys, - convert_value_type(value, option_type), - ) + prepared_values.append( + ( + keys, + convert_value_type(value, option_type), ) - except ruamel.yaml.error.YAMLError as error: - raise ValueError(f'Invalid override "{argument_name}": {error.problem}') + ) return tuple(prepared_values) -def apply_arguments_to_config(config, schema, global_arguments): +def apply_arguments_to_config(config, schema, global_arguments): # pragma: no cover ''' Given a configuration dict, a corresponding configuration schema dict, and global arguments as an argparse.Namespace, set those given argument values into their corresponding configuration @@ -164,6 +170,5 @@ def apply_arguments_to_config(config, schema, global_arguments): configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list element in the configuration. ''' - for keys, value in prepare_arguments_for_config(global_arguments, schema): set_values(config, keys, value) diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 52358a3e9..6981dde5e 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -23,22 +23,27 @@ def get_properties(schema): return schema.get('properties', {}) -def parse_type(schema_type): +def parse_type(schema_type, **overrides): ''' Given a schema type as a string, return the corresponding Python type. + If any overrides are given in the from of a schema type string to a Python type, then override + the default type mapping with them. + Raise ValueError if the schema type is unknown. ''' try: - return { - 'string': str, - 'integer': int, - 'number': decimal.Decimal, - 'boolean': bool, - # This is str instead of list to support specifying a list as a YAML string on the - # command-line. - 'array': str, - }[schema_type] + return dict( + { + 'array': list, + 'boolean': bool, + 'integer': int, + 'number': decimal.Decimal, + 'object': dict, + 'string': str, + }, + **overrides, + )[schema_type] except KeyError: raise ValueError(f'Unknown type in configuration schema: {schema_type}') diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 200368aa1..032b9495c 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -71,9 +71,9 @@ def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option( 'properties': { 'bar': {'type': 'integer', 'description': 'help 1'}, 'baz': {'type': 'string', 'description': 'help 2'}, - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -109,11 +109,11 @@ def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_fl 'type': 'integer', 'description': 'help 1', } - } + }, }, 'description': 'help 2', } - } + }, }, unparsed_arguments=(), ) diff --git a/tests/integration/config/test_arguments.py b/tests/integration/config/test_arguments.py new file mode 100644 index 000000000..449964ee1 --- /dev/null +++ b/tests/integration/config/test_arguments.py @@ -0,0 +1,35 @@ +import pytest +from flexmock import flexmock + +from borgmatic.config import arguments as module + + +def test_convert_value_type_passes_through_non_string_value(): + assert module.convert_value_type([1, 2], 'array') == [1, 2] + + +def test_convert_value_type_passes_through_string_option_type(): + assert module.convert_value_type('foo', 'string') == 'foo' + + +def test_convert_value_type_parses_array_option_type(): + assert module.convert_value_type('[foo, bar]', 'array') == ['foo', 'bar'] + + +def test_convert_value_type_with_array_option_type_and_no_array_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar}', 'array') + + +def test_convert_value_type_parses_object_option_type(): + assert module.convert_value_type('{foo: bar}', 'object') == {'foo': 'bar'} + + +def test_convert_value_type_with_invalid_value_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar', 'object') + + +def test_convert_value_type_with_unknown_option_type_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar}', 'thingy') diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index fcfa9589c..1e5bbe0ba 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -859,8 +859,12 @@ def test_add_arguments_from_schema_with_non_dict_schema_bails(): def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return('help 2') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int).and_return(str) + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( + 'help 2' + ) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return( + int + ).and_return(str) arguments_group.should_receive('add_argument').with_args( '--foo.bar', type=int, @@ -885,9 +889,9 @@ def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option( 'properties': { 'bar': {'type': 'integer'}, 'baz': {'type': 'str'}, - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -914,9 +918,9 @@ def test_add_arguments_from_schema_uses_first_non_null_type_from_multi_type_obje 'type': ['null', 'object', 'boolean'], 'properties': { 'bar': {'type': 'integer'}, - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -939,9 +943,9 @@ def test_add_arguments_from_schema_with_empty_multi_type_raises(): 'type': [], 'properties': { 'bar': {'type': 'integer'}, - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -962,7 +966,7 @@ def test_add_arguments_from_schema_with_propertyless_option_does_not_add_flag(): 'foo': { 'type': 'object', } - } + }, }, unparsed_arguments=(), ) @@ -989,9 +993,9 @@ def test_add_arguments_from_schema_with_array_adds_flag(): 'type': 'array', 'items': { 'type': 'integer', - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -999,8 +1003,12 @@ def test_add_arguments_from_schema_with_array_adds_flag(): def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return('help 2') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int).and_return(str) + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( + 'help 2' + ) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return( + int + ).and_return(str) arguments_group.should_receive('add_argument').with_args( '--foo[0].bar', type=int, @@ -1033,10 +1041,10 @@ def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_fl 'bar': { 'type': 'integer', } - } - } + }, + }, } - } + }, }, unparsed_arguments=(), ) @@ -1063,7 +1071,7 @@ def test_add_arguments_from_schema_with_default_false_boolean_adds_valueless_fla 'type': 'boolean', 'default': False, } - } + }, }, unparsed_arguments=(), ) @@ -1090,7 +1098,7 @@ def test_add_arguments_from_schema_with_default_true_boolean_adds_value_flag(): 'type': 'boolean', 'default': True, } - } + }, }, unparsed_arguments=(), ) @@ -1116,7 +1124,7 @@ def test_add_arguments_from_schema_with_defaultless_boolean_adds_value_flag(): 'foo': { 'type': 'boolean', } - } + }, }, unparsed_arguments=(), ) @@ -1151,7 +1159,7 @@ def test_add_arguments_from_schema_skips_omitted_flag_name(): 'foo': { 'type': 'string', }, - } + }, }, unparsed_arguments=(), ) @@ -1177,7 +1185,7 @@ def test_add_arguments_from_schema_rewrites_option_name_to_flag_name(): 'foo_and_stuff': { 'type': 'string', }, - } + }, }, unparsed_arguments=(), ) diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py new file mode 100644 index 000000000..25c8873f5 --- /dev/null +++ b/tests/unit/config/test_arguments.py @@ -0,0 +1,195 @@ +import pytest +from flexmock import flexmock + +from borgmatic.config import arguments as module + + +def test_set_values_without_keys_bails(): + config = {'option': 'value'} + module.set_values(config=config, keys=(), value=5) + + assert config == {'option': 'value'} + + +def test_set_values_with_keys_adds_them_to_config(): + config = {'option': 'value'} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'option': 'value', 'foo': {'bar': {'baz': 5}}} + + +def test_set_values_with_one_existing_key_adds_others_to_config(): + config = {'foo': {'other': 'value'}} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'foo': {'other': 'value', 'bar': {'baz': 5}}} + + +def test_set_values_with_two_existing_keys_adds_others_to_config(): + config = {'foo': {'bar': {'other': 'value'}}} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'foo': {'bar': {'other': 'value', 'baz': 5}}} + + +def test_set_values_with_list_index_key_adds_it_to_config(): + config = {'foo': {'bar': [{'option': 'value'}, {'other': 'thing'}]}} + + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + assert config == {'foo': {'bar': [{'option': 'value'}, {'other': 'thing', 'baz': 5}]}} + + +def test_set_values_with_list_index_key_out_of_range_raises(): + config = {'foo': {'bar': [{'option': 'value'}]}} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + +def test_set_values_with_list_index_key_missing_list_and_out_of_range_raises(): + config = {'other': 'value'} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + +def test_set_values_with_final_list_index_key_adds_it_to_config(): + config = {'foo': {'bar': [1, 2]}} + + module.set_values(config=config, keys=('foo', 'bar[1]'), value=5) + + assert config == {'foo': {'bar': [1, 5]}} + + +def test_type_for_option_with_option_finds_type(): + assert ( + module.type_for_option( + schema={'type': 'object', 'properties': {'foo': {'type': 'integer'}}}, + option_keys=('foo',), + ) + == 'integer' + ) + + +def test_type_for_option_with_nested_option_finds_type(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': {'type': 'object', 'properties': {'bar': {'type': 'boolean'}}} + }, + }, + option_keys=('foo', 'bar'), + ) + == 'boolean' + ) + + +def test_type_for_option_with_missing_nested_option_finds_nothing(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': {'type': 'object', 'properties': {'other': {'type': 'integer'}}} + }, + }, + option_keys=('foo', 'bar'), + ) + is None + ) + + +def test_type_for_option_with_typeless_nested_option_finds_nothing(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': {'foo': {'type': 'object', 'properties': {'bar': {'example': 5}}}}, + }, + option_keys=('foo', 'bar'), + ) + is None + ) + + +def test_type_for_list_index_option_finds_type(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': {'foo': {'type': 'array', 'items': {'type': 'integer'}}}, + }, + option_keys=('foo[0]',), + ) + == 'integer' + ) + + +def test_type_for_nested_list_index_option_finds_type(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': {'type': 'object', 'properties': {'bar': {'type': 'integer'}}}, + } + }, + }, + option_keys=('foo[0]', 'bar'), + ) + == 'integer' + ) + + +def test_prepare_arguments_for_config_converts_arguments_to_keys(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ( + (('my_option', 'sub_option'), 'value1'), + (('other_option',), 'value2'), + ) + + +def test_prepare_arguments_for_config_skips_option_with_none_value(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': None, 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ( + (('other_option',), 'value2'), + ) + + +def test_prepare_arguments_for_config_skips_option_missing_from_schema(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object'}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ( + (('other_option',), 'value2'), + ) From 975a6e4540203d078a2ffbadf3a8524e90c573a1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Mar 2025 11:37:48 -0700 Subject: [PATCH 32/51] Add additional tests for complete coverage (#303). --- borgmatic/borg/create.py | 2 +- borgmatic/commands/arguments.py | 5 +- borgmatic/config/schema.py | 22 +++--- borgmatic/config/validate.py | 2 +- tests/integration/commands/test_arguments.py | 7 +- tests/integration/config/test_arguments.py | 1 - tests/unit/config/test_arguments.py | 8 +- tests/unit/config/test_generate.py | 1 - tests/unit/config/test_normalize.py | 5 ++ tests/unit/config/test_schema.py | 83 ++++++++++++++++++++ 10 files changed, 113 insertions(+), 23 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 54a7f9ad4..ccdb41a97 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -196,7 +196,7 @@ def check_all_root_patterns_exist(patterns): if missing_paths: raise ValueError( - f"Source directories / root pattern paths do not exist: {', '.join(missing_paths)}" + f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}" ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 140dc1ae9..7ad600d96 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,7 +1,6 @@ import collections import io import itertools -import json import re import sys from argparse import ArgumentParser @@ -377,7 +376,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): registry_name for registry_name, action_type in arguments_group._registries['action'].items() # Not using isinstance() here because we only want an exact match—no parent classes. - if type(argument_action) == action_type + if type(argument_action) is action_type ) except StopIteration: return @@ -492,7 +491,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names # As a UX nicety, allow boolean options that have a default of false to have command-line flags # without values. - if schema_type == 'boolean' and schema.get('default') == False: + if schema_type == 'boolean' and schema.get('default') is False: arguments_group.add_argument( full_flag_name, action='store_true', diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 6981dde5e..3d13c0c25 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -23,6 +23,16 @@ def get_properties(schema): return schema.get('properties', {}) +SCHEMA_TYPE_TO_PYTHON_TYPE = { + 'array': list, + 'boolean': bool, + 'integer': int, + 'number': decimal.Decimal, + 'object': dict, + 'string': str, +} + + def parse_type(schema_type, **overrides): ''' Given a schema type as a string, return the corresponding Python type. @@ -34,14 +44,7 @@ def parse_type(schema_type, **overrides): ''' try: return dict( - { - 'array': list, - 'boolean': bool, - 'integer': int, - 'number': decimal.Decimal, - 'object': dict, - 'string': str, - }, + SCHEMA_TYPE_TO_PYTHON_TYPE, **overrides, )[schema_type] except KeyError: @@ -54,7 +57,8 @@ def compare_types(schema_type, target_types, match=any): target type strings, return whether every schema type is in the set of target types. If the schema type is a list of strings, use the given match function (such as any or all) to - compare elements. + compare elements. For instance, if match is given as all, then every element of the schema_type + list must be in the target types. ''' if isinstance(schema_type, list): if match(element_schema_type in target_types for element_schema_type in schema_type): diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 29215f911..98bf080cd 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -21,7 +21,7 @@ def schema_filename(): return schema_path -def load_schema(schema_path): +def load_schema(schema_path): # pragma: no cover ''' Given a schema filename path, load the schema and return it as a dict. diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 032b9495c..ed6a98922 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -14,7 +14,12 @@ def test_make_argument_description_with_array_adds_example(): }, flag_name='flag', ) - == 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + # Apparently different versions of ruamel.yaml serialize this + # differently. + in ( + 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + 'Thing. Example value: "[1, \'- foo\', {bar: baz}]"' + ) ) diff --git a/tests/integration/config/test_arguments.py b/tests/integration/config/test_arguments.py index 449964ee1..9bcc37ff9 100644 --- a/tests/integration/config/test_arguments.py +++ b/tests/integration/config/test_arguments.py @@ -1,5 +1,4 @@ import pytest -from flexmock import flexmock from borgmatic.config import arguments as module diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py index 25c8873f5..18ad7f569 100644 --- a/tests/unit/config/test_arguments.py +++ b/tests/unit/config/test_arguments.py @@ -175,9 +175,7 @@ def test_prepare_arguments_for_config_skips_option_with_none_value(): 'other_option': {'type': 'string'}, }, }, - ) == ( - (('other_option',), 'value2'), - ) + ) == ((('other_option',), 'value2'),) def test_prepare_arguments_for_config_skips_option_missing_from_schema(): @@ -190,6 +188,4 @@ def test_prepare_arguments_for_config_skips_option_missing_from_schema(): 'other_option': {'type': 'string'}, }, }, - ) == ( - (('other_option',), 'value2'), - ) + ) == ((('other_option',), 'value2'),) diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 885bdcca0..ad90bb5af 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -2,7 +2,6 @@ import pytest from flexmock import flexmock from borgmatic.config import generate as module -import borgmatic.config.schema def test_schema_to_sample_configuration_generates_config_map_with_examples(): diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index abd7e54dc..29fe25e80 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -359,6 +359,11 @@ def test_normalize_commands_moves_individual_command_hooks_to_unified_commands( {'repositories': [{'path': '/repo', 'label': 'foo'}]}, False, ), + ( + {'repositories': [{'path': None, 'label': 'foo'}]}, + {'repositories': []}, + False, + ), ( {'prefix': 'foo'}, {'prefix': 'foo'}, diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index a07b19bab..8af890fb3 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -1,3 +1,5 @@ +import pytest + from borgmatic.config import schema as module @@ -75,3 +77,84 @@ def test_get_properties_interleaves_oneof_list_properties(): ('field3', {'example': 'Example 3'}), ] ) + + +def test_parse_type_maps_schema_type_to_python_type(): + module.parse_type('boolean') == bool + + +def test_parse_type_with_unknown_schema_type_raises(): + with pytest.raises(ValueError): + module.parse_type('what') + + +def test_parse_type_respect_overrides_when_mapping_types(): + module.parse_type('boolean', boolean=int) == int + + +@pytest.mark.parametrize( + 'schema_type,target_types,match,expected_result', + ( + ( + 'string', + {'integer', 'string', 'boolean'}, + None, + True, + ), + ( + 'string', + {'integer', 'boolean'}, + None, + False, + ), + ( + 'string', + {'integer', 'string', 'boolean'}, + all, + True, + ), + ( + 'string', + {'integer', 'boolean'}, + all, + False, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean'}, + None, + True, + ), + ( + ['string', 'array'], + {'integer', 'boolean'}, + None, + False, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean', 'array'}, + all, + True, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean'}, + all, + False, + ), + ( + ['string', 'array'], + {'integer', 'boolean'}, + all, + False, + ), + ), +) +def test_compare_types_returns_whether_schema_type_matches_target_types( + schema_type, target_types, match, expected_result +): + if match: + assert module.compare_types(schema_type, target_types, match) == expected_result + else: + assert module.compare_types(schema_type, target_types) == expected_result From 1c27e0dadc40ce34138a6397243d69bb0a93660e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Mar 2025 13:46:58 -0700 Subject: [PATCH 33/51] Add an end-to-end test for command-line flags of configuration options (#303). --- tests/end-to-end/test_config_flag.py | 51 ++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/end-to-end/test_config_flag.py diff --git a/tests/end-to-end/test_config_flag.py b/tests/end-to-end/test_config_flag.py new file mode 100644 index 000000000..0300ac4e4 --- /dev/null +++ b/tests/end-to-end/test_config_flag.py @@ -0,0 +1,51 @@ +import os +import shlex +import shutil +import subprocess +import tempfile + + +def generate_configuration(config_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing (including injecting the given repository path and tacking on an encryption + passphrase). But don't actually set the repository path, as that's done on the command-line + below. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('- ssh://user@backupserver/./{fqdn}', '') # noqa: FS003 + .replace('- /var/local/backups/local.borg', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + 'encryption_passphrase: "test"' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_config_flags_do_not_error(): + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path) + + subprocess.check_call( + shlex.split(f'borgmatic -v 2 --config {config_path} --repositories "[{{path: {repository_path}, label: repo}}]" repo-create --encryption repokey') + ) + + subprocess.check_call( + shlex.split(f'borgmatic create --config {config_path} --repositories[0].path "{repository_path}"') + ) + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) From 668f767bfc8b48e1c73e2471f2282e9e61643d36 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Mar 2025 23:11:15 -0700 Subject: [PATCH 34/51] Adding some missing tests and fixing related flag vs. config logic (#303). --- borgmatic/actions/compact.py | 12 ++- borgmatic/actions/create.py | 18 +++- borgmatic/actions/export_tar.py | 6 +- borgmatic/actions/extract.py | 6 +- borgmatic/actions/repo_create.py | 18 +++- borgmatic/borg/check.py | 4 +- borgmatic/borg/delete.py | 7 +- borgmatic/borg/prune.py | 11 ++- borgmatic/borg/transfer.py | 9 +- borgmatic/commands/arguments.py | 8 +- borgmatic/config/schema.yaml | 4 +- tests/end-to-end/test_config_flag.py | 8 +- tests/unit/actions/test_compact.py | 78 +++++++++++++++++ tests/unit/actions/test_create.py | 112 +++++++++++++++++++++++++ tests/unit/actions/test_export_tar.py | 80 ++++++++++++++++++ tests/unit/actions/test_extract.py | 78 +++++++++++++++++ tests/unit/actions/test_repo_create.py | 88 +++++++++++++++++++ tests/unit/borg/test_check.py | 39 ++++++++- tests/unit/borg/test_delete.py | 36 +++++++- tests/unit/borg/test_prune.py | 50 ++++++++++- tests/unit/borg/test_transfer.py | 35 +++++++- 21 files changed, 671 insertions(+), 36 deletions(-) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 7386d2309..dbc2e5736 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -37,9 +37,17 @@ def run_compact( global_arguments, local_path=local_path, remote_path=remote_path, - progress=compact_arguments.progress or config.get('progress'), + progress=( + config.get('progress') + if compact_arguments.progress is None + else compact_arguments.progress + ), cleanup_commits=compact_arguments.cleanup_commits, - threshold=compact_arguments.threshold or config.get('compact_threshold'), + threshold=( + config.get('compact_threshold') + if compact_arguments.threshold is None + else compact_arguments.threshold + ), ) else: # pragma: nocover logger.info('Skipping compact (only available/needed in Borg 1.2+)') diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index c771f3d2e..e0ef3b1fd 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -327,10 +327,22 @@ def run_create( borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - progress=create_arguments.progress or config.get('progress'), - stats=create_arguments.stats or config.get('stats'), + progress=( + config.get('progress') + if create_arguments.progress is None + else create_arguments.progress + ), + stats=( + config.get('statistics') + if create_arguments.stats is None + else create_arguments.stats + ), json=create_arguments.json, - list_files=create_arguments.list_files or config.get('list'), + list_files=( + config.get('list_details') + if create_arguments.list_files is None + else create_arguments.list_files + ), stream_processes=stream_processes, ) diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index 1b41548b5..cc18aeeb9 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -43,6 +43,10 @@ def run_export_tar( local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, - list_files=export_tar_arguments.list_files or config.get('list'), + list_files=( + config.get('list_details') + if export_tar_arguments.list_files is None + else export_tar_arguments.list_files + ), strip_components=export_tar_arguments.strip_components, ) diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 83f811764..3e42b57d6 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -45,5 +45,9 @@ def run_extract( remote_path=remote_path, destination_path=extract_arguments.destination, strip_components=extract_arguments.strip_components, - progress=extract_arguments.progress or config.get('progress'), + progress=( + config.get('progress') + if extract_arguments.progress is None + else extract_arguments.progress + ), ) diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index 49f4e6c05..5166b9b83 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -41,9 +41,21 @@ def run_repo_create( encryption_mode, repo_create_arguments.source_repository, repo_create_arguments.copy_crypt_key, - repo_create_arguments.append_only or repository.get('append_only'), - repo_create_arguments.storage_quota or repository.get('storage_quota'), - repo_create_arguments.make_parent_dirs or repository.get('make_parent_dirs'), + ( + repository.get('append_only') + if repo_create_arguments.append_only is None + else repo_create_arguments.append_only + ), + ( + repository.get('storage_quota') + if repo_create_arguments.storage_quota is None + else repo_create_arguments.storage_quota + ), + ( + repository.get('make_parent_dirs') + if repo_create_arguments.make_parent_dirs is None + else repo_create_arguments.make_parent_dirs + ), local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 13f4ffca6..c02b6ec41 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -143,7 +143,9 @@ def check_archives( umask = config.get('umask') borg_exit_codes = config.get('borg_exit_codes') working_directory = borgmatic.config.paths.get_working_directory(config) - progress = check_arguments.progress or config.get('progress') + progress = ( + config.get('progress') if check_arguments.progress is None else check_arguments.progress + ) if 'data' in checks: checks.add('archives') diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index a9c3478cd..f31730ce5 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -35,7 +35,12 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) + borgmatic.borg.flags.make_flags( - 'list', delete_arguments.list_archives or config.get('list') + 'list', + ( + config.get('list_details') + if delete_arguments.list_archives is None + else delete_arguments.list_archives + ), ) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 5eef3a291..b29cbe67e 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -66,7 +66,12 @@ def prune_archives( borgmatic.logger.add_custom_log_levels() umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) - stats = prune_arguments.stats or config.get('stats') + stats = config.get('statistics') if prune_arguments.stats is None else prune_arguments.stats + list_archives = ( + config.get('list_details') + if prune_arguments.list_archives is None + else prune_arguments.list_archives + ) extra_borg_options = config.get('extra_borg_options', {}).get('prune', '') full_command = ( @@ -88,14 +93,14 @@ def prune_archives( prune_arguments, excludes=('repository', 'match_archives', 'stats', 'list_archives'), ) - + (('--list',) if prune_arguments.list_archives or config.get('list') else ()) + + (('--list',) if list_archives else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) - if stats or prune_arguments.list_archives: + if stats or list_archives: output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index ca90063ea..528cd64cb 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -52,13 +52,16 @@ def transfer_archives( + flags.make_flags('other-repo', transfer_arguments.source_repository) + flags.make_flags('dry-run', dry_run) ) + progress = ( + config.get('progress') + if transfer_arguments.progress is None + else transfer_arguments.progress + ) return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=( - DO_NOT_CAPTURE if (transfer_arguments.progress or config.get('progress')) else None - ), + output_file=DO_NOT_CAPTURE if progress else None, environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 7ad600d96..6344a0b23 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -287,7 +287,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'stats', 'list'} +OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'statistics', 'list_files'} def make_argument_description(schema, flag_name): @@ -520,9 +520,9 @@ def make_parsers(schema, unparsed_arguments): config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) - # allow_abbrev=False prevents the global parser from erroring about "ambiguous" options like - # --encryption. Such options are intended for an action parser rather than the global parser, - # and so we don't want to error on them here. + # Using allow_abbrev=False here prevents the global parser from erroring about "ambiguous" + # options like --encryption. Such options are intended for an action parser rather than the + # global parser, and so we don't want to error on them here. global_parser = ArgumentParser(allow_abbrev=False, add_help=False) global_group = global_parser.add_argument_group('global arguments') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index aaeb44763..d0bcaa0fe 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -823,7 +823,7 @@ properties: actions. Defaults to false. default: false example: true - stats: + statistics: type: boolean description: | Display statistics for an archive when running supported actions. @@ -831,7 +831,7 @@ properties: false. default: false example: true - list: + list_details: type: boolean description: | Display details for each file or archive as it is processed when diff --git a/tests/end-to-end/test_config_flag.py b/tests/end-to-end/test_config_flag.py index 0300ac4e4..279a1c6ec 100644 --- a/tests/end-to-end/test_config_flag.py +++ b/tests/end-to-end/test_config_flag.py @@ -40,11 +40,15 @@ def test_config_flags_do_not_error(): generate_configuration(config_path) subprocess.check_call( - shlex.split(f'borgmatic -v 2 --config {config_path} --repositories "[{{path: {repository_path}, label: repo}}]" repo-create --encryption repokey') + shlex.split( + f'borgmatic -v 2 --config {config_path} --repositories "[{{path: {repository_path}, label: repo}}]" repo-create --encryption repokey' + ) ) subprocess.check_call( - shlex.split(f'borgmatic create --config {config_path} --repositories[0].path "{repository_path}"') + shlex.split( + f'borgmatic create --config {config_path} --repositories[0].path "{repository_path}"' + ) ) finally: os.chdir(original_working_directory) diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 14dcfaadc..1531fd59d 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -51,6 +51,84 @@ def test_compact_runs_with_selected_repository(): ) +def test_compact_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').with_args( + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + progress=False, + cleanup_commits=object, + threshold=15, + ).once() + compact_arguments = flexmock( + repository=flexmock(), + progress=False, + cleanup_commits=flexmock(), + threshold=15, + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_compact( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'progress': True, 'compact_threshold': 20}, + local_borg_version=None, + compact_arguments=compact_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + + +def test_compact_favors_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').with_args( + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + progress=True, + cleanup_commits=object, + threshold=20, + ).once() + compact_arguments = flexmock( + repository=flexmock(), + progress=None, + cleanup_commits=flexmock(), + threshold=None, + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_compact( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'progress': True, 'compact_threshold': 20}, + local_borg_version=None, + compact_arguments=compact_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + + def test_compact_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 1f6ca7dce..6f40fb509 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -506,6 +506,118 @@ def test_run_create_runs_with_selected_repository(): ) +def test_run_create_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.create).should_receive('create_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + progress=False, + stats=False, + json=object, + list_files=False, + stream_processes=object, + ).once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).and_return({}) + flexmock(module).should_receive('collect_patterns').and_return(()) + flexmock(module).should_receive('process_patterns').and_return([]) + flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') + create_arguments = flexmock( + repository=flexmock(), + progress=False, + stats=False, + json=False, + list_files=False, + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'progress': True, 'statistics': True, 'list_details': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + +def test_run_create_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.create).should_receive('create_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + progress=True, + stats=True, + json=object, + list_files=True, + stream_processes=object, + ).once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).and_return({}) + flexmock(module).should_receive('collect_patterns').and_return(()) + flexmock(module).should_receive('process_patterns').and_return([]) + flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') + create_arguments = flexmock( + repository=flexmock(), + progress=True, + stats=True, + json=False, + list_files=True, + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'progress': True, 'statistics': True, 'list_details': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + def test_run_create_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index aea54af34..b2e91c388 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -27,3 +27,83 @@ def test_run_export_tar_does_not_raise(): local_path=None, remote_path=None, ) + + +def test_run_export_tar_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.export_tar).should_receive('export_tar_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + tar_filter=object, + list_files=False, + strip_components=object, + ).once() + export_tar_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + paths=flexmock(), + destination=flexmock(), + tar_filter=flexmock(), + list_files=False, + strip_components=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_export_tar( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version=None, + export_tar_arguments=export_tar_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_export_tar_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.export_tar).should_receive('export_tar_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + tar_filter=object, + list_files=True, + strip_components=object, + ).once() + export_tar_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + paths=flexmock(), + destination=flexmock(), + tar_filter=flexmock(), + list_files=None, + strip_components=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_export_tar( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version=None, + export_tar_arguments=export_tar_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index c483adcc5..3707c8878 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -27,3 +27,81 @@ def test_run_extract_calls_hooks(): local_path=None, remote_path=None, ) + + +def test_run_extract_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + destination_path=object, + strip_components=object, + progress=False, + ).once() + extract_arguments = flexmock( + paths=flexmock(), + progress=False, + destination=flexmock(), + strip_components=flexmock(), + archive=flexmock(), + repository='repo', + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_extract( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'repositories': ['repo'], 'progress': True}, + local_borg_version=None, + extract_arguments=extract_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_extract_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + destination_path=object, + strip_components=object, + progress=True, + ).once() + extract_arguments = flexmock( + paths=flexmock(), + progress=None, + destination=flexmock(), + strip_components=flexmock(), + archive=flexmock(), + repository='repo', + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_extract( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'repositories': ['repo'], 'progress': True}, + local_borg_version=None, + extract_arguments=extract_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_repo_create.py b/tests/unit/actions/test_repo_create.py index 035dd6232..665d9db79 100644 --- a/tests/unit/actions/test_repo_create.py +++ b/tests/unit/actions/test_repo_create.py @@ -105,3 +105,91 @@ def test_run_repo_create_bails_if_repository_does_not_match(): local_path=None, remote_path=None, ) + + +def test_run_repo_create_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + append_only=False, + storage_quota=0, + make_parent_dirs=False, + local_path=object, + remote_path=object, + ).once() + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=False, + storage_quota=0, + make_parent_dirs=False, + ) + + module.run_repo_create( + repository={ + 'path': 'repo', + 'append_only': True, + 'storage_quota': '10G', + 'make_parent_dirs': True, + }, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + +def test_run_repo_create_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + append_only=True, + storage_quota='10G', + make_parent_dirs=True, + local_path=object, + remote_path=object, + ).once() + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=None, + storage_quota=None, + make_parent_dirs=None, + ) + + module.run_repo_create( + repository={ + 'path': 'repo', + 'append_only': True, + 'storage_quota': '10G', + 'make_parent_dirs': True, + }, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index cb3ce67cd..80ce0d94a 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -331,8 +331,8 @@ def test_get_repository_id_with_missing_json_keys_raises(): ) -def test_check_archives_with_progress_passes_through_to_borg(): - config = {} +def test_check_archives_favors_progress_flag_over_config(): + config = {'progress': False} flexmock(module).should_receive('make_check_name_flags').with_args( {'repository'}, () ).and_return(()) @@ -366,6 +366,41 @@ def test_check_archives_with_progress_passes_through_to_borg(): ) +def test_check_archives_defaults_to_progress_config(): + config = {'progress': True} + flexmock(module).should_receive('make_check_name_flags').with_args( + {'repository'}, () + ).and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.environment).should_receive('make_environment') + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'check', '--progress', 'repo'), + output_file=module.DO_NOT_CAPTURE, + environment=None, + working_directory=None, + borg_local_path='borg', + borg_exit_codes=None, + ).once() + + module.check_archives( + repository_path='repo', + config=config, + local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, + repair=None, + only_checks=None, + force=None, + match_archives=None, + max_duration=None, + ), + global_arguments=flexmock(log_json=False), + checks={'repository'}, + archive_filter_flags=(), + ) + + def test_check_archives_with_repair_passes_through_to_borg(): config = {} flexmock(module).should_receive('make_check_name_flags').with_args( diff --git a/tests/unit/borg/test_delete.py b/tests/unit/borg/test_delete.py index 6d8614b66..f6ace21d3 100644 --- a/tests/unit/borg/test_delete.py +++ b/tests/unit/borg/test_delete.py @@ -171,7 +171,7 @@ def test_make_delete_command_includes_lock_wait(): assert command == ('borg', 'delete', '--lock-wait', '5', 'repo') -def test_make_delete_command_includes_list(): +def test_make_delete_command_favors_list_flag_over_config(): flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( 'list', True @@ -184,7 +184,7 @@ def test_make_delete_command_includes_list(): command = module.make_delete_command( repository={'path': 'repo'}, - config={}, + config={'list_details': False}, local_borg_version='1.2.3', delete_arguments=flexmock(list_archives=True, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), @@ -195,6 +195,30 @@ def test_make_delete_command_includes_list(): assert command == ('borg', 'delete', '--list', 'repo') +def test_make_delete_command_defaults_to_list_config(): + flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( + 'list', True + ).and_return(('--list',)) + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo',) + ) + + command = module.make_delete_command( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version='1.2.3', + delete_arguments=flexmock(list_archives=None, force=0, match_archives=None, archive=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + remote_path=None, + ) + + assert command == ('borg', 'delete', '--list', 'repo') + + def test_make_delete_command_includes_force(): flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) @@ -287,8 +311,12 @@ def test_make_delete_command_includes_match_archives(): assert command == ('borg', 'delete', '--match-archives', 'sh:foo*', 'repo') +LOGGING_ANSWER = flexmock() + + def test_delete_archives_with_archive_calls_borg_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -308,6 +336,7 @@ def test_delete_archives_with_archive_calls_borg_delete(): def test_delete_archives_with_match_archives_calls_borg_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -328,6 +357,7 @@ def test_delete_archives_with_match_archives_calls_borg_delete(): @pytest.mark.parametrize('argument_name', module.ARCHIVE_RELATED_ARGUMENT_NAMES[2:]) def test_delete_archives_with_archive_related_argument_calls_borg_delete(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -347,6 +377,7 @@ def test_delete_archives_with_archive_related_argument_calls_borg_delete(argumen def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').once() flexmock(module).should_receive('make_delete_command').never() @@ -367,6 +398,7 @@ def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete def test_delete_archives_calls_borg_delete_with_working_directory(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() command = flexmock() flexmock(module).should_receive('make_delete_command').and_return(command) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 3f2da33a1..1d7888a3e 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -361,7 +361,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): ) -def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): +def test_prune_archives_favors_stats_flag_over_config(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -375,14 +375,35 @@ def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_ module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'statistics': False}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_log_level(): +def test_prune_archives_defaults_to_stats_config(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) + insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) + + prune_arguments = flexmock(stats=None, list_archives=False) + module.prune_archives( + dry_run=False, + repository_path='repo', + config={'statistics': True}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + prune_arguments=prune_arguments, + ) + + +def test_prune_archives_favors_list_flag_over_config(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -396,7 +417,28 @@ def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_l module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'list_details': False}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + prune_arguments=prune_arguments, + ) + + +def test_prune_archives_defaults_to_list_config(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) + insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) + + prune_arguments = flexmock(stats=False, list_archives=None) + module.prune_archives( + dry_run=False, + repository_path='repo', + config={'list_details': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index edd2131b0..9e6618e50 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -436,7 +436,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ) -def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): +def test_transfer_archives_favors_progress_flag_over_config(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) @@ -458,7 +458,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - config={}, + config={'progress': False}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=True, match_archives=None, source_repository=None @@ -467,6 +467,37 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): ) +def test_transfer_archives_defaults_to_progress_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--progress', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.DO_NOT_CAPTURE, + environment=None, + working_directory=None, + borg_local_path='borg', + borg_exit_codes=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + config={'progress': True}, + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + global_arguments=flexmock(log_json=False), + ) + + @pytest.mark.parametrize('argument_name', ('upgrader', 'sort_by', 'first', 'last')) def test_transfer_archives_passes_through_arguments_to_borg(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') From f6929f8891aac09d9d62a32b70d542d0177f5ebf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 14:26:54 -0700 Subject: [PATCH 35/51] Add last couple of missing tests after audit (#303). --- NEWS | 4 ++-- borgmatic/commands/completion/flag.py | 4 ++-- tests/unit/commands/completion/test_flag.py | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 tests/unit/commands/completion/test_flag.py diff --git a/NEWS b/NEWS index 0d2507c1e..7af92c69b 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ 2.0.0.dev0 - * #303: Add command-line flags for every borgmatic configuration option. See the - documentation for more information: + * #303: Deprecate the "--override" flag in favor of direct command-line flags for every borgmatic + configuration option. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides * #303: Add configuration options that serve as defaults for some (but not all) command-line action flags. For example, each entry in "repositories:" now has an "encryption" option that diff --git a/borgmatic/commands/completion/flag.py b/borgmatic/commands/completion/flag.py index 6de6517fc..5a6bc7972 100644 --- a/borgmatic/commands/completion/flag.py +++ b/borgmatic/commands/completion/flag.py @@ -1,7 +1,7 @@ def variants(flag_name): ''' - Given an flag name as a string, yield it and any variations that should be complete-able as - well. For instance, for a string like "--foo[0].bar", yield "--foo[0].bar", "--foo[1].bar", ..., + Given a flag name as a string, yield it and any variations that should be complete-able as well. + For instance, for a string like "--foo[0].bar", yield "--foo[0].bar", "--foo[1].bar", ..., "--foo[9].bar". ''' if '[0]' in flag_name: diff --git a/tests/unit/commands/completion/test_flag.py b/tests/unit/commands/completion/test_flag.py new file mode 100644 index 000000000..06e999792 --- /dev/null +++ b/tests/unit/commands/completion/test_flag.py @@ -0,0 +1,20 @@ +from borgmatic.commands.completion import flag as module + + +def test_variants_passes_through_non_list_index_flag_name(): + assert tuple(module.variants('foo')) == ('foo',) + + +def test_variants_broadcasts_list_index_flag_name_with_a_range_of_indices(): + assert tuple(module.variants('foo[0].bar')) == ( + 'foo[0].bar', + 'foo[1].bar', + 'foo[2].bar', + 'foo[3].bar', + 'foo[4].bar', + 'foo[5].bar', + 'foo[6].bar', + 'foo[7].bar', + 'foo[8].bar', + 'foo[9].bar', + ) From 8101e5c56fc1d49024a962effb8fd7a7312bf07a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 15:24:37 -0700 Subject: [PATCH 36/51] Add "list_details" config option support to new "recreate" action (#303). --- borgmatic/borg/recreate.py | 19 +++++++------- borgmatic/commands/arguments.py | 2 +- tests/unit/borg/test_recreate.py | 44 +++++++++++++++++++++++--------- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index bea11b587..15969f5f5 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -22,24 +22,25 @@ def recreate_archive( patterns=None, ): ''' - Given a local or remote repository path, an archive name, a configuration dict, - the local Borg version string, an argparse.Namespace of recreate arguments, - an argparse.Namespace of global arguments, optional local and remote Borg paths. - - Executes the recreate command with the given arguments. + Given a local or remote repository path, an archive name, a configuration dict, the local Borg + version string, an argparse.Namespace of recreate arguments, an argparse.Namespace of global + arguments, optional local and remote Borg paths, executes the recreate command with the given + arguments. ''' - lock_wait = config.get('lock_wait', None) exclude_flags = make_exclude_flags(config) compression = config.get('compression', None) chunker_params = config.get('chunker_params', None) - # Available recompress MODES: 'if-different' (default), 'always', 'never' + # Available recompress MODES: "if-different", "always", "never" (default) recompress = config.get('recompress', None) # Write patterns to a temporary file and use that file with --patterns-from. patterns_file = write_patterns_file( patterns, borgmatic.config.paths.get_working_directory(config) ) + list_files = ( + config.get('list_details') if recreate_arguments.list is None else recreate_arguments.list + ) recreate_command = ( (local_path, 'recreate') @@ -55,10 +56,10 @@ def recreate_archive( '--filter', make_list_filter_flags(local_borg_version, global_arguments.dry_run), ) - if recreate_arguments.list + if list_files else () ) - # Flag --target works only for a single archive + # Flag --target works only for a single archive. + (('--target', recreate_arguments.target) if recreate_arguments.target and archive else ()) + ( ('--comment', shlex.quote(recreate_arguments.comment)) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8e3241d77..b4623a10b 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,7 +288,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'statistics', 'list_files'} +OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'statistics', 'list_details'} def make_argument_description(schema, flag_name): diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index fe2739b75..dd91218d4 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -20,16 +20,6 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes ).once() -def mock_dependencies(): - flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) - flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) - flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') - flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive( - 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - - def test_recreate_archive_dry_run_skips_execution(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) @@ -235,7 +225,7 @@ def test_recreate_with_log_json(): ) -def test_recreate_with_list_filter_flags(): +def test_recreate_archive_favors_list_flag_over_config(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) @@ -250,7 +240,7 @@ def test_recreate_with_list_filter_flags(): module.recreate_archive( repository='repo', archive='archive', - config={}, + config={'list_details': False}, local_borg_version='1.2.3', recreate_arguments=flexmock( list=True, @@ -265,6 +255,36 @@ def test_recreate_with_list_filter_flags(): ) +def test_recreate_archive_defaults_to_list_config(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) + flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-') + insert_execute_command_mock( + ('borg', 'recreate', '--list', '--filter', 'AME+-', 'repo::archive') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'list_details': True}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + def test_recreate_with_patterns_from_flag(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') From fd8c11eb0a02bac3e76ebce235fd9b124e1cf9e8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 21:59:47 -0700 Subject: [PATCH 37/51] Add documentation for "native" command-line overrides without --override (#303). --- docs/how-to/make-per-application-backups.md | 79 ++++++++++++++++++--- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 095fef5f9..db3819996 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -482,16 +482,83 @@ applications, but then set the repository for each application at runtime. Or you might want to try a variant of an option for testing purposes without actually touching your configuration file. +New in version 2.0.0 Whatever the reason, you can override borgmatic configuration options at the -command-line via the `--override` flag. Here's an example: +command-line, as there's a command-line flag corresponding to every +configuration option (with its underscores converted to dashes). + +For instance, to override the `compression` configuration option, use the +corresponding `--compression` flag on the command-line: + +```bash +borgmatic create --compression zstd +``` + +What this does is load your given configuration files and for each one, disregard +the configured value for the `compression` option and use the value given on the +command-line instead—but just for the duration of the borgmatic run. + +You can override nested configuration options too by separating such option +names with a period. For instance: + +```bash +borgmatic create --bootstrap.store-config-files false +``` + +You can even set complex option data structures by using inline YAML syntax. For +example, set the `repositories` option with a YAML list of key/value pairs: + +```bash +borgmatic create --repositories "[{path: /mnt/backup, label: local}]" +``` + +If your override value contains characters like colons or spaces, then you'll +need to use quotes for it to parse correctly. + +You can also set individual nested options within existing list elements: + +```bash +borgmatic create --repositories[0].path /mnt/backup +``` + +This updates the `path` option for the first repository in `repositories`. +Change the `[0]` index as needed to address different list elements. And note +that this only works for elements already set in configuration; you can't append +new list elements from the command-line. + +See the [command-line reference +documentation](https://torsion.org/borgmatic/docs/reference/command-line/) for +the full set of available arguments, including examples of each for the complex +values. + +There are a handful of configuration options that don't have corresponding +command-line flags at the global scope, but instead have flags within individual +borgmatic actions. For instance, the `list_details` option can be overridden by +the `--list` flag that's only present on particular actions. Similarly with +`progress` and `--progress`, `statistics` and `--stats`, and `match_archives` +and `--match-archives`. + +An alternate to command-line overrides is passing in your values via +[environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). + + +### Deprecated overrides + +Prior to version 2.0.0 +Configuration overrides were performed with an `--override` flag. You can still +use `--override` with borgmatic 2.0.0+, but it's deprecated in favor of the new +command-line flags described above. + +Here's an example of `--override`: ```bash borgmatic create --override remote_path=/usr/local/bin/borg1 ``` -What this does is load your configuration files and for each one, disregard -the configured value for the `remote_path` option and use the value of -`/usr/local/bin/borg1` instead. +What this does is load your given configuration files and for each one, disregard +the configured value for the `remote_path` option and use the value given on the +command-line instead—but just for the duration of the borgmatic run. You can even override nested values or multiple values at once. For instance: @@ -540,10 +607,6 @@ reference](https://torsion.org/borgmatic/docs/reference/configuration/) for which options are list types. (YAML list values look like `- this` with an indentation and a leading dash.) -An alternate to command-line overrides is passing in your values via -[environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). - ## Constant interpolation From cf477bdc1c65e6e7210a7ee6049b1752f47f6806 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 31 Mar 2025 11:33:56 -0700 Subject: [PATCH 38/51] Fix broken list_details, progress, and statistics options (#303). --- borgmatic/actions/check.py | 3 +- borgmatic/actions/compact.py | 10 - borgmatic/actions/config/bootstrap.py | 1 - borgmatic/actions/create.py | 25 +-- borgmatic/actions/export_tar.py | 5 - borgmatic/actions/extract.py | 5 - borgmatic/actions/transfer.py | 6 + borgmatic/borg/check.py | 11 +- borgmatic/borg/compact.py | 5 +- borgmatic/borg/create.py | 17 +- borgmatic/borg/delete.py | 21 +- borgmatic/borg/export_tar.py | 5 +- borgmatic/borg/extract.py | 9 +- borgmatic/borg/info.py | 4 +- borgmatic/borg/prune.py | 16 +- borgmatic/borg/recreate.py | 9 +- borgmatic/borg/repo_delete.py | 4 +- borgmatic/borg/repo_list.py | 2 +- borgmatic/borg/transfer.py | 11 +- borgmatic/commands/arguments.py | 79 ++++---- borgmatic/commands/borgmatic.py | 18 +- borgmatic/config/arguments.py | 13 +- borgmatic/config/validate.py | 12 +- tests/integration/commands/test_arguments.py | 38 +--- tests/integration/config/test_validate.py | 24 +-- tests/unit/actions/config/test_bootstrap.py | 3 - tests/unit/actions/test_check.py | 7 +- tests/unit/actions/test_compact.py | 93 ++------- tests/unit/actions/test_create.py | 194 +++++++------------ tests/unit/actions/test_export_tar.py | 8 +- tests/unit/actions/test_extract.py | 2 - tests/unit/actions/test_prune.py | 10 +- tests/unit/actions/test_transfer.py | 21 +- tests/unit/borg/test_check.py | 53 +---- tests/unit/borg/test_compact.py | 26 ++- tests/unit/borg/test_create.py | 28 ++- tests/unit/borg/test_delete.py | 52 ++--- tests/unit/borg/test_export_tar.py | 15 +- tests/unit/borg/test_extract.py | 8 +- tests/unit/borg/test_info.py | 2 +- tests/unit/borg/test_prune.py | 108 ++--------- tests/unit/borg/test_recreate.py | 32 +-- tests/unit/borg/test_repo_delete.py | 26 +-- tests/unit/borg/test_repo_list.py | 2 +- tests/unit/borg/test_transfer.py | 35 +--- tests/unit/commands/test_borgmatic.py | 6 +- tests/unit/config/test_arguments.py | 12 ++ 47 files changed, 357 insertions(+), 739 deletions(-) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index b562d760d..afcc92622 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -372,7 +372,7 @@ def collect_spot_check_source_paths( borgmatic.borg.create.make_base_create_command( dry_run=True, repository_path=repository['path'], - config=config, + config=dict(config, list_details=True), patterns=borgmatic.actions.create.process_patterns( borgmatic.actions.create.collect_patterns(config), working_directory, @@ -382,7 +382,6 @@ def collect_spot_check_source_paths( borgmatic_runtime_directory=borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - list_files=True, stream_processes=stream_processes, ) ) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index dbc2e5736..551c680da 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -37,17 +37,7 @@ def run_compact( global_arguments, local_path=local_path, remote_path=remote_path, - progress=( - config.get('progress') - if compact_arguments.progress is None - else compact_arguments.progress - ), cleanup_commits=compact_arguments.cleanup_commits, - threshold=( - config.get('compact_threshold') - if compact_arguments.threshold is None - else compact_arguments.threshold - ), ) else: # pragma: nocover logger.info('Skipping compact (only available/needed in Borg 1.2+)') diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 4a72b2cdd..74714c7e2 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -127,5 +127,4 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): extract_to_stdout=False, destination_path=bootstrap_arguments.destination, strip_components=bootstrap_arguments.strip_components, - progress=bootstrap_arguments.progress, ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index e0ef3b1fd..a92af261c 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -289,6 +289,16 @@ def run_create( ): return + if config.get('list_details') and config.get('progress'): + raise ValueError( + 'With the create action, only one of --list/--files/list_details and --progress/progress can be used.' + ) + + if config.get('list_details') and create_arguments.json: + raise ValueError( + 'With the create action, only one of --list/--files/list_details and --json can be used.' + ) + logger.info(f'Creating archive{dry_run_label}') working_directory = borgmatic.config.paths.get_working_directory(config) @@ -327,22 +337,7 @@ def run_create( borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - progress=( - config.get('progress') - if create_arguments.progress is None - else create_arguments.progress - ), - stats=( - config.get('statistics') - if create_arguments.stats is None - else create_arguments.stats - ), json=create_arguments.json, - list_files=( - config.get('list_details') - if create_arguments.list_files is None - else create_arguments.list_files - ), stream_processes=stream_processes, ) diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index cc18aeeb9..f04b06ff3 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -43,10 +43,5 @@ def run_export_tar( local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, - list_files=( - config.get('list_details') - if export_tar_arguments.list_files is None - else export_tar_arguments.list_files - ), strip_components=export_tar_arguments.strip_components, ) diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 3e42b57d6..feaaac810 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -45,9 +45,4 @@ def run_extract( remote_path=remote_path, destination_path=extract_arguments.destination, strip_components=extract_arguments.strip_components, - progress=( - config.get('progress') - if extract_arguments.progress is None - else extract_arguments.progress - ), ) diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 8a27d16bd..c96015159 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,7 +17,13 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' + if transfer_arguments.archive and config.get('match_archives'): + raise ValueError( + 'With the transfer action, only one of --archive and --match-archives/match_archives can be used.' + ) + logger.info('Transferring archives to repository') + borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index c02b6ec41..935c52c85 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -32,7 +32,7 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument if prefix else ( flags.make_match_archives_flags( - check_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -143,9 +143,6 @@ def check_archives( umask = config.get('umask') borg_exit_codes = config.get('borg_exit_codes') working_directory = borgmatic.config.paths.get_working_directory(config) - progress = ( - config.get('progress') if check_arguments.progress is None else check_arguments.progress - ) if 'data' in checks: checks.add('archives') @@ -173,7 +170,7 @@ def check_archives( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) @@ -182,7 +179,9 @@ def check_archives( full_command, # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. - output_file=(DO_NOT_CAPTURE if check_arguments.repair or progress else None), + output_file=( + DO_NOT_CAPTURE if check_arguments.repair or config.get('progress') else None + ), environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index fd248cbf1..b443e8c0d 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -15,9 +15,7 @@ def compact_segments( global_arguments, local_path='borg', remote_path=None, - progress=False, cleanup_commits=False, - threshold=None, ): ''' Given dry-run flag, a local or remote repository path, a configuration dict, and the local Borg @@ -26,6 +24,7 @@ def compact_segments( umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) extra_borg_options = config.get('extra_borg_options', {}).get('compact', '') + threshold = config.get('compact_threshold') full_command = ( (local_path, 'compact') @@ -33,7 +32,7 @@ def compact_segments( + (('--umask', str(umask)) if umask else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--cleanup-commits',) if cleanup_commits else ()) + (('--threshold', str(threshold)) if threshold else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index ccdb41a97..de5115dcd 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -213,9 +213,7 @@ def make_base_create_command( borgmatic_runtime_directory, local_path='borg', remote_path=None, - progress=False, json=False, - list_files=False, stream_processes=None, ): ''' @@ -293,7 +291,7 @@ def make_base_create_command( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--list', '--filter', list_filter_flags) - if list_files and not json and not progress + if config.get('list_details') and not json and not config.get('progress') else () ) + (('--dry-run',) if dry_run else ()) @@ -361,10 +359,7 @@ def create_archive( borgmatic_runtime_directory, local_path='borg', remote_path=None, - progress=False, - stats=False, json=False, - list_files=False, stream_processes=None, ): ''' @@ -389,28 +384,26 @@ def create_archive( borgmatic_runtime_directory, local_path, remote_path, - progress, json, - list_files, stream_processes, ) if json: output_log_level = None - elif list_files or (stats and not dry_run): + elif config.get('list_details') or (config.get('statistics') and not dry_run): output_log_level = logging.ANSWER else: output_log_level = logging.INFO # The progress output isn't compatible with captured and logged output, as progress messes with # the terminal directly. - output_file = DO_NOT_CAPTURE if progress else None + output_file = DO_NOT_CAPTURE if config.get('progress') else None create_flags += ( (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ()) - + (('--stats',) if stats and not json and not dry_run else ()) + + (('--stats',) if config.get('statistics') and not json and not dry_run else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--json',) if json else ()) ) borg_exit_codes = config.get('borg_exit_codes') diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index f31730ce5..63ee0ef8b 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -34,14 +34,7 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags( - 'list', - ( - config.get('list_details') - if delete_arguments.list_archives is None - else delete_arguments.list_archives - ), - ) + + borgmatic.borg.flags.make_flags('list', config.get('list_details')) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) if delete_arguments.force @@ -55,9 +48,17 @@ def make_delete_command( local_borg_version=local_borg_version, default_archive_name_format='*', ) + + (('--stats',) if config.get('statistics') else ()) + borgmatic.borg.flags.make_flags_from_arguments( delete_arguments, - excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'), + excludes=( + 'list_details', + 'statistics', + 'force', + 'match_archives', + 'archive', + 'repository', + ), ) + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version) ) @@ -105,7 +106,7 @@ def delete_archives( repo_delete_arguments = argparse.Namespace( repository=repository['path'], - list_archives=delete_arguments.list_archives, + list_details=delete_arguments.list_details, force=delete_arguments.force, cache_only=delete_arguments.cache_only, keep_security_info=delete_arguments.keep_security_info, diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index d8283535c..224c07b16 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -20,7 +20,6 @@ def export_tar_archive( local_path='borg', remote_path=None, tar_filter=None, - list_files=False, strip_components=None, ): ''' @@ -43,7 +42,7 @@ def export_tar_archive( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) - + (('--list',) if list_files else ()) + + (('--list',) if config.get('list_details') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) @@ -57,7 +56,7 @@ def export_tar_archive( + (tuple(paths) if paths else ()) ) - if list_files: + if config.get('list_details'): output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 52d7c35f8..aa354cbcf 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -77,7 +77,6 @@ def extract_archive( remote_path=None, destination_path=None, strip_components=None, - progress=False, extract_to_stdout=False, ): ''' @@ -92,8 +91,8 @@ def extract_archive( umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) - if progress and extract_to_stdout: - raise ValueError('progress and extract_to_stdout cannot both be set') + if config.get('progress') and extract_to_stdout: + raise ValueError('progress and extract to stdout cannot both be set') if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version): numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else () @@ -128,7 +127,7 @@ def extract_archive( + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( # Make the repository path absolute so the destination directory used below via changing @@ -148,7 +147,7 @@ def extract_archive( # The progress output isn't compatible with captured and logged output, as progress messes with # the terminal directly. - if progress: + if config.get('progress'): return execute_command( full_command, output_file=DO_NOT_CAPTURE, diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index bde80a972..2d90ef523 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -48,9 +48,7 @@ def make_info_command( if info_arguments.prefix else ( flags.make_match_archives_flags( - info_arguments.match_archives - or info_arguments.archive - or config.get('match_archives'), + info_arguments.archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index b29cbe67e..34530eeb4 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -41,7 +41,7 @@ def make_prune_flags(config, prune_arguments, local_borg_version): if prefix else ( flags.make_match_archives_flags( - prune_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -66,12 +66,6 @@ def prune_archives( borgmatic.logger.add_custom_log_levels() umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) - stats = config.get('statistics') if prune_arguments.stats is None else prune_arguments.stats - list_archives = ( - config.get('list_details') - if prune_arguments.list_archives is None - else prune_arguments.list_archives - ) extra_borg_options = config.get('extra_borg_options', {}).get('prune', '') full_command = ( @@ -83,7 +77,7 @@ def prune_archives( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--stats',) - if stats + if config.get('statistics') and not dry_run and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version) else () @@ -91,16 +85,16 @@ def prune_archives( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + flags.make_flags_from_arguments( prune_arguments, - excludes=('repository', 'match_archives', 'stats', 'list_archives'), + excludes=('repository', 'match_archives', 'statistics', 'list_details'), ) - + (('--list',) if list_archives else ()) + + (('--list',) if config.get('list_details') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) - if stats or list_archives: + if config.get('statistics') or config.get('list_details'): output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index 15969f5f5..e6f1e04d1 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -38,9 +38,6 @@ def recreate_archive( patterns_file = write_patterns_file( patterns, borgmatic.config.paths.get_working_directory(config) ) - list_files = ( - config.get('list_details') if recreate_arguments.list is None else recreate_arguments.list - ) recreate_command = ( (local_path, 'recreate') @@ -56,7 +53,7 @@ def recreate_archive( '--filter', make_list_filter_flags(local_borg_version, global_arguments.dry_run), ) - if list_files + if config.get('list_details') else () ) # Flag --target works only for a single archive. @@ -71,12 +68,10 @@ def recreate_archive( + (('--chunker-params', chunker_params) if chunker_params else ()) + ( flags.make_match_archives_flags( - recreate_arguments.match_archives or archive or config.get('match_archives'), + archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) - if recreate_arguments.match_archives - else () ) + (('--recompress', recompress) if recompress else ()) + exclude_flags diff --git a/borgmatic/borg/repo_delete.py b/borgmatic/borg/repo_delete.py index fa66d3c05..dee5a1a9f 100644 --- a/borgmatic/borg/repo_delete.py +++ b/borgmatic/borg/repo_delete.py @@ -39,14 +39,14 @@ def make_repo_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', repo_delete_arguments.list_archives) + + borgmatic.borg.flags.make_flags('list', config.get('list_details')) + ( (('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ())) if repo_delete_arguments.force else () ) + borgmatic.borg.flags.make_flags_from_arguments( - repo_delete_arguments, excludes=('list_archives', 'force', 'repository') + repo_delete_arguments, excludes=('list_details', 'force', 'repository') ) + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version) ) diff --git a/borgmatic/borg/repo_list.py b/borgmatic/borg/repo_list.py index 9722758fe..9f05adbff 100644 --- a/borgmatic/borg/repo_list.py +++ b/borgmatic/borg/repo_list.py @@ -113,7 +113,7 @@ def make_repo_list_command( if repo_list_arguments.prefix else ( flags.make_match_archives_flags( - repo_list_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 528cd64cb..5e16efceb 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -40,9 +40,7 @@ def transfer_archives( ) or ( flags.make_match_archives_flags( - transfer_arguments.match_archives - or transfer_arguments.archive - or config.get('match_archives'), + transfer_arguments.archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -52,16 +50,11 @@ def transfer_archives( + flags.make_flags('other-repo', transfer_arguments.source_repository) + flags.make_flags('dry-run', dry_run) ) - progress = ( - config.get('progress') - if transfer_arguments.progress is None - else transfer_arguments.progress - ) return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=DO_NOT_CAPTURE if progress else None, + output_file=DO_NOT_CAPTURE if config.get('progress') else None, environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index b4623a10b..c3b61f423 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -709,7 +709,7 @@ def make_parsers(schema, unparsed_arguments): ) transfer_group.add_argument( '--progress', - default=False, + default=None, action='store_true', help='Display progress as each archive is transferred', ) @@ -776,13 +776,17 @@ def make_parsers(schema, unparsed_arguments): ) prune_group.add_argument( '--stats', - dest='stats', - default=False, + dest='statistics', + default=None, action='store_true', help='Display statistics of the pruned archive [Borg 1 only]', ) prune_group.add_argument( - '--list', dest='list_archives', action='store_true', help='List archives kept/pruned' + '--list', + dest='list_details', + default=None, + action='store_true', + help='List archives kept/pruned', ) prune_group.add_argument( '--oldest', @@ -820,8 +824,7 @@ def make_parsers(schema, unparsed_arguments): ) compact_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress as each segment is compacted', ) @@ -835,7 +838,7 @@ def make_parsers(schema, unparsed_arguments): compact_group.add_argument( '--threshold', type=int, - dest='threshold', + dest='compact_threshold', help='Minimum saved space percentage threshold for compacting a segment, defaults to 10', ) compact_group.add_argument( @@ -856,20 +859,24 @@ def make_parsers(schema, unparsed_arguments): ) create_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is backed up', ) create_group.add_argument( '--stats', - dest='stats', - default=False, + dest='statistics', + default=None, action='store_true', help='Display statistics of archive', ) create_group.add_argument( - '--list', '--files', dest='list_files', action='store_true', help='Show per-file details' + '--list', + '--files', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) create_group.add_argument( '--json', dest='json', default=False, action='store_true', help='Output results as JSON' @@ -890,8 +897,7 @@ def make_parsers(schema, unparsed_arguments): ) check_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is checked', ) @@ -948,12 +954,15 @@ def make_parsers(schema, unparsed_arguments): ) delete_group.add_argument( '--list', - dest='list_archives', + dest='list_details', + default=None, action='store_true', help='Show details for the deleted archives', ) delete_group.add_argument( '--stats', + dest='statistics', + default=None, action='store_true', help='Display statistics for the deleted archives', ) @@ -1058,8 +1067,7 @@ def make_parsers(schema, unparsed_arguments): ) extract_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is extracted', ) @@ -1134,8 +1142,7 @@ def make_parsers(schema, unparsed_arguments): ) config_bootstrap_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is extracted', ) @@ -1228,7 +1235,12 @@ def make_parsers(schema, unparsed_arguments): '--tar-filter', help='Name of filter program to pipe data through' ) export_tar_group.add_argument( - '--list', '--files', dest='list_files', action='store_true', help='Show per-file details' + '--list', + '--files', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) export_tar_group.add_argument( '--strip-components', @@ -1339,7 +1351,8 @@ def make_parsers(schema, unparsed_arguments): ) repo_delete_group.add_argument( '--list', - dest='list_archives', + dest='list_details', + default=None, action='store_true', help='Show details for the archives in the given repository', ) @@ -1770,7 +1783,11 @@ def make_parsers(schema, unparsed_arguments): help='Archive name, hash, or series to recreate', ) recreate_group.add_argument( - '--list', dest='list', action='store_true', help='Show per-file details' + '--list', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) recreate_group.add_argument( '--target', @@ -1865,15 +1882,6 @@ def parse_arguments(schema, *unparsed_arguments): f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}" ) - if 'create' in arguments and arguments['create'].list_files and arguments['create'].progress: - raise ValueError( - 'With the create action, only one of --list (--files) and --progress flags can be used.' - ) - if 'create' in arguments and arguments['create'].list_files and arguments['create'].json: - raise ValueError( - 'With the create action, only one of --list (--files) and --json flags can be used.' - ) - if ( ('list' in arguments and 'repo-info' in arguments and arguments['list'].json) or ('list' in arguments and 'info' in arguments and arguments['list'].json) @@ -1881,15 +1889,6 @@ def parse_arguments(schema, *unparsed_arguments): ): raise ValueError('With the --json flag, multiple actions cannot be used together.') - if ( - 'transfer' in arguments - and arguments['transfer'].archive - and arguments['transfer'].match_archives - ): - raise ValueError( - 'With the transfer action, only one of --archive and --match-archives flags can be used.' - ) - if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives): raise ValueError( 'With the list action, only one of --prefix or --match-archives flags can be used.' diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c457c8beb..28a92b533 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -600,14 +600,14 @@ def run_actions( ) -def load_configurations(config_filenames, global_arguments, overrides=None, resolve_env=True): +def load_configurations(config_filenames, arguments, overrides=None, resolve_env=True): ''' - Given a sequence of configuration filenames, global arguments as an argparse.Namespace, a - sequence of configuration file override strings in the form of "option.suboption=value", and - whether to resolve environment variables, load and validate each configuration file. Return the - results as a tuple of: dict of configuration filename to corresponding parsed configuration, a - sequence of paths for all loaded configuration files (including includes), and a sequence of - logging.LogRecord instances containing any parse errors. + Given a sequence of configuration filenames, arguments as a dict from action name to + argparse.Namespace, a sequence of configuration file override strings in the form of + "option.suboption=value", and whether to resolve environment variables, load and validate each + configuration file. Return the results as a tuple of: dict of configuration filename to + corresponding parsed configuration, a sequence of paths for all loaded configuration files + (including includes), and a sequence of logging.LogRecord instances containing any parse errors. Log records are returned here instead of being logged directly because logging isn't yet initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this @@ -635,7 +635,7 @@ def load_configurations(config_filenames, global_arguments, overrides=None, reso configs[config_filename], paths, parse_logs = validate.parse_configuration( config_filename, validate.schema_filename(), - global_arguments, + arguments, overrides, resolve_env, ) @@ -1010,7 +1010,7 @@ def main(extra_summary_logs=[]): # pragma: no cover config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( config_filenames, - global_arguments, + arguments, global_arguments.overrides, resolve_env=global_arguments.resolve_env and not validate, ) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 85c8eef97..3252d779f 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -160,15 +160,16 @@ def prepare_arguments_for_config(global_arguments, schema): return tuple(prepared_values) -def apply_arguments_to_config(config, schema, global_arguments): # pragma: no cover +def apply_arguments_to_config(config, schema, arguments): ''' - Given a configuration dict, a corresponding configuration schema dict, and global arguments as - an argparse.Namespace, set those given argument values into their corresponding configuration - options in the configuration dict. + Given a configuration dict, a corresponding configuration schema dict, and arguments as a dict + from action name to argparse.Namespace, set those given argument values into their corresponding + configuration options in the configuration dict. This supports argument flags of the from "--foo.bar.baz" where each dotted component is a nested configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list element in the configuration. ''' - for keys, value in prepare_arguments_for_config(global_arguments, schema): - set_values(config, keys, value) + for action_arguments in arguments.values(): + for keys, value in prepare_arguments_for_config(action_arguments, schema): + set_values(config, keys, value) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 98bf080cd..9cb02243c 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -97,14 +97,14 @@ def apply_logical_validation(config_filename, parsed_configuration): def parse_configuration( - config_filename, schema_filename, global_arguments, overrides=None, resolve_env=True + config_filename, schema_filename, arguments, overrides=None, resolve_env=True ): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML - rendition of JSON Schema format, global arguments as an argparse.Namespace, a sequence of - configuration file override strings in the form of "option.suboption=value", and whether to - resolve environment variables, return the parsed configuration as a data structure of nested - dicts and lists corresponding to the schema. Example return value. + rendition of JSON Schema format, arguments as dict from action name to argparse.Namespace, a + sequence of configuration file override strings in the form of "option.suboption=value", and + whether to resolve environment variables, return the parsed configuration as a data structure of + nested dicts and lists corresponding to the schema. Example return value. Example return value: @@ -129,7 +129,7 @@ def parse_configuration( except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) - borgmatic.config.arguments.apply_arguments_to_config(config, schema, global_arguments) + borgmatic.config.arguments.apply_arguments_to_config(config, schema, arguments) override.apply_overrides(config, schema, overrides) constants.apply_constants(config, config.get('constants') if config else {}) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index ed6a98922..a357e1b05 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -271,11 +271,11 @@ def test_parse_arguments_with_no_actions_passes_argument_to_relevant_actions(): arguments = module.parse_arguments({}, '--stats', '--list') assert 'prune' in arguments - assert arguments['prune'].stats - assert arguments['prune'].list_archives + assert arguments['prune'].statistics + assert arguments['prune'].list_details assert 'create' in arguments - assert arguments['create'].stats - assert arguments['create'].list_files + assert arguments['create'].statistics + assert arguments['create'].list_details assert 'check' in arguments @@ -557,20 +557,6 @@ def test_parse_arguments_with_list_flag_but_no_relevant_action_raises_value_erro module.parse_arguments({}, '--list', 'repo-create') -def test_parse_arguments_disallows_list_with_progress_for_create_action(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments({}, 'create', '--list', '--progress') - - -def test_parse_arguments_disallows_list_with_json_for_create_action(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments({}, 'create', '--list', '--json') - - def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) @@ -599,22 +585,6 @@ def test_parse_arguments_disallows_json_with_both_repo_info_and_info(): module.parse_arguments({}, 'repo-info', 'info', '--json') -def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments( - {}, - 'transfer', - '--source-repository', - 'source.borg', - '--archive', - 'foo', - '--match-archives', - 'sh:*bar', - ) - - def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 61d12d7fb..e74989c69 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -59,7 +59,7 @@ def test_parse_configuration_transforms_file_into_mapping(): ) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { @@ -89,7 +89,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): ) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { @@ -123,7 +123,9 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): ''', ) - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock()) + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_inlines_include_inside_deprecated_section(): @@ -150,7 +152,7 @@ def test_parse_configuration_inlines_include_inside_deprecated_section(): builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { @@ -188,7 +190,7 @@ def test_parse_configuration_merges_include(): builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { @@ -205,7 +207,7 @@ def test_parse_configuration_merges_include(): def test_parse_configuration_raises_for_missing_config_file(): with pytest.raises(FileNotFoundError): module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) @@ -219,7 +221,7 @@ def test_parse_configuration_raises_for_missing_schema_file(): with pytest.raises(FileNotFoundError): module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) @@ -228,7 +230,7 @@ def test_parse_configuration_raises_for_syntax_error(): with pytest.raises(ValueError): module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) @@ -243,7 +245,7 @@ def test_parse_configuration_raises_for_validation_error(): with pytest.raises(module.Validation_error): module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) @@ -263,7 +265,7 @@ def test_parse_configuration_applies_overrides(): config, config_paths, logs = module.parse_configuration( '/tmp/config.yaml', '/tmp/schema.yaml', - global_arguments=flexmock(), + arguments={'global': flexmock()}, overrides=['local_path=borg2'], ) @@ -293,7 +295,7 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in flexmock(os).should_receive('getenv').replace_with(lambda variable_name, default: default) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index 7a21e7152..554ad05d6 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -267,7 +267,6 @@ def test_run_bootstrap_does_not_raise(): archive='archive', destination='dest', strip_components=1, - progress=False, user_runtime_directory='/borgmatic', ssh_command=None, local_path='borg7', @@ -307,7 +306,6 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): archive='archive', destination='dest', strip_components=1, - progress=False, user_runtime_directory='/borgmatic', ssh_command='ssh -i key', local_path='borg7', @@ -339,7 +337,6 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): extract_to_stdout=False, destination_path='dest', strip_components=1, - progress=False, local_path='borg7', remote_path='borg8', ).and_return(extract_process).once() diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 6b16a9014..069fae1e7 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -577,7 +577,6 @@ def test_collect_spot_check_source_paths_parses_borg_output(): borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -625,7 +624,6 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false() borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=False, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -673,7 +671,6 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -721,7 +718,6 @@ def test_collect_spot_check_source_paths_skips_directories(): borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -860,14 +856,13 @@ def test_collect_spot_check_source_paths_uses_working_directory(): flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args( dry_run=True, repository_path='repo', - config=object, + config={'working_directory': '/working/dir', 'list_details': True}, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version=object, global_arguments=object, borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 1531fd59d..04a3a25e4 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -9,7 +9,10 @@ def test_compact_actions_calls_hooks_for_configured_repository(): flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() compact_arguments = flexmock( - repository=None, progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=None, + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -34,7 +37,10 @@ def test_compact_runs_with_selected_repository(): flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() compact_arguments = flexmock( - repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -51,84 +57,6 @@ def test_compact_runs_with_selected_repository(): ) -def test_compact_favors_flags_over_config(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive( - 'repositories_match' - ).once().and_return(True) - flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) - flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').with_args( - object, - object, - object, - object, - object, - local_path=object, - remote_path=object, - progress=False, - cleanup_commits=object, - threshold=15, - ).once() - compact_arguments = flexmock( - repository=flexmock(), - progress=False, - cleanup_commits=flexmock(), - threshold=15, - ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - module.run_compact( - config_filename='test.yaml', - repository={'path': 'repo'}, - config={'progress': True, 'compact_threshold': 20}, - local_borg_version=None, - compact_arguments=compact_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - - -def test_compact_favors_defaults_to_config(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive( - 'repositories_match' - ).once().and_return(True) - flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) - flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').with_args( - object, - object, - object, - object, - object, - local_path=object, - remote_path=object, - progress=True, - cleanup_commits=object, - threshold=20, - ).once() - compact_arguments = flexmock( - repository=flexmock(), - progress=None, - cleanup_commits=flexmock(), - threshold=None, - ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - module.run_compact( - config_filename='test.yaml', - repository={'path': 'repo'}, - config={'progress': True, 'compact_threshold': 20}, - local_borg_version=None, - compact_arguments=compact_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - - def test_compact_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) @@ -137,7 +65,10 @@ def test_compact_bails_if_repository_does_not_match(): ).once().and_return(False) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').never() compact_arguments = flexmock( - repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 6f40fb509..6761f8f46 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -443,9 +443,9 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): create_arguments = flexmock( repository=None, progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -484,9 +484,9 @@ def test_run_create_runs_with_selected_repository(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -506,118 +506,6 @@ def test_run_create_runs_with_selected_repository(): ) -def test_run_create_favors_flags_over_config(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive( - 'repositories_match' - ).once().and_return(True) - flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( - flexmock() - ) - flexmock(module.borgmatic.borg.create).should_receive('create_archive').with_args( - object, - object, - object, - object, - object, - object, - object, - local_path=object, - remote_path=object, - progress=False, - stats=False, - json=object, - list_files=False, - stream_processes=object, - ).once() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) - flexmock(module.borgmatic.hooks.dispatch).should_receive( - 'call_hooks_even_if_unconfigured' - ).and_return({}) - flexmock(module).should_receive('collect_patterns').and_return(()) - flexmock(module).should_receive('process_patterns').and_return([]) - flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') - create_arguments = flexmock( - repository=flexmock(), - progress=False, - stats=False, - json=False, - list_files=False, - ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - list( - module.run_create( - config_filename='test.yaml', - repository={'path': 'repo'}, - config={'progress': True, 'statistics': True, 'list_details': True}, - config_paths=['/tmp/test.yaml'], - local_borg_version=None, - create_arguments=create_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - ) - - -def test_run_create_defaults_to_config(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive( - 'repositories_match' - ).once().and_return(True) - flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( - flexmock() - ) - flexmock(module.borgmatic.borg.create).should_receive('create_archive').with_args( - object, - object, - object, - object, - object, - object, - object, - local_path=object, - remote_path=object, - progress=True, - stats=True, - json=object, - list_files=True, - stream_processes=object, - ).once() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) - flexmock(module.borgmatic.hooks.dispatch).should_receive( - 'call_hooks_even_if_unconfigured' - ).and_return({}) - flexmock(module).should_receive('collect_patterns').and_return(()) - flexmock(module).should_receive('process_patterns').and_return([]) - flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') - create_arguments = flexmock( - repository=flexmock(), - progress=True, - stats=True, - json=False, - list_files=True, - ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - list( - module.run_create( - config_filename='test.yaml', - repository={'path': 'repo'}, - config={'progress': True, 'statistics': True, 'list_details': True}, - config_paths=['/tmp/test.yaml'], - local_borg_version=None, - create_arguments=create_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - ) - - def test_run_create_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( @@ -628,9 +516,9 @@ def test_run_create_bails_if_repository_does_not_match(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -650,6 +538,72 @@ def test_run_create_bails_if_repository_does_not_match(): ) +def test_run_create_with_both_list_and_json_errors(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never() + flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + statistics=flexmock(), + json=True, + list_details=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'list_details': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + +def test_run_create_with_both_list_and_progress_errors(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never() + flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + statistics=flexmock(), + json=False, + list_details=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'list_details': True, 'progress': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + def test_run_create_produces_json(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( @@ -673,9 +627,9 @@ def test_run_create_produces_json(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=True, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index b2e91c388..d32aa6fe8 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -13,7 +13,7 @@ def test_run_export_tar_does_not_raise(): paths=flexmock(), destination=flexmock(), tar_filter=flexmock(), - list_files=flexmock(), + list_details=flexmock(), strip_components=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -44,7 +44,6 @@ def test_run_export_tar_favors_flags_over_config(): local_path=object, remote_path=object, tar_filter=object, - list_files=False, strip_components=object, ).once() export_tar_arguments = flexmock( @@ -53,7 +52,7 @@ def test_run_export_tar_favors_flags_over_config(): paths=flexmock(), destination=flexmock(), tar_filter=flexmock(), - list_files=False, + list_details=False, strip_components=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -84,7 +83,6 @@ def test_run_export_tar_defaults_to_config(): local_path=object, remote_path=object, tar_filter=object, - list_files=True, strip_components=object, ).once() export_tar_arguments = flexmock( @@ -93,7 +91,7 @@ def test_run_export_tar_defaults_to_config(): paths=flexmock(), destination=flexmock(), tar_filter=flexmock(), - list_files=None, + list_details=None, strip_components=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index 3707c8878..1504d4cec 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -44,7 +44,6 @@ def test_run_extract_favors_flags_over_config(): remote_path=object, destination_path=object, strip_components=object, - progress=False, ).once() extract_arguments = flexmock( paths=flexmock(), @@ -83,7 +82,6 @@ def test_run_extract_defaults_to_config(): remote_path=object, destination_path=object, strip_components=object, - progress=True, ).once() extract_arguments = flexmock( paths=flexmock(), diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index 762a83c20..b37e50fb5 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -7,7 +7,7 @@ def test_run_prune_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() - prune_arguments = flexmock(repository=None, stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock(repository=None, statistics=flexmock(), list_details=flexmock()) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( @@ -29,7 +29,9 @@ def test_run_prune_runs_with_selected_repository(): 'repositories_match' ).once().and_return(True) flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() - prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock( + repository=flexmock(), statistics=flexmock(), list_details=flexmock() + ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( @@ -51,7 +53,9 @@ def test_run_prune_bails_if_repository_does_not_match(): 'repositories_match' ).once().and_return(False) flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').never() - prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock( + repository=flexmock(), statistics=flexmock(), list_details=flexmock() + ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py index 03d259bec..be4eda257 100644 --- a/tests/unit/actions/test_transfer.py +++ b/tests/unit/actions/test_transfer.py @@ -1,3 +1,4 @@ +import pytest from flexmock import flexmock from borgmatic.actions import transfer as module @@ -6,7 +7,7 @@ from borgmatic.actions import transfer as module def test_run_transfer_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.transfer).should_receive('transfer_archives') - transfer_arguments = flexmock() + transfer_arguments = flexmock(archive=None) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_transfer( @@ -18,3 +19,21 @@ def test_run_transfer_does_not_raise(): local_path=None, remote_path=None, ) + + +def test_run_transfer_with_archive_and_match_archives_raises(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.transfer).should_receive('transfer_archives') + transfer_arguments = flexmock(archive='foo') + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + module.run_transfer( + repository={'path': 'repo'}, + config={'match_archives': 'foo*'}, + local_borg_version=None, + transfer_arguments=transfer_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 80ce0d94a..0a1f5361d 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -155,22 +155,6 @@ def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_arc assert flags == ('--match-archives', 'sh:foo-*') -def test_make_archive_filter_flags_prefers_check_arguments_match_archives_to_config_match_archives(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'baz-*', None, '1.2.3' - ).and_return(('--match-archives', 'sh:baz-*')) - - flags = module.make_archive_filter_flags( - '1.2.3', - {'match_archives': 'bar-{now}', 'prefix': ''}, # noqa: FS003 - ('archives',), - check_arguments=flexmock(match_archives='baz-*'), - ) - - assert flags == ('--match-archives', 'sh:baz-*') - - def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( @@ -331,42 +315,7 @@ def test_get_repository_id_with_missing_json_keys_raises(): ) -def test_check_archives_favors_progress_flag_over_config(): - config = {'progress': False} - flexmock(module).should_receive('make_check_name_flags').with_args( - {'repository'}, () - ).and_return(()) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) - flexmock(module.environment).should_receive('make_environment') - flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'check', '--progress', 'repo'), - output_file=module.DO_NOT_CAPTURE, - environment=None, - working_directory=None, - borg_local_path='borg', - borg_exit_codes=None, - ).once() - - module.check_archives( - repository_path='repo', - config=config, - local_borg_version='1.2.3', - check_arguments=flexmock( - progress=True, - repair=None, - only_checks=None, - force=None, - match_archives=None, - max_duration=None, - ), - global_arguments=flexmock(log_json=False), - checks={'repository'}, - archive_filter_flags=(), - ) - - -def test_check_archives_defaults_to_progress_config(): +def test_check_archives_with_progress_passes_through_to_borg(): config = {'progress': True} flexmock(module).should_receive('make_check_name_flags').with_args( {'repository'}, () diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 882dac691..7fff77286 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -27,7 +27,7 @@ def insert_execute_command_mock( COMPACT_COMMAND = ('borg', 'compact') -def test_compact_segments_calls_borg_with_parameters(): +def test_compact_segments_calls_borg_with_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) @@ -40,7 +40,7 @@ def test_compact_segments_calls_borg_with_parameters(): ) -def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): +def test_compact_segments_with_log_info_calls_borg_with_info_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) @@ -54,7 +54,7 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): ) -def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): +def test_compact_segments_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) @@ -110,7 +110,7 @@ def test_compact_segments_with_exit_codes_calls_borg_using_them(): ) -def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_compact_segments_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) @@ -124,21 +124,20 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter ) -def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): +def test_compact_segments_with_progress_calls_borg_with_progress_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--progress', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, ) -def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_parameter(): +def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--cleanup-commits', 'repo'), logging.INFO) @@ -152,21 +151,20 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p ) -def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): +def test_compact_segments_with_threshold_calls_borg_with_threshold_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--threshold', '20', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - config={}, + config={'compact_threshold': 20}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - threshold=20, ) -def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): +def test_compact_segments_with_umask_calls_borg_with_umask_flags(): config = {'umask': '077'} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.INFO) @@ -180,7 +178,7 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): ) -def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): +def test_compact_segments_with_log_json_calls_borg_with_log_json_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--log-json', 'repo'), logging.INFO) @@ -193,7 +191,7 @@ def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): ) -def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_flags(): config = {'lock_wait': 5} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index ad7c044ea..80b80ee18 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -631,12 +631,12 @@ def test_make_base_create_command_includes_list_flags_in_borg_command(): config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], + 'list_details': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/run/borgmatic', - list_files=True, ) assert create_flags == ('borg', 'create', '--list', '--filter', 'FOO') @@ -962,7 +962,7 @@ def test_make_base_create_command_with_non_existent_directory_and_source_directo ) -def test_create_archive_calls_borg_with_parameters(): +def test_create_archive_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1029,7 +1029,7 @@ def test_create_archive_calls_borg_with_environment(): ) -def test_create_archive_with_log_info_calls_borg_with_info_parameter(): +def test_create_archive_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1096,7 +1096,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): ) -def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): +def test_create_archive_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1196,7 +1196,6 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats(): local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - stats=True, ) @@ -1271,7 +1270,7 @@ def test_create_archive_with_exit_codes_calls_borg_using_them(): ) -def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_output_log_level(): +def test_create_archive_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1296,12 +1295,12 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'statistics': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - stats=True, ) @@ -1334,16 +1333,16 @@ def test_create_archive_with_files_calls_borg_with_answer_output_log_level(): 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'list_details': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - list_files=True, ) -def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list(): +def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_flag_and_no_list(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1369,16 +1368,16 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, ) -def test_create_archive_with_progress_calls_borg_with_progress_parameter(): +def test_create_archive_with_progress_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1403,16 +1402,16 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, ) -def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter(): +def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER processes = flexmock() @@ -1459,12 +1458,12 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, stream_processes=processes, ) @@ -1532,7 +1531,6 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag(): global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', json=True, - stats=True, ) assert json_output == '[]' diff --git a/tests/unit/borg/test_delete.py b/tests/unit/borg/test_delete.py index f6ace21d3..770f411db 100644 --- a/tests/unit/borg/test_delete.py +++ b/tests/unit/borg/test_delete.py @@ -21,7 +21,7 @@ def test_make_delete_command_includes_log_info(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -43,7 +43,7 @@ def test_make_delete_command_includes_log_debug(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -67,7 +67,7 @@ def test_make_delete_command_includes_dry_run(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=True, log_json=False), local_path='borg', remote_path=None, @@ -91,7 +91,7 @@ def test_make_delete_command_includes_remote_path(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path='borg1', @@ -114,7 +114,7 @@ def test_make_delete_command_includes_umask(): repository={'path': 'repo'}, config={'umask': '077'}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -138,7 +138,7 @@ def test_make_delete_command_includes_log_json(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', remote_path=None, @@ -162,7 +162,7 @@ def test_make_delete_command_includes_lock_wait(): repository={'path': 'repo'}, config={'lock_wait': 5}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -171,31 +171,7 @@ def test_make_delete_command_includes_lock_wait(): assert command == ('borg', 'delete', '--lock-wait', '5', 'repo') -def test_make_delete_command_favors_list_flag_over_config(): - flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( - 'list', True - ).and_return(('--list',)) - flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive('make_flags_from_arguments').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo',) - ) - - command = module.make_delete_command( - repository={'path': 'repo'}, - config={'list_details': False}, - local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=True, force=0, match_archives=None, archive=None), - global_arguments=flexmock(dry_run=False, log_json=False), - local_path='borg', - remote_path=None, - ) - - assert command == ('borg', 'delete', '--list', 'repo') - - -def test_make_delete_command_defaults_to_list_config(): +def test_make_delete_command_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( 'list', True @@ -210,7 +186,7 @@ def test_make_delete_command_defaults_to_list_config(): repository={'path': 'repo'}, config={'list_details': True}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=None, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=None, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -231,7 +207,7 @@ def test_make_delete_command_includes_force(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=1, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=1, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -252,7 +228,7 @@ def test_make_delete_command_includes_force_twice(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=2, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=2, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -276,7 +252,7 @@ def test_make_delete_command_includes_archive(): config={}, local_borg_version='1.2.3', delete_arguments=flexmock( - list_archives=False, force=0, match_archives=None, archive='archive' + list_details=False, force=0, match_archives=None, archive='archive' ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -301,7 +277,7 @@ def test_make_delete_command_includes_match_archives(): config={}, local_borg_version='1.2.3', delete_arguments=flexmock( - list_archives=False, force=0, match_archives='sh:foo*', archive='archive' + list_details=False, force=0, match_archives='sh:foo*', archive='archive' ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -390,7 +366,7 @@ def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete config={}, local_borg_version=flexmock(), delete_arguments=flexmock( - list_archives=True, force=False, cache_only=False, keep_security_info=False + list_details=True, force=False, cache_only=False, keep_security_info=False ), global_arguments=flexmock(), ) diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index f45026f35..3fb50129b 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -144,7 +144,7 @@ def test_export_tar_archive_calls_borg_with_umask_flags(): ) -def test_export_tar_archive_calls_borg_with_log_json_parameter(): +def test_export_tar_archive_calls_borg_with_log_json_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -186,7 +186,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_flags(): ) -def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): +def test_export_tar_archive_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -230,7 +230,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_flags(): ) -def test_export_tar_archive_calls_borg_with_dry_run_parameter(): +def test_export_tar_archive_calls_borg_with_dry_run_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -273,7 +273,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_flags(): ) -def test_export_tar_archive_calls_borg_with_list_parameter(): +def test_export_tar_archive_calls_borg_with_list_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -290,14 +290,13 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): archive='archive', paths=None, destination_path='test.tar', - config={}, + config={'list_details': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - list_files=True, ) -def test_export_tar_archive_calls_borg_with_strip_components_parameter(): +def test_export_tar_archive_calls_borg_with_strip_components_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -320,7 +319,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): ) -def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): +def test_export_tar_archive_skips_abspath_for_remote_repository_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index de392f246..a11155f87 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -580,7 +580,7 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): ) -def test_extract_archive_calls_borg_with_progress_parameter(): +def test_extract_archive_calls_borg_with_progress_flag(): flexmock(module.os.path).should_receive('abspath').and_return('repo') flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) @@ -606,10 +606,9 @@ def test_extract_archive_calls_borg_with_progress_parameter(): repository='repo', archive='archive', paths=None, - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, ) @@ -622,10 +621,9 @@ def test_extract_archive_with_progress_and_extract_to_stdout_raises(): repository='repo', archive='archive', paths=None, - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, extract_to_stdout=True, ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index 5866b4abf..23f416bb7 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -380,7 +380,7 @@ def test_make_info_command_with_match_archives_flag_passes_through_to_command(): command = module.make_info_command( repository_path='repo', - config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}', 'match_archives': 'sh:foo-*'}, # noqa: FS003 local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 1d7888a3e..10e8ebf98 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -135,32 +135,6 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): assert result == expected -def test_make_prune_flags_without_prefix_uses_match_archives_flag_instead_of_option(): - config = { - 'archive_name_format': 'bar-{now}', # noqa: FS003 - 'match_archives': 'foo*', - 'keep_daily': 1, - 'prefix': None, - } - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'baz*', 'bar-{now}', '1.2.3' # noqa: FS003 - ).and_return(('--match-archives', 'sh:bar-*')).once() - - result = module.make_prune_flags( - config, flexmock(match_archives='baz*'), local_borg_version='1.2.3' - ) - - expected = ( - '--keep-daily', - '1', - '--match-archives', - 'sh:bar-*', # noqa: FS003 - ) - - assert result == expected - - def test_make_prune_flags_without_prefix_uses_match_archives_option(): config = { 'archive_name_format': 'bar-{now}', # noqa: FS003 @@ -215,7 +189,7 @@ def test_prune_archives_calls_borg_with_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -237,7 +211,7 @@ def test_prune_archives_with_log_info_calls_borg_with_info_flag(): insert_execute_command_mock(PRUNE_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -259,7 +233,7 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_flag(): insert_execute_command_mock(PRUNE_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -280,7 +254,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_flag(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -301,7 +275,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): ).and_return(False) insert_execute_command_mock(('borg1',) + PRUNE_COMMAND[1:] + ('repo',), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -328,7 +302,7 @@ def test_prune_archives_with_exit_codes_calls_borg_using_them(): borg_exit_codes=borg_exit_codes, ) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -349,7 +323,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -361,7 +335,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): ) -def test_prune_archives_favors_stats_flag_over_config(): +def test_prune_archives_with_stats_config_calls_borg_with_stats_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -371,28 +345,7 @@ def test_prune_archives_favors_stats_flag_over_config(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) - prune_arguments = flexmock(stats=True, list_archives=False) - module.prune_archives( - dry_run=False, - repository_path='repo', - config={'statistics': False}, - local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), - prune_arguments=prune_arguments, - ) - - -def test_prune_archives_defaults_to_stats_config(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) - flexmock(module.feature).should_receive('available').with_args( - module.feature.Feature.NO_PRUNE_STATS, '1.2.3' - ).and_return(False) - insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) - - prune_arguments = flexmock(stats=None, list_archives=False) + prune_arguments = flexmock(statistics=None, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -403,7 +356,7 @@ def test_prune_archives_defaults_to_stats_config(): ) -def test_prune_archives_favors_list_flag_over_config(): +def test_prune_archives_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -413,28 +366,7 @@ def test_prune_archives_favors_list_flag_over_config(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) - prune_arguments = flexmock(stats=False, list_archives=True) - module.prune_archives( - dry_run=False, - repository_path='repo', - config={'list_details': False}, - local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), - prune_arguments=prune_arguments, - ) - - -def test_prune_archives_defaults_to_list_config(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) - flexmock(module.feature).should_receive('available').with_args( - module.feature.Feature.NO_PRUNE_STATS, '1.2.3' - ).and_return(False) - insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) - - prune_arguments = flexmock(stats=False, list_archives=None) + prune_arguments = flexmock(statistics=False, list_details=None) module.prune_archives( dry_run=False, repository_path='repo', @@ -456,7 +388,7 @@ def test_prune_archives_with_umask_calls_borg_with_umask_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -477,7 +409,7 @@ def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--log-json', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -499,7 +431,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -520,7 +452,7 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -588,7 +520,7 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag ) prune_arguments = flexmock( - stats=False, list_archives=False, newer='1d', newest='1y', older='1m', oldest='1w' + statistics=False, list_details=False, newer='1d', newest='1y', older='1m', oldest='1w' ) module.prune_archives( dry_run=False, @@ -612,7 +544,7 @@ def test_prune_archives_calls_borg_with_working_directory(): PRUNE_COMMAND + ('repo',), logging.INFO, working_directory='/working/dir' ) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -623,7 +555,7 @@ def test_prune_archives_calls_borg_with_working_directory(): ) -def test_prune_archives_calls_borg_with_flags_and_when_feature_available(): +def test_prune_archives_calls_borg_without_stats_when_feature_is_not_available(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -633,11 +565,11 @@ def test_prune_archives_calls_borg_with_flags_and_when_feature_available(): ).and_return(True) insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.ANSWER) - prune_arguments = flexmock(stats=True, list_archives=False) + prune_arguments = flexmock(statistics=True, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'statistics': True}, local_borg_version='2.0.0b10', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index dd91218d4..c1f135b20 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -225,37 +225,7 @@ def test_recreate_with_log_json(): ) -def test_recreate_archive_favors_list_flag_over_config(): - flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) - flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) - flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive( - 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-') - insert_execute_command_mock( - ('borg', 'recreate', '--list', '--filter', 'AME+-', 'repo::archive') - ) - - module.recreate_archive( - repository='repo', - archive='archive', - config={'list_details': False}, - local_borg_version='1.2.3', - recreate_arguments=flexmock( - list=True, - target=None, - comment=None, - timestamp=None, - match_archives=None, - ), - global_arguments=flexmock(dry_run=False, log_json=False), - local_path='borg', - patterns=None, - ) - - -def test_recreate_archive_defaults_to_list_config(): +def test_recreate_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) diff --git a/tests/unit/borg/test_repo_delete.py b/tests/unit/borg/test_repo_delete.py index 66e737782..4af66085c 100644 --- a/tests/unit/borg/test_repo_delete.py +++ b/tests/unit/borg/test_repo_delete.py @@ -19,7 +19,7 @@ def test_make_repo_delete_command_with_feature_available_runs_borg_repo_delete() repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -40,7 +40,7 @@ def test_make_repo_delete_command_without_feature_available_runs_borg_delete(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -62,7 +62,7 @@ def test_make_repo_delete_command_includes_log_info(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -84,7 +84,7 @@ def test_make_repo_delete_command_includes_log_debug(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -108,7 +108,7 @@ def test_make_repo_delete_command_includes_dry_run(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=True, log_json=False), local_path='borg', remote_path=None, @@ -132,7 +132,7 @@ def test_make_repo_delete_command_includes_remote_path(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path='borg1', @@ -155,7 +155,7 @@ def test_make_repo_delete_command_includes_umask(): repository={'path': 'repo'}, config={'umask': '077'}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -179,7 +179,7 @@ def test_make_repo_delete_command_includes_log_json(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', remote_path=None, @@ -203,7 +203,7 @@ def test_make_repo_delete_command_includes_lock_wait(): repository={'path': 'repo'}, config={'lock_wait': 5}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -225,9 +225,9 @@ def test_make_repo_delete_command_includes_list(): command = module.make_repo_delete_command( repository={'path': 'repo'}, - config={}, + config={'list_details': True}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=True, force=0), + repo_delete_arguments=flexmock(list_details=True, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -248,7 +248,7 @@ def test_make_repo_delete_command_includes_force(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=1), + repo_delete_arguments=flexmock(list_details=False, force=1), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -269,7 +269,7 @@ def test_make_repo_delete_command_includes_force_twice(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=2), + repo_delete_arguments=flexmock(list_details=False, force=2), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, diff --git a/tests/unit/borg/test_repo_list.py b/tests/unit/borg/test_repo_list.py index da750d7ad..73f567f58 100644 --- a/tests/unit/borg/test_repo_list.py +++ b/tests/unit/borg/test_repo_list.py @@ -664,7 +664,7 @@ def test_make_repo_list_command_with_match_archives_calls_borg_with_match_archiv command = module.make_repo_list_command( repository_path='repo', - config={}, + config={'match_archives': 'foo-*'}, local_borg_version='1.2.3', repo_list_arguments=flexmock( archive=None, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 9e6618e50..ced41f3c7 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -193,7 +193,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, repository_path='repo', - config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}', 'match_archives': 'sh:foo*'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None @@ -436,38 +436,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ) -def test_transfer_archives_favors_progress_flag_over_config(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) - flexmock(module.environment).should_receive('make_environment') - flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'transfer', '--progress', '--repo', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, - output_file=module.DO_NOT_CAPTURE, - environment=None, - working_directory=None, - borg_local_path='borg', - borg_exit_codes=None, - ) - - module.transfer_archives( - dry_run=False, - repository_path='repo', - config={'progress': False}, - local_borg_version='2.3.4', - transfer_arguments=flexmock( - archive=None, progress=True, match_archives=None, source_repository=None - ), - global_arguments=flexmock(log_json=False), - ) - - -def test_transfer_archives_defaults_to_progress_flag(): +def test_transfer_archives_with_progress_calls_borg_with_progress_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 18793f19f..57f3cf678 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1578,7 +1578,7 @@ def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env configs, config_paths, logs = tuple( module.load_configurations( ('test.yaml', 'other.yaml'), - global_arguments=flexmock(), + arguments=flexmock(), resolve_env=resolve_env, ) ) @@ -1592,7 +1592,7 @@ def test_load_configurations_logs_warning_for_permission_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(PermissionError) configs, config_paths, logs = tuple( - module.load_configurations(('test.yaml',), global_arguments=flexmock()) + module.load_configurations(('test.yaml',), arguments=flexmock()) ) assert configs == {} @@ -1604,7 +1604,7 @@ def test_load_configurations_logs_critical_for_parse_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError) configs, config_paths, logs = tuple( - module.load_configurations(('test.yaml',), global_arguments=flexmock()) + module.load_configurations(('test.yaml',), arguments=flexmock()) ) assert configs == {} diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py index 18ad7f569..2c07f8156 100644 --- a/tests/unit/config/test_arguments.py +++ b/tests/unit/config/test_arguments.py @@ -189,3 +189,15 @@ def test_prepare_arguments_for_config_skips_option_missing_from_schema(): }, }, ) == ((('other_option',), 'value2'),) + + +def test_apply_arguments_to_config_does_not_raise(): + flexmock(module).should_receive('prepare_arguments_for_config').and_return( + ( + (('foo', 'bar'), 'baz'), + (('one', 'two'), 'three'), + ) + ) + flexmock(module).should_receive('set_values') + + module.apply_arguments_to_config(config={}, schema={}, arguments={'global': flexmock()}) From 10fb02c40ad19762563a8ccd79a84f95efa51c6a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 31 Mar 2025 13:33:39 -0700 Subject: [PATCH 39/51] Fix bootstrap --progress flag (#303). --- borgmatic/actions/config/bootstrap.py | 4 +++- tests/unit/actions/config/test_bootstrap.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 74714c7e2..520bcb1c9 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -119,7 +119,9 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): bootstrap_arguments.repository, archive_name, [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths], - config, + # Only add progress here and not the extract_archive() call above, because progress + # conflicts with extract_to_stdout. + dict(config, progress=bootstrap_arguments.progress or False), local_borg_version, global_arguments, local_path=bootstrap_arguments.local_path, diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index 554ad05d6..ece911977 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -105,7 +105,7 @@ def test_get_config_paths_translates_ssh_command_argument_to_config(): flexmock(module.borgmatic.config.paths).should_receive( 'get_borgmatic_source_directory' ).and_return('/source') - config = flexmock() + config = {} flexmock(module).should_receive('make_bootstrap_config').and_return(config) bootstrap_arguments = flexmock( repository='repo', @@ -271,6 +271,7 @@ def test_run_bootstrap_does_not_raise(): ssh_command=None, local_path='borg7', remote_path='borg8', + progress=None, ) global_arguments = flexmock( dry_run=False, @@ -298,7 +299,7 @@ def test_run_bootstrap_does_not_raise(): def test_run_bootstrap_translates_ssh_command_argument_to_config(): - config = flexmock() + config = {} flexmock(module).should_receive('make_bootstrap_config').and_return(config) flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml']) bootstrap_arguments = flexmock( @@ -310,6 +311,7 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): ssh_command='ssh -i key', local_path='borg7', remote_path='borg8', + progress=None, ) global_arguments = flexmock( dry_run=False, @@ -331,7 +333,7 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): 'repo', 'archive', object, - config, + {'progress': False}, object, object, extract_to_stdout=False, From f166111b9bde9820e40fa1b919d4be6c1022581a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 31 Mar 2025 15:26:24 -0700 Subject: [PATCH 40/51] Fix new "repositories:" sub-options ("append_only", "make_parent_directories", etc.) (#303). --- borgmatic/actions/repo_create.py | 6 +++--- borgmatic/borg/repo_create.py | 4 ++-- borgmatic/commands/arguments.py | 3 +++ borgmatic/config/schema.yaml | 18 +++++++++++++++- tests/unit/actions/test_repo_create.py | 20 ++++++++--------- tests/unit/borg/test_repo_create.py | 30 ++++++++++++++++++++++---- 6 files changed, 61 insertions(+), 20 deletions(-) diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index 5166b9b83..38d35922f 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -52,9 +52,9 @@ def run_repo_create( else repo_create_arguments.storage_quota ), ( - repository.get('make_parent_dirs') - if repo_create_arguments.make_parent_dirs is None - else repo_create_arguments.make_parent_dirs + repository.get('make_parent_directories') + if repo_create_arguments.make_parent_directories is None + else repo_create_arguments.make_parent_directories ), local_path=local_path, remote_path=remote_path, diff --git a/borgmatic/borg/repo_create.py b/borgmatic/borg/repo_create.py index b4cde3946..9ed2619e7 100644 --- a/borgmatic/borg/repo_create.py +++ b/borgmatic/borg/repo_create.py @@ -24,7 +24,7 @@ def create_repository( copy_crypt_key=False, append_only=None, storage_quota=None, - make_parent_dirs=False, + make_parent_directories=False, local_path='borg', remote_path=None, ): @@ -79,7 +79,7 @@ def create_repository( + (('--copy-crypt-key',) if copy_crypt_key else ()) + (('--append-only',) if append_only else ()) + (('--storage-quota', storage_quota) if storage_quota else ()) - + (('--make-parent-dirs',) if make_parent_dirs else ()) + + (('--make-parent-dirs',) if make_parent_directories else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + (('--log-json',) if global_arguments.log_json else ()) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index c3b61f423..6bb4fc466 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -666,6 +666,7 @@ def make_parsers(schema, unparsed_arguments): ) repo_create_group.add_argument( '--append-only', + default=None, action='store_true', help='Create an append-only repository', ) @@ -675,6 +676,8 @@ def make_parsers(schema, unparsed_arguments): ) repo_create_group.add_argument( '--make-parent-dirs', + dest='make_parent_directories', + default=None, action='store_true', help='Create any missing parent directories of the repository directory', ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3242bdbb7..d5c572f07 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -33,6 +33,7 @@ properties: type: object required: - path + additionalProperties: false properties: path: type: string @@ -67,7 +68,7 @@ properties: only used for the repo-create action. Defaults to no quota. example: 5G - make_parent_dirs: + make_parent_directories: type: boolean description: | Whether any missing parent directories of the repository @@ -1187,6 +1188,7 @@ properties: run: [echo Backing up.] bootstrap: type: object + additionalProperties: false properties: store_config_files: type: boolean @@ -1918,6 +1920,7 @@ properties: example: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2 start: type: object + additionalProperties: false properties: title: type: string @@ -1941,6 +1944,7 @@ properties: example: incoming_envelope finish: type: object + additionalProperties: false properties: title: type: string @@ -1964,6 +1968,7 @@ properties: example: incoming_envelope fail: type: object + additionalProperties: false properties: title: type: string @@ -2022,6 +2027,7 @@ properties: example: hwRwoWsXMBWwgrSecfa9EfPey55WSN start: type: object + additionalProperties: false properties: message: type: string @@ -2097,6 +2103,7 @@ properties: example: Pushover Link finish: type: object + additionalProperties: false properties: message: type: string @@ -2172,6 +2179,7 @@ properties: example: Pushover Link fail: type: object + additionalProperties: false properties: message: type: string @@ -2311,6 +2319,7 @@ properties: example: fakekey start: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2319,6 +2328,7 @@ properties: example: STARTED finish: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2327,6 +2337,7 @@ properties: example: FINISH fail: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2358,6 +2369,7 @@ properties: type: array items: type: object + additionalProperties: false required: - url - label @@ -2399,6 +2411,7 @@ properties: start: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2414,6 +2427,7 @@ properties: finish: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2429,6 +2443,7 @@ properties: fail: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2444,6 +2459,7 @@ properties: log: type: object required: ['body'] + additionalProperties: false properties: title: type: string diff --git a/tests/unit/actions/test_repo_create.py b/tests/unit/actions/test_repo_create.py index 665d9db79..0b54818e4 100644 --- a/tests/unit/actions/test_repo_create.py +++ b/tests/unit/actions/test_repo_create.py @@ -15,7 +15,7 @@ def test_run_repo_create_with_encryption_mode_argument_does_not_raise(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -40,7 +40,7 @@ def test_run_repo_create_with_encryption_mode_option_does_not_raise(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -65,7 +65,7 @@ def test_run_repo_create_without_encryption_mode_raises(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) with pytest.raises(ValueError): @@ -93,7 +93,7 @@ def test_run_repo_create_bails_if_repository_does_not_match(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -121,7 +121,7 @@ def test_run_repo_create_favors_flags_over_config(): object, append_only=False, storage_quota=0, - make_parent_dirs=False, + make_parent_directories=False, local_path=object, remote_path=object, ).once() @@ -132,7 +132,7 @@ def test_run_repo_create_favors_flags_over_config(): copy_crypt_key=flexmock(), append_only=False, storage_quota=0, - make_parent_dirs=False, + make_parent_directories=False, ) module.run_repo_create( @@ -140,7 +140,7 @@ def test_run_repo_create_favors_flags_over_config(): 'path': 'repo', 'append_only': True, 'storage_quota': '10G', - 'make_parent_dirs': True, + 'make_parent_directories': True, }, config={}, local_borg_version=None, @@ -165,7 +165,7 @@ def test_run_repo_create_defaults_to_config(): object, append_only=True, storage_quota='10G', - make_parent_dirs=True, + make_parent_directories=True, local_path=object, remote_path=object, ).once() @@ -176,7 +176,7 @@ def test_run_repo_create_defaults_to_config(): copy_crypt_key=flexmock(), append_only=None, storage_quota=None, - make_parent_dirs=None, + make_parent_directories=None, ) module.run_repo_create( @@ -184,7 +184,7 @@ def test_run_repo_create_defaults_to_config(): 'path': 'repo', 'append_only': True, 'storage_quota': '10G', - 'make_parent_dirs': True, + 'make_parent_directories': True, }, config={}, local_borg_version=None, diff --git a/tests/unit/borg/test_repo_create.py b/tests/unit/borg/test_repo_create.py index dc645f12d..9e3a674d0 100644 --- a/tests/unit/borg/test_repo_create.py +++ b/tests/unit/borg/test_repo_create.py @@ -228,7 +228,29 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'append_only': True}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), + encryption_mode='repokey', + append_only=True, + ) + + +def test_create_repository_with_append_only_config_calls_borg_with_append_only_flag(): + insert_repo_info_command_not_found_mock() + insert_repo_create_command_mock(REPO_CREATE_COMMAND + ('--append-only', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + config={'append_only': True}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -252,7 +274,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'storage_quota': '5G'}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -274,11 +296,11 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'make_parent_directories': True}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', - make_parent_dirs=True, + make_parent_directories=True, ) From e96db2e100c20556bac065f6abd76d5ac2cdf3da Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 1 Apr 2025 19:43:56 -0700 Subject: [PATCH 41/51] Fix "progress" option with the "transfer" action (#303). --- borgmatic/borg/transfer.py | 11 +++++++++-- tests/unit/borg/test_transfer.py | 5 ++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 5e16efceb..7b1680353 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -32,11 +32,18 @@ def transfer_archives( + flags.make_flags('remote-path', remote_path) + flags.make_flags('umask', config.get('umask')) + flags.make_flags('log-json', global_arguments.log_json) - + flags.make_flags('lock-wait', config.get('lock_wait', None)) + + flags.make_flags('lock-wait', config.get('lock_wait')) + + flags.make_flags('progress', config.get('progress')) + ( flags.make_flags_from_arguments( transfer_arguments, - excludes=('repository', 'source_repository', 'archive', 'match_archives'), + excludes=( + 'repository', + 'source_repository', + 'archive', + 'match_archives', + 'progress', + ), ) or ( flags.make_match_archives_flags( diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index ced41f3c7..b2b9886d6 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -440,8 +440,11 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('progress', True).and_return( + ('--progress',) + ) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) From 017cbae4f9b3b0435b0b2e59d739f16c58a3a78f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 1 Apr 2025 19:44:47 -0700 Subject: [PATCH 42/51] Fix for the example not showing up in generated config for empty YAML objects (#303). --- borgmatic/config/generate.py | 25 +++++++++++++++---------- tests/unit/config/test_generate.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 5ddc26c04..734457f23 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -53,16 +53,21 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict): source_config = dict(collections.ChainMap(*source_config)) - config = ruamel.yaml.comments.CommentedMap( - [ - ( - field_name, - schema_to_sample_configuration( - sub_schema, (source_config or {}).get(field_name, {}), level + 1 - ), - ) - for field_name, sub_schema in borgmatic.config.schema.get_properties(schema).items() - ] + config = ( + ruamel.yaml.comments.CommentedMap( + [ + ( + field_name, + schema_to_sample_configuration( + sub_schema, (source_config or {}).get(field_name, {}), level + 1 + ), + ) + for field_name, sub_schema in borgmatic.config.schema.get_properties( + schema + ).items() + ] + ) + or example ) indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) add_comments_to_configuration_object( diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index ad90bb5af..ff71b9fb6 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -39,6 +39,35 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): ) +def test_schema_to_sample_configuration_with_empty_object_generates_config_map_with_example(): + schema = { + 'type': 'object', + 'example': { + 'foo': 'Example 1', + 'baz': 'Example 2', + }, + } + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return({}) + flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) + flexmock(module).should_receive('add_comments_to_configuration_object') + + config = module.schema_to_sample_configuration(schema) + + assert config == dict( + [ + ('foo', 'Example 1'), + ('baz', 'Example 2'), + ] + ) + + def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example(): flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') From affe7cdc1b3e8cfa948e10d7dcee6150837a2cfc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 1 Apr 2025 21:05:44 -0700 Subject: [PATCH 43/51] Expose propertyless YAML objects from configuration (e.g. "constants") as command-line flags (#303). --- borgmatic/commands/arguments.py | 10 ++-- tests/integration/commands/test_arguments.py | 16 ++++++ tests/unit/commands/test_arguments.py | 55 +++++++++++++++++--- 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 09ec0baaf..a043ae6ce 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -307,7 +307,7 @@ def make_argument_description(schema, flag_name): if '[0]' in flag_name: description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' - if example and schema_type == 'array': + if example and schema_type in ('array', 'object'): example_buffer = io.StringIO() yaml = ruamel.yaml.YAML(typ='safe') yaml.default_flow_style = True @@ -453,13 +453,15 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names if schema_type == 'object': properties = schema.get('properties') + # If there are child properties, recurse for each one. But if there are no child properties, + # fall through so that a flag gets added below for the (empty) object. if properties: for name, child in properties.items(): add_arguments_from_schema( arguments_group, child, unparsed_arguments, names + (name,) ) - return + return # If this is an "array" type, recurse for each items type child option. Don't return yet so that # a flag also gets added below for the array itself. @@ -485,9 +487,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names metavar = names[-1].upper() description = make_argument_description(schema, flag_name) - # array=str instead of list here to support specifying a list as a YAML string on the + # The ...=str given here is to support specifying an object or an array as a YAML string on the # command-line. - argument_type = borgmatic.config.schema.parse_type(schema_type, array=str) + argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) full_flag_name = f"--{flag_name.replace('_', '-')}" # As a UX nicety, allow boolean options that have a default of false to have command-line flags diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index a357e1b05..1a4ee87d3 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -4,6 +4,22 @@ from flexmock import flexmock from borgmatic.commands import arguments as module +def test_make_argument_description_with_object_adds_example(): + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + 'example': {'bar': 'baz'}, + }, + flag_name='flag', + ) + # Apparently different versions of ruamel.yaml serialize this + # differently. + in ('Thing. Example value: "bar: baz"' 'Thing. Example value: "{bar: baz}"') + ) + + def test_make_argument_description_with_array_adds_example(): assert ( module.make_argument_description( diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 1e5bbe0ba..bf1b4f23c 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -590,6 +590,42 @@ def test_make_argument_description_without_description_bails(): ) +def test_make_argument_description_with_object_adds_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('{foo: example}') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + 'example': {'foo': 'example'}, + }, + flag_name='flag', + ) + == 'Thing. Example value: "{foo: example}"' + ) + + +def test_make_argument_description_with_object_skips_missing_example(): + flexmock(module.ruamel.yaml).should_receive('YAML').never() + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + }, + flag_name='flag', + ) + == 'Thing.' + ) + + def test_make_argument_description_with_array_adds_example(): buffer = flexmock() buffer.should_receive('getvalue').and_return('[example]') @@ -612,9 +648,7 @@ def test_make_argument_description_with_array_adds_example(): def test_make_argument_description_with_array_skips_missing_example(): - yaml = flexmock() - yaml.should_receive('dump').and_return('[example]') - flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + flexmock(module.ruamel.yaml).should_receive('YAML').never() assert ( module.make_argument_description( @@ -951,12 +985,17 @@ def test_add_arguments_from_schema_with_empty_multi_type_raises(): ) -def test_add_arguments_from_schema_with_propertyless_option_does_not_add_flag(): +def test_add_arguments_from_schema_with_propertyless_option_adds_flag(): arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').never() - flexmock(module.borgmatic.config.schema).should_receive('parse_type').never() - arguments_group.should_receive('add_argument').never() - flexmock(module).should_receive('add_array_element_arguments').never() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') module.add_arguments_from_schema( arguments_group=arguments_group, From 4065c5d0f7bb8ca3319c4240e5d1598fc3e32aad Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 1 Apr 2025 23:04:53 -0700 Subject: [PATCH 44/51] Fix use of dashed command-line flags like "--repositories[2].append-only" generated from configuration (#303). --- borgmatic/commands/arguments.py | 35 +++++++++++++------- tests/integration/commands/test_arguments.py | 5 +-- tests/unit/commands/test_arguments.py | 32 ++++++++++++++++++ 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index a043ae6ce..d06f6ec7e 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -359,7 +359,9 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: return - pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + pattern = re.compile( + fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.").replace("_", "-")}$' + ) try: # Find an existing list index flag (and its action) corresponding to the given flag name. @@ -368,7 +370,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): for action in arguments_group._group_actions for action_flag_name in action.option_strings if pattern.match(action_flag_name) - if f'--{flag_name}'.startswith(action_flag_name) + if f'--{flag_name.replace("_", "-")}'.startswith(action_flag_name) ) # Based on the type of the action (e.g. argparse._StoreTrueAction), look up the corresponding @@ -388,16 +390,25 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name: continue - arguments_group.add_argument( - unparsed_flag_name, - action=action_registry_name, - choices=argument_action.choices, - default=argument_action.default, - dest=unparsed_flag_name.lstrip('-'), - nargs=argument_action.nargs, - required=argument_action.nargs, - type=argument_action.type, - ) + if action_registry_name in ('store_true', 'store_false'): + arguments_group.add_argument( + unparsed_flag_name, + action=action_registry_name, + default=argument_action.default, + dest=unparsed_flag_name.lstrip('-'), + required=argument_action.nargs, + ) + else: + arguments_group.add_argument( + unparsed_flag_name, + action=action_registry_name, + choices=argument_action.choices, + default=argument_action.default, + dest=unparsed_flag_name.lstrip('-'), + nargs=argument_action.nargs, + required=argument_action.nargs, + type=argument_action.type, + ) def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 1a4ee87d3..b3ffef0d1 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -51,12 +51,9 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): flexmock(arguments_group).should_receive('add_argument').with_args( '--foo[25].val', action='store_true', - choices=object, - default=object, + default=False, dest='foo[25].val', - nargs=object, required=object, - type=object, ).once() module.add_array_element_arguments( diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index bf1b4f23c..138a3b2e0 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -880,6 +880,38 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_e ) +def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_dashes(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val-and-stuff',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val-and-stuff', + action='store_stuff', + choices=object, + default=object, + dest='foo[25].val-and-stuff', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val-and-stuff', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val_and_stuff', + ) + + def test_add_arguments_from_schema_with_non_dict_schema_bails(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').never() From 364200c65aa2128547bb2ccfd85b4d8020f467d4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Apr 2025 09:37:52 -0700 Subject: [PATCH 45/51] Fix incorrect matching of non-zero array index flags with dashed names (#303). --- borgmatic/commands/arguments.py | 17 ++++++++--------- tests/unit/commands/test_arguments.py | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index d06f6ec7e..9c896c34b 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,7 +288,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'statistics', 'list_details'} +OMITTED_FLAG_NAMES = {'match-archives', 'progress', 'statistics', 'list-details'} def make_argument_description(schema, flag_name): @@ -359,9 +359,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: return - pattern = re.compile( - fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.").replace("_", "-")}$' - ) + pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') try: # Find an existing list index flag (and its action) corresponding to the given flag name. @@ -370,7 +368,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): for action in arguments_group._group_actions for action_flag_name in action.option_strings if pattern.match(action_flag_name) - if f'--{flag_name.replace("_", "-")}'.startswith(action_flag_name) + if f'--{flag_name}'.startswith(action_flag_name) ) # Based on the type of the action (e.g. argparse._StoreTrueAction), look up the corresponding @@ -386,6 +384,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): for unparsed in unparsed_arguments: unparsed_flag_name = unparsed.split('=', 1)[0] + destination_name = unparsed_flag_name.lstrip('-').replace('-', '_') if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name: continue @@ -395,7 +394,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): unparsed_flag_name, action=action_registry_name, default=argument_action.default, - dest=unparsed_flag_name.lstrip('-'), + dest=destination_name, required=argument_action.nargs, ) else: @@ -404,7 +403,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): action=action_registry_name, choices=argument_action.choices, default=argument_action.default, - dest=unparsed_flag_name.lstrip('-'), + dest=destination_name, nargs=argument_action.nargs, required=argument_action.nargs, type=argument_action.type, @@ -488,7 +487,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names names[:-1] + (f'{names[-1]}[0]',) + (name,), ) - flag_name = '.'.join(names) + flag_name = '.'.join(names).replace('_', '-') # Certain options already have corresponding flags on individual actions (like "create # --progress"), so don't bother adding them to the global flags. @@ -501,7 +500,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names # The ...=str given here is to support specifying an object or an array as a YAML string on the # command-line. argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) - full_flag_name = f"--{flag_name.replace('_', '-')}" + full_flag_name = f"--{flag_name}" # As a UX nicety, allow boolean options that have a default of false to have command-line flags # without values. diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 138a3b2e0..1f2457dc7 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -899,7 +899,7 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_d action='store_stuff', choices=object, default=object, - dest='foo[25].val-and-stuff', + dest='foo[25].val_and_stuff', nargs=object, required=object, type=object, @@ -908,7 +908,7 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_d module.add_array_element_arguments( arguments_group=arguments_group, unparsed_arguments=('--foo[25].val-and-stuff', 'fooval', '--bar[1].val', 'barval'), - flag_name='foo[0].val_and_stuff', + flag_name='foo[0].val-and-stuff', ) From bbf6f2771532fd61e09db93a7fca0713d04a91ce Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Apr 2025 17:08:04 -0700 Subject: [PATCH 46/51] For boolean configuration options, add separate "--foo" and "--no-foo" CLI flags (#303). --- borgmatic/commands/arguments.py | 21 +++--- borgmatic/commands/borgmatic.py | 2 +- borgmatic/config/schema.yaml | 36 +---------- borgmatic/logger.py | 9 +-- tests/unit/commands/test_arguments.py | 93 ++++++++++++--------------- tests/unit/test_logger.py | 42 +++++++----- 6 files changed, 86 insertions(+), 117 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 9c896c34b..2e71ed1d2 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -500,20 +500,26 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names # The ...=str given here is to support specifying an object or an array as a YAML string on the # command-line. argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) - full_flag_name = f"--{flag_name}" - # As a UX nicety, allow boolean options that have a default of false to have command-line flags - # without values. - if schema_type == 'boolean' and schema.get('default') is False: + # As a UX nicety, add separate true and false flags for boolean options. + if schema_type == 'boolean': arguments_group.add_argument( - full_flag_name, + f'--{flag_name}', action='store_true', default=None, help=description, ) + no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-') + arguments_group.add_argument( + f'--{no_flag_name}', + dest=flag_name.replace('-', '_'), + action='store_false', + default=None, + help=f'Set the --{flag_name} value to false.', + ) else: arguments_group.add_argument( - full_flag_name, + f'--{flag_name}', type=argument_type, metavar=metavar, help=description, @@ -553,9 +559,6 @@ def make_parsers(schema, unparsed_arguments): action='store_true', help='Go through the motions, but do not actually write to any repositories', ) - global_group.add_argument( - '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output' - ) global_group.add_argument( '-v', '--verbosity', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 097fe488f..5bf41db0f 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -1025,7 +1025,7 @@ def main(extra_summary_logs=[]): # pragma: no cover any_json_flags = any( getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values() ) - color_enabled = should_do_markup(global_arguments.no_color or any_json_flags, configs) + color_enabled = should_do_markup(configs, any_json_flags) try: configure_logging( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d5c572f07..873887350 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -59,7 +59,6 @@ properties: description: | Whether the repository should be created append-only, only used for the repo-create action. Defaults to false. - default: false example: true storage_quota: type: string @@ -74,7 +73,6 @@ properties: Whether any missing parent directories of the repository path should be created, only used for the repo-create action. Defaults to false. - default: false example: true description: | A required list of local or remote repositories with paths and @@ -104,14 +102,12 @@ properties: description: | Stay in same file system; do not cross mount points beyond the given source directories. Defaults to false. - default: false example: true numeric_ids: type: boolean description: | Only store/extract numeric user and group identifiers. Defaults to false. - default: false example: true atime: type: boolean @@ -122,13 +118,11 @@ properties: ctime: type: boolean description: Store ctime into archive. Defaults to true. - default: true example: false birthtime: type: boolean description: | Store birthtime (creation date) into archive. Defaults to true. - default: true example: false read_special: type: boolean @@ -138,14 +132,12 @@ properties: used when backing up special devices such as /dev/zero. Defaults to false. But when a database hook is used, the setting here is ignored and read_special is considered true. - default: false example: true flags: type: boolean description: | Record filesystem flags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true. - default: true example: false files_cache: type: string @@ -219,7 +211,6 @@ properties: Exclude directories that contain a CACHEDIR.TAG file. See http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false. - default: false example: true exclude_if_present: type: array @@ -236,13 +227,11 @@ properties: If true, the exclude_if_present filename is included in backups. Defaults to false, meaning that the exclude_if_present filename is omitted from backups. - default: false example: true exclude_nodump: type: boolean description: | Exclude files with the NODUMP flag. Defaults to false. - default: false example: true borgmatic_source_directory: type: string @@ -274,7 +263,6 @@ properties: description: | If true, then source directories (and root pattern paths) must exist. If they don't, an error is raised. Defaults to false. - default: false example: true encryption_passcommand: type: string @@ -488,21 +476,18 @@ properties: description: | Bypass Borg error about a repository that has been moved. Defaults to false. - default: false example: true unknown_unencrypted_repo_access_is_ok: type: boolean description: | Bypass Borg error about a previously unknown unencrypted repository. Defaults to false. - default: false example: true check_i_know_what_i_am_doing: type: boolean description: | Bypass Borg confirmation about check with repair option. Defaults to false and an interactive prompt from Borg. - default: false example: true extra_borg_options: type: object @@ -828,9 +813,7 @@ properties: color: type: boolean description: | - Apply color to console output. Can be overridden with --no-color - command-line flag. Defaults to true. - default: true + Apply color to console output. Defaults to true. example: false progress: type: boolean @@ -838,7 +821,6 @@ properties: Display progress as each file or archive is processed when running supported actions. Corresponds to the "--progress" flag on those actions. Defaults to false. - default: false example: true statistics: type: boolean @@ -846,7 +828,6 @@ properties: Display statistics for an archive when running supported actions. Corresponds to the "--stats" flag on those actions. Defaults to false. - default: false example: true list_details: type: boolean @@ -854,7 +835,6 @@ properties: Display details for each file or archive as it is processed when running supported actions. Corresponds to the "--list" flag on those actions. Defaults to false. - default: false example: true skip_actions: type: array @@ -1197,7 +1177,6 @@ properties: backup itself. Defaults to true. Changing this to false prevents "borgmatic bootstrap" from extracting configuration files from the backup. - default: true example: false description: | Support for the "borgmatic bootstrap" action, used to extract @@ -1282,7 +1261,6 @@ properties: schema elements. These statements will fail unless the initial connection to the database is made by a superuser. - default: false example: true format: type: string @@ -1521,7 +1499,6 @@ properties: Use the "--add-drop-database" flag with mariadb-dump, causing the database to be dropped right before restore. Defaults to true. - default: true example: false options: type: string @@ -1669,7 +1646,6 @@ properties: Use the "--add-drop-database" flag with mysqldump, causing the database to be dropped right before restore. Defaults to true. - default: true example: false options: type: string @@ -2397,9 +2373,8 @@ properties: send_logs: type: boolean description: | - Send borgmatic logs to Apprise services as part the + Send borgmatic logs to Apprise services as part of the "finish", "fail", and "log" states. Defaults to true. - default: true example: false logs_size_limit: type: integer @@ -2509,14 +2484,12 @@ properties: description: | Verify the TLS certificate of the ping URL host. Defaults to true. - default: true example: false send_logs: type: boolean description: | - Send borgmatic logs to Healthchecks as part the "finish", + Send borgmatic logs to Healthchecks as part of the "finish", "fail", and "log" states. Defaults to true. - default: true example: false ping_body_limit: type: integer @@ -2549,7 +2522,6 @@ properties: the slug URL scheme (https://hc-ping.com// as opposed to https://hc-ping.com/). Defaults to false. - default: false example: true description: | Configuration for a monitoring integration with Healthchecks. Create @@ -2589,7 +2561,6 @@ properties: description: | Verify the TLS certificate of the push URL host. Defaults to true. - default: true example: false description: | Configuration for a monitoring integration with Uptime Kuma using @@ -2626,7 +2597,6 @@ properties: description: | Send borgmatic logs to PagerDuty when a backup errors. Defaults to true. - default: true example: false description: | Configuration for a monitoring integration with PagerDuty. Create an diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 8e327b629..4eb34a34e 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -29,12 +29,13 @@ def interactive_console(): return sys.stderr.isatty() and os.environ.get('TERM') != 'dumb' -def should_do_markup(no_color, configs): +def should_do_markup(configs, json_enabled): ''' - Given the value of the command-line no-color argument, and a dict of configuration filename to - corresponding parsed configuration, determine if we should enable color marking up. + Given a dict of configuration filename to corresponding parsed configuration (which already have + any command-line overrides applied) and whether json is enabled, determine if we should enable + color marking up. ''' - if no_color: + if json_enabled: return False if any(config.get('color', True) is False for config in configs.values()): diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 1f2457dc7..51a647614 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -1121,7 +1121,7 @@ def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_fl ) -def test_add_arguments_from_schema_with_default_false_boolean_adds_valueless_flag(): +def test_add_arguments_from_schema_with_boolean_adds_two_valueless_flags(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help') flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) @@ -1131,59 +1131,12 @@ def test_add_arguments_from_schema_with_default_false_boolean_adds_valueless_fla default=None, help='help', ).once() - flexmock(module).should_receive('add_array_element_arguments') - - module.add_arguments_from_schema( - arguments_group=arguments_group, - schema={ - 'type': 'object', - 'properties': { - 'foo': { - 'type': 'boolean', - 'default': False, - } - }, - }, - unparsed_arguments=(), - ) - - -def test_add_arguments_from_schema_with_default_true_boolean_adds_value_flag(): - arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').and_return('help') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) arguments_group.should_receive('add_argument').with_args( - '--foo', - type=bool, - metavar='FOO', - help='help', - ).once() - flexmock(module).should_receive('add_array_element_arguments') - - module.add_arguments_from_schema( - arguments_group=arguments_group, - schema={ - 'type': 'object', - 'properties': { - 'foo': { - 'type': 'boolean', - 'default': True, - } - }, - }, - unparsed_arguments=(), - ) - - -def test_add_arguments_from_schema_with_defaultless_boolean_adds_value_flag(): - arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').and_return('help') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) - arguments_group.should_receive('add_argument').with_args( - '--foo', - type=bool, - metavar='FOO', - help='help', + '--no-foo', + dest='foo', + action='store_false', + default=None, + help=object, ).once() flexmock(module).should_receive('add_array_element_arguments') @@ -1201,6 +1154,40 @@ def test_add_arguments_from_schema_with_defaultless_boolean_adds_value_flag(): ) +def test_add_arguments_from_schema_with_nested_boolean_adds_two_valueless_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar.baz-quux', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo.bar.no-baz-quux', + dest='foo.bar.baz_quux', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'baz_quux': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + names=('foo', 'bar'), + ) + + def test_add_arguments_from_schema_skips_omitted_flag_name(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help') diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index dffaafc0d..d179735f4 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -44,19 +44,23 @@ def test_interactive_console_true_when_isatty_and_TERM_is_not_dumb(capsys): assert module.interactive_console() is True -def test_should_do_markup_respects_no_color_value(): - flexmock(module.os.environ).should_receive('get').and_return(None) +def test_should_do_markup_respects_json_enabled_value(): + flexmock(module.os.environ).should_receive('get').never() flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=True) is False def test_should_do_markup_respects_config_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': False}}, json_enabled=False) is False + ) flexmock(module).should_receive('interactive_console').and_return(True).once() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': True}}) is True + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': True}}, json_enabled=False) is True + ) def test_should_do_markup_prefers_any_false_config_value(): @@ -65,11 +69,11 @@ def test_should_do_markup_prefers_any_false_config_value(): assert ( module.should_do_markup( - no_color=False, configs={ 'foo.yaml': {'color': True}, 'bar.yaml': {'color': False}, }, + json_enabled=False, ) is False ) @@ -83,14 +87,16 @@ def test_should_do_markup_respects_PY_COLORS_environment_variable(): flexmock(module).should_receive('to_bool').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True -def test_should_do_markup_prefers_no_color_value_to_config_value(): +def test_should_do_markup_prefers_json_enabled_value_to_config_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={'foo.yaml': {'color': True}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': True}}, json_enabled=True) is False + ) def test_should_do_markup_prefers_config_value_to_environment_variables(): @@ -98,7 +104,9 @@ def test_should_do_markup_prefers_config_value_to_environment_variables(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': False}}, json_enabled=False) is False + ) def test_should_do_markup_prefers_no_color_value_to_environment_variables(): @@ -106,14 +114,14 @@ def test_should_do_markup_prefers_no_color_value_to_environment_variables(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_respects_interactive_console_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value(): @@ -124,7 +132,7 @@ def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value(): @@ -132,7 +140,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value(): flexmock(module.os.environ).should_receive('get').with_args('NO_COLOR', None).and_return('True') flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_respects_NO_COLOR_environment_variable(): @@ -140,7 +148,7 @@ def test_should_do_markup_respects_NO_COLOR_environment_variable(): flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_ignores_empty_NO_COLOR_environment_variable(): @@ -148,7 +156,7 @@ def test_should_do_markup_ignores_empty_NO_COLOR_environment_variable(): flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None) flexmock(module).should_receive('interactive_console').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_NO_COLOR_to_PY_COLORS(): @@ -160,7 +168,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_PY_COLORS(): ) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_multi_stream_handler_logs_to_handler_for_log_level(): From d2c3ed26a90877c79fca6756ae9bf0b0a3c3ac26 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Apr 2025 23:15:21 -0700 Subject: [PATCH 47/51] Make a CLI flag for any config option that's a list of scalars (#303). --- borgmatic/commands/arguments.py | 26 +++++---- borgmatic/config/arguments.py | 16 +++--- tests/unit/commands/test_arguments.py | 79 +++++++++++++++++++++------ tests/unit/config/test_arguments.py | 7 +++ 4 files changed, 93 insertions(+), 35 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 2e71ed1d2..c2ebbd885 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -300,12 +300,12 @@ def make_argument_description(schema, flag_name): description = schema.get('description') schema_type = schema.get('type') example = schema.get('example') - - if not description: - return None + pieces = [description] if description else [] if '[0]' in flag_name: - description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + pieces.append( + ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + ) if example and schema_type in ('array', 'object'): example_buffer = io.StringIO() @@ -313,11 +313,9 @@ def make_argument_description(schema, flag_name): yaml.default_flow_style = True yaml.dump(example, example_buffer) - description += f' Example value: "{example_buffer.getvalue().strip()}"' + pieces.append(f'Example value: "{example_buffer.getvalue().strip()}"') - description = description.replace('%', '%%') - - return description + return ' '.join(pieces).replace('%', '%%') def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): @@ -476,7 +474,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names # If this is an "array" type, recurse for each items type child option. Don't return yet so that # a flag also gets added below for the array itself. if schema_type == 'array': - properties = borgmatic.config.schema.get_properties(schema.get('items', {})) + items = schema.get('items', {}) + properties = borgmatic.config.schema.get_properties(items) if properties: for name, child in properties.items(): @@ -486,6 +485,11 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names unparsed_arguments, names[:-1] + (f'{names[-1]}[0]',) + (name,), ) + # If there aren't any children, then this is an array of scalars. Recurse accordingly. + else: + add_arguments_from_schema( + arguments_group, items, unparsed_arguments, names[:-1] + (f'{names[-1]}[0]',) + ) flag_name = '.'.join(names).replace('_', '-') @@ -497,8 +501,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names metavar = names[-1].upper() description = make_argument_description(schema, flag_name) - # The ...=str given here is to support specifying an object or an array as a YAML string on the - # command-line. + # The object=str and array=str given here is to support specifying an object or an array as a + # YAML string on the command-line. argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) # As a UX nicety, add separate true and false flags for boolean options. diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 3252d779f..e23587d73 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -36,15 +36,15 @@ def set_values(config, keys, value): list_key = match.group('list_name') list_index = int(match.group('index')) - if len(keys) == 1: - config[list_key][list_index] = value - - return - - if list_key not in config: - config[list_key] = [] - try: + if len(keys) == 1: + config[list_key][list_index] = value + + return + + if list_key not in config: + config[list_key] = [] + set_values(config[list_key][list_index], keys[1:], value) except IndexError: raise ValueError(f'Argument list index {first_key} is out of range') diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 51a647614..efc437b35 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -577,19 +577,6 @@ def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): module.parse_arguments_for_actions(('config',), action_parsers, global_parser) -def test_make_argument_description_without_description_bails(): - assert ( - module.make_argument_description( - schema={ - 'description': None, - 'type': 'not yours', - }, - flag_name='flag', - ) - is None - ) - - def test_make_argument_description_with_object_adds_example(): buffer = flexmock() buffer.should_receive('getvalue').and_return('{foo: example}') @@ -611,6 +598,26 @@ def test_make_argument_description_with_object_adds_example(): ) +def test_make_argument_description_without_description_and_with_object_sets_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('{foo: example}') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'type': 'object', + 'example': {'foo': 'example'}, + }, + flag_name='flag', + ) + == 'Example value: "{foo: example}"' + ) + + def test_make_argument_description_with_object_skips_missing_example(): flexmock(module.ruamel.yaml).should_receive('YAML').never() @@ -647,6 +654,26 @@ def test_make_argument_description_with_array_adds_example(): ) +def test_make_argument_description_without_description_and_with_array_sets_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('[example]') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) + == 'Example value: "[example]"' + ) + + def test_make_argument_description_with_array_skips_missing_example(): flexmock(module.ruamel.yaml).should_receive('YAML').never() @@ -672,6 +699,15 @@ def test_make_argument_description_with_array_index_in_flag_name_adds_to_descrip ) +def test_make_argument_description_without_description_and_with_array_index_in_flag_name_sets_description(): + assert 'list element' in module.make_argument_description( + schema={ + 'type': 'something', + }, + flag_name='flag[0]', + ) + + def test_make_argument_description_escapes_percent_character(): assert ( module.make_argument_description( @@ -1043,10 +1079,21 @@ def test_add_arguments_from_schema_with_propertyless_option_adds_flag(): ) -def test_add_arguments_from_schema_with_array_adds_flag(): +def test_add_arguments_from_schema_with_array_of_scalars_adds_multiple_flags(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').with_args( + 'integer', object=str, array=str + ).and_return(int) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').with_args( + 'array', object=str, array=str + ).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo[0]', + type=int, + metavar='FOO[0]', + help='help', + ).once() arguments_group.should_receive('add_argument').with_args( '--foo', type=str, @@ -1072,7 +1119,7 @@ def test_add_arguments_from_schema_with_array_adds_flag(): ) -def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): +def test_add_arguments_from_schema_with_array_of_objects_adds_multiple_flags(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( 'help 2' diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py index 2c07f8156..79f0236a8 100644 --- a/tests/unit/config/test_arguments.py +++ b/tests/unit/config/test_arguments.py @@ -50,6 +50,13 @@ def test_set_values_with_list_index_key_out_of_range_raises(): module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) +def test_set_values_with_final_list_index_key_out_of_range_raises(): + config = {'foo': {'bar': [{'option': 'value'}]}} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]'), value=5) + + def test_set_values_with_list_index_key_missing_list_and_out_of_range_raises(): config = {'other': 'value'} From d0a5aa63be3e0b7538a9580dee6440ba26f5c6d3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 09:24:47 -0700 Subject: [PATCH 48/51] Add a TL;DR to NEWS since 2.0.0 is such a huge release and ain't nobody got time for reading a huge changelog. --- NEWS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 723ba5dc8..87c1db3e8 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,7 @@ 2.0.0.dev0 + * TL;DR: More flexible, completely revamped command hooks. All config options settable on the + command-line. Config option defaults for many command-line flags. New "key import" and "recreate" + actions. Almost everything is backwards compatible. * #262: Add a "default_actions" option that supports disabling default actions when borgmatic is run without any command-line arguments. * #303: Deprecate the "--override" flag in favor of direct command-line flags for every borgmatic @@ -32,7 +35,7 @@ * #1048: Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested directories that reside on separate devices/filesystems. * #1050: Fix a failure in the "spot" check when the archive contains a symlink. - * #1051: Add configuration filename to "Successfully ran configuration file" log message. + * #1051: Add configuration filename to the "Successfully ran configuration file" log message. 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the From 9407f246748114fd02fa92c968efd62b02bd2768 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 11:28:32 -0700 Subject: [PATCH 49/51] Fix setting of "--checks" on the command-line (#303). --- borgmatic/actions/check.py | 2 +- borgmatic/config/arguments.py | 7 ++++--- tests/unit/config/test_arguments.py | 28 ++++++++++++++++++++++++++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 1e65eecb0..13023ff99 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -170,7 +170,7 @@ def filter_checks_on_frequency( if calendar.day_name[datetime_now().weekday()] not in days: logger.info( - f"Skipping {check} check due to day of the week; check only runs on {'/'.join(days)} (use --force to check anyway)" + f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)" ) filtered_checks.remove(check) continue diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index e23587d73..d5996f0a9 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -46,7 +46,7 @@ def set_values(config, keys, value): config[list_key] = [] set_values(config[list_key][list_index], keys[1:], value) - except IndexError: + except (IndexError, KeyError): raise ValueError(f'Argument list index {first_key} is out of range') return @@ -75,12 +75,13 @@ def type_for_option(schema, option_keys): for key in option_keys: # Support "name[0]"-style list index syntax. match = LIST_INDEX_KEY_PATTERN.match(key) + properties = borgmatic.config.schema.get_properties(option_schema) try: if match: - option_schema = option_schema['properties'][match.group('list_name')]['items'] + option_schema = properties[match.group('list_name')]['items'] else: - option_schema = option_schema['properties'][key] + option_schema = properties[key] except KeyError: return None diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py index 79f0236a8..fdf5f0f58 100644 --- a/tests/unit/config/test_arguments.py +++ b/tests/unit/config/test_arguments.py @@ -73,6 +73,10 @@ def test_set_values_with_final_list_index_key_adds_it_to_config(): def test_type_for_option_with_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={'type': 'object', 'properties': {'foo': {'type': 'integer'}}}, @@ -83,6 +87,10 @@ def test_type_for_option_with_option_finds_type(): def test_type_for_option_with_nested_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ @@ -98,6 +106,10 @@ def test_type_for_option_with_nested_option_finds_type(): def test_type_for_option_with_missing_nested_option_finds_nothing(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ @@ -113,6 +125,10 @@ def test_type_for_option_with_missing_nested_option_finds_nothing(): def test_type_for_option_with_typeless_nested_option_finds_nothing(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ @@ -125,7 +141,11 @@ def test_type_for_option_with_typeless_nested_option_finds_nothing(): ) -def test_type_for_list_index_option_finds_type(): +def test_type_for_option_with_list_index_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ @@ -138,7 +158,11 @@ def test_type_for_list_index_option_finds_type(): ) -def test_type_for_nested_list_index_option_finds_type(): +def test_type_for_option_with_nested_list_index_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ From 3eabda45f2880c348a7f2ae0076729102f625b70 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 16:21:22 -0700 Subject: [PATCH 50/51] If a boolean option name already starts with "no_", don't add a "--no-no-..." CLI flag (#303). --- borgmatic/commands/arguments.py | 7 +++++- tests/unit/commands/test_arguments.py | 33 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index c2ebbd885..0cdb33576 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -513,7 +513,12 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names default=None, help=description, ) - no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-') + + if names[-1].startswith('no_'): + no_flag_name = '.'.join(names[:-1] + (names[-1][len('no_') :],)).replace('_', '-') + else: + no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-') + arguments_group.add_argument( f'--{no_flag_name}', dest=flag_name.replace('-', '_'), diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index efc437b35..f30942772 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -1235,6 +1235,39 @@ def test_add_arguments_from_schema_with_nested_boolean_adds_two_valueless_flags( ) +def test_add_arguments_from_schema_with_boolean_with_name_prefixed_with_no_adds_two_valueless_flags_and_removes_the_no_for_one(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--no-foo', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + dest='no_foo', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'no_foo': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + ) + + def test_add_arguments_from_schema_skips_omitted_flag_name(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help') From 9ea55d9aa31b82319571f3a88226788d0e643fb4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 16:38:17 -0700 Subject: [PATCH 51/51] Add a documentation note about a limitation: You can't pass flags as values to flags (#303). --- docs/how-to/make-per-application-backups.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index db3819996..a81cd9432 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -538,6 +538,12 @@ the `--list` flag that's only present on particular actions. Similarly with `progress` and `--progress`, `statistics` and `--stats`, and `match_archives` and `--match-archives`. +Also note that if you want to pass a command-line flag itself as a value to one +of these override flags, that may not work. For instance, specifying +`--extra-borg-options.create --no-cache-sync` results in an error, because +`--no-cache-sync` gets interpreted as a borgmatic option (which in this case +doesn't exist) rather than a Borg option. + An alternate to command-line overrides is passing in your values via [environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).