From 0009471f67f5a00f03bcaa925d0d92988f7c85c4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 18:46:13 -0700 Subject: [PATCH 01/68] start work on completion --- borgmatic/commands/arguments.py | 6 ++++++ borgmatic/commands/borgmatic.py | 3 +++ borgmatic/commands/completion.py | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 61b54769..c7198856 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -207,6 +207,12 @@ def make_parsers(): action='store_true', help='Show bash completion script and exit', ) + global_group.add_argument( + '--fish-completion', + default=False, + action='store_true', + help='Show fish completion script and exit', + ) global_group.add_argument( '--version', dest='version', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 999e9d8e..d5fedba7 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -715,6 +715,9 @@ def main(): # pragma: no cover if global_arguments.bash_completion: print(borgmatic.commands.completion.bash_completion()) sys.exit(0) + if global_arguments.fish_completion: + print(borgmatic.commands.completion.fish_completion()) + sys.exit(0) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, parse_logs = load_configurations( diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 1fc976bc..4f262ddd 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -55,3 +55,25 @@ def bash_completion(): '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic', ) ) + +def fish_completion(): + ''' + Return a fish completion script for the borgmatic command. Produce this by introspecting + borgmatic's command-line argument parsers. + ''' + top_level_parser, subparsers = arguments.make_parsers() + global_flags = parser_flags(top_level_parser) + actions = ' '.join(subparsers.choices.keys()) + + # Avert your eyes. + return '\n'.join( + ( + 'function __borgmatic_check_version', + ' set this_script (cat "$BASH_SOURCE" 2> /dev/null)', + ' set installed_script (borgmatic --bash-completion 2> /dev/null)', + ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', + f' echo "{UPGRADE_MESSAGE}"', + ' end', + 'end', + 'function __borgmatic_complete', + )) From 28b152aedd5e460cb16009b339781789650e40e9 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 19:31:42 -0700 Subject: [PATCH 02/68] make upgrade message a template --- borgmatic/commands/completion.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 4f262ddd..13210b02 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,12 +1,13 @@ from borgmatic.commands import arguments -UPGRADE_MESSAGE = ''' -Your bash completions script is from a different version of borgmatic than is +def upgrade_message(language: str, upgrade_command: str, completion_file: str): + return f''' +Your {language} completions script is from a different version of borgmatic than is currently installed. Please upgrade your script so your completions match the command-line flags in your installed borgmatic! Try this to upgrade: - sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE" - source $BASH_SOURCE + {upgrade_command} + source {completion_file} ''' @@ -34,7 +35,7 @@ def bash_completion(): ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - f' then cat << EOF\n{UPGRADE_MESSAGE}\nEOF', + ' then cat << EOF\n{}\nEOF'.format(upgrade_message('bash', 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', '$BASH_SOURCE')), ' fi', '}', 'complete_borgmatic() {', @@ -69,11 +70,11 @@ def fish_completion(): return '\n'.join( ( 'function __borgmatic_check_version', - ' set this_script (cat "$BASH_SOURCE" 2> /dev/null)', - ' set installed_script (borgmatic --bash-completion 2> /dev/null)', + ' set this_script (status current-filename)', + ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - f' echo "{UPGRADE_MESSAGE}"', + ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee (status current-filename)', '(status current-filename)')), ' end', 'end', - 'function __borgmatic_complete', + # 'function __borgmatic_complete', )) From 5678f3a96e9731e3ff747de13440f53aa1572f5c Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 19:44:11 -0700 Subject: [PATCH 03/68] basic working version --- borgmatic/commands/completion.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 13210b02..de0991c9 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,4 +1,5 @@ from borgmatic.commands import arguments +import shlex def upgrade_message(language: str, upgrade_command: str, completion_file: str): return f''' @@ -76,5 +77,12 @@ def fish_completion(): ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee (status current-filename)', '(status current-filename)')), ' end', 'end', - # 'function __borgmatic_complete', - )) + ) + tuple( + '''complete -c borgmatic -n '__borgmatic_check_version' -a '%s' -d %s -f''' + % (action, shlex.quote(subparser.description)) + for action, subparser in subparsers.choices.items() + ) + ( + 'complete -c borgmatic -a "%s" -d "borgmatic actions" -f' % actions, + 'complete -c borgmatic -a "%s" -d "borgmatic global flags" -f' % global_flags, + ) + ) From 25b3db72a0d9bb47df8f44ad86c5a705eb8104ee Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 19:58:22 -0700 Subject: [PATCH 04/68] make more precise, fix the version check fn --- borgmatic/commands/completion.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index de0991c9..da62cccc 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -64,25 +64,29 @@ def fish_completion(): borgmatic's command-line argument parsers. ''' top_level_parser, subparsers = arguments.make_parsers() - global_flags = parser_flags(top_level_parser) actions = ' '.join(subparsers.choices.keys()) # Avert your eyes. return '\n'.join( ( 'function __borgmatic_check_version', - ' set this_script (status current-filename)', + ' set this_script (cat (status current-filename) 2> /dev/null)', ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee (status current-filename)', '(status current-filename)')), ' end', 'end', + '__borgmatic_check_version &', ) + tuple( - '''complete -c borgmatic -n '__borgmatic_check_version' -a '%s' -d %s -f''' + '''complete -c borgmatic -a '%s' -d %s -f''' % (action, shlex.quote(subparser.description)) for action, subparser in subparsers.choices.items() ) + ( 'complete -c borgmatic -a "%s" -d "borgmatic actions" -f' % actions, - 'complete -c borgmatic -a "%s" -d "borgmatic global flags" -f' % global_flags, + ) + tuple( + '''complete -c borgmatic -a '%s' -d %s -f''' + % (option, shlex.quote(action.help)) + for action in top_level_parser._actions + for option in action.option_strings ) ) From 8060586d8b5088c2d96d91a0a34c79eb2bf8763b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 20:05:17 -0700 Subject: [PATCH 05/68] fix the script and drop unneeded options --- borgmatic/commands/completion.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index da62cccc..aadf85e2 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -64,16 +64,16 @@ def fish_completion(): borgmatic's command-line argument parsers. ''' top_level_parser, subparsers = arguments.make_parsers() - actions = ' '.join(subparsers.choices.keys()) # Avert your eyes. return '\n'.join( ( 'function __borgmatic_check_version', - ' set this_script (cat (status current-filename) 2> /dev/null)', + ' set this_filename (status current-filename)', + ' set this_script (cat $this_filename 2> /dev/null)', ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee (status current-filename)', '(status current-filename)')), + ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee $this_filename', '$this_filename')), ' end', 'end', '__borgmatic_check_version &', @@ -81,8 +81,6 @@ def fish_completion(): '''complete -c borgmatic -a '%s' -d %s -f''' % (action, shlex.quote(subparser.description)) for action, subparser in subparsers.choices.items() - ) + ( - 'complete -c borgmatic -a "%s" -d "borgmatic actions" -f' % actions, ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' % (option, shlex.quote(action.help)) From 412d18f2183292c0edd46cd8566aab0328eade50 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 21:31:53 -0700 Subject: [PATCH 06/68] show sub options --- borgmatic/commands/completion.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index aadf85e2..5555c42b 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -79,12 +79,18 @@ def fish_completion(): '__borgmatic_check_version &', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' - % (action, shlex.quote(subparser.description)) - for action, subparser in subparsers.choices.items() + % (actionStr, shlex.quote(subparser.description)) + for actionStr, subparser in subparsers.choices.items() ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' % (option, shlex.quote(action.help)) for action in top_level_parser._actions for option in action.option_strings + ) + tuple( + '''complete -c borgmatic -a '%s' -d %s -f -n "__fish_seen_subcommand_from %s"''' + % (option, shlex.quote(action.help), actionStr) + for actionStr, subparser in subparsers.choices.items() + for action in subparser._actions + for option in action.option_strings ) ) From 2e658cfa5687b90d867ef69baa8be94cc29a802d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 21:57:50 -0700 Subject: [PATCH 07/68] only allow one parser --- borgmatic/commands/completion.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 5555c42b..225fffa7 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -65,6 +65,8 @@ def fish_completion(): ''' top_level_parser, subparsers = arguments.make_parsers() + all_subparsers = ' '.join(action for action in subparsers.choices.keys()) + # Avert your eyes. return '\n'.join( ( @@ -78,8 +80,8 @@ def fish_completion(): 'end', '__borgmatic_check_version &', ) + tuple( - '''complete -c borgmatic -a '%s' -d %s -f''' - % (actionStr, shlex.quote(subparser.description)) + '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' + % (actionStr, shlex.quote(subparser.description), all_subparsers) for actionStr, subparser in subparsers.choices.items() ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' From 77c3161c7774bec3a094da17986e395a638b215c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Apr 2023 08:36:03 -0700 Subject: [PATCH 08/68] Fix canonical home link in README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2af792ba..079ec8ae 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ borgmatic is simple, configuration-driven backup software for servers and workstations. Protect your files with client-side encryption. Backup your databases too. Monitor it all with integrated third-party services. -The canonical home of borgmatic is at https://torsion.org/borgmatic. +The canonical home of borgmatic is at https://torsion.org/borgmatic/ Here's an example configuration file: From d265b6ed6f316a16e9c945829a8da1596841f1e2 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 11:57:16 -0700 Subject: [PATCH 09/68] add comments in generated files --- borgmatic/commands/completion.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 225fffa7..56109f37 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -79,15 +79,21 @@ def fish_completion(): ' end', 'end', '__borgmatic_check_version &', + ) + ( + '# subparser completions', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' % (actionStr, shlex.quote(subparser.description), all_subparsers) for actionStr, subparser in subparsers.choices.items() + ) + ( + '# global flags', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' % (option, shlex.quote(action.help)) for action in top_level_parser._actions for option in action.option_strings + ) + ( + '# subparser flags', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "__fish_seen_subcommand_from %s"''' % (option, shlex.quote(action.help), actionStr) From 23f478ce7454fc715e23dc695abd38884854c0b8 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 12:13:08 -0700 Subject: [PATCH 10/68] use less completion lines --- borgmatic/commands/completion.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 56109f37..a256ad51 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -80,25 +80,24 @@ def fish_completion(): 'end', '__borgmatic_check_version &', ) + ( - '# subparser completions', + '\n# subparser completions', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' % (actionStr, shlex.quote(subparser.description), all_subparsers) for actionStr, subparser in subparsers.choices.items() ) + ( - '# global flags', + '\n# global flags', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' % (option, shlex.quote(action.help)) for action in top_level_parser._actions for option in action.option_strings ) + ( - '# subparser flags', + '\n# subparser flags', ) + tuple( - '''complete -c borgmatic -a '%s' -d %s -f -n "__fish_seen_subcommand_from %s"''' - % (option, shlex.quote(action.help), actionStr) + '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" -f''' + % (' '.join(action.option_strings), shlex.quote(action.help), actionStr) for actionStr, subparser in subparsers.choices.items() for action in subparser._actions - for option in action.option_strings ) ) From 9c77ebb01600b7caa093a4818b3fed662df28183 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 12:15:01 -0700 Subject: [PATCH 11/68] continue deduping --- borgmatic/commands/completion.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index a256ad51..e711b789 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -89,9 +89,8 @@ def fish_completion(): '\n# global flags', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' - % (option, shlex.quote(action.help)) + % (' '.join(action.option_strings), shlex.quote(action.help)) for action in top_level_parser._actions - for option in action.option_strings ) + ( '\n# subparser flags', ) + tuple( From 98e3a81fcf6d5608ce1ebb7d27fcdefc2910d6d8 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 12:42:26 -0700 Subject: [PATCH 12/68] allow file completions as applicable --- borgmatic/commands/completion.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e711b789..e62de01c 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,3 +1,4 @@ +from argparse import Action from borgmatic.commands import arguments import shlex @@ -58,6 +59,15 @@ def bash_completion(): ) ) +def build_fish_flags(action: Action): + ''' + Given an argparse.Action instance, return a string containing the fish flags for that action. + ''' + if action.metavar and action.metavar == 'PATH' or action.metavar == 'FILENAME': + return '-r -F' + else: + return '-f' + def fish_completion(): ''' Return a fish completion script for the borgmatic command. Produce this by introspecting @@ -88,14 +98,14 @@ def fish_completion(): ) + ( '\n# global flags', ) + tuple( - '''complete -c borgmatic -a '%s' -d %s -f''' - % (' '.join(action.option_strings), shlex.quote(action.help)) + '''complete -c borgmatic -a '%s' -d %s %s''' + % (' '.join(action.option_strings), shlex.quote(action.help), build_fish_flags(action)) for action in top_level_parser._actions ) + ( '\n# subparser flags', ) + tuple( - '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" -f''' - % (' '.join(action.option_strings), shlex.quote(action.help), actionStr) + '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" %s''' + % (' '.join(action.option_strings), shlex.quote(action.help), actionStr, build_fish_flags(action)) for actionStr, subparser in subparsers.choices.items() for action in subparser._actions ) From f7e4024fcae5fc1aa836f0cb90f379df65cdb251 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 14:02:06 -0700 Subject: [PATCH 13/68] add to readme --- docs/how-to/set-up-backups.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index be229cb2..098f0139 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -334,10 +334,13 @@ Access](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/293). ### Shell completion -borgmatic includes a shell completion script (currently only for Bash) to +borgmatic includes a shell completion script (currently only for Bash and Fish) to support tab-completing borgmatic command-line actions and flags. Depending on -how you installed borgmatic, this may be enabled by default. But if it's not, -start by installing the `bash-completion` Linux package or the +how you installed borgmatic, this may be enabled by default. + +#### Bash + +If completions aren't enabled, start by installing the `bash-completion` Linux package or the [`bash-completion@2`](https://formulae.brew.sh/formula/bash-completion@2) macOS Homebrew formula. Then, install the shell completion script globally: @@ -362,6 +365,14 @@ borgmatic --bash-completion > ~/.local/share/bash-completion/completions/borgmat Finally, restart your shell (`exit` and open a new shell) so the completions take effect. +#### Fish + +To add completions for fish, install the completions file globally: + +```fish +borgmatic --fish-completion | sudo tee /usr/share/fish/vendor_completions.d/borgmatic.fish +source /usr/share/fish/vendor_completions.d/borgmatic.fish +``` ### Colored output From a60d7fd173a4fa045ac3b17bae864269cb88a59a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 30 Apr 2023 15:43:41 -0700 Subject: [PATCH 14/68] Run "borgmatic borg" action without capturing output so interactive prompts and flags like "--progress" still work. --- NEWS | 2 ++ borgmatic/borg/borg.py | 4 ++-- docs/how-to/monitor-your-backups.md | 2 +- docs/how-to/run-arbitrary-borg-commands.md | 6 +++-- tests/unit/borg/test_borg.py | 28 +++++++++++----------- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/NEWS b/NEWS index 20214674..e196a70f 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,8 @@ * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. + * Run "borgmatic borg" action without capturing output so interactive prompts and flags like + "--progress" still work. 1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index f19d6555..1c41b8ec 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -2,7 +2,7 @@ import logging import borgmatic.logger from borgmatic.borg import environment, flags -from borgmatic.execute import execute_command +from borgmatic.execute import DO_NOT_CAPTURE, execute_command logger = logging.getLogger(__name__) @@ -62,7 +62,7 @@ def run_arbitrary_borg( return execute_command( full_command, - output_log_level=logging.ANSWER, + output_file=DO_NOT_CAPTURE, borg_local_path=local_path, extra_environment=environment.make_environment(storage_config), ) diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index eb7a6200..517f9c79 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -73,7 +73,7 @@ from borgmatic for a configured interval. ### Consistency checks -While not strictly part of monitoring, if you really want confidence that your +While not strictly part of monitoring, if you want confidence that your backups are not only running but are restorable as well, you can configure particular [consistency checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index 0777ebba..ea265eaa 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -89,8 +89,10 @@ borgmatic's `borg` action is not without limitations: * Unlike normal borgmatic actions that support JSON, the `borg` action will not disable certain borgmatic logs to avoid interfering with JSON output. * Unlike other borgmatic actions, the `borg` action captures (and logs) all - output, so interactive prompts or flags like `--progress` will not work as - expected. + output, so interactive prompts and flags like `--progress` will not work as + expected. New in version + 1.7.13 borgmatic now runs the `borg` action without capturing output, + so interactive prompts work. In general, this `borgmatic borg` feature should be considered an escape valve—a feature of second resort. In the long run, it's preferable to wrap diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 5b735960..4c71ce1a 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -15,7 +15,7 @@ def test_run_arbitrary_borg_calls_borg_with_parameters(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -36,7 +36,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--info'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -58,7 +58,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--debug', '--show-rc'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -83,7 +83,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters( flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--lock-wait', '5'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -106,7 +106,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo::archive'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -128,7 +128,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg1', 'break-lock', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg1', extra_environment=None, ) @@ -152,7 +152,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--remote-path', 'borg1'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -174,7 +174,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo', '--progress'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -195,7 +195,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -216,7 +216,7 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg',), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -237,7 +237,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'key', 'export', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -258,7 +258,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository() flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'debug', 'dump-manifest', 'repo', 'path'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -279,7 +279,7 @@ def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repositor flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'debug', 'info'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -300,7 +300,7 @@ def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_bor flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'debug', 'convert-profile', 'in', 'out'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) From 0b397a5bf93e5255f515a59ea49d091db51e90d9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 30 Apr 2023 16:24:10 -0700 Subject: [PATCH 15/68] Fix borgmatic error when not finding the configuration schema for certain "pip install --editable" development installs (#687). --- NEWS | 2 ++ borgmatic/config/validate.py | 5 ++++- tests/unit/config/test_validate.py | 5 ++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index e196a70f..30a2b9b0 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,8 @@ * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. + * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install + --editable" development installs. * Run "borgmatic borg" action without capturing output so interactive prompts and flags like "--progress" still work. diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 537f4bee..b39199fe 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -8,6 +8,7 @@ try: except ModuleNotFoundError: # pragma: nocover import importlib.metadata as importlib_metadata +import borgmatic.config from borgmatic.config import environment, load, normalize, override @@ -25,7 +26,9 @@ def schema_filename(): if path.match('config/schema.yaml') ) except StopIteration: - raise FileNotFoundError('Configuration file schema could not be found') + # If the schema wasn't found in the package's files, this is probably a pip editable + # install, so try a different approach to get the schema. + return os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml') def format_json_error_path_element(path_element): diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index e2b9f98f..e81f2b02 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -16,14 +16,13 @@ def test_schema_filename_finds_schema_path(): assert module.schema_filename() == schema_path -def test_schema_filename_with_missing_schema_path_raises(): +def test_schema_filename_with_missing_schema_path_in_package_still_finds_it_in_config_directory(): flexmock(module.importlib_metadata).should_receive('files').and_return( flexmock(match=lambda path: False, locate=lambda: None), flexmock(match=lambda path: False, locate=lambda: None), ) - with pytest.raises(FileNotFoundError): - assert module.schema_filename() + assert module.schema_filename().endswith('/borgmatic/config/schema.yaml') def test_format_json_error_path_element_formats_array_index(): From 359afe531803938205a648f8f1093343c2360f16 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 3 May 2023 17:16:36 -0700 Subject: [PATCH 16/68] Error if --list is used with --json for create action (#680). --- borgmatic/commands/arguments.py | 4 ++++ tests/integration/commands/test_arguments.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 61b54769..3a8ef2e2 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -878,6 +878,10 @@ def parse_arguments(*unparsed_arguments): raise ValueError( 'With the create action, only one of --list (--files) and --progress flags can be used.' ) + if 'create' in arguments and arguments['create'].list_files and arguments['create'].json: + raise ValueError( + 'With the create action, only one of --list (--files) and --json flags can be used.' + ) if ( ('list' in arguments and 'rinfo' in arguments and arguments['list'].json) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 1fc8f8c8..4990fc4f 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -422,6 +422,13 @@ def test_parse_arguments_disallows_list_with_progress_for_create_action(): module.parse_arguments('create', '--list', '--progress') +def test_parse_arguments_disallows_list_with_json_for_create_action(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('create', '--list', '--json') + + def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) From 9ff5ea52409b8e426600996f9a398fcb386aee88 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:20:01 -0700 Subject: [PATCH 17/68] add a unit test, fix isort and black --- borgmatic/commands/completion.py | 49 +++++++++++++------ tests/integration/commands/test_completion.py | 4 ++ 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e62de01c..352dc365 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,6 +1,8 @@ -from argparse import Action -from borgmatic.commands import arguments import shlex +from argparse import Action + +from borgmatic.commands import arguments + def upgrade_message(language: str, upgrade_command: str, completion_file: str): return f''' @@ -37,7 +39,13 @@ def bash_completion(): ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - ' then cat << EOF\n{}\nEOF'.format(upgrade_message('bash', 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', '$BASH_SOURCE')), + ' then cat << EOF\n{}\nEOF'.format( + upgrade_message( + 'bash', + 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', + '$BASH_SOURCE', + ) + ), ' fi', '}', 'complete_borgmatic() {', @@ -59,6 +67,7 @@ def bash_completion(): ) ) + def build_fish_flags(action: Action): ''' Given an argparse.Action instance, return a string containing the fish flags for that action. @@ -68,6 +77,7 @@ def build_fish_flags(action: Action): else: return '-f' + def fish_completion(): ''' Return a fish completion script for the borgmatic command. Produce this by introspecting @@ -85,27 +95,38 @@ def fish_completion(): ' set this_script (cat $this_filename 2> /dev/null)', ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee $this_filename', '$this_filename')), + ' echo "{}"'.format( + upgrade_message( + 'fish', + 'borgmatic --fish-completion | sudo tee $this_filename', + '$this_filename', + ) + ), ' end', 'end', '__borgmatic_check_version &', - ) + ( - '\n# subparser completions', - ) + tuple( + ) + + ('\n# subparser completions',) + + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' % (actionStr, shlex.quote(subparser.description), all_subparsers) for actionStr, subparser in subparsers.choices.items() - ) + ( - '\n# global flags', - ) + tuple( + ) + + ('\n# global flags',) + + tuple( '''complete -c borgmatic -a '%s' -d %s %s''' % (' '.join(action.option_strings), shlex.quote(action.help), build_fish_flags(action)) for action in top_level_parser._actions - ) + ( - '\n# subparser flags', - ) + tuple( + ) + + ('\n# subparser flags',) + + tuple( '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" %s''' - % (' '.join(action.option_strings), shlex.quote(action.help), actionStr, build_fish_flags(action)) + % ( + ' '.join(action.option_strings), + shlex.quote(action.help), + actionStr, + build_fish_flags(action), + ) for actionStr, subparser in subparsers.choices.items() for action in subparser._actions ) diff --git a/tests/integration/commands/test_completion.py b/tests/integration/commands/test_completion.py index a3b0b9c2..9a118abf 100644 --- a/tests/integration/commands/test_completion.py +++ b/tests/integration/commands/test_completion.py @@ -3,3 +3,7 @@ from borgmatic.commands import completion as module def test_bash_completion_does_not_raise(): assert module.bash_completion() + + +def test_fish_completion_does_not_raise(): + assert module.fish_completion() From ca689505e57261fb35c6229014b6a0a7e089c87c Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:27:00 -0700 Subject: [PATCH 18/68] add e2e fish test --- scripts/run-full-tests | 6 +++--- tests/end-to-end/test_completion.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/run-full-tests b/scripts/run-full-tests index bf26c212..a7a49a2a 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -10,7 +10,7 @@ set -e -if [ -z "$TEST_CONTAINER" ] ; then +if [ -z "$TEST_CONTAINER" ]; then echo "This script is designed to work inside a test container and is not intended to" echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute" echo "scripts/run-end-to-end-dev-tests instead." @@ -18,14 +18,14 @@ if [ -z "$TEST_CONTAINER" ] ; then fi apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \ - py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite + py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite fish # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 pip3 install --ignore-installed tox==3.25.1 export COVERAGE_FILE=/tmp/.coverage -if [ "$1" != "--end-to-end-only" ] ; then +if [ "$1" != "--end-to-end-only" ]; then tox --workdir /tmp/.tox --sitepackages fi diff --git a/tests/end-to-end/test_completion.py b/tests/end-to-end/test_completion.py index e4037ece..7d6af4ce 100644 --- a/tests/end-to-end/test_completion.py +++ b/tests/end-to-end/test_completion.py @@ -3,3 +3,7 @@ import subprocess def test_bash_completion_runs_without_error(): subprocess.check_call('borgmatic --bash-completion | bash', shell=True) + + +def test_fish_completion_runs_without_error(): + subprocess.check_call('borgmatic --fish-completion | fish', shell=True) From b7fe2a503173614361cd94e946647790f57ec7a5 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:27:57 -0700 Subject: [PATCH 19/68] lowercase fish in docs --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 098f0139..de5bf8b9 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -365,7 +365,7 @@ borgmatic --bash-completion > ~/.local/share/bash-completion/completions/borgmat Finally, restart your shell (`exit` and open a new shell) so the completions take effect. -#### Fish +#### fish To add completions for fish, install the completions file globally: From 062453af51b7c99401e1508f989a116d01b20a09 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:29:25 -0700 Subject: [PATCH 20/68] replace actionStr with action_name --- borgmatic/commands/completion.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 352dc365..92a221d4 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -109,8 +109,8 @@ def fish_completion(): + ('\n# subparser completions',) + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' - % (actionStr, shlex.quote(subparser.description), all_subparsers) - for actionStr, subparser in subparsers.choices.items() + % (action_name, shlex.quote(subparser.description), all_subparsers) + for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) + tuple( @@ -124,10 +124,10 @@ def fish_completion(): % ( ' '.join(action.option_strings), shlex.quote(action.help), - actionStr, + action_name, build_fish_flags(action), ) - for actionStr, subparser in subparsers.choices.items() + for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) ) From f04036e4a76dc99b4764f33b067f2d9c7f5f0fbc Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:33:21 -0700 Subject: [PATCH 21/68] use fstring to produce completion lines --- borgmatic/commands/completion.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 92a221d4..6a686d2b 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -108,25 +108,17 @@ def fish_completion(): ) + ('\n# subparser completions',) + tuple( - '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' - % (action_name, shlex.quote(subparser.description), all_subparsers) + f'''complete -c borgmatic -a '{action_name}' -d {shlex.quote(subparser.description)} -f -n "not __fish_seen_subcommand_from {all_subparsers}"''' for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) + tuple( - '''complete -c borgmatic -a '%s' -d %s %s''' - % (' '.join(action.option_strings), shlex.quote(action.help), build_fish_flags(action)) + f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} {build_fish_flags(action)}''' for action in top_level_parser._actions ) + ('\n# subparser flags',) + tuple( - '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" %s''' - % ( - ' '.join(action.option_strings), - shlex.quote(action.help), - action_name, - build_fish_flags(action), - ) + f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}" {build_fish_flags(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) From 700f8e9d9c1ccd7e67a05a0d63647245a057272b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:39:48 -0700 Subject: [PATCH 22/68] replace .format with fstring --- borgmatic/commands/completion.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 6a686d2b..e206df73 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -39,13 +39,11 @@ def bash_completion(): ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - ' then cat << EOF\n{}\nEOF'.format( - upgrade_message( + f''' then cat << EOF\n{upgrade_message( 'bash', 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', '$BASH_SOURCE', - ) - ), + )}\nEOF''', ' fi', '}', 'complete_borgmatic() {', @@ -95,13 +93,11 @@ def fish_completion(): ' set this_script (cat $this_filename 2> /dev/null)', ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - ' echo "{}"'.format( - upgrade_message( + f''' echo "{upgrade_message( 'fish', 'borgmatic --fish-completion | sudo tee $this_filename', '$this_filename', - ) - ), + )}"''', ' end', 'end', '__borgmatic_check_version &', From f1fd2e88dd5d947341e37bbe2352ac1399ab5151 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:49:29 -0700 Subject: [PATCH 23/68] drop blank completion --- borgmatic/commands/completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e206df73..ec4d8d15 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -111,6 +111,7 @@ def fish_completion(): + tuple( f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} {build_fish_flags(action)}''' for action in top_level_parser._actions + if len(action.option_strings) > 0 ) + ('\n# subparser flags',) + tuple( From 28efc8566075a53eff78783129d26111b0b804d8 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 18:11:13 -0700 Subject: [PATCH 24/68] rearrange to improve legability of the file --- borgmatic/commands/completion.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index ec4d8d15..c8696e60 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -102,9 +102,10 @@ def fish_completion(): 'end', '__borgmatic_check_version &', ) + + (f'''set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}"''',) + ('\n# subparser completions',) + tuple( - f'''complete -c borgmatic -a '{action_name}' -d {shlex.quote(subparser.description)} -f -n "not __fish_seen_subcommand_from {all_subparsers}"''' + f'''complete -c borgmatic -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)} -f''' for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) From f12a10d888c2cdc5b5eb581dc40938605fb09fe2 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 19:50:49 -0700 Subject: [PATCH 25/68] start work on conditional file completion --- borgmatic/commands/completion.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index c8696e60..5ae7a30a 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,5 +1,6 @@ import shlex from argparse import Action +from textwrap import dedent from borgmatic.commands import arguments @@ -66,14 +67,20 @@ def bash_completion(): ) -def build_fish_flags(action: Action): +def conditionally_emit_file_completion(action: Action): ''' - Given an argparse.Action instance, return a string containing the fish flags for that action. + Given an argparse.Action instance, return a completion invocation that forces file completion + if the action takes a file argument and was the last action on the command line. + + Otherwise, return an empty string. ''' - if action.metavar and action.metavar == 'PATH' or action.metavar == 'FILENAME': - return '-r -F' - else: - return '-f' + if not action.metavar: + return '' + + args = ' '.join(action.option_strings) + + return dedent(f''' + complete -c borgmatic -a {args} -Fr -n "__borgmatic_last_arg {args}"''') def fish_completion(): @@ -105,18 +112,18 @@ def fish_completion(): + (f'''set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}"''',) + ('\n# subparser completions',) + tuple( - f'''complete -c borgmatic -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)} -f''' + f'''complete -c borgmatic -f -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) + tuple( - f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} {build_fish_flags(action)}''' + f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{conditionally_emit_file_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 ) + ('\n# subparser flags',) + tuple( - f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}" {build_fish_flags(action)}''' + f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{conditionally_emit_file_completion(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) From 639e88262e3d0f7a27fdb7359bec44f9fa22d318 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 20:17:26 -0700 Subject: [PATCH 26/68] create working file completion --- borgmatic/commands/completion.py | 56 +++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 5ae7a30a..9a21334d 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -76,11 +76,17 @@ def conditionally_emit_file_completion(action: Action): ''' if not action.metavar: return '' - + args = ' '.join(action.option_strings) - - return dedent(f''' - complete -c borgmatic -a {args} -Fr -n "__borgmatic_last_arg {args}"''') + + return dedent( + f''' + complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' + ) + + +def dedent_strip_as_tuple(string: str): + return (dedent(string).strip("\n"),) def fish_completion(): @@ -94,22 +100,40 @@ def fish_completion(): # Avert your eyes. return '\n'.join( - ( - 'function __borgmatic_check_version', - ' set this_filename (status current-filename)', - ' set this_script (cat $this_filename 2> /dev/null)', - ' set installed_script (borgmatic --fish-completion 2> /dev/null)', - ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - f''' echo "{upgrade_message( + dedent_strip_as_tuple( + f''' + function __borgmatic_check_version + set this_filename (status current-filename) + set this_script (cat $this_filename 2> /dev/null) + set installed_script (borgmatic --fish-completion 2> /dev/null) + if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] + echo "{upgrade_message( 'fish', 'borgmatic --fish-completion | sudo tee $this_filename', '$this_filename', - )}"''', - ' end', - 'end', - '__borgmatic_check_version &', + )}" + end + end + __borgmatic_check_version & + + function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line' + set -l all_args (commandline -poc) + # premature optimization to avoid iterating all args if there aren't enough + # to have a last arg beyond borgmatic + if [ (count $all_args) -lt 2 ] + return 1 + end + for arg in $argv + if [ "$arg" = "$all_args[-1]" ] + return 0 + end + end + return 1 + end + + set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}" + ''' ) - + (f'''set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}"''',) + ('\n# subparser completions',) + tuple( f'''complete -c borgmatic -f -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' From bbc3e9d7173784058646ee3c8325b976a2b04034 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 21:12:24 -0700 Subject: [PATCH 27/68] show possible choices --- borgmatic/commands/completion.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 9a21334d..fb82e2a4 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -66,6 +66,15 @@ def bash_completion(): ) ) +file_metavars = ( + 'FILENAME', + 'PATH', +) + +file_destinations = ( + 'config_paths' +) + def conditionally_emit_file_completion(action: Action): ''' @@ -74,15 +83,22 @@ def conditionally_emit_file_completion(action: Action): Otherwise, return an empty string. ''' - if not action.metavar: - return '' args = ' '.join(action.option_strings) - return dedent( - f''' - complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' - ) + if action.metavar in file_metavars or action.dest in file_destinations: + return dedent( + f''' + complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' + ) + + if action.choices: + return dedent( + f''' + complete -c borgmatic -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' + ) + + return '' def dedent_strip_as_tuple(string: str): From 193731a017a87b267f126fe4fb02a0bc52eff36d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 21:14:48 -0700 Subject: [PATCH 28/68] rename function --- borgmatic/commands/completion.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index fb82e2a4..be94dd0f 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -66,20 +66,20 @@ def bash_completion(): ) ) + file_metavars = ( 'FILENAME', 'PATH', ) -file_destinations = ( - 'config_paths' -) +file_destinations = 'config_paths' -def conditionally_emit_file_completion(action: Action): +def conditionally_emit_arg_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation that forces file completion - if the action takes a file argument and was the last action on the command line. + Given an argparse.Action instance, return a completion invocation + that forces file completion or options completion, if the action + takes such an argument and was the last action on the command line. Otherwise, return an empty string. ''' @@ -91,7 +91,7 @@ def conditionally_emit_file_completion(action: Action): f''' complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' ) - + if action.choices: return dedent( f''' @@ -157,13 +157,13 @@ def fish_completion(): ) + ('\n# global flags',) + tuple( - f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{conditionally_emit_file_completion(action)}''' + f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{conditionally_emit_arg_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 ) + ('\n# subparser flags',) + tuple( - f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{conditionally_emit_file_completion(action)}''' + f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{conditionally_emit_arg_completion(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) From d962376a9dd3b1c643bbe3c22de88d329eafe577 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 21:58:30 -0700 Subject: [PATCH 29/68] refactor to only show specific options if possible --- borgmatic/commands/completion.py | 42 ++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index be94dd0f..0b1deb4f 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -75,7 +75,11 @@ file_metavars = ( file_destinations = 'config_paths' -def conditionally_emit_arg_completion(action: Action): +def has_exact_options(action: Action): + return action.metavar in file_metavars or action.dest in file_destinations or action.choices + + +def exact_options_completion(action: Action): ''' Given an argparse.Action instance, return a completion invocation that forces file completion or options completion, if the action @@ -84,21 +88,20 @@ def conditionally_emit_arg_completion(action: Action): Otherwise, return an empty string. ''' + if not has_exact_options(action): + return '' + args = ' '.join(action.option_strings) if action.metavar in file_metavars or action.dest in file_destinations: - return dedent( - f''' - complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' - ) + return f'''\ncomplete -c borgmatic -Fr -a '{args}' -n "__borgmatic_last_arg {args}"''' if action.choices: - return dedent( - f''' - complete -c borgmatic -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' - ) + return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' - return '' + raise RuntimeError( + f'Unexpected action: {action} passes has_exact_options but has no choices produced' + ) def dedent_strip_as_tuple(string: str): @@ -114,6 +117,18 @@ def fish_completion(): all_subparsers = ' '.join(action for action in subparsers.choices.keys()) + exact_option_args = tuple( + ' '.join(action.option_strings) + for subparser in subparsers.choices.values() + for action in subparser._actions + if has_exact_options(action) + ) + tuple( + ' '.join(action.option_strings) + for action in top_level_parser._actions + if len(action.option_strings) > 0 + if has_exact_options(action) + ) + # Avert your eyes. return '\n'.join( dedent_strip_as_tuple( @@ -148,22 +163,23 @@ def fish_completion(): end set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}" + set --local exact_option_condition "not __borgmatic_last_arg {' '.join(exact_option_args)}" ''' ) + ('\n# subparser completions',) + tuple( - f'''complete -c borgmatic -f -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' + f'''complete -c borgmatic -f -n "$subparser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) + tuple( - f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{conditionally_emit_arg_completion(action)}''' + f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 ) + ('\n# subparser flags',) + tuple( - f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{conditionally_emit_arg_completion(action)}''' + f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) From b4a38d8be9428a49eb3d47ef3fbda896d81eed95 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:06:11 -0700 Subject: [PATCH 30/68] fix flag showing up for paths --- borgmatic/commands/completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 0b1deb4f..0f6c3c20 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -94,7 +94,7 @@ def exact_options_completion(action: Action): args = ' '.join(action.option_strings) if action.metavar in file_metavars or action.dest in file_destinations: - return f'''\ncomplete -c borgmatic -Fr -a '{args}' -n "__borgmatic_last_arg {args}"''' + return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_last_arg {args}"''' if action.choices: return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' From 8f3039be2332c348810f330ede2466b529db0290 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:23:29 -0700 Subject: [PATCH 31/68] handle the expanding filters better --- borgmatic/commands/completion.py | 38 +++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 0f6c3c20..c22e50fd 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -67,16 +67,35 @@ def bash_completion(): ) -file_metavars = ( - 'FILENAME', - 'PATH', -) +# fish section -file_destinations = 'config_paths' +def has_file_options(action: Action): + return action.metavar in ( + 'FILENAME', + 'PATH', + ) or action.dest in ('config_paths',) + + +def has_choice_options(action: Action): + return action.choices is not None + + +def has_required_param_options(action: Action): + return ( + action.nargs + in ( + "+", + "*", + ) + or '--archive' in action.option_strings + or action.metavar in ('PATTERN', 'KEYS', 'N') + ) def has_exact_options(action: Action): - return action.metavar in file_metavars or action.dest in file_destinations or action.choices + return ( + has_file_options(action) or has_choice_options(action) or has_required_param_options(action) + ) def exact_options_completion(action: Action): @@ -93,12 +112,15 @@ def exact_options_completion(action: Action): args = ' '.join(action.option_strings) - if action.metavar in file_metavars or action.dest in file_destinations: + if has_file_options(action): return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_last_arg {args}"''' - if action.choices: + if has_choice_options(action): return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' + if has_required_param_options(action): + return f'''\ncomplete -c borgmatic -x -n "__borgmatic_last_arg {args}"''' + raise RuntimeError( f'Unexpected action: {action} passes has_exact_options but has no choices produced' ) From 3592ec3ddf44dd45f256d25b239bf0450a03bd99 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:32:09 -0700 Subject: [PATCH 32/68] dont show deprecated options --- borgmatic/commands/completion.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index c22e50fd..eb45799f 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -198,11 +198,13 @@ def fish_completion(): f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 + if 'Deprecated' not in action.help ) + ('\n# subparser flags',) + tuple( f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions + if 'Deprecated' not in action.help ) ) From 16ac4824a51965d55162044e43edcba8f01b6102 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:42:04 -0700 Subject: [PATCH 33/68] handle typed without default params --- borgmatic/commands/completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index eb45799f..d24c0430 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -89,6 +89,7 @@ def has_required_param_options(action: Action): ) or '--archive' in action.option_strings or action.metavar in ('PATTERN', 'KEYS', 'N') + or (action.type is not None and action.default is None) ) From d59b9b817f3ebf8ab4f60b664c6fbbfbc49da9fd Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:44:54 -0700 Subject: [PATCH 34/68] support required actions --- borgmatic/commands/completion.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index d24c0430..ee83c28e 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -69,6 +69,7 @@ def bash_completion(): # fish section + def has_file_options(action: Action): return action.metavar in ( 'FILENAME', @@ -82,7 +83,8 @@ def has_choice_options(action: Action): def has_required_param_options(action: Action): return ( - action.nargs + action.required is True + or action.nargs in ( "+", "*", From b557d635fd18c6c952ff26432f396e9f6cfef37d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:57:37 -0700 Subject: [PATCH 35/68] async validity check --- borgmatic/commands/completion.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index ee83c28e..2dc63e67 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -159,18 +159,20 @@ def fish_completion(): dedent_strip_as_tuple( f''' function __borgmatic_check_version - set this_filename (status current-filename) - set this_script (cat $this_filename 2> /dev/null) - set installed_script (borgmatic --fish-completion 2> /dev/null) - if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] - echo "{upgrade_message( - 'fish', - 'borgmatic --fish-completion | sudo tee $this_filename', - '$this_filename', - )}" - end + set -fx this_filename (status current-filename) + fish -c ' + set this_script (cat $this_filename 2> /dev/null) + set installed_script (borgmatic --fish-completion 2> /dev/null) + if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] + echo "{upgrade_message( + 'fish', + 'borgmatic --fish-completion | sudo tee $this_filename', + '$this_filename', + )}" + end + ' & end - __borgmatic_check_version & + __borgmatic_check_version function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line' set -l all_args (commandline -poc) From 5a7a1747f29a16e8b3c2508ee0dcf2def5c58b85 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 5 May 2023 00:01:45 -0700 Subject: [PATCH 36/68] add safety check to avoid infinite cat hang --- borgmatic/commands/completion.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 2dc63e67..3ce5c03e 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -161,14 +161,16 @@ def fish_completion(): function __borgmatic_check_version set -fx this_filename (status current-filename) fish -c ' - set this_script (cat $this_filename 2> /dev/null) - set installed_script (borgmatic --fish-completion 2> /dev/null) - if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] - echo "{upgrade_message( - 'fish', - 'borgmatic --fish-completion | sudo tee $this_filename', - '$this_filename', - )}" + if test -f "$this_filename" + set this_script (cat $this_filename 2> /dev/null) + set installed_script (borgmatic --fish-completion 2> /dev/null) + if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] + echo "{upgrade_message( + 'fish', + 'borgmatic --fish-completion | sudo tee $this_filename', + '$this_filename', + )}" + end end ' & end From 59a6ce1462d87dea5ebabbecf4bb4e18f3e80d44 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 5 May 2023 00:03:43 -0700 Subject: [PATCH 37/68] replace double quotes with single quotes --- borgmatic/commands/completion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 3ce5c03e..e99f3903 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -86,8 +86,8 @@ def has_required_param_options(action: Action): action.required is True or action.nargs in ( - "+", - "*", + '+', + '*', ) or '--archive' in action.option_strings or action.metavar in ('PATTERN', 'KEYS', 'N') @@ -130,7 +130,7 @@ def exact_options_completion(action: Action): def dedent_strip_as_tuple(string: str): - return (dedent(string).strip("\n"),) + return (dedent(string).strip('\n'),) def fish_completion(): From 469e0ccace89270ff3763614ad125ccacd956702 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 10:42:06 -0700 Subject: [PATCH 38/68] create doccomments, start writing unit tests --- borgmatic/commands/completion.py | 11 ++++++++--- tests/unit/commands/test_completions.py | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 tests/unit/commands/test_completions.py diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e99f3903..e160a8fc 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -71,6 +71,9 @@ def bash_completion(): def has_file_options(action: Action): + ''' + Given an argparse.Action instance, return True if it takes a file argument. + ''' return action.metavar in ( 'FILENAME', 'PATH', @@ -78,6 +81,9 @@ def has_file_options(action: Action): def has_choice_options(action: Action): + ''' + Given an argparse.Action instance, return True if it takes one of a predefined set of arguments. + ''' return action.choices is not None @@ -103,9 +109,8 @@ def has_exact_options(action: Action): def exact_options_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation - that forces file completion or options completion, if the action - takes such an argument and was the last action on the command line. + Given an argparse.Action instance, return a completion invocation that forces file completion or options + completion, if the action takes such an argument and was the last action on the command line. Otherwise, return an empty string. ''' diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py new file mode 100644 index 00000000..4cc1f456 --- /dev/null +++ b/tests/unit/commands/test_completions.py @@ -0,0 +1,21 @@ +from argparse import Action + +import pytest + +from borgmatic.commands.completion import has_exact_options, has_file_options + +file_options_test_data = [ + (Action('--flag', 'flag'), False), + (Action('--flag', 'flag', metavar='FILENAME'), True), + (Action('--flag', 'flag', metavar='PATH'), True), + (Action('--flag', dest='config_paths'), True), + (Action('--flag', 'flag', metavar='OTHER'), False), +] + + +@pytest.mark.parametrize('action, expected', file_options_test_data) +def test_has_file_options_detects_file_options(action: Action, expected: bool): + assert has_file_options(action) == expected + # if has_file_options(action) was true, has_exact_options(action) should also be true + if expected: + assert has_exact_options(action) From 372622fbb107c4bbfee99c4e4f0d57e3e3a6e281 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 10:46:27 -0700 Subject: [PATCH 39/68] add more doccomments, drop a check --- borgmatic/commands/completion.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e160a8fc..702a09dd 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -87,7 +87,13 @@ def has_choice_options(action: Action): return action.choices is not None -def has_required_param_options(action: Action): +def has_unknown_required_param_options(action: Action): + ''' + A catch-all for options that take a required parameter, but we don't know what the parameter is. + This should be used last. These are actions that take something like a glob, a list of numbers, or a string. + There is no way to know what the valid options are, but we need to prevent another argument from being shown, + and let the user know that they need to provide a parameter. + ''' return ( action.required is True or action.nargs @@ -95,7 +101,6 @@ def has_required_param_options(action: Action): '+', '*', ) - or '--archive' in action.option_strings or action.metavar in ('PATTERN', 'KEYS', 'N') or (action.type is not None and action.default is None) ) @@ -103,7 +108,9 @@ def has_required_param_options(action: Action): def has_exact_options(action: Action): return ( - has_file_options(action) or has_choice_options(action) or has_required_param_options(action) + has_file_options(action) + or has_choice_options(action) + or has_unknown_required_param_options(action) ) @@ -126,7 +133,7 @@ def exact_options_completion(action: Action): if has_choice_options(action): return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' - if has_required_param_options(action): + if has_unknown_required_param_options(action): return f'''\ncomplete -c borgmatic -x -n "__borgmatic_last_arg {args}"''' raise RuntimeError( From e623f401b90031c7bcb948cad9f25574c614196b Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 10:56:54 -0700 Subject: [PATCH 40/68] write more unit tests --- tests/unit/commands/test_completions.py | 42 ++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 4cc1f456..4de34fe7 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -2,7 +2,12 @@ from argparse import Action import pytest -from borgmatic.commands.completion import has_exact_options, has_file_options +from borgmatic.commands.completion import ( + has_choice_options, + has_exact_options, + has_file_options, + has_unknown_required_param_options, +) file_options_test_data = [ (Action('--flag', 'flag'), False), @@ -19,3 +24,38 @@ def test_has_file_options_detects_file_options(action: Action, expected: bool): # if has_file_options(action) was true, has_exact_options(action) should also be true if expected: assert has_exact_options(action) + + +choices_test_data = [ + (Action('--flag', 'flag'), False), + (Action('--flag', 'flag', choices=['a', 'b']), True), + (Action('--flag', 'flag', choices=None), False), +] + + +@pytest.mark.parametrize('action, expected', choices_test_data) +def test_has_choice_options_detects_choice_options(action: Action, expected: bool): + assert has_choice_options(action) == expected + # if has_choice_options(action) was true, has_exact_options(action) should also be true + if expected: + assert has_exact_options(action) + + +unknown_required_param_test_data = [ + (Action('--flag', 'flag'), False), + (Action('--flag', 'flag', required=True), True), + *((Action('--flag', 'flag', nargs=nargs), True) for nargs in ('+', '*')), + *((Action('--flag', 'flag', metavar=metavar), True) for metavar in ('PATTERN', 'KEYS', 'N')), + *((Action('--flag', 'flag', type=type, default=None), True) for type in (int, str)), + (Action('--flag', 'flag', type=int, default=1), False), +] + + +@pytest.mark.parametrize('action, expected', unknown_required_param_test_data) +def test_has_unknown_required_param_options_detects_unknown_required_param_options( + action: Action, expected: bool +): + assert has_unknown_required_param_options(action) == expected + # if has_unknown_required_param_options(action) was true, has_exact_options(action) should also be true + if expected: + assert has_exact_options(action) From 77dbb5c499d19ca40f98f68d8f91acb9c3939e8a Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:16:45 -0700 Subject: [PATCH 41/68] create way for test cases to be shared --- tests/unit/commands/test_completions.py | 129 +++++++++++++++++------- 1 file changed, 94 insertions(+), 35 deletions(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 4de34fe7..51ab29ac 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -1,4 +1,6 @@ from argparse import Action +from collections import namedtuple +from typing import Tuple import pytest @@ -9,53 +11,110 @@ from borgmatic.commands.completion import ( has_unknown_required_param_options, ) -file_options_test_data = [ - (Action('--flag', 'flag'), False), - (Action('--flag', 'flag', metavar='FILENAME'), True), - (Action('--flag', 'flag', metavar='PATH'), True), - (Action('--flag', dest='config_paths'), True), - (Action('--flag', 'flag', metavar='OTHER'), False), +OptionType = namedtuple('OptionType', ['file', 'choice', 'unknown_required']) +TestCase = Tuple[Action, OptionType] + +test_data: list[TestCase] = [ + (Action('--flag', 'flag'), OptionType(file=False, choice=False, unknown_required=False)), + *( + ( + Action('--flag', 'flag', metavar=metavar), + OptionType(file=True, choice=False, unknown_required=False), + ) + for metavar in ('FILENAME', 'PATH') + ), + ( + Action('--flag', dest='config_paths'), + OptionType(file=True, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', metavar='OTHER'), + OptionType(file=False, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', choices=['a', 'b']), + OptionType(file=False, choice=True, unknown_required=False), + ), + ( + Action('--flag', 'flag', choices=['a', 'b'], type=str), + OptionType(file=False, choice=True, unknown_required=True), + ), + ( + Action('--flag', 'flag', choices=None), + OptionType(file=False, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', required=True), + OptionType(file=False, choice=False, unknown_required=True), + ), + *( + ( + Action('--flag', 'flag', nargs=nargs), + OptionType(file=False, choice=False, unknown_required=True), + ) + for nargs in ('+', '*') + ), + *( + ( + Action('--flag', 'flag', metavar=metavar), + OptionType(file=False, choice=False, unknown_required=True), + ) + for metavar in ('PATTERN', 'KEYS', 'N') + ), + *( + ( + Action('--flag', 'flag', type=type, default=None), + OptionType(file=False, choice=False, unknown_required=True), + ) + for type in (int, str) + ), + ( + Action('--flag', 'flag', type=int, default=1), + OptionType(file=False, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', type=str, required=True, metavar='PATH'), + OptionType(file=True, choice=False, unknown_required=True), + ), + ( + Action('--flag', 'flag', type=str, required=True, metavar='PATH', default='/dev/null'), + OptionType(file=True, choice=False, unknown_required=True), + ), + ( + Action('--flag', 'flag', type=str, required=False, metavar='PATH', default='/dev/null'), + OptionType(file=True, choice=False, unknown_required=False), + ), ] -@pytest.mark.parametrize('action, expected', file_options_test_data) -def test_has_file_options_detects_file_options(action: Action, expected: bool): - assert has_file_options(action) == expected +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_file_options_detects_file_options(action: Action, option_type: OptionType): + assert has_file_options(action) == option_type.file # if has_file_options(action) was true, has_exact_options(action) should also be true - if expected: + if option_type.file: assert has_exact_options(action) -choices_test_data = [ - (Action('--flag', 'flag'), False), - (Action('--flag', 'flag', choices=['a', 'b']), True), - (Action('--flag', 'flag', choices=None), False), -] - - -@pytest.mark.parametrize('action, expected', choices_test_data) -def test_has_choice_options_detects_choice_options(action: Action, expected: bool): - assert has_choice_options(action) == expected +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_choice_options_detects_choice_options(action: Action, option_type: OptionType): + assert has_choice_options(action) == option_type.choice # if has_choice_options(action) was true, has_exact_options(action) should also be true - if expected: + if option_type.choice: assert has_exact_options(action) -unknown_required_param_test_data = [ - (Action('--flag', 'flag'), False), - (Action('--flag', 'flag', required=True), True), - *((Action('--flag', 'flag', nargs=nargs), True) for nargs in ('+', '*')), - *((Action('--flag', 'flag', metavar=metavar), True) for metavar in ('PATTERN', 'KEYS', 'N')), - *((Action('--flag', 'flag', type=type, default=None), True) for type in (int, str)), - (Action('--flag', 'flag', type=int, default=1), False), -] - - -@pytest.mark.parametrize('action, expected', unknown_required_param_test_data) +@pytest.mark.parametrize('action, option_type', test_data) def test_has_unknown_required_param_options_detects_unknown_required_param_options( - action: Action, expected: bool + action: Action, option_type: OptionType ): - assert has_unknown_required_param_options(action) == expected + assert has_unknown_required_param_options(action) == option_type.unknown_required # if has_unknown_required_param_options(action) was true, has_exact_options(action) should also be true - if expected: + if option_type.unknown_required: assert has_exact_options(action) + + +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): + assert has_exact_options(action) == ( + option_type.file or option_type.choice or option_type.unknown_required + ) From aa564ac5fef2be1c13d1178c9b482f326c053baf Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:25:15 -0700 Subject: [PATCH 42/68] fix the error thrown, unit test for it, and add string explanations --- borgmatic/commands/completion.py | 2 +- tests/unit/commands/test_completions.py | 35 ++++++++++++++++--------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 702a09dd..65ce415b 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -136,7 +136,7 @@ def exact_options_completion(action: Action): if has_unknown_required_param_options(action): return f'''\ncomplete -c borgmatic -x -n "__borgmatic_last_arg {args}"''' - raise RuntimeError( + raise ValueError( f'Unexpected action: {action} passes has_exact_options but has no choices produced' ) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 51ab29ac..73623096 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -5,6 +5,7 @@ from typing import Tuple import pytest from borgmatic.commands.completion import ( + exact_options_completion, has_choice_options, has_exact_options, has_file_options, @@ -89,32 +90,40 @@ test_data: list[TestCase] = [ @pytest.mark.parametrize('action, option_type', test_data) def test_has_file_options_detects_file_options(action: Action, option_type: OptionType): - assert has_file_options(action) == option_type.file - # if has_file_options(action) was true, has_exact_options(action) should also be true - if option_type.file: - assert has_exact_options(action) + assert ( + has_file_options(action) == option_type.file + ), f'Action: {action} should be file={option_type.file}' @pytest.mark.parametrize('action, option_type', test_data) def test_has_choice_options_detects_choice_options(action: Action, option_type: OptionType): - assert has_choice_options(action) == option_type.choice - # if has_choice_options(action) was true, has_exact_options(action) should also be true - if option_type.choice: - assert has_exact_options(action) + assert ( + has_choice_options(action) == option_type.choice + ), f'Action: {action} should be choice={option_type.choice}' @pytest.mark.parametrize('action, option_type', test_data) def test_has_unknown_required_param_options_detects_unknown_required_param_options( action: Action, option_type: OptionType ): - assert has_unknown_required_param_options(action) == option_type.unknown_required - # if has_unknown_required_param_options(action) was true, has_exact_options(action) should also be true - if option_type.unknown_required: - assert has_exact_options(action) + assert ( + has_unknown_required_param_options(action) == option_type.unknown_required + ), f'Action: {action} should be unknown_required={option_type.unknown_required}' @pytest.mark.parametrize('action, option_type', test_data) def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): assert has_exact_options(action) == ( option_type.file or option_type.choice or option_type.unknown_required - ) + ), f'Action: {action} should have exact options given {option_type}' + + +@pytest.mark.parametrize('action, option_type', test_data) +def test_produce_exact_options_completion(action: Action, option_type: OptionType): + try: + completion = exact_options_completion(action) + assert ( + type(completion) == str + ), f'Completion should be a string, got {completion} of type {type(completion)}' + except ValueError as value_error: + assert False, f'exact_options_completion raised ValueError: {value_error}' From ccfdd6806f51fed91c6778eb3f66ab6313f4c7d8 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:29:14 -0700 Subject: [PATCH 43/68] test the value of completions --- tests/unit/commands/test_completions.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 73623096..74502775 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -114,7 +114,7 @@ def test_has_unknown_required_param_options_detects_unknown_required_param_optio @pytest.mark.parametrize('action, option_type', test_data) def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): assert has_exact_options(action) == ( - option_type.file or option_type.choice or option_type.unknown_required + True in option_type ), f'Action: {action} should have exact options given {option_type}' @@ -122,8 +122,12 @@ def test_has_exact_options_detects_exact_options(action: Action, option_type: Op def test_produce_exact_options_completion(action: Action, option_type: OptionType): try: completion = exact_options_completion(action) - assert ( - type(completion) == str - ), f'Completion should be a string, got {completion} of type {type(completion)}' + if True in option_type: + assert completion.startswith( + '\ncomplete -c borgmatic' + ), f'Completion should start with "complete -c borgmatic", got {completion}' + else: + assert completion == '', f'Completion should be empty, got {completion}' + except ValueError as value_error: assert False, f'exact_options_completion raised ValueError: {value_error}' From d7320599795bac4f6a2605fa7f630a5282e6f50d Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:32:10 -0700 Subject: [PATCH 44/68] fix rotted comments --- borgmatic/commands/completion.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 65ce415b..e39f742d 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -116,8 +116,9 @@ def has_exact_options(action: Action): def exact_options_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation that forces file completion or options - completion, if the action takes such an argument and was the last action on the command line. + Given an argparse.Action instance, return a completion invocation that forces file completions, options completion, + or just that some value follow the action, if the action takes such an argument and was the last action on the + command line prior to the cursor. Otherwise, return an empty string. ''' @@ -188,7 +189,7 @@ def fish_completion(): end __borgmatic_check_version - function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line' + function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line before the cursor' set -l all_args (commandline -poc) # premature optimization to avoid iterating all args if there aren't enough # to have a last arg beyond borgmatic From a047f856a13c7cce4f7463ad2d9fa0a01c695d76 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:37:38 -0700 Subject: [PATCH 45/68] tweak docstring, add comment --- borgmatic/commands/completion.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e39f742d..555ffcf9 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -91,8 +91,8 @@ def has_unknown_required_param_options(action: Action): ''' A catch-all for options that take a required parameter, but we don't know what the parameter is. This should be used last. These are actions that take something like a glob, a list of numbers, or a string. - There is no way to know what the valid options are, but we need to prevent another argument from being shown, - and let the user know that they need to provide a parameter. + + Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid. ''' return ( action.required is True @@ -215,6 +215,7 @@ def fish_completion(): ) + ('\n# global flags',) + tuple( + # -n is checked in order, so put faster / more likely to be true checks first f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 From c8f4344f8968aa9c61a8c7f047b091f6f9854868 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:39:02 -0700 Subject: [PATCH 46/68] add more justification to checks --- borgmatic/commands/completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 555ffcf9..affe5f97 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -218,6 +218,7 @@ def fish_completion(): # -n is checked in order, so put faster / more likely to be true checks first f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' for action in top_level_parser._actions + # ignore the noargs action, as this is an impossible completion for fish if len(action.option_strings) > 0 if 'Deprecated' not in action.help ) From efb81fc2c1ea61fdabf77ecbba27214a359a0ec9 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:42:32 -0700 Subject: [PATCH 47/68] rename last arg helper function to current arg for clarity --- borgmatic/commands/completion.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index affe5f97..d0ce7452 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -129,13 +129,13 @@ def exact_options_completion(action: Action): args = ' '.join(action.option_strings) if has_file_options(action): - return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_last_arg {args}"''' + return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_current_arg {args}"''' if has_choice_options(action): - return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' + return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_current_arg {args}"''' if has_unknown_required_param_options(action): - return f'''\ncomplete -c borgmatic -x -n "__borgmatic_last_arg {args}"''' + return f'''\ncomplete -c borgmatic -x -n "__borgmatic_current_arg {args}"''' raise ValueError( f'Unexpected action: {action} passes has_exact_options but has no choices produced' @@ -189,7 +189,7 @@ def fish_completion(): end __borgmatic_check_version - function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line before the cursor' + function __borgmatic_current_arg --description 'Check if any of the given arguments are the last on the command line before the cursor' set -l all_args (commandline -poc) # premature optimization to avoid iterating all args if there aren't enough # to have a last arg beyond borgmatic @@ -205,7 +205,7 @@ def fish_completion(): end set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}" - set --local exact_option_condition "not __borgmatic_last_arg {' '.join(exact_option_args)}" + set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}" ''' ) + ('\n# subparser completions',) From 43c532bc577eb7169c5700fb42d74b805025ae66 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:51:35 -0700 Subject: [PATCH 48/68] add test for dedent strip --- borgmatic/commands/completion.py | 4 ++++ tests/unit/commands/test_completions.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index d0ce7452..4b2f17f3 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -143,6 +143,10 @@ def exact_options_completion(action: Action): def dedent_strip_as_tuple(string: str): + ''' + Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string. + Makes it easier to write multiline strings for completions when you join them with a tuple. + ''' return (dedent(string).strip('\n'),) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 74502775..69110af6 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -5,6 +5,7 @@ from typing import Tuple import pytest from borgmatic.commands.completion import ( + dedent_strip_as_tuple, exact_options_completion, has_choice_options, has_exact_options, @@ -131,3 +132,12 @@ def test_produce_exact_options_completion(action: Action, option_type: OptionTyp except ValueError as value_error: assert False, f'exact_options_completion raised ValueError: {value_error}' + + +def test_dedent_strip_as_tuple(): + dedent_strip_as_tuple( + ''' + a + b + ''' + ) From 0657106893b729a35227a8be7198b90982b214d9 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:46:15 -0700 Subject: [PATCH 49/68] clarify dedent test name --- tests/unit/commands/test_completions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 69110af6..b6d3103f 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -134,7 +134,7 @@ def test_produce_exact_options_completion(action: Action, option_type: OptionTyp assert False, f'exact_options_completion raised ValueError: {value_error}' -def test_dedent_strip_as_tuple(): +def test_dedent_strip_as_tuple_does_not_raise(): dedent_strip_as_tuple( ''' a From 453b78c852fb36186dc3a6cc0a3f00681e4757ee Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:49:07 -0700 Subject: [PATCH 50/68] drop messages --- tests/unit/commands/test_completions.py | 32 +++++++------------------ 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index b6d3103f..ba562209 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -91,47 +91,33 @@ test_data: list[TestCase] = [ @pytest.mark.parametrize('action, option_type', test_data) def test_has_file_options_detects_file_options(action: Action, option_type: OptionType): - assert ( - has_file_options(action) == option_type.file - ), f'Action: {action} should be file={option_type.file}' + assert has_file_options(action) == option_type.file @pytest.mark.parametrize('action, option_type', test_data) def test_has_choice_options_detects_choice_options(action: Action, option_type: OptionType): - assert ( - has_choice_options(action) == option_type.choice - ), f'Action: {action} should be choice={option_type.choice}' + assert has_choice_options(action) == option_type.choice @pytest.mark.parametrize('action, option_type', test_data) def test_has_unknown_required_param_options_detects_unknown_required_param_options( action: Action, option_type: OptionType ): - assert ( - has_unknown_required_param_options(action) == option_type.unknown_required - ), f'Action: {action} should be unknown_required={option_type.unknown_required}' + assert has_unknown_required_param_options(action) == option_type.unknown_required @pytest.mark.parametrize('action, option_type', test_data) def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): - assert has_exact_options(action) == ( - True in option_type - ), f'Action: {action} should have exact options given {option_type}' + assert has_exact_options(action) == (True in option_type) @pytest.mark.parametrize('action, option_type', test_data) def test_produce_exact_options_completion(action: Action, option_type: OptionType): - try: - completion = exact_options_completion(action) - if True in option_type: - assert completion.startswith( - '\ncomplete -c borgmatic' - ), f'Completion should start with "complete -c borgmatic", got {completion}' - else: - assert completion == '', f'Completion should be empty, got {completion}' - - except ValueError as value_error: - assert False, f'exact_options_completion raised ValueError: {value_error}' + completion = exact_options_completion(action) + if True in option_type: + assert completion.startswith('\ncomplete -c borgmatic') + else: + assert completion == '' def test_dedent_strip_as_tuple_does_not_raise(): From aa770b98f90652a3579cf4b21ae1a5b370a3c4b2 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:50:37 -0700 Subject: [PATCH 51/68] follow unit test module convention --- tests/unit/commands/test_completions.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index ba562209..64ea3c7e 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -4,14 +4,7 @@ from typing import Tuple import pytest -from borgmatic.commands.completion import ( - dedent_strip_as_tuple, - exact_options_completion, - has_choice_options, - has_exact_options, - has_file_options, - has_unknown_required_param_options, -) +from borgmatic.commands import completion as module OptionType = namedtuple('OptionType', ['file', 'choice', 'unknown_required']) TestCase = Tuple[Action, OptionType] @@ -91,29 +84,29 @@ test_data: list[TestCase] = [ @pytest.mark.parametrize('action, option_type', test_data) def test_has_file_options_detects_file_options(action: Action, option_type: OptionType): - assert has_file_options(action) == option_type.file + assert module.has_file_options(action) == option_type.file @pytest.mark.parametrize('action, option_type', test_data) def test_has_choice_options_detects_choice_options(action: Action, option_type: OptionType): - assert has_choice_options(action) == option_type.choice + assert module.has_choice_options(action) == option_type.choice @pytest.mark.parametrize('action, option_type', test_data) def test_has_unknown_required_param_options_detects_unknown_required_param_options( action: Action, option_type: OptionType ): - assert has_unknown_required_param_options(action) == option_type.unknown_required + assert module.has_unknown_required_param_options(action) == option_type.unknown_required @pytest.mark.parametrize('action, option_type', test_data) def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): - assert has_exact_options(action) == (True in option_type) + assert module.has_exact_options(action) == (True in option_type) @pytest.mark.parametrize('action, option_type', test_data) def test_produce_exact_options_completion(action: Action, option_type: OptionType): - completion = exact_options_completion(action) + completion = module.exact_options_completion(action) if True in option_type: assert completion.startswith('\ncomplete -c borgmatic') else: @@ -121,7 +114,7 @@ def test_produce_exact_options_completion(action: Action, option_type: OptionTyp def test_dedent_strip_as_tuple_does_not_raise(): - dedent_strip_as_tuple( + module.dedent_strip_as_tuple( ''' a b From 614c1bf2e41ee189188a5a8c9ec85079a2d98989 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:52:42 -0700 Subject: [PATCH 52/68] rename test to make function under test clearer --- tests/unit/commands/test_completions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 64ea3c7e..878e8204 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -105,7 +105,7 @@ def test_has_exact_options_detects_exact_options(action: Action, option_type: Op @pytest.mark.parametrize('action, option_type', test_data) -def test_produce_exact_options_completion(action: Action, option_type: OptionType): +def test_exact_options_completion_produces_reasonable_completions(action: Action, option_type: OptionType): completion = module.exact_options_completion(action) if True in option_type: assert completion.startswith('\ncomplete -c borgmatic') From 66964f613c1c8cd6e7fd3dece33cde430c8ed6b6 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:56:50 -0700 Subject: [PATCH 53/68] formatting! --- tests/unit/commands/test_completions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 878e8204..acb01f6a 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -105,7 +105,9 @@ def test_has_exact_options_detects_exact_options(action: Action, option_type: Op @pytest.mark.parametrize('action, option_type', test_data) -def test_exact_options_completion_produces_reasonable_completions(action: Action, option_type: OptionType): +def test_exact_options_completion_produces_reasonable_completions( + action: Action, option_type: OptionType +): completion = module.exact_options_completion(action) if True in option_type: assert completion.startswith('\ncomplete -c borgmatic') From 1a956e8b05a4700fecd503bfcc2e86be3dc3cb7b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 6 May 2023 16:04:15 -0700 Subject: [PATCH 54/68] Add fish shell completions to NEWS (#686). --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 30a2b9b0..9971da7e 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,9 @@ * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. + * #686: Add fish shell completion script so you can tab-complete on the borgmatic command-line. See + the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install --editable" development installs. * Run "borgmatic borg" action without capturing output so interactive prompts and flags like From e84bac29e580d782c54980e86a6c90f93d8be318 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 6 May 2023 16:18:37 -0700 Subject: [PATCH 55/68] Remove value type for compatibility with Python 3.8 (#686). --- tests/unit/commands/test_completions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index acb01f6a..ec596988 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -9,7 +9,7 @@ from borgmatic.commands import completion as module OptionType = namedtuple('OptionType', ['file', 'choice', 'unknown_required']) TestCase = Tuple[Action, OptionType] -test_data: list[TestCase] = [ +test_data = [ (Action('--flag', 'flag'), OptionType(file=False, choice=False, unknown_required=False)), *( ( From 15ef37d89fe5866ec2f248014b29a25393976c7a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 6 May 2023 16:25:26 -0700 Subject: [PATCH 56/68] Add test coverage for exact_options_completion() raising (#686). --- tests/unit/commands/test_completions.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index ec596988..12829d5f 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -3,6 +3,7 @@ from collections import namedtuple from typing import Tuple import pytest +from flexmock import flexmock from borgmatic.commands import completion as module @@ -115,6 +116,16 @@ def test_exact_options_completion_produces_reasonable_completions( assert completion == '' +def test_exact_options_completion_raises_for_unexpected_action(): + flexmock(module).should_receive('has_exact_options').and_return(True) + flexmock(module).should_receive('has_file_options').and_return(False) + flexmock(module).should_receive('has_choice_options').and_return(False) + flexmock(module).should_receive('has_unknown_required_param_options').and_return(False) + + with pytest.raises(ValueError): + module.exact_options_completion(Action('--unknown', dest='unknown')) + + def test_dedent_strip_as_tuple_does_not_raise(): module.dedent_strip_as_tuple( ''' From b3b08ee6d776afbb2e6f05d6623108c3c2305739 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 7 May 2023 21:21:35 -0700 Subject: [PATCH 57/68] Fix error in "borgmatic restore" action when the configured repository path is relative (#691). --- NEWS | 2 ++ borgmatic/borg/extract.py | 5 +++- tests/unit/borg/test_extract.py | 45 +++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 9971da7e..8be142b9 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,8 @@ https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install --editable" development installs. + * #691: Fix error in "borgmatic restore" action when the configured repository path is relative + instead of absolute. * Run "borgmatic borg" action without capturing output so interactive prompts and flags like "--progress" still work. diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index f9471416..1b10ba26 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -2,6 +2,7 @@ import logging import os import subprocess +import borgmatic.config.validate from borgmatic.borg import environment, feature, flags, rlist from borgmatic.execute import DO_NOT_CAPTURE, execute_command @@ -109,7 +110,9 @@ def extract_archive( + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( - repository, + # Make the repository path absolute so the working directory changes below don't + # prevent Borg from finding the repo. + borgmatic.config.validate.normalize_repository_path(repository), archive, local_borg_version, ) diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index 26fd7380..6517379e 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -121,6 +121,9 @@ def test_extract_archive_calls_borg_with_path_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -140,6 +143,9 @@ def test_extract_archive_calls_borg_with_remote_path_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -167,6 +173,9 @@ def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -186,6 +195,9 @@ def test_extract_archive_calls_borg_with_umask_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -205,6 +217,9 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -225,6 +240,9 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -247,6 +265,9 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -266,6 +287,9 @@ def test_extract_archive_calls_borg_with_dry_run_parameter(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=True, @@ -285,6 +309,9 @@ def test_extract_archive_calls_borg_with_destination_path(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -305,6 +332,9 @@ def test_extract_archive_calls_borg_with_strip_components(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -335,6 +365,9 @@ def test_extract_archive_calls_borg_with_strip_components_calculated_from_all(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -354,6 +387,9 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') flexmock(module).should_receive('execute_command').never() with pytest.raises(ValueError): @@ -382,6 +418,9 @@ def test_extract_archive_calls_borg_with_progress_parameter(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -427,6 +466,9 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') assert ( module.extract_archive( @@ -455,6 +497,9 @@ def test_extract_archive_skips_abspath_for_remote_repository(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('server:repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, From 92a2230a07e9e6e3a6024c3e86dbc3cfe7208187 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 8 May 2023 23:00:49 -0700 Subject: [PATCH 58/68] Add support for logging each log line as a JSON object via global "--log-json" flag (#680). --- NEWS | 1 + borgmatic/actions/borg.py | 2 + borgmatic/actions/break_lock.py | 2 + borgmatic/actions/check.py | 1 + borgmatic/actions/compact.py | 1 + borgmatic/actions/create.py | 1 + borgmatic/actions/export_tar.py | 2 + borgmatic/actions/extract.py | 2 + borgmatic/actions/info.py | 9 +- borgmatic/actions/list.py | 9 +- borgmatic/actions/mount.py | 3 + borgmatic/actions/prune.py | 1 + borgmatic/actions/rcreate.py | 1 + borgmatic/actions/restore.py | 11 +- borgmatic/actions/rinfo.py | 2 + borgmatic/actions/rlist.py | 2 + borgmatic/actions/transfer.py | 1 + borgmatic/borg/break_lock.py | 6 +- borgmatic/borg/check.py | 11 +- borgmatic/borg/compact.py | 2 + borgmatic/borg/create.py | 2 + borgmatic/borg/export_tar.py | 2 + borgmatic/borg/extract.py | 24 +++-- borgmatic/borg/info.py | 8 +- borgmatic/borg/list.py | 24 +++-- borgmatic/borg/mount.py | 6 +- borgmatic/borg/prune.py | 2 + borgmatic/borg/rcreate.py | 5 + borgmatic/borg/rinfo.py | 8 +- borgmatic/borg/rlist.py | 30 +++--- borgmatic/borg/transfer.py | 5 +- borgmatic/commands/arguments.py | 5 + borgmatic/commands/borgmatic.py | 11 +- tests/integration/borg/test_commands.py | 26 ++++- tests/unit/actions/test_borg.py | 1 + tests/unit/actions/test_break_lock.py | 1 + tests/unit/actions/test_info.py | 1 + tests/unit/actions/test_list.py | 1 + tests/unit/actions/test_mount.py | 1 + tests/unit/actions/test_restore.py | 3 + tests/unit/actions/test_rinfo.py | 1 + tests/unit/actions/test_rlist.py | 1 + tests/unit/borg/test_break_lock.py | 18 ++++ tests/unit/borg/test_check.py | 46 ++++++++ tests/unit/borg/test_compact.py | 45 +++++++- tests/unit/borg/test_create.py | 89 +++++++++++++++ tests/unit/borg/test_export_tar.py | 33 ++++++ tests/unit/borg/test_extract.py | 98 ++++++++++++++--- tests/unit/borg/test_info.py | 45 ++++++++ tests/unit/borg/test_list.py | 62 ++++++++++- tests/unit/borg/test_mount.py | 31 ++++++ tests/unit/borg/test_prune.py | 46 ++++++-- tests/unit/borg/test_rcreate.py | 57 ++++++++++ tests/unit/borg/test_rinfo.py | 49 +++++++-- tests/unit/borg/test_rlist.py | 138 +++++++++++++++++++++--- tests/unit/borg/test_transfer.py | 44 ++++++++ 56 files changed, 934 insertions(+), 105 deletions(-) diff --git a/NEWS b/NEWS index 8be142b9..17560d7e 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,7 @@ commands with arguments. * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break database dumping. + * #680: Add support for logging each log line as a JSON object via global "--log-json" flag. * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index 3d2998b6..ec445fbb 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -12,6 +12,7 @@ def run_borg( storage, local_borg_version, borg_arguments, + global_arguments, local_path, remote_path, ): @@ -27,6 +28,7 @@ def run_borg( borg_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index 2174161c..f049e772 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -11,6 +11,7 @@ def run_break_lock( storage, local_borg_version, break_lock_arguments, + global_arguments, local_path, remote_path, ): @@ -25,6 +26,7 @@ def run_break_lock( repository['path'], storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 1696e07d..aac536e3 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -44,6 +44,7 @@ def run_check( storage, consistency, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, progress=check_arguments.progress, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 95334c52..24b30c0e 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -45,6 +45,7 @@ def run_compact( repository['path'], storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, progress=compact_arguments.progress, diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 3fbe31e6..a3f8da57 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -67,6 +67,7 @@ def run_create( location, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, progress=create_arguments.progress, diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index ff9f31ba..798bd418 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -33,6 +33,7 @@ def run_export_tar( export_tar_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ), @@ -40,6 +41,7 @@ def run_export_tar( export_tar_arguments.destination, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index cc1516ce..1f4317cd 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -44,6 +44,7 @@ def run_extract( extract_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ), @@ -51,6 +52,7 @@ def run_extract( location, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, destination_path=extract_arguments.destination, diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index 54023127..d138dbd4 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -13,6 +13,7 @@ def run_info( storage, local_borg_version, info_arguments, + global_arguments, local_path, remote_path, ): @@ -31,6 +32,7 @@ def run_info( info_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -38,9 +40,10 @@ def run_info( repository['path'], storage, local_borg_version, - info_arguments=info_arguments, - local_path=local_path, - remote_path=remote_path, + info_arguments, + global_arguments, + local_path, + remote_path, ) if json_output: # pragma: nocover yield json.loads(json_output) diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 359f3b67..548f1979 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -12,6 +12,7 @@ def run_list( storage, local_borg_version, list_arguments, + global_arguments, local_path, remote_path, ): @@ -33,6 +34,7 @@ def run_list( list_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -40,9 +42,10 @@ def run_list( repository['path'], storage, local_borg_version, - list_arguments=list_arguments, - local_path=local_path, - remote_path=remote_path, + list_arguments, + global_arguments, + local_path, + remote_path, ) if json_output: # pragma: nocover yield json.loads(json_output) diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index 72e321a0..60f7f23c 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -12,6 +12,7 @@ def run_mount( storage, local_borg_version, mount_arguments, + global_arguments, local_path, remote_path, ): @@ -33,6 +34,7 @@ def run_mount( mount_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ), @@ -42,6 +44,7 @@ def run_mount( mount_arguments.options, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 9a5d936b..2e25264b 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -44,6 +44,7 @@ def run_prune( storage, retention, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, stats=prune_arguments.stats, diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index 62206318..a3015c61 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -29,6 +29,7 @@ def run_rcreate( repository['path'], storage, local_borg_version, + global_arguments, rcreate_arguments.encryption_mode, rcreate_arguments.source_repository, rcreate_arguments.copy_crypt_key, diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index f061dca8..246c11a6 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -93,6 +93,7 @@ def restore_single_database( location_config=location, storage_config=storage, local_borg_version=local_borg_version, + global_arguments=global_arguments, local_path=local_path, remote_path=remote_path, destination_path='/', @@ -119,14 +120,15 @@ def collect_archive_database_names( location, storage, local_borg_version, + global_arguments, local_path, remote_path, ): ''' Given a local or remote repository path, a resolved archive name, a location configuration dict, - a storage configuration dict, the local Borg version, and local and remote Borg paths, query the - archive for the names of databases it contains and return them as a dict from hook name to a - sequence of database names. + a storage configuration dict, the local Borg version, global_arguments an argparse.Namespace, + and local and remote Borg paths, query the archive for the names of databases it contains and + return them as a dict from hook name to a sequence of database names. ''' borgmatic_source_directory = os.path.expanduser( location.get( @@ -141,6 +143,7 @@ def collect_archive_database_names( archive, storage, local_borg_version, + global_arguments, list_path=parent_dump_path, local_path=local_path, remote_path=remote_path, @@ -279,6 +282,7 @@ def run_restore( restore_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -288,6 +292,7 @@ def run_restore( location, storage, local_borg_version, + global_arguments, local_path, remote_path, ) diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 0947ec3d..279cd0e7 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -12,6 +12,7 @@ def run_rinfo( storage, local_borg_version, rinfo_arguments, + global_arguments, local_path, remote_path, ): @@ -31,6 +32,7 @@ def run_rinfo( storage, local_borg_version, rinfo_arguments=rinfo_arguments, + global_arguments=global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index 10d06a51..50c59b6f 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -12,6 +12,7 @@ def run_rlist( storage, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ): @@ -31,6 +32,7 @@ def run_rlist( storage, local_borg_version, rlist_arguments=rlist_arguments, + global_arguments=global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 8089fd4e..36ac166d 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -24,6 +24,7 @@ def run_transfer( storage, local_borg_version, transfer_arguments, + global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 7099af83..3c361956 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -10,13 +10,14 @@ def break_lock( repository_path, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage configuration dict, the local Borg version, - and optional local and remote Borg paths, break any repository and cache locks leftover from Borg - aborting. + an argparse.Namespace of global arguments, and optional local and remote Borg paths, break any + repository and cache locks leftover from Borg aborting. ''' umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) @@ -25,6 +26,7 @@ def break_lock( (local_path, 'break-lock') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index cee9d923..52d5208c 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -259,6 +259,7 @@ def check_archives( storage_config, consistency_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, progress=None, @@ -283,6 +284,7 @@ def check_archives( storage_config, local_borg_version, argparse.Namespace(json=True), + global_arguments, local_path, remote_path, ) @@ -317,6 +319,7 @@ def check_archives( + (('--repair',) if repair else ()) + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix) + (('--remote-path', remote_path) if remote_path else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags + (('--progress',) if progress else ()) @@ -340,6 +343,12 @@ def check_archives( if 'extract' in checks: extract.extract_last_archive_dry_run( - storage_config, local_borg_version, repository_path, lock_wait, local_path, remote_path + storage_config, + local_borg_version, + global_arguments, + repository_path, + lock_wait, + local_path, + remote_path, ) write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract')) diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index 0e9d3e89..24f37ee3 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -11,6 +11,7 @@ def compact_segments( repository_path, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, progress=False, @@ -29,6 +30,7 @@ def compact_segments( (local_path, 'compact') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--progress',) if progress else ()) + (('--cleanup-commits',) if cleanup_commits else ()) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 7413dfc5..e3b70eb5 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -326,6 +326,7 @@ def create_archive( location_config, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, progress=False, @@ -438,6 +439,7 @@ def create_archive( + (('--files-cache', files_cache) if files_cache else ()) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--list', '--filter', list_filter_flags) diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index a624f07d..b6d9a04c 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -15,6 +15,7 @@ def export_tar_archive( destination_path, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, tar_filter=None, @@ -38,6 +39,7 @@ def export_tar_archive( (local_path, 'export-tar') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--list',) if list_files else ()) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 1b10ba26..d5465bb9 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -12,6 +12,7 @@ logger = logging.getLogger(__name__) def extract_last_archive_dry_run( storage_config, local_borg_version, + global_arguments, repository_path, lock_wait=None, local_path='borg', @@ -21,8 +22,6 @@ def extract_last_archive_dry_run( Perform an extraction dry-run of the most recent archive. If there are no archives, skip the dry-run. ''' - remote_path_flags = ('--remote-path', remote_path) if remote_path else () - lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else () verbosity_flags = () if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') @@ -31,7 +30,13 @@ def extract_last_archive_dry_run( try: last_archive_name = rlist.resolve_archive_name( - repository_path, 'latest', storage_config, local_borg_version, local_path, remote_path + repository_path, + 'latest', + storage_config, + local_borg_version, + global_arguments, + local_path, + remote_path, ) except ValueError: logger.warning('No archives found. Skipping extract consistency check.') @@ -41,8 +46,9 @@ def extract_last_archive_dry_run( borg_environment = environment.make_environment(storage_config) full_extract_command = ( (local_path, 'extract', '--dry-run') - + remote_path_flags - + lock_wait_flags + + (('--remote-path', remote_path) if remote_path else ()) + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags + list_flag + flags.make_repository_archive_flags( @@ -63,6 +69,7 @@ def extract_archive( location_config, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, destination_path=None, @@ -72,9 +79,9 @@ def extract_archive( ): ''' Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to - restore from the archive, the local Borg version string, location/storage configuration dicts, - optional local and remote Borg paths, and an optional destination path to extract to, extract - the archive into the current directory. + restore from the archive, the local Borg version string, an argparse.Namespace of global + arguments, location/storage configuration dicts, optional local and remote Borg paths, and an + optional destination path to extract to, extract the archive into the current directory. If extract to stdout is True, then start the extraction streaming to stdout, and return that extract process as an instance of subprocess.Popen. @@ -102,6 +109,7 @@ def extract_archive( + (('--remote-path', remote_path) if remote_path else ()) + numeric_ids_flags + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index ef2c0c44..91520e00 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -12,13 +12,14 @@ def display_archives_info( storage_config, local_borg_version, info_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, and the - arguments to the info action, display summary information for Borg archives in the repository or - return JSON summary information. + Given a local or remote repository path, a storage config dict, the local Borg version, global + arguments as an argparse.Namespace, and the arguments to the info action, display summary + information for Borg archives in the repository or return JSON summary information. ''' borgmatic.logger.add_custom_log_levels() lock_wait = storage_config.get('lock_wait', None) @@ -36,6 +37,7 @@ def display_archives_info( else () ) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', lock_wait) + ( ( diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 908f8fef..96a6a87f 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -25,6 +25,7 @@ def make_list_command( storage_config, local_borg_version, list_arguments, + global_arguments, local_path='borg', remote_path=None, ): @@ -48,6 +49,7 @@ def make_list_command( else () ) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', lock_wait) + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES) + ( @@ -90,14 +92,16 @@ def capture_archive_listing( archive, storage_config, local_borg_version, + global_arguments, list_path=None, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, an archive name, a storage config dict, the local Borg - version, the archive path in which to list files, and local and remote Borg paths, capture the - output of listing that archive and return it as a list of file paths. + version, global arguments as an argparse.Namespace, the archive path in which to list files, and + local and remote Borg paths, capture the output of listing that archive and return it as a list + of file paths. ''' borg_environment = environment.make_environment(storage_config) @@ -115,6 +119,7 @@ def capture_archive_listing( json=None, format='{path}{NL}', # noqa: FS003 ), + global_arguments, local_path, remote_path, ), @@ -130,15 +135,17 @@ def list_archive( storage_config, local_borg_version, list_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, the - arguments to the list action, and local and remote Borg paths, display the output of listing - the files of a Borg archive (or return JSON output). If list_arguments.find_paths are given, - list the files by searching across multiple archives. If neither find_paths nor archive name - are given, instead list the archives in the given repository. + Given a local or remote repository path, a storage config dict, the local Borg version, global + arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace, + and local and remote Borg paths, display the output of listing the files of a Borg archive (or + return JSON output). If list_arguments.find_paths are given, list the files by searching across + multiple archives. If neither find_paths nor archive name are given, instead list the archives + in the given repository. ''' borgmatic.logger.add_custom_log_levels() @@ -164,6 +171,7 @@ def list_archive( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ) @@ -205,6 +213,7 @@ def list_archive( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ), @@ -233,6 +242,7 @@ def list_archive( storage_config, local_borg_version, archive_arguments, + global_arguments, local_path, remote_path, ) + make_find_paths(list_arguments.find_paths) diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 07a6c632..6ce01a87 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -15,14 +15,15 @@ def mount_archive( options, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, an optional archive name, a filesystem mount point, zero or more paths to mount from the archive, extra Borg mount options, a storage configuration - dict, the local Borg version, and optional local and remote Borg paths, mount the archive onto - the mount point. + dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional + local and remote Borg paths, mount the archive onto the mount point. ''' umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) @@ -31,6 +32,7 @@ def mount_archive( (local_path, 'mount') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 3f06dc2d..b6be75ba 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -53,6 +53,7 @@ def prune_archives( storage_config, retention_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, stats=False, @@ -73,6 +74,7 @@ def prune_archives( + make_prune_flags(storage_config, retention_config, local_borg_version) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--stats',) if stats and not dry_run else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) diff --git a/borgmatic/borg/rcreate.py b/borgmatic/borg/rcreate.py index 7510529d..54a865c5 100644 --- a/borgmatic/borg/rcreate.py +++ b/borgmatic/borg/rcreate.py @@ -16,6 +16,7 @@ def create_repository( repository_path, storage_config, local_borg_version, + global_arguments, encryption_mode, source_repository=None, copy_crypt_key=False, @@ -37,6 +38,7 @@ def create_repository( storage_config, local_borg_version, argparse.Namespace(json=True), + global_arguments, local_path, remote_path, ) @@ -46,6 +48,7 @@ def create_repository( if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE: raise + lock_wait = storage_config.get('lock_wait') extra_borg_options = storage_config.get('extra_borg_options', {}).get('rcreate', '') rcreate_command = ( @@ -63,6 +66,8 @@ def create_repository( + (('--make-parent-dirs',) if make_parent_dirs else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--remote-path', remote_path) if remote_path else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index 97d7a666..e1542d28 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -12,13 +12,14 @@ def display_repository_info( storage_config, local_borg_version, rinfo_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, and the - arguments to the rinfo action, display summary information for the Borg repository or return - JSON summary information. + Given a local or remote repository path, a storage config dict, the local Borg version, the + arguments to the rinfo action, and global arguments as an argparse.Namespace, display summary + information for the Borg repository or return JSON summary information. ''' borgmatic.logger.add_custom_log_levels() lock_wait = storage_config.get('lock_wait', None) @@ -41,6 +42,7 @@ def display_repository_info( else () ) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', lock_wait) + (('--json',) if rinfo_arguments.json else ()) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 7f468705..ba45aa0a 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -12,28 +12,29 @@ def resolve_archive_name( archive, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, an archive name, a storage config dict, a local Borg - path, and a remote Borg path, return the archive name. But if the archive name is "latest", - then instead introspect the repository for the latest archive and return its name. + Given a local or remote repository path, an archive name, a storage config dict, the local Borg + version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path, + return the archive name. But if the archive name is "latest", then instead introspect the + repository for the latest archive and return its name. Raise ValueError if "latest" is given but there are no archives in the repository. ''' if archive != 'latest': return archive - lock_wait = storage_config.get('lock_wait', None) - full_command = ( ( local_path, 'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list', ) + flags.make_flags('remote-path', remote_path) - + flags.make_flags('lock-wait', lock_wait) + + flags.make_flags('log-json', global_arguments.log_json) + + flags.make_flags('lock-wait', storage_config.get('lock_wait')) + flags.make_flags('last', 1) + ('--short',) + flags.make_repository_flags(repository_path, local_borg_version) @@ -61,16 +62,15 @@ def make_rlist_command( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage config dict, the local Borg version, the - arguments to the rlist action, and local and remote Borg paths, return a command as a tuple to - list archives with a repository. + arguments to the rlist action, global arguments as an argparse.Namespace instance, and local and + remote Borg paths, return a command as a tuple to list archives with a repository. ''' - lock_wait = storage_config.get('lock_wait', None) - return ( ( local_path, @@ -87,7 +87,8 @@ def make_rlist_command( else () ) + flags.make_flags('remote-path', remote_path) - + flags.make_flags('lock-wait', lock_wait) + + flags.make_flags('log-json', global_arguments.log_json) + + flags.make_flags('lock-wait', storage_config.get('lock_wait')) + ( ( flags.make_flags('match-archives', f'sh:{rlist_arguments.prefix}*') @@ -113,13 +114,15 @@ def list_repository( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage config dict, the local Borg version, the - arguments to the list action, and local and remote Borg paths, display the output of listing - Borg archives in the given repository (or return JSON output). + arguments to the list action, global arguments as an argparse.Namespace instance, and local and + remote Borg paths, display the output of listing Borg archives in the given repository (or + return JSON output). ''' borgmatic.logger.add_custom_log_levels() borg_environment = environment.make_environment(storage_config) @@ -129,6 +132,7 @@ def list_repository( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 9fd05b76..d8f3978f 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -13,12 +13,14 @@ def transfer_archives( storage_config, local_borg_version, transfer_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg - version, and the arguments to the transfer action, transfer archives to the given repository. + version, the arguments to the transfer action, and global arguments as an argparse.Namespace + instance, transfer archives to the given repository. ''' borgmatic.logger.add_custom_log_levels() @@ -27,6 +29,7 @@ def transfer_archives( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', storage_config.get('lock_wait', None)) + ( flags.make_flags_from_arguments( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e5b99ec5..0812edea 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -187,6 +187,11 @@ def make_parsers(): type=str, help='Log format string used for log messages written to the log file', ) + global_group.add_argument( + '--log-json', + action='store_true', + help='Write log messages and console output as one JSON object per log line instead of formatted text', + ) global_group.add_argument( '--override', metavar='SECTION.OPTION=VALUE', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index d5fedba7..44396cd4 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -398,7 +398,8 @@ def run_actions( repository, storage, local_borg_version, - arguments['mount'], + action_arguments, + global_arguments, local_path, remote_path, ) @@ -420,6 +421,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -429,6 +431,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -438,6 +441,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -447,6 +451,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -455,7 +460,8 @@ def run_actions( repository, storage, local_borg_version, - arguments['break-lock'], + action_arguments, + global_arguments, local_path, remote_path, ) @@ -465,6 +471,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 1afb0e0f..44403193 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -1,3 +1,4 @@ +import argparse import copy from flexmock import flexmock @@ -58,7 +59,12 @@ def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): continue borgmatic.borg.transfer.transfer_archives( - False, 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + False, + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + global_arguments=flexmock(log_json=False), ) @@ -70,7 +76,11 @@ def test_make_list_command_does_not_duplicate_flags_or_raise(): continue command = borgmatic.borg.list.make_list_command( - 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + argparse.Namespace(log_json=False), ) assert_command_does_not_duplicate_flags(command) @@ -84,7 +94,11 @@ def test_make_rlist_command_does_not_duplicate_flags_or_raise(): continue command = borgmatic.borg.rlist.make_rlist_command( - 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + global_arguments=flexmock(log_json=True), ) assert_command_does_not_duplicate_flags(command) @@ -104,5 +118,9 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): continue borgmatic.borg.info.display_archives_info( - 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + argparse.Namespace(log_json=False), ) diff --git a/tests/unit/actions/test_borg.py b/tests/unit/actions/test_borg.py index f597acbf..2e03ec9c 100644 --- a/tests/unit/actions/test_borg.py +++ b/tests/unit/actions/test_borg.py @@ -16,6 +16,7 @@ def test_run_borg_does_not_raise(): repository={'path': 'repos'}, storage={}, local_borg_version=None, + global_arguments=flexmock(log_json=False), borg_arguments=borg_arguments, local_path=None, remote_path=None, diff --git a/tests/unit/actions/test_break_lock.py b/tests/unit/actions/test_break_lock.py index 6dc2470e..5949d7c1 100644 --- a/tests/unit/actions/test_break_lock.py +++ b/tests/unit/actions/test_break_lock.py @@ -14,6 +14,7 @@ def test_run_break_lock_does_not_raise(): storage={}, local_borg_version=None, break_lock_arguments=break_lock_arguments, + global_arguments=flexmock(), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py index a4f1d544..97161968 100644 --- a/tests/unit/actions/test_info.py +++ b/tests/unit/actions/test_info.py @@ -18,6 +18,7 @@ def test_run_info_does_not_raise(): storage={}, local_borg_version=None, info_arguments=info_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py index bfdfd010..5ee72251 100644 --- a/tests/unit/actions/test_list.py +++ b/tests/unit/actions/test_list.py @@ -18,6 +18,7 @@ def test_run_list_does_not_raise(): storage={}, local_borg_version=None, list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_mount.py b/tests/unit/actions/test_mount.py index 7eadfca1..743747d2 100644 --- a/tests/unit/actions/test_mount.py +++ b/tests/unit/actions/test_mount.py @@ -21,6 +21,7 @@ def test_run_mount_does_not_raise(): storage={}, local_borg_version=None, mount_arguments=mount_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 16fe2920..4bad6f82 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -72,6 +72,7 @@ def test_collect_archive_database_names_parses_archive_paths(): location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), local_path=flexmock(), remote_path=flexmock(), ) @@ -97,6 +98,7 @@ def test_collect_archive_database_names_parses_directory_format_archive_paths(): location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), local_path=flexmock(), remote_path=flexmock(), ) @@ -118,6 +120,7 @@ def test_collect_archive_database_names_skips_bad_archive_paths(): location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), local_path=flexmock(), remote_path=flexmock(), ) diff --git a/tests/unit/actions/test_rinfo.py b/tests/unit/actions/test_rinfo.py index 133e61ac..7b2371a3 100644 --- a/tests/unit/actions/test_rinfo.py +++ b/tests/unit/actions/test_rinfo.py @@ -15,6 +15,7 @@ def test_run_rinfo_does_not_raise(): storage={}, local_borg_version=None, rinfo_arguments=rinfo_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_rlist.py b/tests/unit/actions/test_rlist.py index 7f8b58aa..4a59dc30 100644 --- a/tests/unit/actions/test_rlist.py +++ b/tests/unit/actions/test_rlist.py @@ -15,6 +15,7 @@ def test_run_rlist_does_not_raise(): storage={}, local_borg_version=None, rlist_arguments=rlist_arguments, + global_arguments=flexmock(), local_path=None, remote_path=None, ) diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 509fc1b8..3dc55672 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -24,6 +24,7 @@ def test_break_lock_calls_borg_with_required_flags(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -35,6 +36,7 @@ def test_break_lock_calls_borg_with_remote_path_flags(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -47,6 +49,19 @@ def test_break_lock_calls_borg_with_umask_flags(): repository_path='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_break_lock_calls_borg_with_log_json_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(('borg', 'break-lock', '--log-json', 'repo')) + + module.break_lock( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -58,6 +73,7 @@ def test_break_lock_calls_borg_with_lock_wait_flags(): repository_path='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -70,6 +86,7 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -82,4 +99,5 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 1f992d3d..4cd6aa77 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -421,6 +421,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -451,6 +452,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), repair=True, ) @@ -490,6 +492,7 @@ def test_check_archives_calls_borg_with_parameters(checks): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -510,6 +513,7 @@ def test_check_archives_with_json_error_raises(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -528,6 +532,7 @@ def test_check_archives_with_missing_json_keys_raises(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -552,6 +557,7 @@ def test_check_archives_with_extract_check_calls_extract_only(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -576,6 +582,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -600,6 +607,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -618,6 +626,7 @@ def test_check_archives_without_any_checks_bails(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -648,6 +657,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -679,10 +689,43 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) +def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): + checks = ('repository',) + check_last = flexmock() + storage_config = {} + consistency_config = {'check_last': check_last} + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module.rinfo).should_receive('display_repository_info').and_return( + '{"repository": {"id": "repo"}}' + ) + flexmock(module).should_receive('make_check_flags').with_args( + '1.2.3', + storage_config, + checks, + check_last, + None, + ).and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(('borg', 'check', '--log-json', 'repo')) + flexmock(module).should_receive('make_check_time_path') + flexmock(module).should_receive('write_check_time') + + module.check_archives( + repository_path='repo', + location_config={}, + storage_config=storage_config, + consistency_config=consistency_config, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + + def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): checks = ('repository',) check_last = flexmock() @@ -711,6 +754,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): storage_config=storage_config, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -738,6 +782,7 @@ def test_check_archives_with_retention_prefix(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -761,4 +806,5 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): storage_config={'extra_borg_options': {'check': '--extra --options'}}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 60447db6..beacf547 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -25,7 +25,11 @@ def test_compact_segments_calls_borg_with_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) module.compact_segments( - dry_run=False, repository_path='repo', storage_config={}, local_borg_version='1.2.3' + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -35,7 +39,11 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.compact_segments( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + dry_run=False, ) @@ -45,7 +53,11 @@ def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.compact_segments( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + dry_run=False, ) @@ -53,7 +65,11 @@ def test_compact_segments_with_dry_run_skips_borg_call(): flexmock(module).should_receive('execute_command').never() module.compact_segments( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=True + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + dry_run=True, ) @@ -66,6 +82,7 @@ def test_compact_segments_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -79,6 +96,7 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -92,6 +110,7 @@ def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -105,6 +124,7 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), cleanup_commits=True, ) @@ -118,6 +138,7 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), threshold=20, ) @@ -132,6 +153,20 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(COMPACT_COMMAND + ('--log-json', 'repo'), logging.INFO) + + module.compact_segments( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -145,6 +180,7 @@ def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -157,4 +193,5 @@ def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options( repository_path='repo', storage_config={'extra_borg_options': {'compact': '--extra --options'}}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 3728a0bf..e0462e2d 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -492,6 +492,7 @@ def test_create_archive_calls_borg_with_parameters(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -535,6 +536,7 @@ def test_create_archive_calls_borg_with_environment(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -580,6 +582,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -625,6 +628,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -668,6 +672,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -708,6 +713,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, ) @@ -752,6 +758,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -792,6 +799,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, ) @@ -835,6 +843,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -880,6 +889,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stats=True, ) @@ -923,6 +933,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte }, storage_config={'checkpoint_interval': 600}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -965,6 +976,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume }, storage_config={'checkpoint_volume': 1024}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1007,6 +1019,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param }, storage_config={'chunker_params': '1,2,3,4'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1049,6 +1062,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( }, storage_config={'compression': 'rle'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1097,6 +1111,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ }, storage_config={'upload_rate_limit': 100}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1142,6 +1157,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1185,6 +1201,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1234,6 +1251,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1287,6 +1305,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1342,6 +1361,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1396,6 +1416,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1450,6 +1471,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1493,6 +1515,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1535,6 +1558,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -1578,6 +1602,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -1621,6 +1646,50 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): }, storage_config={'umask': 740}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_create_archive_with_log_json_calls_borg_with_log_json_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('map_directories_to_devices').and_return({}) + flexmock(module).should_receive('expand_directories').and_return(()) + flexmock(module).should_receive('pattern_root_directories').and_return([]) + flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) + flexmock(module).should_receive('expand_home_directories').and_return(()) + flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module).should_receive('ensure_files_readable') + flexmock(module).should_receive('make_pattern_flags').and_return(()) + flexmock(module).should_receive('make_exclude_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'create', '--log-json') + REPO_ARCHIVE_WITH_PATHS, + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=None, + ) + + module.create_archive( + dry_run=False, + repository_path='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -1663,6 +1732,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): }, storage_config={'lock_wait': 5}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1705,6 +1775,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stats=True, ) @@ -1748,6 +1819,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), list_files=True, ) @@ -1797,6 +1869,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -1840,6 +1913,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -1900,6 +1974,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, stream_processes=processes, ) @@ -1964,6 +2039,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2031,6 +2107,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2095,6 +2172,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2135,6 +2213,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, ) @@ -2177,6 +2256,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, stats=True, ) @@ -2224,6 +2304,7 @@ def test_create_archive_with_source_directories_glob_expands(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2267,6 +2348,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2309,6 +2391,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2351,6 +2434,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): }, storage_config={'archive_name_format': 'ARCHIVE_NAME'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2394,6 +2478,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2437,6 +2522,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2479,6 +2565,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): }, storage_config={'extra_borg_options': {'create': '--extra --options'}}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2539,6 +2626,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2564,6 +2652,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index 92776dd4..5fb7bff2 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -38,6 +38,7 @@ def test_export_tar_archive_calls_borg_with_path_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -59,6 +60,7 @@ def test_export_tar_archive_calls_borg_with_local_path_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -81,6 +83,7 @@ def test_export_tar_archive_calls_borg_with_remote_path_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -103,6 +106,27 @@ def test_export_tar_archive_calls_borg_with_umask_parameters(): destination_path='test.tar', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_export_tar_archive_calls_borg_with_log_json_parameter(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + insert_execute_command_mock(('borg', 'export-tar', '--log-json', 'repo::archive', 'test.tar')) + + module.export_tar_archive( + dry_run=False, + repository_path='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -124,6 +148,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_parameters(): destination_path='test.tar', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -144,6 +169,7 @@ def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -166,6 +192,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -185,6 +212,7 @@ def test_export_tar_archive_calls_borg_with_dry_run_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -206,6 +234,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), tar_filter='bzip2', ) @@ -229,6 +258,7 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), list_files=True, ) @@ -251,6 +281,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components=5, ) @@ -271,6 +302,7 @@ def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -290,4 +322,5 @@ def test_export_tar_archive_calls_borg_with_stdout_destination_path(): destination_path='-', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index 6517379e..a4032f6c 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -25,7 +25,11 @@ def test_extract_last_archive_dry_run_calls_borg_with_last_archive(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -34,7 +38,11 @@ def test_extract_last_archive_dry_run_without_any_archives_should_not_raise(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(('repo',)) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -47,7 +55,11 @@ def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_paramet ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -62,7 +74,11 @@ def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_param ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -76,13 +92,14 @@ def test_extract_last_archive_dry_run_calls_borg_via_local_path(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), repository_path='repo', lock_wait=None, local_path='borg1', ) -def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): +def test_extract_last_archive_dry_run_calls_borg_with_remote_path_flags(): flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') insert_execute_command_mock( ('borg', 'extract', '--dry-run', '--remote-path', 'borg1', 'repo::archive') @@ -94,13 +111,30 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), repository_path='repo', lock_wait=None, remote_path='borg1', ) -def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): +def test_extract_last_archive_dry_run_calls_borg_with_log_json_flag(): + flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') + insert_execute_command_mock(('borg', 'extract', '--dry-run', '--log-json', 'repo::archive')) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + + module.extract_last_archive_dry_run( + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + repository_path='repo', + lock_wait=None, + ) + + +def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_flags(): flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') insert_execute_command_mock( ('borg', 'extract', '--dry-run', '--lock-wait', '5', 'repo::archive') @@ -110,11 +144,15 @@ def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=5 + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=5, ) -def test_extract_archive_calls_borg_with_path_parameters(): +def test_extract_archive_calls_borg_with_path_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', 'repo::archive', 'path1', 'path2')) flexmock(module.feature).should_receive('available').and_return(True) @@ -133,10 +171,11 @@ def test_extract_archive_calls_borg_with_path_parameters(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_calls_borg_with_remote_path_parameters(): +def test_extract_archive_calls_borg_with_remote_path_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', '--remote-path', 'borg1', 'repo::archive')) flexmock(module.feature).should_receive('available').and_return(True) @@ -155,6 +194,7 @@ def test_extract_archive_calls_borg_with_remote_path_parameters(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -185,10 +225,11 @@ def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available location_config={'numeric_ids': True}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_calls_borg_with_umask_parameters(): +def test_extract_archive_calls_borg_with_umask_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', '--umask', '0770', 'repo::archive')) flexmock(module.feature).should_receive('available').and_return(True) @@ -207,10 +248,31 @@ def test_extract_archive_calls_borg_with_umask_parameters(): location_config={}, storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_calls_borg_with_lock_wait_parameters(): +def test_extract_archive_calls_borg_with_log_json_flags(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock(('borg', 'extract', '--log-json', 'repo::archive')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + + module.extract_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + location_config={}, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + + +def test_extract_archive_calls_borg_with_lock_wait_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', '--lock-wait', '5', 'repo::archive')) flexmock(module.feature).should_receive('available').and_return(True) @@ -229,6 +291,7 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters(): location_config={}, storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -252,10 +315,11 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters(): +def test_extract_archive_with_log_debug_calls_borg_with_debug_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock( ('borg', 'extract', '--debug', '--list', '--show-rc', 'repo::archive') @@ -277,6 +341,7 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -299,6 +364,7 @@ def test_extract_archive_calls_borg_with_dry_run_parameter(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -321,6 +387,7 @@ def test_extract_archive_calls_borg_with_destination_path(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), destination_path='/dest', ) @@ -344,6 +411,7 @@ def test_extract_archive_calls_borg_with_strip_components(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components=5, ) @@ -377,6 +445,7 @@ def test_extract_archive_calls_borg_with_strip_components_calculated_from_all(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components='all', ) @@ -401,6 +470,7 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components='all', ) @@ -430,6 +500,7 @@ def test_extract_archive_calls_borg_with_progress_parameter(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -446,6 +517,7 @@ def test_extract_archive_with_progress_and_extract_to_stdout_raises(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, extract_to_stdout=True, ) @@ -479,6 +551,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), extract_to_stdout=True, ) == process @@ -509,4 +582,5 @@ def test_extract_archive_skips_abspath_for_remote_repository(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index 2eed4fea..112ef4c5 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -29,6 +29,7 @@ def test_display_archives_info_calls_borg_with_parameters(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -54,6 +55,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -78,6 +80,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) @@ -106,6 +109,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -130,6 +134,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) @@ -155,6 +160,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) @@ -182,6 +188,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive='archive', json=False, prefix=None, match_archives=None), ) @@ -207,6 +214,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), local_path='borg1', ) @@ -236,11 +244,41 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), remote_path='borg1', ) +def test_display_archives_info_with_log_json_calls_borg_with_log_json_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('log-json', True).and_return( + ('--log-json',) + ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--log-json', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=True), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), + ) + + def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -266,6 +304,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -294,6 +333,7 @@ def test_display_archives_info_transforms_prefix_into_match_archives_parameters( repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix='foo'), ) @@ -322,6 +362,7 @@ def test_display_archives_info_prefers_prefix_over_archive_name_format(): repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix='foo'), ) @@ -347,6 +388,7 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -375,6 +417,7 @@ def test_display_archives_with_match_archives_option_calls_borg_with_match_archi 'match_archives': 'sh:foo-*', }, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -400,6 +443,7 @@ def test_display_archives_with_match_archives_flag_calls_borg_with_match_archive repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), ) @@ -429,6 +473,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock( archive=None, json=False, prefix=None, match_archives=None, **{argument_name: 'value'} ), diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 0a7db4cc..4e3a5f7c 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -20,6 +20,7 @@ def test_make_list_command_includes_log_info(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--info', 'repo') @@ -36,6 +37,7 @@ def test_make_list_command_includes_json_but_not_info(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -52,6 +54,7 @@ def test_make_list_command_includes_log_debug(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') @@ -68,6 +71,7 @@ def test_make_list_command_includes_json_but_not_debug(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -83,11 +87,28 @@ def test_make_list_command_includes_json(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') +def test_make_list_command_includes_log_json(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(('--log-json',)) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_list_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=True), + ) + + assert command == ('borg', 'list', '--log-json', 'repo') + + def test_make_list_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') @@ -100,6 +121,7 @@ def test_make_list_command_includes_lock_wait(): storage_config={'lock_wait': 5}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') @@ -117,6 +139,7 @@ def test_make_list_command_includes_archive(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', 'repo::archive') @@ -134,6 +157,7 @@ def test_make_list_command_includes_archive_and_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', 'repo::archive', 'var/lib') @@ -149,6 +173,7 @@ def test_make_list_command_includes_local_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), local_path='borg2', ) @@ -156,9 +181,13 @@ def test_make_list_command_includes_local_path(): def test_make_list_command_includes_remote_path(): - flexmock(module.flags).should_receive('make_flags').and_return( - ('--remote-path', 'borg2') - ).and_return(()) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args( + 'remote-path', 'borg2' + ).and_return(('--remote-path', 'borg2')) + flexmock(module.flags).should_receive('make_flags').with_args('log-json', True).and_return( + ('--log-json') + ) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -167,6 +196,7 @@ def test_make_list_command_includes_remote_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), remote_path='borg2', ) @@ -183,6 +213,7 @@ def test_make_list_command_includes_short(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False, short=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--short', 'repo') @@ -221,6 +252,7 @@ def test_make_list_command_includes_additional_flags(argument_name): format=None, **{argument_name: 'value'}, ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') @@ -263,10 +295,11 @@ def test_capture_archive_listing_does_not_raise(): archive='archive', storage_config=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), ) -def test_list_archive_calls_borg_with_parameters(): +def test_list_archive_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.logger).answer = lambda message: None @@ -281,6 +314,7 @@ def test_list_archive_calls_borg_with_parameters(): first=None, last=None, ) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -288,6 +322,7 @@ def test_list_archive_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -305,6 +340,7 @@ def test_list_archive_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, ) @@ -322,6 +358,7 @@ def test_list_archive_with_archive_and_json_errors(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -340,6 +377,7 @@ def test_list_archive_calls_borg_with_local_path(): first=None, last=None, ) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -347,6 +385,7 @@ def test_list_archive_calls_borg_with_local_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg2', remote_path=None, ).and_return(('borg2', 'list', 'repo::archive')) @@ -364,6 +403,7 @@ def test_list_archive_calls_borg_with_local_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg2', ) @@ -413,6 +453,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -431,6 +472,7 @@ def test_list_archive_calls_borg_with_archive(): first=None, last=None, ) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -438,6 +480,7 @@ def test_list_archive_calls_borg_with_archive(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -455,6 +498,7 @@ def test_list_archive_calls_borg_with_archive(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, ) @@ -485,6 +529,7 @@ def test_list_archive_without_archive_delegates_to_list_repository(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -515,6 +560,7 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -534,6 +580,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.logger).answer = lambda message: None + global_arguments = flexmock(log_json=False) default_filter_flags = { 'prefix': None, 'match_archives': None, @@ -553,6 +600,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( list_arguments=argparse.Namespace( archive='archive', paths=None, json=False, find_paths=None, **default_filter_flags ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -572,6 +620,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( list_arguments=argparse.Namespace( archive='archive', paths=None, json=False, find_paths=None, **altered_filter_flags ), + global_arguments=global_arguments, ) @@ -600,6 +649,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes } altered_filter_flags = {**default_filter_flags, **{archive_filter_flag: 'foo'}} glob_paths = ('**/*foo.txt*/**',) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.rlist).should_receive('make_rlist_command').with_args( @@ -609,6 +659,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes rlist_arguments=argparse.Namespace( repository='repo', short=True, format=None, json=None, **altered_filter_flags ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'rlist', '--repo', 'repo')) @@ -632,6 +683,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes find_paths=['foo.txt'], **default_filter_flags, ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', '--repo', 'repo', 'archive1')) @@ -650,6 +702,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes find_paths=['foo.txt'], **default_filter_flags, ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', '--repo', 'repo', 'archive2')) @@ -683,4 +736,5 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes find_paths=['foo.txt'], **altered_filter_flags, ), + global_arguments=global_arguments, ) diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 658b2e52..cd177511 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -30,6 +30,7 @@ def test_mount_archive_calls_borg_with_required_flags(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -54,6 +55,7 @@ def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_a options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -73,6 +75,7 @@ def test_mount_archive_without_archive_calls_borg_with_repository_flags_only(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -92,6 +95,7 @@ def test_mount_archive_calls_borg_with_path_flags(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -113,6 +117,7 @@ def test_mount_archive_calls_borg_with_remote_path_flags(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -133,6 +138,27 @@ def test_mount_archive_calls_borg_with_umask_flags(): options=None, storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_mount_archive_calls_borg_with_log_json_flags(): + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + insert_execute_command_mock(('borg', 'mount', '--log-json', 'repo::archive', '/mnt')) + + module.mount_archive( + repository_path='repo', + archive='archive', + mount_point='/mnt', + paths=None, + foreground=False, + options=None, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -152,6 +178,7 @@ def test_mount_archive_calls_borg_with_lock_wait_flags(): options=None, storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -172,6 +199,7 @@ def test_mount_archive_with_log_info_calls_borg_with_info_parameter(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -192,6 +220,7 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_flags(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -217,6 +246,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -236,4 +266,5 @@ def test_mount_archive_calls_borg_with_options_flags(): options='super_mount', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 128bdc0a..9028a0c7 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -110,7 +110,7 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') -def test_prune_archives_calls_borg_with_parameters(): +def test_prune_archives_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -123,10 +123,11 @@ def test_prune_archives_calls_borg_with_parameters(): storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): +def test_prune_archives_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -140,10 +141,11 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): dry_run=False, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): +def test_prune_archives_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -157,10 +159,11 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): dry_run=False, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): +def test_prune_archives_with_dry_run_calls_borg_with_dry_run_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -173,6 +176,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): dry_run=True, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -189,11 +193,12 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) -def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -206,11 +211,12 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters( storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) -def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_output_log_level(): +def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -223,11 +229,12 @@ def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_ou storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stats=True, ) -def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_output_log_level(): +def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -240,11 +247,12 @@ def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_out storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), list_archives=True, ) -def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): +def test_prune_archives_with_umask_calls_borg_with_umask_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'umask': '077'} @@ -258,10 +266,28 @@ def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(PRUNE_COMMAND + ('--log-json', 'repo'), logging.INFO) + + module.prune_archives( + dry_run=False, + repository_path='repo', + storage_config={}, + retention_config=flexmock(), + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + + +def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} @@ -275,6 +301,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -291,4 +318,5 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): storage_config={'extra_borg_options': {'prune': '--extra --options'}}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index 4da04dff..2f71a8ff 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -48,6 +48,7 @@ def test_create_repository_calls_borg_with_flags(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -68,6 +69,7 @@ def test_create_repository_with_dry_run_skips_borg_call(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -92,6 +94,7 @@ def test_create_repository_raises_for_borg_rcreate_error(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -111,6 +114,7 @@ def test_create_repository_skips_creation_when_repository_already_exists(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -126,6 +130,7 @@ def test_create_repository_raises_for_unknown_rinfo_command_error(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -146,6 +151,7 @@ def test_create_repository_with_source_repository_calls_borg_with_other_repo_fla repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', source_repository='other.borg', ) @@ -167,6 +173,7 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', copy_crypt_key=True, ) @@ -188,6 +195,7 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', append_only=True, ) @@ -209,6 +217,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', storage_quota='5G', ) @@ -230,6 +239,7 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', make_parent_dirs=True, ) @@ -252,6 +262,7 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -273,6 +284,49 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), + encryption_mode='repokey', + ) + + +def test_create_repository_with_log_json_calls_borg_with_log_json_flag(): + insert_rinfo_command_not_found_mock() + insert_rcreate_command_mock(RCREATE_COMMAND + ('--log-json', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=True), + encryption_mode='repokey', + ) + + +def test_create_repository_with_lock_wait_calls_borg_with_lock_wait_flag(): + insert_rinfo_command_not_found_mock() + insert_rcreate_command_mock(RCREATE_COMMAND + ('--lock-wait', '5', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + storage_config={'lock_wait': 5}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -293,6 +347,7 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', local_path='borg1', ) @@ -314,6 +369,7 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', remote_path='borg1', ) @@ -335,5 +391,6 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options repository_path='repo', storage_config={'extra_borg_options': {'rcreate': '--extra --options'}}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index 979b253e..a6e3f08c 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -7,7 +7,7 @@ from borgmatic.borg import rinfo as module from ..test_verbosity import insert_logging_mock -def test_display_repository_info_calls_borg_with_parameters(): +def test_display_repository_info_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -30,6 +30,7 @@ def test_display_repository_info_calls_borg_with_parameters(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) @@ -51,10 +52,11 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) -def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): +def test_display_repository_info_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -77,6 +79,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) @@ -102,12 +105,13 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), + global_arguments=flexmock(log_json=False), ) assert json_output == '[]' -def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter(): +def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -131,6 +135,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter( storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) @@ -156,12 +161,13 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), + global_arguments=flexmock(log_json=False), ) assert json_output == '[]' -def test_display_repository_info_with_json_calls_borg_with_json_parameter(): +def test_display_repository_info_with_json_calls_borg_with_json_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -182,6 +188,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_parameter(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), + global_arguments=flexmock(log_json=False), ) assert json_output == '[]' @@ -210,11 +217,12 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), local_path='borg1', ) -def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -237,11 +245,39 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_pa storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), remote_path='borg1', ) -def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'rinfo', '--log-json', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_repository_info( + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=True), + ) + + +def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} @@ -265,4 +301,5 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parame storage_config=storage_config, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index b83ba615..e1e04f8b 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -20,12 +20,18 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name(): archive = 'myhost-2030-01-01T14:41:17.647620' assert ( - module.resolve_archive_name('repo', archive, storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + archive, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == archive ) -def test_resolve_archive_name_calls_borg_with_parameters(): +def test_resolve_archive_name_calls_borg_with_flags(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -34,12 +40,18 @@ def test_resolve_archive_name_calls_borg_with_parameters(): ).and_return(expected_archive + '\n') assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == expected_archive ) -def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter(): +def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -49,12 +61,18 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter(): insert_logging_mock(logging.INFO) assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == expected_archive ) -def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter(): +def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -64,7 +82,13 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter( insert_logging_mock(logging.DEBUG) assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == expected_archive ) @@ -79,13 +103,18 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): assert ( module.resolve_archive_name( - 'repo', 'latest', storage_config={}, local_borg_version='1.2.3', local_path='borg1' + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + local_path='borg1', ) == expected_archive ) -def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -95,7 +124,12 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param assert ( module.resolve_archive_name( - 'repo', 'latest', storage_config={}, local_borg_version='1.2.3', remote_path='borg1' + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + remote_path='borg1', ) == expected_archive ) @@ -109,10 +143,37 @@ def test_resolve_archive_name_without_archives_raises(): ).and_return('') with pytest.raises(ValueError): - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) -def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): + expected_archive = 'archive-name' + + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ('borg', 'list', '--log-json') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, + ).and_return(expected_archive + '\n') + + assert ( + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + == expected_archive + ) + + +def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') @@ -123,7 +184,11 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter assert ( module.resolve_archive_name( - 'repo', 'latest', storage_config={'lock_wait': 'okay'}, local_borg_version='1.2.3' + 'repo', + 'latest', + storage_config={'lock_wait': 'okay'}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) == expected_archive ) @@ -145,6 +210,7 @@ def test_make_rlist_command_includes_log_info(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--info', 'repo') @@ -166,6 +232,7 @@ def test_make_rlist_command_includes_json_but_not_info(): rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -187,6 +254,7 @@ def test_make_rlist_command_includes_log_debug(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') @@ -208,6 +276,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -228,11 +297,35 @@ def test_make_rlist_command_includes_json(): rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') +def test_make_rlist_command_includes_log_json(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( + ('--log-json',) + ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), + global_arguments=flexmock(log_json=True), + ) + + assert command == ('borg', 'list', '--log-json', 'repo') + + def test_make_rlist_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') @@ -250,6 +343,7 @@ def test_make_rlist_command_includes_lock_wait(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') @@ -270,6 +364,7 @@ def test_make_rlist_command_includes_local_path(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), local_path='borg2', ) @@ -293,6 +388,7 @@ def test_make_rlist_command_includes_remote_path(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), remote_path='borg2', ) @@ -314,6 +410,7 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') @@ -332,6 +429,7 @@ def test_make_rlist_command_prefers_prefix_over_archive_name_format(): storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') @@ -352,6 +450,7 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives() rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') @@ -372,6 +471,7 @@ def test_make_rlist_command_includes_short(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None, short=True ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--short', 'repo') @@ -413,12 +513,13 @@ def test_make_rlist_command_includes_additional_flags(argument_name): format=None, **{argument_name: 'value'}, ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') -def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_parameters(): +def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_flags(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( None, None, '1.2.3' @@ -444,15 +545,17 @@ def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_p find_paths=None, format=None, ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'foo-*', 'repo') -def test_list_repository_calls_borg_with_parameters(): +def test_list_repository_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER rlist_arguments = argparse.Namespace(json=False) + global_arguments = flexmock() flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( @@ -460,6 +563,7 @@ def test_list_repository_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'rlist', 'repo')) @@ -476,6 +580,7 @@ def test_list_repository_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, ) @@ -483,6 +588,7 @@ def test_list_repository_with_json_returns_borg_output(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER rlist_arguments = argparse.Namespace(json=True) + global_arguments = flexmock() json_output = flexmock() flexmock(module.feature).should_receive('available').and_return(False) @@ -491,6 +597,7 @@ def test_list_repository_with_json_returns_borg_output(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'rlist', 'repo')) @@ -503,6 +610,7 @@ def test_list_repository_with_json_returns_borg_output(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, ) == json_output ) diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 8f41bf5a..e85729ea 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -32,6 +32,7 @@ def test_transfer_archives_calls_borg_with_flags(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -62,6 +63,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -89,6 +91,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -117,6 +120,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -146,6 +150,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): transfer_arguments=flexmock( archive='archive', progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -175,6 +180,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -204,6 +210,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -231,6 +238,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), local_path='borg2', ) @@ -262,10 +270,42 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), remote_path='borg2', ) +def test_transfer_archives_with_log_json_calls_borg_with_log_json_flags(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('log-json', True).and_return( + ('--log-json',) + ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--log-json', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, + borg_local_path='borg', + extra_environment=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + global_arguments=flexmock(log_json=True), + ) + + def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -294,6 +334,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -321,6 +362,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): transfer_arguments=flexmock( archive=None, progress=True, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -356,6 +398,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): source_repository=None, **{argument_name: 'value'}, ), + global_arguments=flexmock(log_json=False), ) @@ -385,4 +428,5 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository='other' ), + global_arguments=flexmock(log_json=False), ) From 403ae0f698d8f612832912e32f778f9245eff781 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 9 May 2023 10:14:03 -0700 Subject: [PATCH 59/68] Clarify configuration comment about source_directories also accepting files (#693). --- borgmatic/config/schema.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 0cf02b25..903c0432 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -30,8 +30,8 @@ properties: items: type: string description: | - List of source directories to backup. Globs and tildes are - expanded. Do not backslash spaces in path names. + List of source directories and files to backup. Globs and + tildes are expanded. Do not backslash spaces in path names. example: - /home - /etc From 62b11ba16b974511e9505dea90822b7524538669 Mon Sep 17 00:00:00 2001 From: ennui Date: Sat, 13 May 2023 11:20:47 +0000 Subject: [PATCH 60/68] Docs: add Gentoo Linux to other ways to install --- docs/how-to/set-up-backups.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index de5bf8b9..d428ddc3 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -87,6 +87,7 @@ installing borgmatic: * [Debian](https://tracker.debian.org/pkg/borgmatic) * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic) * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic) + * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic) * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) From c6126a9226f223ea2fbf41667cfbf5b9550a024b Mon Sep 17 00:00:00 2001 From: ennui Date: Sat, 13 May 2023 11:22:47 +0000 Subject: [PATCH 61/68] Docs: add Gentoo Linux to other ways to install --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index d428ddc3..515f017e 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -87,8 +87,8 @@ installing borgmatic: * [Debian](https://tracker.debian.org/pkg/borgmatic) * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic) * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic) - * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic) * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) + * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic) * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) * [OpenBSD](https://openports.pl/path/sysutils/borgmatic) From 8eb05b840a9a07655f0652feb39a15822bcbd3a4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 14 May 2023 09:59:28 -0700 Subject: [PATCH 62/68] Log a warning when "borgmatic borg" is run with an action that borgmatic natively supports (#694). --- NEWS | 2 +- borgmatic/borg/borg.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 17560d7e..ac555822 100644 --- a/NEWS +++ b/NEWS @@ -18,7 +18,7 @@ --editable" development installs. * #691: Fix error in "borgmatic restore" action when the configured repository path is relative instead of absolute. - * Run "borgmatic borg" action without capturing output so interactive prompts and flags like + * #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like "--progress" still work. 1.7.12 diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index 1c41b8ec..82fecc1a 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -1,5 +1,6 @@ import logging +import borgmatic.commands.arguments import borgmatic.logger from borgmatic.borg import environment, flags from borgmatic.execute import DO_NOT_CAPTURE, execute_command @@ -36,6 +37,14 @@ def run_arbitrary_borg( command_options_start_index = 2 if options[0] in BORG_SUBCOMMANDS_WITH_SUBCOMMANDS else 1 borg_command = tuple(options[:command_options_start_index]) command_options = tuple(options[command_options_start_index:]) + + if ( + borg_command + and borg_command[0] in borgmatic.commands.arguments.SUBPARSER_ALIASES.keys() + ): + logger.warning( + f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}" + ) except IndexError: borg_command = () command_options = () From 645d29b040c58dce7d971df84ab40a2e1678bb6a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 15 May 2023 23:17:45 -0700 Subject: [PATCH 63/68] Fix archive checks being skipped even when particular archives haven't been checked recently (#688). --- NEWS | 3 + borgmatic/borg/check.py | 230 ++++++-- docs/how-to/deal-with-very-large-backups.md | 1 + docs/how-to/make-per-application-backups.md | 3 + tests/unit/borg/test_borg.py | 18 +- tests/unit/borg/test_check.py | 602 +++++++++++++------- 6 files changed, 587 insertions(+), 270 deletions(-) diff --git a/NEWS b/NEWS index ac555822..bcfe22b2 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,9 @@ https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install --editable" development installs. + * #688: Fix archive checks being skipped even when particular archives haven't been checked + recently. This occurred when using multiple borgmatic configuration files with different + "archive_name_format"s, for instance. * #691: Fix error in "borgmatic restore" action when the configured repository path is relative instead of absolute. * #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 52d5208c..63acbe26 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -1,5 +1,7 @@ import argparse import datetime +import hashlib +import itertools import json import logging import os @@ -88,12 +90,18 @@ def parse_frequency(frequency): def filter_checks_on_frequency( - location_config, consistency_config, borg_repository_id, checks, force + location_config, + consistency_config, + borg_repository_id, + checks, + force, + archives_check_id=None, ): ''' Given a location config, a consistency config with a "checks" sequence of dicts, a Borg - repository ID, a sequence of checks, and whether to force checks to run, filter down those - checks based on the configured "frequency" for each check as compared to its check time file. + repository ID, a sequence of checks, whether to force checks to run, and an ID for the archives + check potentially being run (if any), filter down those checks based on the configured + "frequency" for each check as compared to its check time file. In other words, a check whose check time file's timestamp is too new (based on the configured frequency) will get cut from the returned sequence of checks. Example: @@ -127,8 +135,8 @@ def filter_checks_on_frequency( if not frequency_delta: continue - check_time = read_check_time( - make_check_time_path(location_config, borg_repository_id, check) + check_time = probe_for_check_time( + location_config, borg_repository_id, check, archives_check_id ) if not check_time: continue @@ -145,36 +153,19 @@ def filter_checks_on_frequency( return tuple(filtered_checks) -def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None): +def make_archive_filter_flags( + local_borg_version, storage_config, checks, check_last=None, prefix=None +): ''' Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the check last value, and a consistency check prefix, transform the checks into tuple of - command-line flags. + command-line flags for filtering archives in a check command. - For example, given parsed checks of: - - ('repository',) - - This will be returned as: - - ('--repository-only',) - - However, if both "repository" and "archives" are in checks, then omit them from the returned - flags because Borg does both checks by default. If "data" is in checks, that implies "archives". - - Additionally, if a check_last value is given and "archives" is in checks, then include a - "--last" flag. And if a prefix value is given and "archives" is in checks, then include a - "--match-archives" flag. + If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if + a prefix value is given and "archives" is in checks, then include a "--match-archives" flag. ''' - if 'data' in checks: - data_flags = ('--verify-data',) - checks += ('archives',) - else: - data_flags = () - - if 'archives' in checks: - last_flags = ('--last', str(check_last)) if check_last else () - match_archives_flags = ( + if 'archives' in checks or 'data' in checks: + return (('--last', str(check_last)) if check_last else ()) + ( ( ('--match-archives', f'sh:{prefix}*') if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) @@ -189,19 +180,53 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None ) ) ) - else: - last_flags = () - match_archives_flags = () - if check_last: - logger.warning( - 'Ignoring check_last option, as "archives" or "data" are not in consistency checks' - ) - if prefix: - logger.warning( - 'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks' - ) - common_flags = last_flags + match_archives_flags + data_flags + if check_last: + logger.warning( + 'Ignoring check_last option, as "archives" or "data" are not in consistency checks' + ) + if prefix: + logger.warning( + 'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks' + ) + + return () + + +def make_archives_check_id(archive_filter_flags): + ''' + Given a sequence of flags to filter archives, return a unique hash corresponding to those + particular flags. If there are no flags, return None. + ''' + if not archive_filter_flags: + return None + + return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest() + + +def make_check_flags(checks, archive_filter_flags): + ''' + Given a parsed sequence of checks and a sequence of flags to filter archives, transform the + checks into tuple of command-line check flags. + + For example, given parsed checks of: + + ('repository',) + + This will be returned as: + + ('--repository-only',) + + However, if both "repository" and "archives" are in checks, then omit them from the returned + flags because Borg does both checks by default. If "data" is in checks, that implies "archives". + ''' + if 'data' in checks: + data_flags = ('--verify-data',) + checks += ('archives',) + else: + data_flags = () + + common_flags = archive_filter_flags + data_flags if {'repository', 'archives'}.issubset(set(checks)): return common_flags @@ -212,18 +237,27 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None ) -def make_check_time_path(location_config, borg_repository_id, check_type): +def make_check_time_path(location_config, borg_repository_id, check_type, archives_check_id=None): ''' - Given a location configuration dict, a Borg repository ID, and the name of a check type - ("repository", "archives", etc.), return a path for recording that check's time (the time of - that check last occurring). + Given a location configuration dict, a Borg repository ID, the name of a check type + ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a + path for recording that check's time (the time of that check last occurring). ''' + borgmatic_source_directory = os.path.expanduser( + location_config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY) + ) + + if check_type in ('archives', 'data'): + return os.path.join( + borgmatic_source_directory, + 'checks', + borg_repository_id, + check_type, + archives_check_id if archives_check_id else 'all', + ) + return os.path.join( - os.path.expanduser( - location_config.get( - 'borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY - ) - ), + borgmatic_source_directory, 'checks', borg_repository_id, check_type, @@ -253,6 +287,74 @@ def read_check_time(path): return None +def probe_for_check_time(location_config, borg_repository_id, check, archives_check_id): + ''' + Given a location configuration dict, a Borg repository ID, the name of a check type + ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a + the corresponding check time or None if such a check time does not exist. + + When the check type is "archives" or "data", this function probes two different paths to find + the check time, e.g.: + + ~/.borgmatic/checks/1234567890/archives/9876543210 + ~/.borgmatic/checks/1234567890/archives/all + + ... and returns the modification time of the first file found (if any). The first path + represents a more specific archives check time (a check on a subset of archives), and the second + is a fallback to the last "all" archives check. + + For other check types, this function reads from a single check time path, e.g.: + + ~/.borgmatic/checks/1234567890/repository + ''' + check_times = ( + read_check_time(group[0]) + for group in itertools.groupby( + ( + make_check_time_path(location_config, borg_repository_id, check, archives_check_id), + make_check_time_path(location_config, borg_repository_id, check), + ) + ) + ) + + try: + return next(check_time for check_time in check_times if check_time) + except StopIteration: + return None + + +def upgrade_check_times(location_config, borg_repository_id): + ''' + Given a location configuration dict and a Borg repository ID, upgrade any corresponding check + times on disk from old-style paths to new-style paths. + + Currently, the only upgrade performed is renaming an archive or data check path that looks like: + + ~/.borgmatic/checks/1234567890/archives + + to: + + ~/.borgmatic/checks/1234567890/archives/all + ''' + for check_type in ('archives', 'data'): + new_path = make_check_time_path(location_config, borg_repository_id, check_type, 'all') + old_path = os.path.dirname(new_path) + temporary_path = f'{old_path}.temp' + + if not os.path.isfile(old_path) and not os.path.isfile(temporary_path): + return + + logger.debug(f'Upgrading archives check time from {old_path} to {new_path}') + + try: + os.rename(old_path, temporary_path) + except FileNotFoundError: + pass + + os.mkdir(old_path) + os.rename(temporary_path, new_path) + + def check_archives( repository_path, location_config, @@ -292,16 +394,26 @@ def check_archives( except (json.JSONDecodeError, KeyError): raise ValueError(f'Cannot determine Borg repository ID for {repository_path}') + upgrade_check_times(location_config, borg_repository_id) + + check_last = consistency_config.get('check_last', None) + prefix = consistency_config.get('prefix') + configured_checks = parse_checks(consistency_config, only_checks) + lock_wait = None + extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') + archive_filter_flags = make_archive_filter_flags( + local_borg_version, storage_config, configured_checks, check_last, prefix + ) + archives_check_id = make_archives_check_id(archive_filter_flags) + checks = filter_checks_on_frequency( location_config, consistency_config, borg_repository_id, - parse_checks(consistency_config, only_checks), + configured_checks, force, + archives_check_id, ) - check_last = consistency_config.get('check_last', None) - lock_wait = None - extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') if set(checks).intersection({'repository', 'archives', 'data'}): lock_wait = storage_config.get('lock_wait') @@ -312,12 +424,10 @@ def check_archives( if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') - prefix = consistency_config.get('prefix') - full_command = ( (local_path, 'check') + (('--repair',) if repair else ()) - + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix) + + make_check_flags(checks, archive_filter_flags) + (('--remote-path', remote_path) if remote_path else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) @@ -339,7 +449,9 @@ def check_archives( execute_command(full_command, extra_environment=borg_environment) for check in checks: - write_check_time(make_check_time_path(location_config, borg_repository_id, check)) + write_check_time( + make_check_time_path(location_config, borg_repository_id, check, archives_check_id) + ) if 'extract' in checks: extract.extract_last_archive_dry_run( diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index e5962c1e..5beb9f24 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -95,6 +95,7 @@ See [Borg's check documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html) for more information. + ### Check frequency New in version 1.6.2 You can diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index f2ddf012..7832dc43 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -81,6 +81,9 @@ If `archive_name_format` is unspecified, the default is `{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a timestamp in a particular format. + +### Achive filtering + New in version 1.7.11 borgmatic uses the `archive_name_format` option to automatically limit which archives get used for actions operating on multiple archives. This prevents, for diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 4c71ce1a..5ae013f8 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -7,7 +7,7 @@ from borgmatic.borg import borg as module from ..test_verbosity import insert_logging_mock -def test_run_arbitrary_borg_calls_borg_with_parameters(): +def test_run_arbitrary_borg_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -28,7 +28,7 @@ def test_run_arbitrary_borg_calls_borg_with_parameters(): ) -def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): +def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -50,7 +50,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): ) -def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): +def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -72,7 +72,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): ) -def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} @@ -96,7 +96,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters( ) -def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter(): +def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -142,7 +142,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): ) -def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -166,7 +166,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet ) -def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): +def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -187,7 +187,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): ) -def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): +def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -208,7 +208,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): ) -def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): +def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').never() diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 4cd6aa77..aad973bd 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -96,7 +96,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks(): module.datetime.timedelta(weeks=4) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return(None) + flexmock(module).should_receive('probe_for_check_time').and_return(None) assert module.filter_checks_on_frequency( location_config={}, @@ -104,6 +104,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks(): borg_repository_id='repo', checks=('repository', 'archives'), force=False, + archives_check_id='1234', ) == ('repository', 'archives') @@ -126,6 +127,7 @@ def test_filter_checks_on_frequency_retains_check_without_frequency(): borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == ('archives',) @@ -134,7 +136,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency(): module.datetime.timedelta(hours=1) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return( + flexmock(module).should_receive('probe_for_check_time').and_return( module.datetime.datetime(year=module.datetime.MINYEAR, month=1, day=1) ) @@ -144,6 +146,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency(): borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == ('archives',) @@ -152,7 +155,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file() module.datetime.timedelta(hours=1) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return(None) + flexmock(module).should_receive('probe_for_check_time').and_return(None) assert module.filter_checks_on_frequency( location_config={}, @@ -160,6 +163,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file() borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == ('archives',) @@ -168,7 +172,9 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency(): module.datetime.timedelta(hours=1) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return(module.datetime.datetime.now()) + flexmock(module).should_receive('probe_for_check_time').and_return( + module.datetime.datetime.now() + ) assert ( module.filter_checks_on_frequency( @@ -177,6 +183,7 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency(): borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == () ) @@ -189,32 +196,177 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_ borg_repository_id='repo', checks=('archives',), force=True, + archives_check_id='1234', ) == ('archives',) -def test_make_check_flags_with_repository_check_returns_flag(): +def test_make_archive_filter_flags_with_default_checks_and_prefix_returns_default_flags(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('repository',)) + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives'), + prefix='foo', + ) + + assert flags == ('--match-archives', 'sh:foo*') + + +def test_make_archive_filter_flags_with_all_checks_and_prefix_returns_default_flags(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', + ) + + assert flags == ('--match-archives', 'sh:foo*') + + +def test_make_archive_filter_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', + ) + + assert flags == ('--glob-archives', 'foo*') + + +def test_make_archive_filter_flags_with_archives_check_and_last_includes_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), check_last=3) + + assert flags == ('--last', '3') + + +def test_make_archive_filter_flags_with_data_check_and_last_includes_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), check_last=3) + + assert flags == ('--last', '3') + + +def test_make_archive_filter_flags_with_repository_check_and_last_omits_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), check_last=3) + + assert flags == () + + +def test_make_archive_filter_flags_with_default_checks_and_last_includes_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) + + assert flags == ('--last', '3') + + +def test_make_archive_filter_flags_with_archives_check_and_prefix_includes_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix='foo-') + + assert flags == ('--match-archives', 'sh:foo-*') + + +def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), prefix='foo-') + + assert flags == ('--match-archives', 'sh:foo-*') + + +def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + + flags = module.make_archive_filter_flags( + '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 + ) + + assert flags == ('--match-archives', 'sh:bar-*') + + +def test_make_archive_filter_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix=None) + + assert flags == () + + +def test_make_archive_filter_flags_with_repository_check_and_prefix_omits_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), prefix='foo-') + + assert flags == () + + +def test_make_archive_filter_flags_with_default_checks_and_prefix_includes_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') + + assert flags == ('--match-archives', 'sh:foo-*') + + +def test_make_archives_check_id_with_flags_returns_a_value_and_does_not_raise(): + assert module.make_archives_check_id(('--match-archives', 'sh:foo-*')) + + +def test_make_archives_check_id_with_empty_flags_returns_none(): + assert module.make_archives_check_id(()) is None + + +def test_make_check_flags_with_repository_check_returns_flag(): + flags = module.make_check_flags(('repository',), ()) assert flags == ('--repository-only',) def test_make_check_flags_with_archives_check_returns_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('archives',)) + flags = module.make_check_flags(('archives',), ()) assert flags == ('--archives-only',) +def test_make_check_flags_with_archive_filtler_flags_includes_those_flags(): + flags = module.make_check_flags(('archives',), ('--match-archives', 'sh:foo-*')) + + assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') + + def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('data',)) + flags = module.make_check_flags(('data',), ()) assert flags == ( '--archives-only', @@ -226,7 +378,7 @@ def test_make_check_flags_with_extract_omits_extract_flag(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('extract',)) + flags = module.make_check_flags(('extract',), ()) assert flags == () @@ -236,151 +388,66 @@ def test_make_check_flags_with_repository_and_data_checks_does_not_return_reposi flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', - {}, ( 'repository', 'data', ), + (), ) assert flags == ('--verify-data',) -def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags( - '1.2.3', - {}, - ('repository', 'archives'), - prefix='foo', +def test_make_check_time_path_with_borgmatic_source_directory_includes_it(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return( + '/home/user/.borgmatic' ) - assert flags == ('--match-archives', 'sh:foo*') - - -def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags( - '1.2.3', - {}, - ('repository', 'archives', 'extract'), - prefix='foo', + assert ( + module.make_check_time_path( + {'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'archives', '5678' + ) + == '/home/user/.borgmatic/checks/1234/archives/5678' ) - assert flags == ('--match-archives', 'sh:foo*') +def test_make_check_time_path_without_borgmatic_source_directory_uses_default(): + flexmock(module.os.path).should_receive('expanduser').with_args( + module.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY + ).and_return('/home/user/.borgmatic') -def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): - flexmock(module.feature).should_receive('available').and_return(False) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags( - '1.2.3', - {}, - ('repository', 'archives', 'extract'), - prefix='foo', + assert ( + module.make_check_time_path({}, '1234', 'archives', '5678') + == '/home/user/.borgmatic/checks/1234/archives/5678' ) - assert flags == ('--glob-archives', 'foo*') - -def test_make_check_flags_with_archives_check_and_last_includes_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('archives',), check_last=3) - - assert flags == ('--archives-only', '--last', '3') - - -def test_make_check_flags_with_data_check_and_last_includes_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('data',), check_last=3) - - assert flags == ('--archives-only', '--last', '3', '--verify-data') - - -def test_make_check_flags_with_repository_check_and_last_omits_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository',), check_last=3) - - assert flags == ('--repository-only',) - - -def test_make_check_flags_with_default_checks_and_last_includes_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) - - assert flags == ('--last', '3') - - -def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix='foo-') - - assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') - - -def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('data',), prefix='foo-') - - assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data') - - -def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, 'bar-{now}', '1.2.3' # noqa: FS003 - ).and_return(('--match-archives', 'sh:bar-*')) - - flags = module.make_check_flags( - '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 +def test_make_check_time_path_with_archives_check_and_no_archives_check_id_defaults_to_all(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return( + '/home/user/.borgmatic' ) - assert flags == ('--archives-only', '--match-archives', 'sh:bar-*') + assert ( + module.make_check_time_path( + {'borgmatic_source_directory': '~/.borgmatic'}, + '1234', + 'archives', + ) + == '/home/user/.borgmatic/checks/1234/archives/all' + ) -def test_make_check_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) +def test_make_check_time_path_with_repositories_check_ignores_archives_check_id(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return( + '/home/user/.borgmatic' + ) - flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix=None) - - assert flags == ('--archives-only',) - - -def test_make_check_flags_with_repository_check_and_prefix_omits_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository',), prefix='foo-') - - assert flags == ('--repository-only',) - - -def test_make_check_flags_with_default_checks_and_prefix_includes_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') - - assert flags == ('--match-archives', 'sh:foo-*') + assert ( + module.make_check_time_path( + {'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'repository', '5678' + ) + == '/home/user/.borgmatic/checks/1234/repository' + ) def test_read_check_time_does_not_raise(): @@ -395,14 +462,135 @@ def test_read_check_time_on_missing_file_does_not_raise(): assert module.read_check_time('/path') is None +def test_probe_for_check_time_uses_first_of_multiple_check_times(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(1).and_return(2) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + + +def test_probe_for_check_time_deduplicates_identical_check_time_paths(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/5678') + flexmock(module).should_receive('read_check_time').and_return(1).once() + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + + +def test_probe_for_check_time_skips_none_check_time(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(None).and_return(2) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2 + + +def test_probe_for_check_time_uses_single_check_time(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(1).and_return(None) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + + +def test_probe_for_check_time_returns_none_when_no_check_time_found(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(None).and_return(None) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) is None + + +def test_upgrade_check_times_renames_old_check_paths_to_all(): + base_path = '~/.borgmatic/checks/1234' + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'archives', 'all' + ).and_return(f'{base_path}/archives/all') + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'data', 'all' + ).and_return(f'{base_path}/data/all') + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return( + True + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/archives.temp' + ).and_return(False) + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/data.temp' + ).and_return(False) + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives', f'{base_path}/archives.temp' + ).once() + flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once() + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives.temp', f'{base_path}/archives/all' + ).once() + + module.upgrade_check_times(flexmock(), flexmock()) + + +def test_upgrade_check_times_skips_missing_check_paths(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/all' + ) + flexmock(module.os.path).should_receive('isfile').and_return(False) + flexmock(module.os).should_receive('rename').never() + flexmock(module.os).should_receive('mkdir').never() + + module.upgrade_check_times(flexmock(), flexmock()) + + +def test_upgrade_check_times_renames_stale_temporary_check_path(): + base_path = '~/.borgmatic/checks/1234' + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'archives', 'all' + ).and_return(f'{base_path}/archives/all') + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'data', 'all' + ).and_return(f'{base_path}/data/all') + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/archives.temp' + ).and_return(True) + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/data.temp' + ).and_return(False) + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives', f'{base_path}/archives.temp' + ).and_raise(FileNotFoundError) + flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once() + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives.temp', f'{base_path}/archives/all' + ).once() + + module.upgrade_check_times(flexmock(), flexmock()) + + def test_check_archives_with_progress_calls_borg_with_progress_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module).should_receive('execute_command').never() flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -429,11 +617,14 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): def test_check_archives_with_repair_calls_borg_with_repair_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module).should_receive('execute_command').never() flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -469,18 +660,15 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): def test_check_archives_calls_borg_with_parameters(checks): check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - {}, - checks, - check_last, - prefix=None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -500,11 +688,14 @@ def test_check_archives_with_json_error_raises(): checks = ('archives',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"unexpected": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) with pytest.raises(ValueError): module.check_archives( @@ -521,9 +712,12 @@ def test_check_archives_with_missing_json_keys_raises(): checks = ('archives',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return('{invalid JSON') + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) with pytest.raises(ValueError): module.check_archives( @@ -540,11 +734,14 @@ def test_check_archives_with_extract_check_calls_extract_only(): checks = ('extract',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').never() flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.extract).should_receive('extract_last_archive_dry_run').once() @@ -564,11 +761,14 @@ def test_check_archives_with_extract_check_calls_extract_only(): def test_check_archives_with_log_info_calls_borg_with_info_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_logging_mock(logging.INFO) @@ -589,11 +789,14 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_logging_mock(logging.DEBUG) @@ -613,11 +816,14 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): def test_check_archives_without_any_checks_bails(): consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(()) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(()) insert_execute_command_never() module.check_archives( @@ -634,18 +840,15 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): checks = ('repository',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - {}, - checks, - check_last, - prefix=None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1', 'check', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -666,18 +869,15 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( checks = ('repository',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - {}, - checks, - check_last, - prefix=None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -699,18 +899,15 @@ def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): check_last = flexmock() storage_config = {} consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - storage_config, - checks, - check_last, - None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--log-json', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -731,18 +928,15 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): check_last = flexmock() storage_config = {'lock_wait': 5} consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - storage_config, - checks, - check_last, - None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -763,14 +957,15 @@ def test_check_archives_with_retention_prefix(): check_last = flexmock() prefix = 'foo-' consistency_config = {'check_last': check_last, 'prefix': prefix} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', {}, checks, check_last, prefix - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -789,11 +984,14 @@ def test_check_archives_with_retention_prefix(): def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--extra', '--options', 'repo')) From ba845d40081c31e646faa2add2b57c12e3811e24 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 15 May 2023 23:25:13 -0700 Subject: [PATCH 64/68] Codespell saves the day. --- docs/how-to/make-per-application-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 7832dc43..b565d668 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -82,7 +82,7 @@ If `archive_name_format` is unspecified, the default is timestamp in a particular format. -### Achive filtering +### Archive filtering New in version 1.7.11 borgmatic uses the `archive_name_format` option to automatically limit which archives From b45e45f1615897c3c60c7433f92fe37b049c1e07 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 16 May 2023 09:36:50 -0700 Subject: [PATCH 65/68] Partial conversion of showing repository labels in logs instead of paths (part of #635). --- borgmatic/actions/borg.py | 4 +++- borgmatic/actions/break_lock.py | 4 +++- borgmatic/actions/check.py | 2 +- borgmatic/actions/compact.py | 8 ++++++-- borgmatic/actions/create.py | 2 +- borgmatic/actions/extract.py | 4 +++- borgmatic/actions/info.py | 4 +++- borgmatic/actions/list.py | 4 ++-- borgmatic/actions/mount.py | 6 ++++-- borgmatic/actions/prune.py | 2 +- borgmatic/actions/rcreate.py | 2 +- borgmatic/actions/restore.py | 16 +++++++++------- borgmatic/actions/rinfo.py | 4 +++- borgmatic/actions/rlist.py | 2 +- borgmatic/actions/transfer.py | 4 +++- borgmatic/commands/borgmatic.py | 15 ++++++++++----- 16 files changed, 54 insertions(+), 29 deletions(-) diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index ec445fbb..44ffc951 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -22,7 +22,9 @@ def run_borg( if borg_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, borg_arguments.repository ): - logger.info(f'{repository["path"]}: Running arbitrary Borg command') + logger.info( + f'{repository.get("label", repository["path"])}: Running arbitrary Borg command' + ) archive_name = borgmatic.borg.rlist.resolve_archive_name( repository['path'], borg_arguments.archive, diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index f049e772..a00d5785 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -21,7 +21,9 @@ def run_break_lock( if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, break_lock_arguments.repository ): - logger.info(f'{repository["path"]}: Breaking repository and cache locks') + logger.info( + f'{repository.get("label", repository["path"])}: Breaking repository and cache locks' + ) borgmatic.borg.break_lock.break_lock( repository['path'], storage, diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index aac536e3..610d41ee 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -37,7 +37,7 @@ def run_check( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository["path"]}: Running consistency checks') + logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks') borgmatic.borg.check.check_archives( repository['path'], location, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 24b30c0e..ad680d21 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -39,7 +39,9 @@ def run_compact( **hook_context, ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): - logger.info(f'{repository["path"]}: Compacting segments{dry_run_label}') + logger.info( + f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}' + ) borgmatic.borg.compact.compact_segments( global_arguments.dry_run, repository['path'], @@ -53,7 +55,9 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover - logger.info(f'{repository["path"]}: Skipping compact (only available/needed in Borg 1.2+)') + logger.info( + f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)' + ) borgmatic.hooks.command.execute_hook( hooks.get('after_compact'), hooks.get('umask'), diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index a3f8da57..cb8b1cf4 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -42,7 +42,7 @@ def run_create( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository["path"]}: Creating archive{dry_run_label}') + logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}') borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 1f4317cd..0bb1efb7 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -35,7 +35,9 @@ def run_extract( if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): - logger.info(f'{repository["path"]}: Extracting archive {extract_arguments.archive}') + logger.info( + f'{repository.get("label", repository["path"])}: Extracting archive {extract_arguments.archive}' + ) borgmatic.borg.extract.extract_archive( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index d138dbd4..91699623 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -26,7 +26,9 @@ def run_info( repository, info_arguments.repository ): if not info_arguments.json: # pragma: nocover - logger.answer(f'{repository["path"]}: Displaying archive summary information') + logger.answer( + f'{repository.get("label", repository["path"])}: Displaying archive summary information' + ) info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( repository['path'], info_arguments.archive, diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 548f1979..720fab1c 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -26,9 +26,9 @@ def run_list( ): if not list_arguments.json: # pragma: nocover if list_arguments.find_paths: - logger.answer(f'{repository["path"]}: Searching archives') + logger.answer(f'{repository.get("label", repository["path"])}: Searching archives') elif not list_arguments.archive: - logger.answer(f'{repository["path"]}: Listing archives') + logger.answer(f'{repository.get("label", repository["path"])}: Listing archives') list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( repository['path'], list_arguments.archive, diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index 60f7f23c..a72701e5 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -23,9 +23,11 @@ def run_mount( repository, mount_arguments.repository ): if mount_arguments.archive: - logger.info(f'{repository["path"]}: Mounting archive {mount_arguments.archive}') + logger.info( + f'{repository.get("label", repository["path"])}: Mounting archive {mount_arguments.archive}' + ) else: # pragma: nocover - logger.info(f'{repository["path"]}: Mounting repository') + logger.info(f'{repository.get("label", repository["path"])}: Mounting repository') borgmatic.borg.mount.mount_archive( repository['path'], diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 2e25264b..422a9d46 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -37,7 +37,7 @@ def run_prune( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository["path"]}: Pruning archives{dry_run_label}') + logger.info(f'{repository.get("label", repository["path"])}: Pruning archives{dry_run_label}') borgmatic.borg.prune.prune_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index a3015c61..1bfc489b 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -23,7 +23,7 @@ def run_rcreate( ): return - logger.info(f'{repository["path"]}: Creating repository') + logger.info(f'{repository.get("label", repository["path"])}: Creating repository') borgmatic.borg.rcreate.create_repository( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 246c11a6..ded83f4f 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -73,12 +73,14 @@ def restore_single_database( Given (among other things) an archive name, a database hook name, and a configured database configuration dict, restore that database from the archive. ''' - logger.info(f'{repository}: Restoring database {database["name"]}') + logger.info( + f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}' + ) dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, database['name'], @@ -87,7 +89,7 @@ def restore_single_database( # Kick off a single database extract to stdout. extract_process = borgmatic.borg.extract.extract_archive( dry_run=global_arguments.dry_run, - repository=repository, + repository=repository['path'], archive=archive_name, paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]), location_config=location, @@ -106,7 +108,7 @@ def restore_single_database( borgmatic.hooks.dispatch.call_hooks( 'restore_database_dump', {hook_name: [database]}, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, @@ -265,7 +267,7 @@ def run_restore( return logger.info( - f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}' + f'{repository.get("label", repository["path"])}: Restoring databases from archive {restore_arguments.archive}' ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( @@ -314,7 +316,7 @@ def run_restore( found_names.add(database_name) restore_single_database( - repository['path'], + repository, location, storage, hooks, @@ -343,7 +345,7 @@ def run_restore( database['name'] = database_name restore_single_database( - repository['path'], + repository, location, storage, hooks, diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 279cd0e7..7756efd0 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -25,7 +25,9 @@ def run_rinfo( repository, rinfo_arguments.repository ): if not rinfo_arguments.json: # pragma: nocover - logger.answer(f'{repository["path"]}: Displaying repository summary information') + logger.answer( + f'{repository.get("label", repository["path"])}: Displaying repository summary information' + ) json_output = borgmatic.borg.rinfo.display_repository_info( repository['path'], diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index 50c59b6f..a9dee21d 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -25,7 +25,7 @@ def run_rlist( repository, rlist_arguments.repository ): if not rlist_arguments.json: # pragma: nocover - logger.answer(f'{repository["path"]}: Listing repository') + logger.answer(f'{repository.get("label", repository["path"])}: Listing repository') json_output = borgmatic.borg.rlist.list_repository( repository['path'], diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 36ac166d..df481e4d 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,7 +17,9 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' - logger.info(f'{repository["path"]}: Transferring archives to repository') + logger.info( + f'{repository.get("label", repository["path"])}: Transferring archives to repository' + ) borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 44396cd4..965f3931 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -113,10 +113,14 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository, retry_num = repo_queue.get() - logger.debug(f'{repository["path"]}: Running actions for repository') + logger.debug( + f'{repository.get("label", repository["path"])}: Running actions for repository' + ) timeout = retry_num * retry_wait if timeout: - logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') + logger.warning( + f'{repository.get("label", repository["path"])}: Sleeping {timeout}s before next retry' + ) time.sleep(timeout) try: yield from run_actions( @@ -139,14 +143,14 @@ def run_configuration(config_filename, config, arguments): ) tuple( # Consume the generator so as to trigger logging. log_error_records( - f'{repository["path"]}: Error running actions for repository', + f'{repository.get("label", repository["path"])}: Error running actions for repository', error, levelno=logging.WARNING, log_command_error_output=True, ) ) logger.warning( - f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}' + f'{repository.get("label", repository["path"])}: Retrying... attempt {retry_num + 1}/{retries}' ) continue @@ -154,7 +158,8 @@ def run_configuration(config_filename, config, arguments): return yield from log_error_records( - f'{repository["path"]}: Error running actions for repository', error + f'{repository.get("label", repository["path"])}: Error running actions for repository', + error, ) encountered_error = error error_repository = repository['path'] From 79b094d035d34844012347982b6c8aa5c2e070c8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 16 May 2023 09:59:09 -0700 Subject: [PATCH 66/68] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index bcfe22b2..21aec8a0 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.13.dev0 +1.7.13 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas diff --git a/setup.py b/setup.py index ce6b78e5..283089a4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.13.dev0' +VERSION = '1.7.13' setup( From e3425f48beca920f9845dfb31f644c8cd7c42367 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 16 May 2023 10:20:52 -0700 Subject: [PATCH 67/68] Instead of taking the first check time found, take the maximum value (#688) --- borgmatic/borg/check.py | 6 +++--- tests/unit/borg/test_check.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 63acbe26..930c82b6 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -299,7 +299,7 @@ def probe_for_check_time(location_config, borg_repository_id, check, archives_ch ~/.borgmatic/checks/1234567890/archives/9876543210 ~/.borgmatic/checks/1234567890/archives/all - ... and returns the modification time of the first file found (if any). The first path + ... and returns the maximum modification time of the files found (if any). The first path represents a more specific archives check time (a check on a subset of archives), and the second is a fallback to the last "all" archives check. @@ -318,8 +318,8 @@ def probe_for_check_time(location_config, borg_repository_id, check, archives_ch ) try: - return next(check_time for check_time in check_times if check_time) - except StopIteration: + return max(check_time for check_time in check_times if check_time) + except ValueError: return None diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index aad973bd..89db5d20 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -462,13 +462,13 @@ def test_read_check_time_on_missing_file_does_not_raise(): assert module.read_check_time('/path') is None -def test_probe_for_check_time_uses_first_of_multiple_check_times(): +def test_probe_for_check_time_uses_maximum_of_multiple_check_times(): flexmock(module).should_receive('make_check_time_path').and_return( '~/.borgmatic/checks/1234/archives/5678' ).and_return('~/.borgmatic/checks/1234/archives/all') flexmock(module).should_receive('read_check_time').and_return(1).and_return(2) - assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2 def test_probe_for_check_time_deduplicates_identical_check_time_paths(): From 833796d1c466eb89deca796cf2d4371cc1e389d8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 17 May 2023 08:48:54 -0700 Subject: [PATCH 68/68] Add archive check probing logic tweak to NEWS (#688). --- NEWS | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 21aec8a0..9308507a 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.7.14.dev0 + * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist. + 1.7.13 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" flag. See the documentation for more information: diff --git a/setup.py b/setup.py index 283089a4..2665af85 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.13' +VERSION = '1.7.14.dev0' setup(