Deprecate generate-borgmatic-config in favor if new "config generate" action (#529).
continuous-integration/drone/push Build is failing Details

This commit is contained in:
Dan Helfman 2023-06-21 12:19:49 -07:00
parent 803fc25848
commit 1b90da5bf1
16 changed files with 317 additions and 153 deletions

5
NEWS
View File

@ -2,8 +2,9 @@
* #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors.
* #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style
configuration.
* #697, #712: Extract borgmatic configuration from backup via "bootstrap" action—even when
borgmatic has no configuration yet!
* #529: Deprecate generate-borgmatic-config in favor if new "config generate" action.
* #697, #712: Extract borgmatic configuration from backup via new "config bootstrap" action—even
when borgmatic has no configuration yet!
* #669: Add sample systemd user service for running borgmatic as a non-root user.
* #711, #713: Fix an error when "data" check time files are accessed without getting upgraded
first.

View File

@ -0,0 +1,39 @@
import logging
import borgmatic.config.generate
import borgmatic.config.validate
logger = logging.getLogger(__name__)
def run_generate(generate_arguments, global_arguments):
dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
logger.answer(
f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
)
borgmatic.config.generate.generate_sample_configuration(
global_arguments.dry_run,
generate_arguments.source_filename,
generate_arguments.destination_filename,
borgmatic.config.validate.schema_filename(),
overwrite=generate_arguments.overwrite,
)
if generate_arguments.source_filename:
logger.answer(
f'''
Merged in the contents of configuration file at: {generate_arguments.source_filename}
To review the changes made, run:
diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
)
logger.answer(
'''
This includes all available configuration options with example values, the few
required options as indicated. Please edit the file to suit your needs.
If you ever need help: https://torsion.org/borgmatic/#issues'''
)

View File

@ -695,14 +695,12 @@ def make_parsers():
config_parsers = config_parser.add_subparsers(
title='config sub-actions',
description='Valid sub-actions for config',
help='Additional help',
)
config_bootstrap_parser = config_parsers.add_parser(
'bootstrap',
help='Extract the config files used to create a borgmatic repository',
description='Extract config files that were used to create a borgmatic repository during the "create" action',
help='Extract the borgmatic config files from a named archive',
description='Extract the borgmatic config files from a named archive',
add_help=False,
)
config_bootstrap_group = config_bootstrap_parser.add_argument_group(
@ -746,6 +744,36 @@ def make_parsers():
'-h', '--help', action='help', help='Show this help message and exit'
)
config_generate_parser = config_parsers.add_parser(
'generate',
help='Generate a sample borgmatic configuration file',
description='Generate a sample borgmatic configuration file',
add_help=False,
)
config_generate_group = config_generate_parser.add_argument_group('config generate arguments')
config_generate_group.add_argument(
'-s',
'--source',
dest='source_filename',
help='Optional configuration file to merge into the generated configuration, useful for upgrading your configuration',
)
config_generate_group.add_argument(
'-d',
'--destination',
dest='destination_filename',
default=config_paths[0],
help=f'Destination configuration file, default: {unexpanded_config_paths[0]}',
)
config_generate_group.add_argument(
'--overwrite',
default=False,
action='store_true',
help='Whether to overwrite any existing destination file, defaults to false',
)
config_generate_group.add_argument(
'-h', '--help', action='help', help='Show this help message and exit'
)
export_tar_parser = action_parsers.add_parser(
'export-tar',
aliases=ACTION_ALIASES['export-tar'],
@ -1170,10 +1198,11 @@ def parse_arguments(*unparsed_arguments):
unparsed_arguments, action_parsers.choices
)
if 'bootstrap' in arguments.keys() and len(arguments.keys()) > 1:
raise ValueError(
'The bootstrap action cannot be combined with other actions. Please run it separately.'
)
for action_name in ('bootstrap', 'generate', 'validate'):
if action_name in arguments.keys() and len(arguments.keys()) > 1:
raise ValueError(
'The {action_name} action cannot be combined with other actions. Please run it separately.'
)
arguments['global'] = top_level_parser.parse_args(remaining_arguments)

View File

@ -19,6 +19,7 @@ import borgmatic.actions.break_lock
import borgmatic.actions.check
import borgmatic.actions.compact
import borgmatic.actions.config.bootstrap
import borgmatic.actions.config.generate
import borgmatic.actions.create
import borgmatic.actions.export_tar
import borgmatic.actions.extract
@ -602,19 +603,24 @@ def get_local_path(configs):
return next(iter(configs.values())).get('location', {}).get('local_path', 'borg')
def collect_configuration_run_summary_logs(configs, arguments):
def collect_highlander_action_summary_logs(configs, arguments):
'''
Given a dict of configuration filename to corresponding parsed configuration, and parsed
Given a dict of configuration filename to corresponding parsed configuration and parsed
command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
each configuration file and yield a series of logging.LogRecord instances containing summary
information about each run.
a highlander action specified in the arguments, if any, and yield a series of logging.LogRecord
instances containing summary information.
As a side effect of running through these configuration files, output their JSON results, if
any, to stdout.
A highlander action is an action that cannot coexist with other actions on the borgmatic
command-line, and borgmatic exits after processing such an action.
'''
if 'bootstrap' in arguments:
# No configuration file is needed for bootstrap.
local_borg_version = borg_version.local_borg_version({}, 'borg')
try:
# No configuration file is needed for bootstrap.
local_borg_version = borg_version.local_borg_version({}, 'borg')
except (OSError, CalledProcessError, ValueError) as error:
yield from log_error_records('Error getting local Borg version', error)
return
try:
borgmatic.actions.config.bootstrap.run_bootstrap(
arguments['bootstrap'], arguments['global'], local_borg_version
@ -622,7 +628,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
yield logging.makeLogRecord(
dict(
levelno=logging.ANSWER,
levelname='INFO',
levelname='ANSWER',
msg='Bootstrap successful',
)
)
@ -635,6 +641,38 @@ def collect_configuration_run_summary_logs(configs, arguments):
return
if 'generate' in arguments:
try:
borgmatic.actions.config.generate.run_generate(
arguments['generate'], arguments['global']
)
yield logging.makeLogRecord(
dict(
levelno=logging.ANSWER,
levelname='ANSWER',
msg='Generate successful',
)
)
except (
CalledProcessError,
ValueError,
OSError,
) as error:
yield from log_error_records(error)
return
def collect_configuration_run_summary_logs(configs, arguments):
'''
Given a dict of configuration filename to corresponding parsed configuration and parsed
command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
each configuration file and yield a series of logging.LogRecord instances containing summary
information about each run.
As a side effect of running through these configuration files, output their JSON results, if
any, to stdout.
'''
# Run cross-file validation checks.
repository = None
@ -730,7 +768,7 @@ def exit_with_help_link(): # pragma: no cover
sys.exit(1)
def main(): # pragma: no cover
def main(extra_summary_logs=[]): # pragma: no cover
configure_signals()
try:
@ -786,7 +824,14 @@ def main(): # pragma: no cover
logger.debug('Ensuring legacy configuration is upgraded')
summary_logs = parse_logs + list(collect_configuration_run_summary_logs(configs, arguments))
summary_logs = (
parse_logs
+ (
list(collect_highlander_action_summary_logs(configs, arguments))
or list(collect_configuration_run_summary_logs(configs, arguments))
)
+ extra_summary_logs
)
summary_logs_max_level = max(log.levelno for log in summary_logs)
for message in ('', 'summary:'):

View File

@ -167,6 +167,6 @@ def fish_completion():
f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}'''
for action_name, subparser in subparsers.choices.items()
for action in subparser._actions
if 'Deprecated' not in action.help
if 'Deprecated' not in (action.help or ())
)
)

View File

@ -1,63 +1,17 @@
import logging
import sys
from argparse import ArgumentParser
from borgmatic.config import generate, validate
DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
import borgmatic.commands.borgmatic
def parse_arguments(*arguments):
'''
Given command-line arguments with which this script was invoked, parse the arguments and return
them as an ArgumentParser instance.
'''
parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
parser.add_argument(
'-s',
'--source',
dest='source_filename',
help='Optional YAML configuration file to merge into the generated configuration, useful for upgrading your configuration',
)
parser.add_argument(
'-d',
'--destination',
dest='destination_filename',
default=DEFAULT_DESTINATION_CONFIG_FILENAME,
help=f'Destination YAML configuration file, default: {DEFAULT_DESTINATION_CONFIG_FILENAME}',
)
parser.add_argument(
'--overwrite',
default=False,
action='store_true',
help='Whether to overwrite any existing destination file, defaults to false',
)
return parser.parse_args(arguments)
def main(): # pragma: no cover
try:
args = parse_arguments(*sys.argv[1:])
generate.generate_sample_configuration(
args.source_filename,
args.destination_filename,
validate.schema_filename(),
overwrite=args.overwrite,
def main():
warning_log = logging.makeLogRecord(
dict(
levelno=logging.WARNING,
levelname='WARNING',
msg='generate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config generate" instead.',
)
)
print(f'Generated a sample configuration file at {args.destination_filename}.')
print()
if args.source_filename:
print(f'Merged in the contents of configuration file at {args.source_filename}.')
print('To review the changes made, run:')
print()
print(f' diff --unified {args.source_filename} {args.destination_filename}')
print()
print('This includes all available configuration options with example values. The few')
print('required options are indicated. Please edit the file to suit your needs.')
print()
print('If you ever need help: https://torsion.org/borgmatic/#issues')
except (ValueError, OSError) as error:
print(error, file=sys.stderr)
sys.exit(1)
sys.argv = ['borgmatic', 'config', 'generate'] + sys.argv[1:]
borgmatic.commands.borgmatic.main([warning_log])

View File

@ -267,7 +267,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
def generate_sample_configuration(
source_filename, destination_filename, schema_filename, overwrite=False
dry_run, source_filename, destination_filename, schema_filename, overwrite=False
):
'''
Given an optional source configuration filename, and a required destination configuration
@ -287,6 +287,9 @@ def generate_sample_configuration(
_schema_to_sample_configuration(schema), source_config
)
if dry_run:
return
write_configuration(
destination_filename,
_comment_out_optional_configuration(render_configuration(destination_config)),

View File

@ -4,7 +4,7 @@ COPY . /app
RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
RUN borgmatic --help > /command-line.txt \
&& for action in rcreate transfer create prune compact check extract config "config bootstrap" export-tar mount umount restore rlist list rinfo info break-lock borg; do \
&& for action in rcreate transfer create prune compact check extract config "config bootstrap" "config generate" export-tar mount umount restore rlist list rinfo info break-lock borg; do \
echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
&& borgmatic $action --help >> /command-line.txt; done

View File

@ -20,18 +20,22 @@ instance, for applications:
```bash
sudo mkdir /etc/borgmatic.d
sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
sudo borgmatic config generate --destination /etc/borgmatic.d/app1.yaml
sudo borgmatic config generate --destination /etc/borgmatic.d/app2.yaml
```
Or, for repositories:
```bash
sudo mkdir /etc/borgmatic.d
sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo1.yaml
sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo2.yaml
sudo borgmatic config generate --destination /etc/borgmatic.d/repo1.yaml
sudo borgmatic config generate --destination /etc/borgmatic.d/repo2.yaml
```
<span class="minilink minilink-addedin">Prior to version 1.7.15</span> The
command to generate configuation files was `generate-borgmatic-config` instead
of `borgmatic config generate`.
When you set up multiple configuration files like this, borgmatic will run
each one in turn from a single borgmatic invocation. This includes, by
default, the traditional `/etc/borgmatic/config.yaml` as well.

View File

@ -120,16 +120,24 @@ offerings, but do not currently fund borgmatic development or hosting.
After you install borgmatic, generate a sample configuration file:
```bash
sudo borgmatic config generate
```
<span class="minilink minilink-addedin">Prior to version 1.7.15</span>
Generate a configuation file with this command instead:
```bash
sudo generate-borgmatic-config
```
If that command is not found, then it may be installed in a location that's
not in your system `PATH` (see above). Try looking in `~/.local/bin/`.
If neither command is found, then borgmatic may be installed in a location
that's not in your system `PATH` (see above). Try looking in `~/.local/bin/`.
This generates a sample configuration file at `/etc/borgmatic/config.yaml` by
default. If you'd like to use another path, use the `--destination` flag, for
instance: `--destination ~/.config/borgmatic/config.yaml`.
The command generates a sample configuration file at
`/etc/borgmatic/config.yaml` by default. If you'd like to use another path,
use the `--destination` flag, for instance: `--destination
~/.config/borgmatic/config.yaml`.
You should edit the configuration file to suit your needs, as the generated
values are only representative. All options are optional except where

View File

@ -29,29 +29,33 @@ configuration options. This is completely optional. If you prefer, you can add
new configuration options manually.
If you do want to upgrade your configuration file to include new options, use
the `generate-borgmatic-config` script with its optional `--source` flag that
the `borgmatic config generate` action with its optional `--source` flag that
takes the path to your original configuration file. If provided with this
path, `generate-borgmatic-config` merges your original configuration into the
path, `borgmatic config generate` merges your original configuration into the
generated configuration file, so you get all the newest options and comments.
Here's an example:
```bash
generate-borgmatic-config --source config.yaml --destination config-new.yaml
borgmatic config generate --source config.yaml --destination config-new.yaml
```
<span class="minilink minilink-addedin">Prior to version 1.7.15</span> The
command to generate configuation files was `generate-borgmatic-config` instead
of `borgmatic config generate`.
New options start as commented out, so you can edit the file and decide
whether you want to use each one.
There are a few caveats to this process. First, when generating the new
configuration file, `generate-borgmatic-config` replaces any comments you've
configuration file, `borgmatic config generate` replaces any comments you've
written in your original configuration file with the newest generated
comments. Second, the script adds back any options you had originally deleted,
although it does so with the options commented out. And finally, any YAML
includes you've used in the source configuration get flattened out into a
single generated file.
As a safety measure, `generate-borgmatic-config` refuses to modify
As a safety measure, `borgmatic config generate` refuses to modify
configuration files in-place. So it's up to you to review the generated file
and, if desired, replace your original configuration file with it.

View File

@ -1,25 +1,9 @@
from flexmock import flexmock
from borgmatic.commands import generate_config as module
def test_parse_arguments_with_no_arguments_uses_default_destination():
parser = module.parse_arguments()
def test_main_does_not_raise():
flexmock(module.borgmatic.commands.borgmatic).should_receive('main')
assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
def test_parse_arguments_with_destination_argument_overrides_default():
parser = module.parse_arguments('--destination', 'config.yaml')
assert parser.destination_filename == 'config.yaml'
def test_parse_arguments_parses_source():
parser = module.parse_arguments('--source', 'source.yaml', '--destination', 'config.yaml')
assert parser.source_filename == 'source.yaml'
def test_parse_arguments_parses_overwrite():
parser = module.parse_arguments('--destination', 'config.yaml', '--overwrite')
assert parser.overwrite
module.main()

View File

@ -210,7 +210,7 @@ def test_generate_sample_configuration_does_not_raise():
flexmock(module).should_receive('_comment_out_optional_configuration')
flexmock(module).should_receive('write_configuration')
module.generate_sample_configuration(None, 'dest.yaml', 'schema.yaml')
module.generate_sample_configuration(False, None, 'dest.yaml', 'schema.yaml')
def test_generate_sample_configuration_with_source_filename_does_not_raise():
@ -225,4 +225,17 @@ def test_generate_sample_configuration_with_source_filename_does_not_raise():
flexmock(module).should_receive('_comment_out_optional_configuration')
flexmock(module).should_receive('write_configuration')
module.generate_sample_configuration('source.yaml', 'dest.yaml', 'schema.yaml')
module.generate_sample_configuration(False, 'source.yaml', 'dest.yaml', 'schema.yaml')
def test_generate_sample_configuration_with_dry_run_does_not_write_file():
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('schema.yaml').and_return('')
flexmock(module.yaml).should_receive('round_trip_load')
flexmock(module).should_receive('_schema_to_sample_configuration')
flexmock(module).should_receive('merge_source_configuration_into_destination')
flexmock(module).should_receive('render_configuration')
flexmock(module).should_receive('_comment_out_optional_configuration')
flexmock(module).should_receive('write_configuration').never()
module.generate_sample_configuration(True, None, 'dest.yaml', 'schema.yaml')

View File

@ -124,4 +124,5 @@ def test_run_bootstrap_does_not_raise():
flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
'archive'
)
module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version)

View File

@ -0,0 +1,39 @@
from flexmock import flexmock
from borgmatic.actions.config import generate as module
def test_run_bootstrap_does_not_raise():
generate_arguments = flexmock(
source_filename=None,
destination_filename='destination.yaml',
overwrite=False,
)
global_arguments = flexmock(dry_run=False)
flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration')
module.run_generate(generate_arguments, global_arguments)
def test_run_bootstrap_with_dry_run_does_not_raise():
generate_arguments = flexmock(
source_filename=None,
destination_filename='destination.yaml',
overwrite=False,
)
global_arguments = flexmock(dry_run=True)
flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration')
module.run_generate(generate_arguments, global_arguments)
def test_run_bootstrap_with_source_filename_does_not_raise():
generate_arguments = flexmock(
source_filename='source.yaml',
destination_filename='destination.yaml',
overwrite=False,
)
global_arguments = flexmock(dry_run=False)
flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration')
module.run_generate(generate_arguments, global_arguments)

View File

@ -962,6 +962,81 @@ def test_get_local_path_without_local_path_defaults_to_borg():
assert module.get_local_path({'test.yaml': {'location': {}}}) == 'borg'
def test_collect_highlander_action_summary_logs_info_for_success_with_bootstrap():
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap')
arguments = {
'bootstrap': flexmock(repository='repo'),
'global': flexmock(dry_run=False),
}
logs = tuple(
module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.ANSWER}
def test_collect_highlander_action_summary_logs_error_on_bootstrap_failure():
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').and_raise(
ValueError
)
arguments = {
'bootstrap': flexmock(repository='repo'),
'global': flexmock(dry_run=False),
}
logs = tuple(
module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_collect_highlander_action_summary_logs_error_on_bootstrap_local_borg_version_failure():
flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError)
flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').never()
arguments = {
'bootstrap': flexmock(repository='repo'),
'global': flexmock(dry_run=False),
}
logs = tuple(
module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_collect_highlander_action_summary_logs_info_for_success_with_generate():
flexmock(module.borgmatic.actions.config.generate).should_receive('run_generate')
arguments = {
'generate': flexmock(destination='test.yaml'),
'global': flexmock(dry_run=False),
}
logs = tuple(
module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.ANSWER}
def test_collect_highlander_action_summary_logs_error_on_generate_failure():
flexmock(module.borgmatic.actions.config.generate).should_receive('run_generate').and_raise(
ValueError
)
arguments = {
'generate': flexmock(destination='test.yaml'),
'global': flexmock(dry_run=False),
}
logs = tuple(
module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_collect_configuration_run_summary_logs_info_for_success():
flexmock(module.command).should_receive('execute_hook').never()
flexmock(module.validate).should_receive('guard_configuration_contains_repository')
@ -1000,41 +1075,6 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
assert {log.levelno for log in logs} == {logging.INFO}
def test_collect_configuration_run_summary_logs_info_for_success_with_bootstrap():
flexmock(module.validate).should_receive('guard_single_repository_selected').never()
flexmock(module.validate).should_receive('guard_configuration_contains_repository').never()
flexmock(module).should_receive('run_configuration').never()
flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap')
arguments = {
'bootstrap': flexmock(repository='repo'),
'global': flexmock(monitoring_verbosity=1, dry_run=False),
}
logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.ANSWER}
def test_collect_configuration_run_summary_logs_error_on_bootstrap_failure():
flexmock(module.validate).should_receive('guard_single_repository_selected').never()
flexmock(module.validate).should_receive('guard_configuration_contains_repository').never()
flexmock(module).should_receive('run_configuration').never()
flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').and_raise(
ValueError
)
arguments = {
'bootstrap': flexmock(repository='repo'),
'global': flexmock(monitoring_verbosity=1, dry_run=False),
}
logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_collect_configuration_run_summary_logs_extract_with_repository_error():
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError