From 7ed5b33db597bbb4a782056a17b81339d3dc111e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 25 Jul 2017 21:18:51 -0700 Subject: [PATCH] #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run. --- NEWS | 5 ++ README.md | 23 +++++++- borgmatic/commands/borgmatic.py | 53 +++++++++-------- borgmatic/config/collect.py | 27 +++++++++ borgmatic/config/convert.py | 11 +++- .../integration/commands/test_borgmatic.py | 15 +++-- borgmatic/tests/unit/config/test_collect.py | 58 +++++++++++++++++++ borgmatic/tests/unit/config/test_convert.py | 16 +++-- setup.py | 2 +- 9 files changed, 172 insertions(+), 38 deletions(-) create mode 100644 borgmatic/config/collect.py create mode 100644 borgmatic/tests/unit/config/test_collect.py diff --git a/NEWS b/NEWS index a4b6c8a..3bc8885 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.1.3.dev0 + + * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run. + * Fix for generate-borgmatic-config writing config with invalid one_file_system value. + 1.1.2 * #32: Fix for passing check_last as integer to subprocess when calling Borg. diff --git a/README.md b/README.md index 0bd5cda..8ea9a89 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,9 @@ To install borgmatic, run the following command to download and install it: Make sure you're using Python 3, as borgmatic does not support Python 2. (You may have to use "pip3" or similar instead of "pip".) -Then, generate a sample configuration file: +## Configuration + +After you install borgmatic, generate a sample configuration file: sudo generate-borgmatic-config @@ -78,6 +80,25 @@ representative. All fields are optional except where indicated, so feel free to remove anything you don't need. +### Multiple configuration files + +A more advanced usage is to create multiple separate configuration files and +place each one in a /etc/borgmatic.d directory. For instance: + + sudo mkdir /etc/borgmatic.d + sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml + sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml + +With this approach, you can have entirely different backup policies for +different applications on your system. For instance, you may want one backup +configuration for your database data directory, and a different configuration +for your user home directories. + +When you set up multiple configuration files like this, borgmatic will run +each one in turn from a single borgmatic invocation. This includes, by +default, the traditional /etc/borgmatic/config.yaml as well. + + ## Upgrading In general, all you should need to do to upgrade borgmatic is run the diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a5f39a0..a26c30c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -5,12 +5,12 @@ from subprocess import CalledProcessError import sys from borgmatic import borg -from borgmatic.config import convert, validate +from borgmatic.config import collect, convert, validate -LEGACY_CONFIG_FILENAME = '/etc/borgmatic/config' -DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' -DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' +LEGACY_CONFIG_PATH = '/etc/borgmatic/config' +DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d'] +DEFAULT_EXCLUDES_PATH = '/etc/borgmatic/excludes' def parse_arguments(*arguments): @@ -21,9 +21,10 @@ def parse_arguments(*arguments): parser = ArgumentParser() parser.add_argument( '-c', '--config', - dest='config_filename', - default=DEFAULT_CONFIG_FILENAME, - help='Configuration filename', + nargs='+', + dest='config_paths', + default=DEFAULT_CONFIG_PATHS, + help='Configuration filenames or directories, defaults to: {}'.format(' '.join(DEFAULT_CONFIG_PATHS)), ) parser.add_argument( '--excludes', @@ -42,25 +43,31 @@ def parse_arguments(*arguments): def main(): # pragma: no cover try: args = parse_arguments(*sys.argv[1:]) - convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename) - config = validate.parse_configuration(args.config_filename, validate.schema_filename()) - (location, storage, retention, consistency) = ( - config.get(section_name, {}) - for section_name in ('location', 'storage', 'retention', 'consistency') - ) - remote_path = location.get('remote_path') + config_filenames = tuple(collect.collect_config_filenames(args.config_paths)) + convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames) - borg.initialize(storage) + if len(config_filenames) == 0: + raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths))) - for repository in location['repositories']: - borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) - borg.create_archive( - args.verbosity, - repository, - location, - storage, + for config_filename in config_filenames: + config = validate.parse_configuration(config_filename, validate.schema_filename()) + (location, storage, retention, consistency) = ( + config.get(section_name, {}) + for section_name in ('location', 'storage', 'retention', 'consistency') ) - borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) + remote_path = location.get('remote_path') + + borg.initialize(storage) + + for repository in location['repositories']: + borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + borg.create_archive( + args.verbosity, + repository, + location, + storage, + ) + borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py new file mode 100644 index 0000000..dbe6f2f --- /dev/null +++ b/borgmatic/config/collect.py @@ -0,0 +1,27 @@ +import os + + +def collect_config_filenames(config_paths): + ''' + Given a sequence of config paths, both filenames and directories, resolve that to just an + iterable of files. Accomplish this by listing any given directories looking for contained config + files. This is non-recursive, so any directories within the given directories are ignored. + + Return paths even if they don't exist on disk, so the user can find out about missing + configuration paths. However, skip /etc/borgmatic.d if it's missing, so the user doesn't have to + create it unless they need it. + ''' + for path in config_paths: + exists = os.path.exists(path) + + if os.path.realpath(path) == '/etc/borgmatic.d' and not exists: + continue + + if not os.path.isdir(path) or not exists: + yield path + continue + + for filename in os.listdir(path): + full_filename = os.path.join(path, filename) + if not os.path.isdir(full_filename): + yield full_filename diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 923ec48..97e7571 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -77,14 +77,19 @@ instead of the old one.''' ) -def guard_configuration_upgraded(source_config_filename, destination_config_filename): +def guard_configuration_upgraded(source_config_filename, destination_config_filenames): ''' - If legacy souce configuration exists but destination upgraded config doesn't, raise + If legacy source configuration exists but no destination upgraded configs do, raise LegacyConfigurationNotUpgraded. The idea is that we want to alert the user about upgrading their config if they haven't already. ''' - if os.path.exists(source_config_filename) and not os.path.exists(destination_config_filename): + destination_config_exists = any( + os.path.exists(filename) + for filename in destination_config_filenames + ) + + if os.path.exists(source_config_filename) and not destination_config_exists: raise LegacyConfigurationNotUpgraded() diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py index f1cc43a..0334341 100644 --- a/borgmatic/tests/integration/commands/test_borgmatic.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -9,23 +9,30 @@ from borgmatic.commands import borgmatic as module def test_parse_arguments_with_no_arguments_uses_defaults(): parser = module.parse_arguments() - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME + assert parser.config_paths == module.DEFAULT_CONFIG_PATHS assert parser.excludes_filename == None assert parser.verbosity is None -def test_parse_arguments_with_filename_arguments_overrides_defaults(): +def test_parse_arguments_with_path_arguments_overrides_defaults(): parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes') - assert parser.config_filename == 'myconfig' + assert parser.config_paths == ['myconfig'] assert parser.excludes_filename == 'myexcludes' assert parser.verbosity is None +def test_parse_arguments_with_multiple_config_paths_parses_as_list(): + parser = module.parse_arguments('--config', 'myconfig', 'otherconfig') + + assert parser.config_paths == ['myconfig', 'otherconfig'] + assert parser.verbosity is None + + def test_parse_arguments_with_verbosity_flag_overrides_default(): parser = module.parse_arguments('--verbosity', '1') - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME + assert parser.config_paths == module.DEFAULT_CONFIG_PATHS assert parser.excludes_filename == None assert parser.verbosity == 1 diff --git a/borgmatic/tests/unit/config/test_collect.py b/borgmatic/tests/unit/config/test_collect.py new file mode 100644 index 0000000..2adee63 --- /dev/null +++ b/borgmatic/tests/unit/config/test_collect.py @@ -0,0 +1,58 @@ +from flexmock import flexmock + +from borgmatic.config import collect as module + + +def test_collect_config_filenames_collects_given_files(): + config_paths = ('config.yaml', 'other.yaml') + flexmock(module.os.path).should_receive('isdir').and_return(False) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == config_paths + + +def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories(): + config_paths = ('config.yaml', '/etc/borgmatic.d') + mock_path = flexmock(module.os.path) + mock_path.should_receive('exists').and_return(True) + mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False) + flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar', 'baz.yaml']) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == ( + 'config.yaml', + '/etc/borgmatic.d/foo.yaml', + '/etc/borgmatic.d/baz.yaml', + ) + + +def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist(): + config_paths = ('config.yaml', '/etc/borgmatic.d') + mock_path = flexmock(module.os.path) + mock_path.should_receive('exists').with_args('config.yaml').and_return(True) + mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False) + mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == ('config.yaml',) + + +def test_collect_config_filenames_includes_directory_if_it_does_not_exist(): + config_paths = ('config.yaml', '/my/directory') + mock_path = flexmock(module.os.path) + mock_path.should_receive('exists').with_args('config.yaml').and_return(True) + mock_path.should_receive('exists').with_args('/my/directory').and_return(False) + mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/my/directory').and_return(True) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == config_paths diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py index 58693ab..995fc8a 100644 --- a/borgmatic/tests/unit/config/test_convert.py +++ b/borgmatic/tests/unit/config/test_convert.py @@ -79,30 +79,34 @@ def test_convert_legacy_parsed_config_splits_space_separated_values(): def test_guard_configuration_upgraded_raises_when_only_source_config_present(): flexmock(os.path).should_receive('exists').with_args('config').and_return(True) flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False) with pytest.raises(module.LegacyConfigurationNotUpgraded): - module.guard_configuration_upgraded('config', 'config.yaml') + module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present(): flexmock(os.path).should_receive('exists').with_args('config').and_return(False) - flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True) + flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True) - module.guard_configuration_upgraded('config', 'config.yaml') + module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present(): flexmock(os.path).should_receive('exists').with_args('config').and_return(True) - flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True) + flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True) - module.guard_configuration_upgraded('config', 'config.yaml') + module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present(): flexmock(os.path).should_receive('exists').with_args('config').and_return(False) flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False) - module.guard_configuration_upgraded('config', 'config.yaml') + module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) def test_guard_excludes_filename_omitted_raises_when_filename_provided(): diff --git a/setup.py b/setup.py index 6d4cc7c..143736b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.2' +VERSION = '1.1.3.dev0' setup(