From 499f8aa0a47fed8d7c841ceaabd8ca83571fa677 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 22:56:46 -0700 Subject: [PATCH] Support for backing up to multiple repositories. --- NEWS | 1 + README.md | 5 +- borgmatic/borg.py | 10 +- borgmatic/commands/borgmatic.py | 20 ++-- borgmatic/config/convert.py | 9 +- borgmatic/config/schema.yaml | 12 +- .../tests/integration/config/test_validate.py | 13 ++- borgmatic/tests/unit/config/test_convert.py | 5 +- borgmatic/tests/unit/test_borg.py | 103 ++++++++++++------ 9 files changed, 115 insertions(+), 63 deletions(-) diff --git a/NEWS b/NEWS index 47e53617c..447317047 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,7 @@ * Dropped Python 2 support. Now Python 3 only. * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. + * Support for backing up to multiple repositories. * To free up space, now pruning backups prior to creating a new backup. * Enabled test coverage output during tox runs. * Added logo. diff --git a/README.md b/README.md index 0d2593692..0bd5cda64 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,9 @@ location: - /etc - /var/log/syslog* - # Path to local or remote repository. - repository: user@backupserver:sourcehostname.borg + # Paths to local or remote repositories. + repositories: + - user@backupserver:sourcehostname.borg # Any paths matching these patterns are excluded from backups. exclude_patterns: diff --git a/borgmatic/borg.py b/borgmatic/borg.py index 22d3032f7..f42353d21 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -39,8 +39,7 @@ def _write_exclude_file(exclude_patterns=None): def create_archive( - verbosity, storage_config, source_directories, repository, exclude_patterns=None, - command=COMMAND, one_file_system=None, remote_path=None, + verbosity, repository, location_config, storage_config, command=COMMAND, ): ''' Given a vebosity flag, a storage config dict, a list of source directories, a local or remote @@ -49,17 +48,18 @@ def create_archive( sources = tuple( itertools.chain.from_iterable( glob.glob(directory) or [directory] - for directory in source_directories + for directory in location_config['source_directories'] ) ) - exclude_file = _write_exclude_file(exclude_patterns) + exclude_file = _write_exclude_file(location_config.get('exclude_patterns')) exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () umask = storage_config.get('umask', None) umask_flags = ('--umask', str(umask)) if umask else () - one_file_system_flags = ('--one-file-system',) if one_file_system else () + one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else () + remote_path = location_config.get('remote_path') remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--info', '--stats',), diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index f1f9c31d4..a5f39a024 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -44,17 +44,23 @@ def main(): # pragma: no cover args = parse_arguments(*sys.argv[1:]) convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename) config = validate.parse_configuration(args.config_filename, validate.schema_filename()) - repository = config['location']['repository'] - remote_path = config['location']['remote_path'] - (storage, retention, consistency) = ( + (location, storage, retention, consistency) = ( config.get(section_name, {}) - for section_name in ('storage', 'retention', 'consistency') + for section_name in ('location', 'storage', 'retention', 'consistency') ) + remote_path = location.get('remote_path') borg.initialize(storage) - borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) - borg.create_archive(args.verbosity, storage, **config['location']) - borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) + + for repository in location['repositories']: + borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + borg.create_archive( + args.verbosity, + repository, + location, + storage, + ) + borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 1aaf7aa29..832d7d310 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -32,9 +32,12 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): for section_name, section_config in source_config._asdict().items() ]) - # Split space-seperated values into actual lists, and merge in excludes. - destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ') - destination_config['location']['exclude_patterns'] = source_excludes + # Split space-seperated values into actual lists, make "repository" into a list, and merge in + # excludes. + location = destination_config['location'] + location['source_directories'] = source_config.location['source_directories'].split(' ') + location['repositories'] = [location.pop('repository')] + location['exclude_patterns'] = source_excludes if source_config.consistency['checks']: destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index dfd0c55bb..64fed0794 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -25,11 +25,15 @@ map: type: scalar desc: Alternate Borg remote executable. Defaults to "borg". example: borg1 - repository: + repositories: required: True - type: scalar - desc: Path to local or remote repository (required). - example: user@backupserver:sourcehostname.borg + seq: + - type: scalar + desc: | + Paths to local or remote repositories (required). Multiple repositories are + backed up to in sequence. + example: + - user@backupserver:sourcehostname.borg exclude_patterns: seq: - type: scalar diff --git a/borgmatic/tests/integration/config/test_validate.py b/borgmatic/tests/integration/config/test_validate.py index 90e223d13..9b63ccc89 100644 --- a/borgmatic/tests/integration/config/test_validate.py +++ b/borgmatic/tests/integration/config/test_validate.py @@ -35,7 +35,8 @@ def test_parse_configuration_transforms_file_into_mapping(): - /home - /etc - repository: hostname.borg + repositories: + - hostname.borg retention: keep_daily: 7 @@ -50,7 +51,7 @@ def test_parse_configuration_transforms_file_into_mapping(): result = module.parse_configuration('config.yaml', 'schema.yaml') assert result == { - 'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, + 'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']}, 'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}, } @@ -65,7 +66,8 @@ def test_parse_configuration_passes_through_quoted_punctuation(): source_directories: - /home - repository: "{}.borg" + repositories: + - "{}.borg" '''.format(escaped_punctuation) ) @@ -74,7 +76,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): assert result == { 'location': { 'source_directories': ['/home'], - 'repository': '{}.borg'.format(string.punctuation), + 'repositories': ['{}.borg'.format(string.punctuation)], }, } @@ -105,7 +107,8 @@ def test_parse_configuration_raises_for_validation_error(): ''' location: source_directories: yes - repository: hostname.borg + repositories: + - hostname.borg ''' ) diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py index 39f0cee0e..827ecdbc2 100644 --- a/borgmatic/tests/unit/config/test_convert.py +++ b/borgmatic/tests/unit/config/test_convert.py @@ -28,7 +28,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): 'location', OrderedDict([ ('source_directories', ['/home']), - ('repository', 'hostname.borg'), + ('repositories', ['hostname.borg']), ('exclude_patterns', ['/var']), ]), ), @@ -41,7 +41,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): def test_convert_legacy_parsed_config_splits_space_separated_values(): flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) source_config = Parsed_config( - location=OrderedDict([('source_directories', '/home /etc')]), + location=OrderedDict([('source_directories', '/home /etc'), ('repository', 'hostname.borg')]), storage=OrderedDict(), retention=OrderedDict(), consistency=OrderedDict([('checks', 'repository archives')]), @@ -56,6 +56,7 @@ def test_convert_legacy_parsed_config_splits_space_separated_values(): 'location', OrderedDict([ ('source_directories', ['/home', '/etc']), + ('repositories', ['hostname.borg']), ('exclude_patterns', ['/var']), ]), ), diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 57e75a312..36cc3121f 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -77,11 +77,14 @@ def test_create_archive_should_call_borg_with_parameters(): insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -93,11 +96,14 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): insert_datetime_mock() module.create_archive( - exclude_patterns=['exclude'], verbosity=None, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': ['exclude'], + }, + storage_config={}, command='borg', ) @@ -109,11 +115,14 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=VERBOSITY_SOME, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -125,11 +134,14 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=VERBOSITY_LOTS, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -141,11 +153,14 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={'compression': 'rle'}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'compression': 'rle'}, command='borg', ) @@ -157,13 +172,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'one_file_system': True, + 'exclude_patterns': None, + }, + storage_config={}, command='borg', - one_file_system=True, ) @@ -174,13 +192,16 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'remote_path': 'borg1', + 'exclude_patterns': None, + }, + storage_config={}, command='borg', - remote_path='borg1', ) @@ -191,11 +212,14 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={'umask': 740}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'umask': 740}, command='borg', ) @@ -208,11 +232,14 @@ def test_create_archive_with_source_directories_glob_expands(): flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo*'], repository='repo', + location_config={ + 'source_directories': ['foo*'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -225,11 +252,14 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([]) module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo*'], repository='repo', + location_config={ + 'source_directories': ['foo*'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -242,11 +272,14 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo*'], repository='repo', + location_config={ + 'source_directories': ['foo*'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', )