From 7dee6194a2d530a30ed38d587130c70bf4c1069e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 18 Aug 2022 23:06:51 -0700 Subject: [PATCH] Add new "transfer" action for Borg 2 (#557). --- NEWS | 16 ++-- borgmatic/borg/compact.py | 17 ++-- borgmatic/borg/rcreate.py | 19 +++-- borgmatic/borg/transfer.py | 45 ++++++++++ borgmatic/commands/arguments.py | 65 +++++++++++++-- borgmatic/commands/borgmatic.py | 15 +++- docs/how-to/upgrade.md | 88 +++++++++++++++++++- tests/integration/commands/test_arguments.py | 9 -- tests/unit/borg/test_rcreate.py | 53 ++++++++++-- tests/unit/commands/test_borgmatic.py | 26 +++++- 10 files changed, 307 insertions(+), 46 deletions(-) create mode 100644 borgmatic/borg/transfer.py diff --git a/NEWS b/NEWS index f22d8563..27470538 100644 --- a/NEWS +++ b/NEWS @@ -1,11 +1,14 @@ 1.7.0.dev0 * #557: Support for Borg 2 while still working with Borg 1. This includes new borgmatic actions - like "rcreate" (replaces "init"), "rlist" (list archives in repository), and "rinfo" (show - repository info). For the most part, borgmatic tries to smooth over differences between Borg 1 - and 2 to make your upgrade process easier. However, there are still a few cases where Borg made - breaking changes. See the Borg 2.0 changelog for more information - (https://www.borgbackup.org/releases/borg-2.0.html). If you install Borg 2, you'll need to - manually "borg transfer" or "borgmatic transfer" your existing Borg 1 repositories before use. + like "rcreate" (replaces "init"), "rlist" (list archives in repository), "rinfo" (show repository + info), and "transfer" (for upgrading Borg repositories). For the most part, borgmatic tries to + smooth over differences between Borg 1 and 2 to make your upgrade process easier. However, there + are still a few cases where Borg made breaking changes. See the Borg 2.0 changelog for more + information: https://www.borgbackup.org/releases/borg-2.0.html + * #557: If you install Borg 2, you'll need to manually upgrade your existing Borg 1 repositories + before use. Note that Borg 2 stable is not yet released as of this borgmatic release, so don't + use Borg 2 for production until it is! See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-borg * #557: Rename several configuration options to match Borg 2: "remote_rate_limit" is now "upload_rate_limit", "numeric_owner" is "numeric_ids", and "bsd_flags" is "flags". borgmatic still works with the old options. @@ -14,6 +17,7 @@ Borg 2. * #557: Omitting the "--archive" flag on the "list" action is deprecated when using Borg 2. Use the new "rlist" action instead. + * #557: The "--dry-run" flag can now be used with the "rcreate"/"init" action. * #565: Fix handling of "repository" and "data" consistency checks to prevent invalid Borg flags. * #566: Modify "mount" and "extract" actions to require the "--repository" flag when multiple repositories are configured. diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index c9770e09..847ed26b 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -39,10 +39,13 @@ def compact_segments( + flags.make_repository_flags(repository, local_borg_version) ) - if not dry_run: - execute_command( - full_command, - output_log_level=logging.INFO, - borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), - ) + if dry_run: + logging.info(f'{repository}: Skipping compact (dry run)') + return + + execute_command( + full_command, + output_log_level=logging.INFO, + borg_local_path=local_path, + extra_environment=environment.make_environment(storage_config), + ) diff --git a/borgmatic/borg/rcreate.py b/borgmatic/borg/rcreate.py index 81be8646..d3a8f7aa 100644 --- a/borgmatic/borg/rcreate.py +++ b/borgmatic/borg/rcreate.py @@ -12,11 +12,12 @@ RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2 def create_repository( + dry_run, repository, storage_config, local_borg_version, encryption_mode, - key_repository=None, + source_repository=None, copy_crypt_key=False, append_only=None, storage_quota=None, @@ -25,10 +26,10 @@ def create_repository( remote_path=None, ): ''' - Given a local or remote repository path, a storage configuration dict, the local Borg version, a - Borg encryption mode, the path to another repo whose key material should be reused, whether the - repository should be append-only, and the storage quota to use, create the repository. If the - repository already exists, then log and skip creation. + Given a dry-run flag, a local or remote repository path, a storage configuration dict, the local + Borg version, a Borg encryption mode, the path to another repo whose key material should be + reused, whether the repository should be append-only, and the storage quota to use, create the + repository. If the repository already exists, then log and skip creation. ''' try: rinfo.display_repository_info( @@ -39,7 +40,7 @@ def create_repository( local_path, remote_path, ) - logger.info('Repository already exists. Skipping creation.') + logger.info(f'{repository}: Repository already exists. Skipping creation.') return except subprocess.CalledProcessError as error: if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE: @@ -55,7 +56,7 @@ def create_repository( else ('init',) ) + (('--encryption', encryption_mode) if encryption_mode else ()) - + (('--other-repo', key_repository) if key_repository else ()) + + (('--other-repo', source_repository) if source_repository else ()) + (('--copy-crypt-key',) if copy_crypt_key else ()) + (('--append-only',) if append_only else ()) + (('--storage-quota', storage_quota) if storage_quota else ()) @@ -67,6 +68,10 @@ def create_repository( + flags.make_repository_flags(repository, local_borg_version) ) + if dry_run: + logging.info(f'{repository}: Skipping repository creation (dry run)') + return + # Do not capture output here, so as to support interactive prompts. execute_command( rcreate_command, diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py new file mode 100644 index 00000000..8647fb9b --- /dev/null +++ b/borgmatic/borg/transfer.py @@ -0,0 +1,45 @@ +import logging + +from borgmatic.borg import environment, flags +from borgmatic.execute import execute_command + +logger = logging.getLogger(__name__) + + +def transfer_archives( + dry_run, + repository, + storage_config, + local_borg_version, + transfer_arguments, + local_path='borg', + remote_path=None, +): + ''' + Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg + version, and the arguments to the transfer action, transfer archives to the given repository. + ''' + full_command = ( + (local_path, 'transfer') + + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + flags.make_flags('remote-path', remote_path) + + flags.make_flags('lock-wait', storage_config.get('lock_wait', None)) + + flags.make_flags( + 'glob-archives', transfer_arguments.glob_archives or transfer_arguments.archive + ) + + flags.make_flags_from_arguments( + transfer_arguments, + excludes=('repository', 'source_repository', 'archive', 'glob_archives'), + ) + + flags.make_repository_flags(repository, local_borg_version) + + flags.make_flags('other-repo', transfer_arguments.source_repository) + + flags.make_flags('dry-run', dry_run) + ) + + return execute_command( + full_command, + output_log_level=logging.WARNING, + borg_local_path=local_path, + extra_environment=environment.make_environment(storage_config), + ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index a5b8afcf..bb3ed4b7 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -241,15 +241,15 @@ def make_parsers(): required=True, ) rcreate_group.add_argument( - '--key-repository', + '--source-repository', '--other-repo', - metavar='SOURCE_REPOSITORY', + metavar='KEY_REPOSITORY', help='Path to an existing Borg repository whose key material should be reused (Borg 2.x+ only)', ) rcreate_group.add_argument( '--copy-crypt-key', action='store_true', - help='Copy the crypt key used for authenticated encryption from the key repository, defaults to a new random key (Borg 2.x+ only)', + help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key (Borg 2.x+ only)', ) rcreate_group.add_argument( '--append-only', action='store_true', help='Create an append-only repository', @@ -266,6 +266,53 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + transfer_parser = subparsers.add_parser( + 'transfer', + aliases=SUBPARSER_ALIASES['transfer'], + help='Transfer archives from one repository to another, optionally upgrading the transferred data', + description='Transfer archives from one repository to another, optionally upgrading the transferred data', + add_help=False, + ) + transfer_group = transfer_parser.add_argument_group('transfer arguments') + transfer_group.add_argument( + '--repository', + help='Path of existing destination repository to transfer archives to, defaults to the configured repository if there is only one', + ) + transfer_group.add_argument( + '--source-repository', + help='Path of existing source repository to transfer archives from', + required=True, + ) + transfer_group.add_argument( + '--archive', + help='Name of single archive to transfer (or "latest"), defaults to transferring all archives', + ) + transfer_group.add_argument( + '--upgrader', + help='Upgrader type used to convert the transfered data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion', + required=True, + ) + transfer_group.add_argument( + '-a', + '--glob-archives', + metavar='GLOB', + help='Only transfer archives with names matching this glob', + ) + transfer_group.add_argument( + '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys' + ) + transfer_group.add_argument( + '--first', + metavar='N', + help='Only transfer first N archives after other filters are applied', + ) + transfer_group.add_argument( + '--last', metavar='N', help='Only transfer last N archives after other filters are applied' + ) + transfer_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) + prune_parser = subparsers.add_parser( 'prune', aliases=SUBPARSER_ALIASES['prune'], @@ -760,9 +807,6 @@ def parse_arguments(*unparsed_arguments): 'The --excludes flag has been replaced with exclude_patterns in configuration.' ) - if 'rcreate' in arguments and arguments['global'].dry_run: - raise ValueError('The rcreate/init action cannot be used with the --dry-run flag.') - if ( ('list' in arguments and 'rinfo' in arguments and arguments['list'].json) or ('list' in arguments and 'info' in arguments and arguments['list'].json) @@ -770,6 +814,15 @@ def parse_arguments(*unparsed_arguments): ): raise ValueError('With the --json flag, multiple actions cannot be used together.') + if ( + 'transfer' in arguments + and arguments['transfer'].archive + and arguments['transfer'].glob_archives + ): + raise ValueError( + 'With the transfer action, only one of --archive and --glob-archives flags can be used.' + ) + if 'info' in arguments and ( (arguments['info'].archive and arguments['info'].prefix) or (arguments['info'].archive and arguments['info'].glob_archives) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a905713c..c03fc441 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -26,6 +26,7 @@ from borgmatic.borg import prune as borg_prune from borgmatic.borg import rcreate as borg_rcreate from borgmatic.borg import rinfo as borg_rinfo from borgmatic.borg import rlist as borg_rlist +from borgmatic.borg import transfer as borg_transfer from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments @@ -254,11 +255,12 @@ def run_actions( if 'rcreate' in arguments: logger.info('{}: Creating repository'.format(repository)) borg_rcreate.create_repository( + global_arguments.dry_run, repository, storage, local_borg_version, arguments['rcreate'].encryption_mode, - arguments['rcreate'].key_repository, + arguments['rcreate'].source_repository, arguments['rcreate'].copy_crypt_key, arguments['rcreate'].append_only, arguments['rcreate'].storage_quota, @@ -266,6 +268,17 @@ def run_actions( local_path=local_path, remote_path=remote_path, ) + if 'transfer' in arguments: + logger.info(f'{repository}: Transferring archives to repository') + borg_transfer.transfer_archives( + global_arguments.dry_run, + repository, + storage, + local_borg_version, + transfer_arguments=arguments['transfer'], + local_path=local_path, + remote_path=remote_path, + ) if 'prune' in arguments: command.execute_hook( hooks.get('before_prune'), diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index cdcdefe5..28955c49 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -1,11 +1,11 @@ --- -title: How to upgrade borgmatic +title: How to upgrade borgmatic and Borg eleventyNavigation: - key: 📦 Upgrade borgmatic + key: 📦 Upgrade borgmatic/Borg parent: How-to guides order: 12 --- -## Upgrading +## Upgrading borgmatic In general, all you should need to do to upgrade borgmatic is run the following: @@ -115,3 +115,85 @@ sudo pip3 install --user borgmatic That's it! borgmatic will continue using your /etc/borgmatic configuration files. + + +## Upgrading Borg + +To upgrade to a new version of Borg, you can generally install a new version +the same way you installed the previous version, paying attention to any +instructions included with each Borg release changelog linked from the +[releases page](https://github.com/borgbackup/borg/releases). However, some +more major Borg releases require additional steps that borgmatic can help +with. + + +### Borg 1.2 to 2.0 + +New in borgmatic version 1.7.0 +Upgrading Borg from 1.2 to 2.0 requires manually upgrading your existing Borg +1 repositories before use with Borg or borgmatic. Here's how you can +accomplish that. + +Start by upgrading borgmatic as described above to at least version 1.7.0 and +Borg to 2.0. Then, rename your repository in borgmatic's configuration file to +a new repository path. The repository upgrade process does not occur +in-place; you'll create a new repository with a copy of your old repository's +data. + +Let's say your original borgmatic repository configuration file looks something +like this: + +```yaml +location: + repositories: + - original.borg +``` + +Change it to a new (not yet created) repository path: + +```yaml +location: + repositories: + - upgraded.borg +``` + +Then, run the `rcreate` action (formerly `init`) to create that new Borg 2 +repository: + +```bash +borgmatic rcreate --verbosity 1 --encryption repokey-aes-ocb \ + --source-repository original.borg --repository upgraded.borg +``` + +(Note that `repokey-chacha20-poly1305` may be faster than `repokey-aes-ocb` on +certain platforms like ARM64.) + +This creates an empty repository and doesn't actually transfer any data yet. +The `--source-repository` flag is necessary to reuse key material from your +Borg 1 repository so that the subsequent data transfer can work. + +To transfer data from your original Borg 1 repository to your newly created +Borg 2 repository: + +```bash +borgmatic transfer --verbosity 1 --upgrader From12To20 --source-repository \ + original.borg --repository upgraded.borg --dry-run +borgmatic transfer --verbosity 1 --upgrader From12To20 --source-repository \ + original.borg --repository upgraded.borg +borgmatic transfer --verbosity 1 --upgrader From12To20 --source-repository \ + original.borg --repository upgraded.borg --dry-run +``` + +The first command with `--dry-run` tells you what Borg is going to do during +the transfer, the second command actually performs the transfer/upgrade (this +might take a while), and the final command with `--dry-run` again provides +confirmation of success—or tells you if something hasn't been transferred yet. + +Note that by omitting the `--upgrader` flag, you can also do archive transfers +between Borg 2 repositories without upgrading, even down to individual +archives. For more on that functionality, see the [Borg transfer +documentation](https://borgbackup.readthedocs.io/en/2.0.0b1/usage/transfer.html). + +That's it! Now you can use your new Borg 2 repository as normal with +borgmatic. If you've got multiple repositories, repeat the above process for +each. diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index b7013563..eb2a1f9e 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -287,15 +287,6 @@ def test_parse_arguments_allows_init_and_create(): module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') -def test_parse_arguments_disallows_init_and_dry_run(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments( - '--config', 'myconfig', 'init', '--encryption', 'repokey', '--dry-run' - ) - - def test_parse_arguments_disallows_repository_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index 18166a9f..612ec11c 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -39,7 +39,26 @@ def test_create_repository_calls_borg_with_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( - repository='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey' + dry_run=False, + repository='repo', + storage_config={}, + local_borg_version='2.3.4', + encryption_mode='repokey', + ) + + +def test_create_repository_with_dry_run_skips_borg_call(): + insert_rinfo_command_not_found_mock() + flexmock(module).should_receive('execute_command').never() + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + + module.create_repository( + dry_run=True, + repository='repo', + storage_config={}, + local_borg_version='2.3.4', + encryption_mode='repokey', ) @@ -54,6 +73,7 @@ def test_create_repository_raises_for_borg_rcreate_error(): with pytest.raises(subprocess.CalledProcessError): module.create_repository( + dry_run=False, repository='repo', storage_config={}, local_borg_version='2.3.4', @@ -67,7 +87,11 @@ def test_create_repository_skips_creation_when_repository_already_exists(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( - repository='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey' + dry_run=False, + repository='repo', + storage_config={}, + local_borg_version='2.3.4', + encryption_mode='repokey', ) @@ -78,6 +102,7 @@ def test_create_repository_raises_for_unknown_rinfo_command_error(): with pytest.raises(subprocess.CalledProcessError): module.create_repository( + dry_run=False, repository='repo', storage_config={}, local_borg_version='2.3.4', @@ -85,18 +110,19 @@ def test_create_repository_raises_for_unknown_rinfo_command_error(): ) -def test_create_repository_with_key_repository_calls_borg_with_other_repo_flag(): +def test_create_repository_with_source_repository_calls_borg_with_other_repo_flag(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--other-repo', 'other.borg', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( + dry_run=False, repository='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', - key_repository='other.borg', + source_repository='other.borg', ) @@ -107,6 +133,7 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( + dry_run=False, repository='repo', storage_config={}, local_borg_version='2.3.4', @@ -122,6 +149,7 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( + dry_run=False, repository='repo', storage_config={}, local_borg_version='2.3.4', @@ -137,6 +165,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( + dry_run=False, repository='repo', storage_config={}, local_borg_version='2.3.4', @@ -152,6 +181,7 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( + dry_run=False, repository='repo', storage_config={}, local_borg_version='2.3.4', @@ -168,7 +198,11 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( - repository='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey' + dry_run=False, + repository='repo', + storage_config={}, + local_borg_version='2.3.4', + encryption_mode='repokey', ) @@ -180,7 +214,11 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( - repository='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey' + dry_run=False, + repository='repo', + storage_config={}, + local_borg_version='2.3.4', + encryption_mode='repokey', ) @@ -191,6 +229,7 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( + dry_run=False, repository='repo', storage_config={}, local_borg_version='2.3.4', @@ -206,6 +245,7 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( + dry_run=False, repository='repo', storage_config={}, local_borg_version='2.3.4', @@ -221,6 +261,7 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) module.create_repository( + dry_run=False, repository='repo', storage_config={'extra_borg_options': {'rcreate': '--extra --options'}}, local_borg_version='2.3.4', diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 4b9c43ea..5a49aff8 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -346,7 +346,7 @@ def test_run_actions_does_not_raise_for_rcreate_action(): 'global': flexmock(monitoring_verbosity=1, dry_run=False), 'rcreate': flexmock( encryption_mode=flexmock(), - key_repository=flexmock(), + source_repository=flexmock(), copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), @@ -371,6 +371,30 @@ def test_run_actions_does_not_raise_for_rcreate_action(): ) +def test_run_actions_does_not_raise_for_transfer_action(): + flexmock(module.borg_transfer).should_receive('transfer_archives') + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False), + 'transfer': flexmock(), + } + + list( + module.run_actions( + arguments=arguments, + config_filename='test.yaml', + location={'repositories': ['repo']}, + storage={}, + retention={}, + consistency={}, + hooks={}, + local_path=None, + remote_path=None, + local_borg_version=None, + repository_path='repo', + ) + ) + + def test_run_actions_calls_hooks_for_prune_action(): flexmock(module.borg_prune).should_receive('prune_archives') flexmock(module.command).should_receive('execute_hook').twice()