From 58c95d8015344fe442d7cad1036c73573a93dd52 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 20 Mar 2023 02:43:23 +0530 Subject: [PATCH 1/3] feat: file:// URLs support --- borgmatic/borg/export_tar.py | 6 +++++- borgmatic/borg/extract.py | 6 +++++- borgmatic/config/normalize.py | 28 ++++++++++++++++------------ borgmatic/config/validate.py | 11 +++++++---- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 5ed494c8..28a876ef 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -47,7 +47,11 @@ def export_tar_archive( + (('--tar-filter', tar_filter) if tar_filter else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) + flags.make_repository_archive_flags( - repository if ':' in repository else os.path.abspath(repository), + os.path.abspath(repository) + if ':' not in repository + else os.path.abspath(repository[7:]) + if repository.startswith('file://') + else repository, archive, local_borg_version, ) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 8ea8bbbd..be3c5b76 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -100,7 +100,11 @@ def extract_archive( + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( - repository if ':' in repository else os.path.abspath(repository), + os.path.abspath(repository) + if ':' not in repository + else os.path.abspath(repository[7:]) + if repository.startswith('file://') + else repository, archive, local_borg_version, ) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 574a48a1..7a250148 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -1,4 +1,5 @@ import logging +import os def normalize(config_filename, config): @@ -68,20 +69,23 @@ def normalize(config_filename, config): ) ) ) - if ':' in repository and not repository.startswith('ssh://'): - rewritten_repository = ( - f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" - ) - logs.append( - logging.makeLogRecord( - dict( - levelno=logging.WARNING, - levelname='WARNING', - msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository}" as "{rewritten_repository}"', + if ':' in repository: + if repository.startswith('file://'): + config['location']['repositories'].append(os.path.abspath(repository[7:])) + elif repository.startswith('ssh://'): + config['location']['repositories'].append(repository) + else: + rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository}" as "{rewritten_repository}"', + ) ) ) - ) - config['location']['repositories'].append(rewritten_repository) + config['location']['repositories'].append(rewritten_repository) else: config['location']['repositories'].append(repository) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 3a2807f9..d61f7f3a 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -126,12 +126,15 @@ def normalize_repository_path(repository): ''' Given a repository path, return the absolute path of it (for local repositories). ''' - # A colon in the repository indicates it's a remote repository. Bail. - if ':' in repository: + # A colon in the repository could mean that it's either a file:// URL or a remote repository. + # If it's a remote repository, we don't want to normalize it. If it's a file:// URL, we do. + if ':' not in repository: + return os.path.abspath(repository) + elif repository.startswith('file://'): + return os.path.abspath(repository[7:]) + else: return repository - return os.path.abspath(repository) - def repositories_match(first, second): ''' From 86587ab2dcd561527f7c79b4e6b76b0d117196db Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 20 Mar 2023 21:51:45 +0530 Subject: [PATCH 2/3] send repo directly to extract and export_tar --- borgmatic/borg/export_tar.py | 6 +----- borgmatic/borg/extract.py | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 28a876ef..a3f05d5c 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -47,11 +47,7 @@ def export_tar_archive( + (('--tar-filter', tar_filter) if tar_filter else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) + flags.make_repository_archive_flags( - os.path.abspath(repository) - if ':' not in repository - else os.path.abspath(repository[7:]) - if repository.startswith('file://') - else repository, + repository, archive, local_borg_version, ) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index be3c5b76..fccb0171 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -100,11 +100,7 @@ def extract_archive( + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( - os.path.abspath(repository) - if ':' not in repository - else os.path.abspath(repository[7:]) - if repository.startswith('file://') - else repository, + repository, archive, local_borg_version, ) From 39ad8f64c4d9038f3ca16d6902420fe15579bba0 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 21 Mar 2023 17:06:03 +0530 Subject: [PATCH 3/3] add tests and remove magic number --- borgmatic/borg/export_tar.py | 6 +----- borgmatic/borg/extract.py | 6 +----- borgmatic/config/normalize.py | 4 +++- borgmatic/config/validate.py | 2 +- tests/unit/config/test_normalize.py | 5 +++++ tests/unit/config/test_validate.py | 7 +++++++ 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index a3f05d5c..a5b752c3 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -46,11 +46,7 @@ def export_tar_archive( + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + flags.make_repository_archive_flags( - repository, - archive, - local_borg_version, - ) + + flags.make_repository_archive_flags(repository, archive, local_borg_version,) + (destination_path,) + (tuple(paths) if paths else ()) ) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 24014c91..6c32f7f0 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -106,11 +106,7 @@ def extract_archive( + (('--strip-components', str(strip_components)) if strip_components else ()) + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) - + flags.make_repository_archive_flags( - repository, - archive, - local_borg_version, - ) + + flags.make_repository_archive_flags(repository, archive, local_borg_version,) + (tuple(paths) if paths else ()) ) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 7a250148..a143a192 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -71,7 +71,9 @@ def normalize(config_filename, config): ) if ':' in repository: if repository.startswith('file://'): - config['location']['repositories'].append(os.path.abspath(repository[7:])) + config['location']['repositories'].append( + os.path.abspath(repository.partition('file://')[-1]) + ) elif repository.startswith('ssh://'): config['location']['repositories'].append(repository) else: diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index d25fb562..5828380e 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -131,7 +131,7 @@ def normalize_repository_path(repository): if ':' not in repository: return os.path.abspath(repository) elif repository.startswith('file://'): - return os.path.abspath(repository[7:]) + return os.path.abspath(repository.partition('file://')[-1]) else: return repository diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 384dd7e3..821c3202 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -87,6 +87,11 @@ from borgmatic.config import normalize as module {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, False, ), + ( + {'location': {'repositories': ['file:///repo']}}, + {'location': {'repositories': ['/repo']}}, + False, + ), ), ) def test_normalize_applies_hard_coded_normalization_to_config( diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 713ecc7a..6a9f4a4e 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -83,6 +83,13 @@ def test_normalize_repository_path_passes_through_remote_repository(): module.normalize_repository_path(repository) == repository +def test_normalize_repository_path_passes_through_file_repository(): + repository = 'file:///foo/bar/test.borg' + flexmock(module.os.path).should_receive('abspath').and_return('/foo/bar/test.borg') + + module.normalize_repository_path(repository) == '/foo/bar/test.borg' + + def test_normalize_repository_path_passes_through_absolute_repository(): repository = '/foo/bar/test.borg' flexmock(module.os.path).should_receive('abspath').and_return(repository)