Follow symlinks when backing up borgmatic configuration files to support the "bootstrap" action (#1270).

NEWS

@@ -4,6 +4,8 @@
    information:
    https://torsion.org/borgmatic/reference/command-line/actions/config-show/
  * #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto".
+ * #1270: Follow symlinks when backing up borgmatic configuration files to support the "bootstrap"
+   action.
  * Add a policy about the use of generative AI in the borgmatic codebase:
    https://torsion.org/borgmatic/how-to/develop-on-borgmatic/#use-of-generative-ai
 

borgmatic/actions/config/bootstrap.py

@@ -120,7 +120,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
             borgmatic_runtime_directory,
         )
 
-    logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
+    logger.info(f"Bootstrapping configuration paths: {', '.join(manifest_config_paths)}")
 
     borgmatic.borg.extract.extract_archive(
         global_arguments.dry_run,

borgmatic/config/collect.py

@@ -47,5 +47,6 @@ def collect_config_filenames(config_paths):
         for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
             matching_filetype = full_filename.endswith(('.yaml', '.yml'))
+
             if matching_filetype and not os.path.isdir(full_filename):
                 yield os.path.abspath(full_filename)

borgmatic/hooks/data_source/bootstrap.py

@@ -1,6 +1,7 @@
 import contextlib
 import glob
 import importlib
+import itertools
 import json
 import logging
 import os
@@ -19,6 +20,29 @@ def use_streaming(hook_config, config):  # pragma: no cover
     return False
 
 
+MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW = 10
+
+
+def resolve_config_path_symlinks(path):
+    '''
+    Given a path, resolve and yield each successive symlink until the final non-symlink target. If
+    the given path isn't a symlink, then just yield it.
+
+    Raise ValueError if we have to follow too many symlinks without getting to the final target.
+    '''
+    original_path = path
+
+    for _ in range(MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW):
+        yield os.path.abspath(path)
+
+        if not os.path.islink(path):
+            return
+
+        path = os.readlink(path)
+
+    raise ValueError(f'Too many symlinks to follow for configuration path: {original_path}')
+
+
 def dump_data_sources(
     hook_config,
     config,
@@ -34,6 +58,9 @@ def dump_data_sources(
     the archive. But skip this if the bootstrap store_config_files option is False or if this is a
     dry run.
 
+    If any configuration paths are symlinks, then store each symlink along with any destination
+    paths as well.
+
     Return an empty sequence, since there are no ongoing dump processes from this hook.
     '''
     if hook_config and hook_config.get('store_config_files') is False:
@@ -45,6 +72,10 @@ def dump_data_sources(
         'manifest.json',
     )
 
+    resolved_config_paths = tuple(
+        itertools.chain.from_iterable(resolve_config_path_symlinks(path) for path in config_paths)
+    )
+
     if dry_run:
         return []
 
@@ -54,7 +85,7 @@ def dump_data_sources(
         json.dump(
             {
                 'borgmatic_version': importlib.metadata.version('borgmatic'),
-                'config_paths': config_paths,
+                'config_paths': resolved_config_paths,
             },
             manifest_file,
         )
@@ -67,7 +98,7 @@ def dump_data_sources(
         ),
     )
 
-    for config_path in config_paths:
+    for config_path in resolved_config_paths:
         borgmatic.hooks.data_source.config.inject_pattern(
             patterns,
             borgmatic.borg.pattern.Pattern(

tests/unit/hooks/data_source/test_bootstrap.py

@@ -1,13 +1,55 @@
 import sys
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.hooks.data_source import bootstrap as module
 
 
-def test_dump_data_sources_creates_manifest_file():
-    flexmock(module.os).should_receive('makedirs')
+def test_resolve_config_path_symlinks_passes_through_non_symlink():
+    flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
+    flexmock(module.os.path).should_receive('islink').and_return(False)
 
+    assert tuple(module.resolve_config_path_symlinks('test.yaml')) == ('test.yaml',)
+
+
+def test_resolve_config_path_symlinks_follows_each_symlink():
+    flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
+    flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True)
+    flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True)
+    flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(False)
+    flexmock(module.os).should_receive('readlink').with_args('test.yaml').and_return('dest1.yaml')
+    flexmock(module.os).should_receive('readlink').with_args('dest1.yaml').and_return('dest2.yaml')
+    flexmock(module.os).should_receive('readlink').with_args('dest2.yaml').never()
+
+    assert tuple(module.resolve_config_path_symlinks('test.yaml')) == (
+        'test.yaml',
+        'dest1.yaml',
+        'dest2.yaml',
+    )
+
+
+def test_resolve_config_path_symlinks_with_too_many_symlinks_raises():
+    flexmock(module).MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW = 2
+    flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
+    flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True)
+    flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True)
+    flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(True)
+    flexmock(module.os.path).should_receive('islink').with_args('dest3.yaml').never()
+    flexmock(module.os).should_receive('readlink').with_args('test.yaml').and_return('dest1.yaml')
+    flexmock(module.os).should_receive('readlink').with_args('dest1.yaml').and_return('dest2.yaml')
+    flexmock(module.os).should_receive('readlink').with_args('dest2.yaml').and_return('dest3.yaml')
+    flexmock(module.os).should_receive('readlink').with_args('dest3.yaml').never()
+
+    with pytest.raises(ValueError):
+        assert tuple(module.resolve_config_path_symlinks('test.yaml'))
+
+
+def test_dump_data_sources_creates_manifest_file():
+    flexmock(module).should_receive('resolve_config_path_symlinks').and_yield(
+        'test.yaml', 'linkdest.yaml'
+    )
+    flexmock(module.os).should_receive('makedirs')
     flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0')
     manifest_file = flexmock(
         __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
@@ -19,7 +61,7 @@ def test_dump_data_sources_creates_manifest_file():
         encoding='utf-8',
     ).and_return(manifest_file)
     flexmock(module.json).should_receive('dump').with_args(
-        {'borgmatic_version': '1.0.0', 'config_paths': ('test.yaml',)},
+        {'borgmatic_version': '1.0.0', 'config_paths': ('test.yaml', 'linkdest.yaml')},
         manifest_file,
     ).once()
     flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
@@ -34,6 +76,12 @@ def test_dump_data_sources_creates_manifest_file():
             'test.yaml', source=module.borgmatic.borg.pattern.Pattern_source.HOOK
         ),
     ).once()
+    flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
+        object,
+        module.borgmatic.borg.pattern.Pattern(
+            'linkdest.yaml', source=module.borgmatic.borg.pattern.Pattern_source.HOOK
+        ),
+    ).once()
 
     module.dump_data_sources(
         hook_config=None,
@@ -46,6 +94,7 @@ def test_dump_data_sources_creates_manifest_file():
 
 
 def test_dump_data_sources_with_store_config_files_false_does_not_create_manifest_file():
+    flexmock(module).should_receive('resolve_config_path_symlinks').and_yield('test.yaml')
     flexmock(module.os).should_receive('makedirs').never()
     flexmock(module.json).should_receive('dump').never()
     flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
@@ -62,6 +111,7 @@ def test_dump_data_sources_with_store_config_files_false_does_not_create_manifes
 
 
 def test_dump_data_sources_with_dry_run_does_not_create_manifest_file():
+    flexmock(module).should_receive('resolve_config_path_symlinks').and_yield('test.yaml')
     flexmock(module.os).should_receive('makedirs').never()
     flexmock(module.json).should_receive('dump').never()
     flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()