Fix a "list" action error when the "encryption_passcommand" option is set (#987).

NEWS

@@ -1,3 +1,6 @@
+1.9.10.dev0
+ * #987: Fix a "list" action error when the "encryption_passcommand" option is set.
+
 1.9.9
  * #635: Log the repository path or label on every relevant log message, not just some logs.
  * #961: When the "encryption_passcommand" option is set, call the command once from borgmatic to

borgmatic/actions/check.py

@@ -386,13 +386,12 @@ def collect_spot_check_source_paths(
             stream_processes=stream_processes,
         )
     )
-    borg_environment = borgmatic.borg.environment.make_environment(config)
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
     paths_output = borgmatic.execute.execute_command_and_capture_output(
         create_flags + create_positional_arguments,
         capture_stderr=True,
-        extra_environment=borg_environment,
+        extra_environment=borgmatic.borg.environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/break_lock.py

@@ -34,10 +34,9 @@ def break_lock(
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    borg_environment = environment.make_environment(config)
     execute_command(
         full_command,
-        extra_environment=borg_environment,
+        extra_environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/check.py

@@ -152,7 +152,6 @@ def check_archives(
     max_duration = check_arguments.max_duration or repository_check_config.get('max_duration')
     umask = config.get('umask')
 
-    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
     full_command = (
@@ -178,7 +177,7 @@ def check_archives(
         execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -186,7 +185,7 @@ def check_archives(
     else:
         execute_command(
             full_command,
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/create.py

@@ -122,15 +122,14 @@ def collect_special_file_paths(
     config,
     local_path,
     working_directory,
-    borg_environment,
     borgmatic_runtime_directory,
 ):
     '''
     Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
-    a working directory, a dict of environment variables to pass to Borg, and the borgmatic runtime
-    directory, collect the paths for any special files (character devices, block devices, and named
-    pipes / FIFOs) that Borg would encounter during a create. These are all paths that could cause
-    Borg to hang if its --read-special flag is used.
+    a working directory, and the borgmatic runtime directory, collect the paths for any special
+    files (character devices, block devices, and named pipes / FIFOs) that Borg would encounter
+    during a create. These are all paths that could cause Borg to hang if its --read-special flag is
+    used.
 
     Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
     its own special files there for database dumps. And if the borgmatic runtime directory is
@@ -144,7 +143,7 @@ def collect_special_file_paths(
         + ('--dry-run', '--list'),
         capture_stderr=True,
         working_directory=working_directory,
-        extra_environment=borg_environment,
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
@@ -299,7 +298,6 @@ def make_base_create_command(
         logger.warning(
             'Ignoring configured "read_special" value of false, as true is needed for database hooks.'
         )
-        borg_environment = environment.make_environment(config)
         working_directory = borgmatic.config.paths.get_working_directory(config)
 
         logger.debug('Collecting special file paths')
@@ -309,7 +307,6 @@ def make_base_create_command(
             config,
             local_path,
             working_directory,
-            borg_environment,
             borgmatic_runtime_directory=borgmatic_runtime_directory,
         )
 
@@ -396,8 +393,6 @@ def create_archive(
     # the terminal directly.
     output_file = DO_NOT_CAPTURE if progress else None
 
-    borg_environment = environment.make_environment(config)
-
     create_flags += (
         (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
         + (('--stats',) if stats and not json and not dry_run else ())
@@ -414,7 +409,7 @@ def create_archive(
             output_log_level,
             output_file,
             working_directory=working_directory,
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -422,7 +417,7 @@ def create_archive(
         return execute_command_and_capture_output(
             create_flags + create_positional_arguments,
             working_directory=working_directory,
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -432,7 +427,7 @@ def create_archive(
             output_log_level,
             output_file,
             working_directory=working_directory,
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/environment.py

@@ -28,6 +28,9 @@ def make_environment(config):
     '''
     Given a borgmatic configuration dict, return its options converted to a Borg environment
     variable dict.
+
+    Do not reuse this environment across multiple Borg invocations, because it can include
+    references to resources like anonymous pipes for passphrases—which can only be consumed once.
     '''
     environment = {}
 

borgmatic/borg/extract.py

@@ -44,7 +44,6 @@ def extract_last_archive_dry_run(
         return
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
-    borg_environment = environment.make_environment(config)
     full_extract_command = (
         (local_path, 'extract', '--dry-run')
         + (('--remote-path', remote_path) if remote_path else ())
@@ -59,7 +58,7 @@ def extract_last_archive_dry_run(
 
     execute_command(
         full_extract_command,
-        extra_environment=borg_environment,
+        extra_environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
@@ -144,7 +143,6 @@ def extract_archive(
         + (tuple(paths) if paths else ())
     )
 
-    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
     full_destination_path = (
         os.path.join(working_directory or '', destination_path) if destination_path else None
@@ -156,7 +154,7 @@ def extract_archive(
         return execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             working_directory=full_destination_path,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -168,7 +166,7 @@ def extract_archive(
             full_command,
             output_file=subprocess.PIPE,
             run_to_completion=False,
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             working_directory=full_destination_path,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -178,7 +176,7 @@ def extract_archive(
     # if the restore paths don't exist in the archive.
     execute_command(
         full_command,
-        extra_environment=borg_environment,
+        extra_environment=environment.make_environment(config),
         working_directory=full_destination_path,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,

borgmatic/borg/list.py

@@ -106,8 +106,6 @@ def capture_archive_listing(
     format to use for the output, and local and remote Borg paths, capture the
     output of listing that archive and return it as a list of file paths.
     '''
-    borg_environment = environment.make_environment(config)
-
     return tuple(
         execute_command_and_capture_output(
             make_list_command(
@@ -126,7 +124,7 @@ def capture_archive_listing(
                 local_path,
                 remote_path,
             ),
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             working_directory=borgmatic.config.paths.get_working_directory(config),
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
@@ -194,7 +192,6 @@ def list_archive(
             'The --json flag on the list action is not supported when using the --archive/--find flags.'
         )
 
-    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
     # If there are any paths to find (and there's not a single archive already selected), start by
@@ -224,7 +221,7 @@ def list_archive(
                     local_path,
                     remote_path,
                 ),
-                extra_environment=borg_environment,
+                extra_environment=environment.make_environment(config),
                 working_directory=borgmatic.config.paths.get_working_directory(config),
                 borg_local_path=local_path,
                 borg_exit_codes=borg_exit_codes,
@@ -260,7 +257,7 @@ def list_archive(
         execute_command(
             main_command,
             output_log_level=logging.ANSWER,
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             working_directory=borgmatic.config.paths.get_working_directory(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/mount.py

@@ -59,7 +59,6 @@ def mount_archive(
         + (tuple(mount_arguments.paths) if mount_arguments.paths else ())
     )
 
-    borg_environment = environment.make_environment(config)
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
@@ -67,7 +66,7 @@ def mount_archive(
         execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            extra_environment=borg_environment,
+            extra_environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
@@ -76,7 +75,7 @@ def mount_archive(
 
     execute_command(
         full_command,
-        extra_environment=borg_environment,
+        extra_environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/repo_info.py

@@ -50,14 +50,13 @@ def display_repository_info(
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    extra_environment = environment.make_environment(config)
     working_directory = borgmatic.config.paths.get_working_directory(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
     if repo_info_arguments.json:
         return execute_command_and_capture_output(
             full_command,
-            extra_environment=extra_environment,
+            extra_environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -66,7 +65,7 @@ def display_repository_info(
         execute_command(
             full_command,
             output_log_level=logging.ANSWER,
-            extra_environment=extra_environment,
+            extra_environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/repo_list.py

@@ -140,7 +140,6 @@ def list_repository(
     return JSON output).
     '''
     borgmatic.logger.add_custom_log_levels()
-    borg_environment = environment.make_environment(config)
 
     main_command = make_repo_list_command(
         repository_path,
@@ -165,7 +164,7 @@ def list_repository(
 
     json_listing = execute_command_and_capture_output(
         json_command,
-        extra_environment=borg_environment,
+        extra_environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
@@ -179,7 +178,7 @@ def list_repository(
     execute_command(
         main_command,
         output_log_level=logging.ANSWER,
-        extra_environment=borg_environment,
+        extra_environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "borgmatic"
-version = "1.9.9"
+version = "1.9.10.dev0"
 authors = [
   { name="Dan Helfman", email="witten@torsion.org" },
 ]

tests/unit/borg/test_create.py

@@ -185,6 +185,7 @@ def test_any_parent_directories_treats_unrelated_paths_as_non_match():
 
 
 def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_list():
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         'Processing files ...\n- /foo\n+ /bar\n- /baz'
     )
@@ -198,12 +199,12 @@ def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_
         config={},
         local_path=None,
         working_directory=None,
-        borg_environment=None,
         borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/bar', '/baz')
 
 
 def test_collect_special_file_paths_skips_borgmatic_runtime_directory():
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n- /run/borgmatic/bar\n- /baz'
     )
@@ -225,12 +226,12 @@ def test_collect_special_file_paths_skips_borgmatic_runtime_directory():
         config={},
         local_path=None,
         working_directory=None,
-        borg_environment=None,
         borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/baz')
 
 
 def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_from_paths_output_errors():
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n- /bar\n- /baz'
     )
@@ -245,12 +246,12 @@ def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_fro
             config={},
             local_path=None,
             working_directory=None,
-            borg_environment=None,
             borgmatic_runtime_directory='/run/borgmatic',
         )
 
 
 def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory_missing_from_paths_output_does_not_raise():
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n- /bar\n- /baz'
     )
@@ -264,12 +265,12 @@ def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory
         config={},
         local_path=None,
         working_directory=None,
-        borg_environment=None,
         borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/bar', '/baz')
 
 
 def test_collect_special_file_paths_excludes_non_special_files():
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n+ /bar\n+ /baz'
     )
@@ -285,12 +286,12 @@ def test_collect_special_file_paths_excludes_non_special_files():
         config={},
         local_path=None,
         working_directory=None,
-        borg_environment=None,
         borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/baz')
 
 
 def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command():
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create', '--dry-run', '--list'),
         capture_stderr=True,
@@ -309,7 +310,6 @@ def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command():
         config={},
         local_path='borg',
         working_directory=None,
-        borg_environment=None,
         borgmatic_runtime_directory='/run/borgmatic',
     )