Fix a unicode error when backing up a non-UTF-8 source filename with a corresponding system locale (#1281).

NEWS

@@ -20,6 +20,8 @@
    action.
  * #1274: Add an optional override for the documentation development listen port and use Podman
    Compose if present.
+ * #1281: Fix a unicode error when backing up a non-UTF-8 source filename with a
+   corresponding system locale.
  * Add a policy about the use of generative AI in the borgmatic codebase:
    https://torsion.org/borgmatic/how-to/develop-on-borgmatic/#use-of-generative-ai
 

borgmatic/actions/restore.py

@@ -1,4 +1,5 @@
 import collections
+import locale
 import logging
 import os
 import pathlib
@@ -317,7 +318,7 @@ def collect_dumps_from_archive(
                 extract_to_stdout=True,
             )
             .stdout.read()
-            .decode(),
+            .decode(locale.getpreferredencoding()),
             dumps_metadata_entry['path'],
         ):
             dumps_from_archive[dump] = None

borgmatic/execute.py

@@ -2,6 +2,7 @@ import collections
 import contextlib
 import enum
 import json
+import locale
 import logging
 import os
 import select
@@ -249,6 +250,7 @@ def read_lines(buffer, process, line_separator='\n'):
     data = b''
     encoded_separator = line_separator.encode()
     separator_size = len(encoded_separator)
+    encoding = locale.getpreferredencoding()
 
     while True:
         chunk = os.read(buffer.fileno(), READ_CHUNK_SIZE)
@@ -271,14 +273,14 @@ def read_lines(buffer, process, line_separator='\n'):
             if separator_position == -1:
                 break
 
-            lines.append(data[:separator_position].decode())
+            lines.append(data[:separator_position].decode(encoding))
             data = data[separator_position + separator_size :]
 
         yield tuple(lines)
 
     # Yield any leftover data from the end of the buffer.
     if data:
-        yield (data.decode().rstrip(),)
+        yield (data.decode(encoding).rstrip(),)
 
 
 Buffer_reader = collections.namedtuple(
@@ -661,7 +663,7 @@ def execute_command_and_capture_output(
             raise
 
         if error.output is not None:
-            yield from iter(error.output.decode().splitlines())
+            yield from iter(error.output.decode(locale.getpreferredencoding()).splitlines())
 
         return
 

tests/integration/test_execute.py

@@ -561,3 +561,10 @@ def test_log_outputs_with_unfinished_process_re_polls():
         )
         == ()
     )
+
+
+def test_read_lines_uses_system_locale_when_decoding_output():
+    flexmock(module.locale).should_receive('getpreferredencoding').and_return('ISO-8859-1')
+    process = subprocess.Popen(['echo', b'\xc4pple'], stdout=subprocess.PIPE)
+
+    assert tuple(module.read_lines(process.stdout, process)) == (('\xc4pple',),)

tests/unit/actions/test_restore.py

@@ -521,6 +521,7 @@ def test_collect_dumps_from_archive_with_dumps_metadata_parses_it():
     flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
         flexmock(stdout=flexmock(read=lambda: b''))
     )
+    flexmock(module.locale).should_receive('getpreferredencoding').and_return('UTF-8')
     dumps_metadata = [
         module.Dump('postgresql_databases', 'foo'),
         module.Dump('postgresql_databases', 'bar', 'host', 1234),

tests/unit/test_execute.py

@@ -1510,6 +1510,7 @@ def test_execute_command_and_capture_output_returns_output_when_process_error_is
     flexmock(module).should_receive('interpret_exit_code').and_return(
         module.Exit_status.SUCCESS,
     ).once()
+    flexmock(module.locale).should_receive('getpreferredencoding').and_return('UTF-8')
 
     output_lines = tuple(module.execute_command_and_capture_output(full_command))
 
@@ -1533,6 +1534,7 @@ def test_execute_command_and_capture_output_raises_when_command_errors():
     flexmock(module).should_receive('interpret_exit_code').and_return(
         module.Exit_status.ERROR,
     ).once()
+    flexmock(module.locale).should_receive('getpreferredencoding').and_return('UTF-8')
 
     with pytest.raises(subprocess.CalledProcessError):
         tuple(module.execute_command_and_capture_output(full_command))