Merge branch 'main' into config-command-line

NEWS

@@ -5,6 +5,7 @@
  * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more
    flexible "commands:". See the documentation for more information:
    https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+ * #836: Add a custom command option for the SQLite hook.
  * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune".
  * #1020: Document a database use case involving a temporary database client container:
    https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers

borgmatic/config/schema.yaml

@@ -1279,11 +1279,11 @@ properties:
                         Command to use instead of "pg_dump" or "pg_dumpall".
                         This can be used to run a specific pg_dump version
                         (e.g., one inside a running container). If you run it
-                        from within a container, make sure to mount your
-                        host's ".borgmatic" folder into the container using
-                        the same directory structure. Defaults to "pg_dump"
-                        for single database dump or "pg_dumpall" to dump all
-                        databases.
+                        from within a container, make sure to mount the path in
+                        the "user_runtime_directory" option from the host into
+                        the container at the same location. Defaults to
+                        "pg_dump" for single database dump or "pg_dumpall" to
+                        dump all databases.
                     example: docker exec my_pg_container pg_dump
                 pg_restore_command:
                     type: string
@@ -1423,10 +1423,11 @@ properties:
                     description: |
                         Command to use instead of "mariadb-dump". This can be
                         used to run a specific mariadb_dump version (e.g., one
-                        inside a running container). If you run it from within
-                        a container, make sure to mount your host's
-                        ".borgmatic" folder into the container using the same
-                        directory structure. Defaults to "mariadb-dump".
+                        inside a running container). If you run it from within a
+                        container, make sure to mount the path in the
+                        "user_runtime_directory" option from the host into the
+                        container at the same location. Defaults to
+                        "mariadb-dump".
                     example: docker exec mariadb_container mariadb-dump
                 mariadb_command:
                     type: string
@@ -1568,12 +1569,12 @@ properties:
                 mysql_dump_command:
                     type: string
                     description: |
-                        Command to use instead of "mysqldump". This can be
-                        used to run a specific mysql_dump version (e.g., one
-                        inside a running container). If you run it from within
-                        a container, make sure to mount your host's
-                        ".borgmatic" folder into the container using the same
-                        directory structure. Defaults to "mysqldump".
+                        Command to use instead of "mysqldump". This can be used
+                        to run a specific mysql_dump version (e.g., one inside a
+                        running container). If you run it from within a
+                        container, make sure to mount the path in the
+                        "user_runtime_directory" option from the host into the
+                        container at the same location. Defaults to "mysqldump".
                     example: docker exec mysql_container mysqldump
                 mysql_command:
                     type: string
@@ -1663,6 +1664,24 @@ properties:
                         Path to the SQLite database file to restore to. Defaults
                         to the "path" option.
                     example: /var/lib/sqlite/users.db
+                sqlite_command:
+                    type: string
+                    description: |
+                        Command to use instead of "sqlite3". This can be used to
+                        run a specific sqlite3 version (e.g., one inside a
+                        running container). If you run it from within a
+                        container, make sure to mount the path in the
+                        "user_runtime_directory" option from the host into the
+                        container at the same location. Defaults to "sqlite3".
+                    example: docker exec sqlite_container sqlite3
+                sqlite_restore_command:
+                    type: string
+                    description: |
+                        Command to run when restoring a database instead
+                        of "sqlite3". This can be used to run a specific 
+                        sqlite3 version (e.g., one inside a running container). 
+                        Defaults to "sqlite3".
+                    example: docker exec sqlite_container sqlite3
         description: |
             List of one or more SQLite databases to dump before creating a
             backup, run once per configuration file. The database dumps are

borgmatic/hooks/data_source/sqlite.py

@@ -79,13 +79,17 @@ def dump_data_sources(
                 )
                 continue
 
-            command = (
-                'sqlite3',
+            sqlite_command = tuple(
+                shlex.quote(part)
+                for part in shlex.split(database.get('sqlite_command') or 'sqlite3')
+            )
+            command = sqlite_command + (
                 shlex.quote(database_path),
                 '.dump',
                 '>',
                 shlex.quote(dump_filename),
             )
+
             logger.debug(
                 f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}'
             )
@@ -168,11 +172,11 @@ def restore_data_source_dump(
     except FileNotFoundError:  # pragma: no cover
         pass
 
-    restore_command = (
-        'sqlite3',
-        database_path,
+    sqlite_restore_command = tuple(
+        shlex.quote(part)
+        for part in shlex.split(data_source.get('sqlite_restore_command') or 'sqlite3')
     )
-
+    restore_command = sqlite_restore_command + (shlex.quote(database_path),)
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
     execute_command_with_processes(

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -78,7 +78,8 @@ commands:
 
 This command hook has the following options:
 
- * `before` or `after`: `dump_data_sources`
+ * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after:
+   * `dump_data_sources` runs before or after data sources are dumped (databases dumped or filesystems snapshotted) for each hook named in `hooks`.
  * `hooks`: Names of other hooks that this command hook applies to, e.g. `postgresql`, `mariadb`, `zfs`, `btrfs`, etc. Defaults to all hooks of the relevant type.
  * `run`: One or more shell commands or scripts to run when this command hook is triggered.
 

tests/unit/hooks/data_source/test_sqlite.py

@@ -116,6 +116,51 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped():
     )
 
 
+def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_gets_escaped():
+    flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return(
+        flexmock()
+    )
+    databases = [
+        {
+            'path': '/path/to/database1; naughty-command',
+            'name': 'database1',
+            'sqlite_command': 'custom_sqlite *',
+        },
+    ]
+    processes = [flexmock()]
+
+    flexmock(module).should_receive('make_dump_path').and_return('/run/borgmatic')
+    flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
+        '/run/borgmatic/database'
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'custom_sqlite',  # custom sqlite command
+            "'*'",  # Should get shell escaped to prevent injection attacks.
+            "'/path/to/database1; naughty-command'",
+            '.dump',
+            '>',
+            '/run/borgmatic/database',
+        ),
+        shell=True,
+        run_to_completion=False,
+    ).and_return(processes[0])
+
+    assert (
+        module.dump_data_sources(
+            databases,
+            {},
+            config_paths=('test.yaml',),
+            borgmatic_runtime_directory='/run/borgmatic',
+            patterns=[],
+            dry_run=False,
+        )
+        == processes
+    )
+
+
 def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database():
     flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return(
         flexmock()
@@ -234,6 +279,41 @@ def test_restore_data_source_dump_restores_database():
     )
 
 
+def test_restore_data_source_dump_runs_non_default_sqlite_restores_database():
+    hook_config = [
+        {
+            'path': '/path/to/database',
+            'name': 'database',
+            'sqlite_restore_command': 'custom_sqlite *',
+        },
+        {'name': 'other'},
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'custom_sqlite',
+            "'*'",  # Should get shell escaped to prevent injection attacks.
+            '/path/to/database',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    ).once()
+
+    flexmock(module.os).should_receive('remove').once()
+
+    module.restore_data_source_dump(
+        hook_config,
+        {},
+        data_source=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={'restore_path': None},
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
 def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore():
     hook_config = [
         {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
@@ -263,6 +343,38 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
     )
 
 
+def test_restore_data_source_dump_runs_non_default_sqlite_with_connection_params_uses_connection_params_for_restore():
+    hook_config = [
+        {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'custom_sqlite',
+            'cli/path/to/database',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    ).once()
+
+    flexmock(module.os).should_receive('remove').once()
+
+    module.restore_data_source_dump(
+        hook_config,
+        {},
+        data_source={
+            'name': 'database',
+            'sqlite_restore_command': 'custom_sqlite',
+        },
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={'restore_path': 'cli/path/to/database'},
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
 def test_restore_data_source_dump_without_connection_params_uses_restore_params_in_config_for_restore():
     hook_config = [
         {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
@@ -292,6 +404,40 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
     )
 
 
+def test_restore_data_source_dump_runs_non_default_sqlite_without_connection_params_uses_restore_params_in_config_for_restore():
+    hook_config = [
+        {
+            'path': '/path/to/database',
+            'name': 'database',
+            'sqlite_restore_command': 'custom_sqlite',
+            'restore_path': 'config/path/to/database',
+        }
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'custom_sqlite',
+            'config/path/to/database',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    ).once()
+
+    flexmock(module.os).should_receive('remove').once()
+
+    module.restore_data_source_dump(
+        hook_config,
+        {},
+        data_source=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={'restore_path': None},
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
 def test_restore_data_source_dump_does_not_restore_database_if_dry_run():
     hook_config = [{'path': '/path/to/database', 'name': 'database'}]
     extract_process = flexmock(stdout=flexmock())