diff --git a/NEWS b/NEWS index 0216afecf..f4c3d6697 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,9 @@ * #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout. * #622: Fix traceback when include merging configuration files on ARM64. * #629: Skip warning about excluded special files when no special files have been excluded. + * #630: Add configuration options for database command customization: "list_options", + "restore_options", and "analyze_options" for PostgreSQL, "restore_options" for MySQL, and + "restore_options" for MongoDB. 1.7.5 * #311: Override PostgreSQL dump/restore commands via configuration options. diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 82a9d89bd..2a46da937 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -806,6 +806,30 @@ properties: any validation on them. See pg_dump documentation for details. example: --role=someone + list_options: + type: string + description: | + Additional psql options to pass directly to the + psql command that lists available databases, + without performing any validation on them. See + psql documentation for details. + example: --role=someone + restore_options: + type: string + description: | + Additional pg_restore/psql options to pass + directly to the restore command, without + performing any validation on them. See + pg_restore/psql documentation for details. + example: --role=someone + analyze_options: + type: string + description: | + Additional psql options to pass directly to the + analyze command run after a restore, without + performing any validation on them. See psql + documentation for details. + example: --role=someone description: | List of one or more PostgreSQL databases to dump before creating a backup, run once per configuration file. The @@ -868,14 +892,6 @@ properties: file of that format, allowing more convenient restores of individual databases. example: directory - list_options: - type: string - description: | - Additional mysql options to pass directly to - the mysql command that lists available - databases, without performing any validation on - them. See mysql documentation for details. - example: --defaults-extra-file=my.cnf options: type: string description: | @@ -884,6 +900,22 @@ properties: validation on them. See mysqldump documentation for details. example: --skip-comments + list_options: + type: string + description: | + Additional mysql options to pass directly to + the mysql command that lists available + databases, without performing any validation on + them. See mysql documentation for details. + example: --defaults-extra-file=my.cnf + restore_options: + type: string + description: | + Additional mysql options to pass directly to + the mysql command that restores database dumps, + without performing any validation on them. See + mysql documentation for details. + example: --defaults-extra-file=my.cnf description: | List of one or more MySQL/MariaDB databases to dump before creating a backup, run once per configuration file. The @@ -956,7 +988,15 @@ properties: directly to the dump command, without performing any validation on them. See mongodump documentation for details. - example: --role=someone + example: --dumpDbUsersAndRoles + restore_options: + type: string + description: | + Additional mongorestore options to pass + directly to the dump command, without performing + any validation on them. See mongorestore + documentation for details. + example: --restoreDbUsersAndRoles description: | List of one or more MongoDB databases to dump before creating a backup, run once per configuration file. The diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index 83b052f6f..cb1ec94fc 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -160,4 +160,6 @@ def build_restore_command(extract_process, database, dump_filename): command.extend(('--password', database['password'])) if 'authentication_database' in database: command.extend(('--authenticationDatabase', database['authentication_database'])) + if 'restore_options' in database: + command.extend(database['restore_options'].split(' ')) return command diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 41786983a..9c7871cbc 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -197,6 +197,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, database = database_config[0] restore_command = ( ('mysql', '--batch') + + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 9e3af435c..d8ed9195e 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -62,7 +62,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) - + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) ) logger.debug( '{}: Querying for "all" PostgreSQL databases to dump{}'.format(log_prefix, dry_run_label) @@ -204,6 +204,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) + (('--dbname', database['name']) if not all_databases else ()) + + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + ('--command', 'ANALYZE') ) pg_restore_command = database.get('pg_restore_command') or 'pg_restore' @@ -217,6 +218,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) + + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) ) extra_environment = make_extra_environment(database) diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index ea383f3bc..ff48ca6f7 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -76,6 +76,11 @@ hooks: options: "--ssl" ``` +See your [borgmatic configuration +file](https://torsion.org/borgmatic/docs/reference/configuration/) for +additional customization of the options passed to database commands (when +listing databases, restoring databases, etc.). + ### All databases diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 515fe9625..44cd34a84 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -256,6 +256,24 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): ) +def test_restore_database_dump_runs_mongorestore_with_options(): + database_config = [{'name': 'foo', 'restore_options': '--harder',}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ['mongorestore', '--archive', '--drop', '--db', 'foo', '--harder',], + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + ) + + def test_restore_database_dump_runs_psql_for_all_database_dump(): database_config = [{'name': 'all'}] extract_process = flexmock(stdout=flexmock()) diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index d9e605b19..9722bdf56 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -336,6 +336,23 @@ def test_restore_database_dump_errors_on_multiple_database_config(): ) +def test_restore_database_dump_runs_mysql_with_options(): + database_config = [{'name': 'foo', 'restore_options': '--harder'}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('mysql', '--batch', '--harder'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment=None, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + ) + + def test_restore_database_dump_runs_mysql_with_hostname_and_port(): database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 5c5818282..ce7a0250b 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -36,6 +36,55 @@ def test_database_names_to_dump_with_all_and_format_lists_databases(): ) +def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port(): + database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234} + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'psql', + '--list', + '--no-password', + '--csv', + '--tuples-only', + '--host', + 'localhost', + '--port', + '1234', + ), + extra_environment=object, + ).and_return('foo,test,\nbar,test,"stuff and such"') + + assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ( + 'foo', + 'bar', + ) + + +def test_database_names_to_dump_with_all_and_format_lists_databases_with_username(): + database = {'name': 'all', 'format': 'custom', 'username': 'postgres'} + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ('psql', '--list', '--no-password', '--csv', '--tuples-only', '--username', 'postgres'), + extra_environment=object, + ).and_return('foo,test,\nbar,test,"stuff and such"') + + assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ( + 'foo', + 'bar', + ) + + +def test_database_names_to_dump_with_all_and_format_lists_databases_with_options(): + database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'} + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ('psql', '--list', '--no-password', '--csv', '--tuples-only', '--harder'), + extra_environment=object, + ).and_return('foo,test,\nbar,test,"stuff and such"') + + assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ( + 'foo', + 'bar', + ) + + def test_database_names_to_dump_with_all_and_format_excludes_particular_databases(): database = {'name': 'all', 'format': 'custom'} flexmock(module).should_receive('execute_command_and_capture_output').and_return( @@ -90,7 +139,7 @@ def test_dump_databases_raises_when_no_database_names_to_dump(): module.dump_databases(databases, 'test.yaml', {}, dry_run=False) -def test_dump_databases_with_dupliate_dump_skips_pg_dump(): +def test_dump_databases_with_duplicate_dump_skips_pg_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -480,6 +529,50 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): ) +def test_restore_database_dump_runs_pg_restore_with_options(): + database_config = [ + {'name': 'foo', 'restore_options': '--harder', 'analyze_options': '--smarter'} + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'pg_restore', + '--no-password', + '--if-exists', + '--exit-on-error', + '--clean', + '--dbname', + 'foo', + '--harder', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ( + 'psql', + '--no-password', + '--quiet', + '--dbname', + 'foo', + '--smarter', + '--command', + 'ANALYZE', + ), + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + ) + + def test_restore_database_dump_runs_psql_for_all_database_dump(): database_config = [{'name': 'all'}] extract_process = flexmock(stdout=flexmock())