From 193dd93de2b7497f05e82c79674b08ee65560cc4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 4 Aug 2023 13:22:44 -0700 Subject: [PATCH] Fork a MariaDB database hook from the MySQL database hook (#727). --- .drone.yml | 16 +- NEWS | 5 + borgmatic/config/schema.yaml | 151 ++++++- borgmatic/hooks/dispatch.py | 2 + borgmatic/hooks/dump.py | 3 +- borgmatic/hooks/mariadb.py | 242 ++++++++++ borgmatic/hooks/mongodb.py | 37 +- docs/how-to/backup-your-databases.md | 71 ++- setup.py | 2 +- tests/end-to-end/docker-compose.yaml | 20 +- tests/end-to-end/test_database.py | 37 +- tests/unit/hooks/test_mariadb.py | 644 +++++++++++++++++++++++++++ tests/unit/hooks/test_mongodb.py | 16 +- 13 files changed, 1150 insertions(+), 96 deletions(-) create mode 100644 borgmatic/hooks/mariadb.py create mode 100644 tests/unit/hooks/test_mariadb.py diff --git a/.drone.yml b/.drone.yml index e353b3c7..ed1762ca 100644 --- a/.drone.yml +++ b/.drone.yml @@ -16,16 +16,16 @@ services: POSTGRES_USER: postgres2 commands: - docker-entrypoint.sh -p 5433 - - name: mysql - image: docker.io/mariadb:10.5 + - name: mariadb + image: docker.io/mariadb:10.11.4 environment: - MYSQL_ROOT_PASSWORD: test - MYSQL_DATABASE: test - - name: mysql2 - image: docker.io/mariadb:10.5 + MARIADB_ROOT_PASSWORD: test + MARIADB_DATABASE: test + - name: mariadb2 + image: docker.io/mariadb:10.11.4 environment: - MYSQL_ROOT_PASSWORD: test2 - MYSQL_DATABASE: test + MARIADB_ROOT_PASSWORD: test2 + MARIADB_DATABASE: test commands: - docker-entrypoint.sh --port=3307 - name: mongodb diff --git a/NEWS b/NEWS index f7c0cac3..59b81f1a 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.8.2.dev0 + * #727: Add a MariaDB database hook that uses native MariaDB commands instead of the deprecated + MySQL ones. Be aware though that any existing backups made with the "mysql_databases:" hook are + only restorable with a "mysql_databases:" configuration. + 1.8.1 * #326: Add documentation for restoring a database to an alternate host: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-to-an-alternate-host diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index fdd7df74..289feb5e 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -841,10 +841,121 @@ properties: description: | List of one or more PostgreSQL databases to dump before creating a backup, run once per configuration file. The database dumps are - added to your source directories at runtime, backed up, and removed - afterwards. Requires pg_dump/pg_dumpall/pg_restore commands. See + added to your source directories at runtime and streamed directly + to Borg. Requires pg_dump/pg_dumpall/pg_restore commands. See https://www.postgresql.org/docs/current/app-pgdump.html and - https://www.postgresql.org/docs/current/libpq-ssl.html for details. + https://www.postgresql.org/docs/current/libpq-ssl.html for + details. + mariadb_databases: + type: array + items: + type: object + required: ['name'] + additionalProperties: false + properties: + name: + type: string + description: | + Database name (required if using this hook). Or "all" to + dump all databases on the host. Note that using this + database hook implicitly enables both read_special and + one_file_system (see above) to support dump and restore + streaming. + example: users + hostname: + type: string + description: | + Database hostname to connect to. Defaults to connecting + via local Unix socket. + example: database.example.org + restore_hostname: + type: string + description: | + Database hostname to restore to. Defaults to the + "hostname" option. + example: database.example.org + port: + type: integer + description: Port to connect to. Defaults to 3306. + example: 3307 + restore_port: + type: integer + description: | + Port to restore to. Defaults to the "port" option. + example: 5433 + username: + type: string + description: | + Username with which to connect to the database. Defaults + to the username of the current user. + example: dbuser + restore_username: + type: string + description: | + Username with which to restore the database. Defaults to + the "username" option. + example: dbuser + password: + type: string + description: | + Password with which to connect to the database. Omitting + a password will only work if MariaDB is configured to + trust the configured username without a password. + example: trustsome1 + restore_password: + type: string + description: | + Password with which to connect to the restore database. + Defaults to the "password" option. + example: trustsome1 + format: + type: string + enum: ['sql'] + description: | + Database dump output format. Currently only "sql" is + supported. Defaults to "sql" for a single database. Or, + when database name is "all" and format is blank, dumps + all databases to a single file. But if a format is + specified with an "all" database name, dumps each + database to a separate file of that format, allowing + more convenient restores of individual databases. + example: directory + add_drop_database: + type: boolean + description: | + Use the "--add-drop-database" flag with mariadb-dump, + causing the database to be dropped right before restore. + Defaults to true. + example: false + options: + type: string + description: | + Additional mariadb-dump options to pass directly to the + dump command, without performing any validation on them. + See mariadb-dump documentation for details. + example: --skip-comments + list_options: + type: string + description: | + Additional options to pass directly to the mariadb + command that lists available databases, without + performing any validation on them. See mariadb command + documentation for details. + example: --defaults-extra-file=mariadb.cnf + restore_options: + type: string + description: | + Additional options to pass directly to the mariadb + command that restores database dumps, without + performing any validation on them. See mariadb command + documentation for details. + example: --defaults-extra-file=mariadb.cnf + description: | + List of one or more MariaDB databases to dump before creating a + backup, run once per configuration file. The database dumps are + added to your source directories at runtime and streamed directly + to Borg. Requires mariadb-dump/mariadb commands. See + https://mariadb.com/kb/en/library/mysqldump/ for details. mysql_databases: type: array items: @@ -893,7 +1004,7 @@ properties: description: | Username with which to restore the database. Defaults to the "username" option. - example: dbuser + example: dbuser password: type: string description: | @@ -906,7 +1017,7 @@ properties: description: | Password with which to connect to the restore database. Defaults to the "password" option. - example: trustsome1 + example: trustsome1 format: type: string enum: ['sql'] @@ -936,26 +1047,26 @@ properties: list_options: type: string description: | - Additional mysql options to pass directly to the mysql + Additional options to pass directly to the mysql command that lists available databases, without - performing any validation on them. See mysql + performing any validation on them. See mysql command documentation for details. example: --defaults-extra-file=my.cnf restore_options: type: string description: | - Additional mysql options to pass directly to the mysql - command that restores database dumps, without performing - any validation on them. See mysql documentation for - details. + Additional options to pass directly to the mysql + command that restores database dumps, without + performing any validation on them. See mysql command + documentation for details. example: --defaults-extra-file=my.cnf description: | - List of one or more MySQL/MariaDB databases to dump before creating - a backup, run once per configuration file. The database dumps are - added to your source directories at runtime, backed up, and removed - afterwards. Requires mysqldump/mysql commands (from either MySQL or - MariaDB). See https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html - or https://mariadb.com/kb/en/library/mysqldump/ for details. + List of one or more MySQL databases to dump before creating a + backup, run once per configuration file. The database dumps are + added to your source directories at runtime and streamed directly + to Borg. Requires mysqldump/mysql commands. See + https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for + details. sqlite_databases: type: array items: @@ -1033,7 +1144,7 @@ properties: description: | Username with which to restore the database. Defaults to the "username" option. - example: dbuser + example: dbuser password: type: string description: | @@ -1080,8 +1191,8 @@ properties: description: | List of one or more MongoDB databases to dump before creating a backup, run once per configuration file. The database dumps are - added to your source directories at runtime, backed up, and removed - afterwards. Requires mongodump/mongorestore commands. See + added to your source directories at runtime and streamed directly + to Borg. Requires mongodump/mongorestore commands. See https://docs.mongodb.com/database-tools/mongodump/ and https://docs.mongodb.com/database-tools/mongorestore/ for details. ntfy: diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index d98473ab..0c003e33 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -4,6 +4,7 @@ from borgmatic.hooks import ( cronhub, cronitor, healthchecks, + mariadb, mongodb, mysql, ntfy, @@ -18,6 +19,7 @@ HOOK_NAME_TO_MODULE = { 'cronhub': cronhub, 'cronitor': cronitor, 'healthchecks': healthchecks, + 'mariadb_databases': mariadb, 'mongodb_databases': mongodb, 'mysql_databases': mysql, 'ntfy': ntfy, diff --git a/borgmatic/hooks/dump.py b/borgmatic/hooks/dump.py index 015ed696..be5431ac 100644 --- a/borgmatic/hooks/dump.py +++ b/borgmatic/hooks/dump.py @@ -7,9 +7,10 @@ from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY logger = logging.getLogger(__name__) DATABASE_HOOK_NAMES = ( - 'postgresql_databases', + 'mariadb_databases', 'mysql_databases', 'mongodb_databases', + 'postgresql_databases', 'sqlite_databases', ) diff --git a/borgmatic/hooks/mariadb.py b/borgmatic/hooks/mariadb.py new file mode 100644 index 00000000..13246287 --- /dev/null +++ b/borgmatic/hooks/mariadb.py @@ -0,0 +1,242 @@ +import copy +import logging +import os + +from borgmatic.execute import ( + execute_command, + execute_command_and_capture_output, + execute_command_with_processes, +) +from borgmatic.hooks import dump + +logger = logging.getLogger(__name__) + + +def make_dump_path(config): # pragma: no cover + ''' + Make the dump path from the given configuration dict and the name of this hook. + ''' + return dump.make_database_dump_path( + config.get('borgmatic_source_directory'), 'mariadb_databases' + ) + + +SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') + + +def database_names_to_dump(database, extra_environment, log_prefix, dry_run): + ''' + Given a requested database config, return the corresponding sequence of database names to dump. + In the case of "all", query for the names of databases on the configured host and return them, + excluding any system databases that will cause problems during restore. + ''' + if database['name'] != 'all': + return (database['name'],) + if dry_run: + return () + + show_command = ( + ('mariadb',) + + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--port', str(database['port'])) if 'port' in database else ()) + + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--user', database['username']) if 'username' in database else ()) + + ('--skip-column-names', '--batch') + + ('--execute', 'show schemas') + ) + logger.debug(f'{log_prefix}: Querying for "all" MariaDB databases to dump') + show_output = execute_command_and_capture_output( + show_command, extra_environment=extra_environment + ) + + return tuple( + show_name + for show_name in show_output.strip().splitlines() + if show_name not in SYSTEM_DATABASE_NAMES + ) + + +def execute_dump_command( + database, log_prefix, dump_path, database_names, extra_environment, dry_run, dry_run_label +): + ''' + Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named + pipe constructed from the given dump path and database names. Use the given log prefix in any + log entries. + + Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if + this is a dry run, then don't actually dump anything and return None. + ''' + database_name = database['name'] + dump_filename = dump.make_database_dump_filename( + dump_path, database['name'], database.get('hostname') + ) + if os.path.exists(dump_filename): + logger.warning( + f'{log_prefix}: Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}' + ) + return None + + dump_command = ( + ('mariadb-dump',) + + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--port', str(database['port'])) if 'port' in database else ()) + + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--user', database['username']) if 'username' in database else ()) + + ('--databases',) + + database_names + + ('--result-file', dump_filename) + ) + + logger.debug( + f'{log_prefix}: Dumping MariaDB database "{database_name}" to {dump_filename}{dry_run_label}' + ) + if dry_run: + return None + + dump.create_named_pipe_for_dump(dump_filename) + + return execute_command( + dump_command, + extra_environment=extra_environment, + run_to_completion=False, + ) + + +def dump_databases(databases, config, log_prefix, dry_run): + ''' + Dump the given MariaDB databases to a named pipe. The databases are supplied as a sequence of + dicts, one dict describing each database as per the configuration schema. Use the given + configuration dict to construct the destination path and the given log prefix in any log + entries. + + Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named + pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence. + ''' + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + processes = [] + + logger.info(f'{log_prefix}: Dumping MariaDB databases{dry_run_label}') + + for database in databases: + dump_path = make_dump_path(config) + extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None + dump_database_names = database_names_to_dump( + database, extra_environment, log_prefix, dry_run + ) + + if not dump_database_names: + if dry_run: + continue + + raise ValueError('Cannot find any MariaDB databases to dump.') + + if database['name'] == 'all' and database.get('format'): + for dump_name in dump_database_names: + renamed_database = copy.copy(database) + renamed_database['name'] = dump_name + processes.append( + execute_dump_command( + renamed_database, + log_prefix, + dump_path, + (dump_name,), + extra_environment, + dry_run, + dry_run_label, + ) + ) + else: + processes.append( + execute_dump_command( + database, + log_prefix, + dump_path, + dump_database_names, + extra_environment, + dry_run, + dry_run_label, + ) + ) + + return [process for process in processes if process] + + +def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover + ''' + Remove all database dump files for this hook regardless of the given databases. Use the given + configuration dict to construct the destination path and the log prefix in any log entries. If + this is a dry run, then don't actually remove anything. + ''' + dump.remove_database_dumps(make_dump_path(config), 'MariaDB', log_prefix, dry_run) + + +def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover + ''' + Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a + database name to match, return the corresponding glob patterns to match the database dump in an + archive. + ''' + return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*') + + +def restore_database_dump( + databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params +): + ''' + Restore the given MariaDB database from an extract stream. The databases are supplied as a + sequence containing one dict describing each database (as per the configuration schema), but + only the database corresponding to the given database name is restored. Use the given log prefix + in any log entries. If this is a dry run, then don't actually restore anything. Trigger the + given active extract process (an instance of subprocess.Popen) to produce output to consume. + ''' + dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' + + try: + database = next( + database_config + for database_config in databases_config + if database_config.get('name') == database_name + ) + except StopIteration: + raise ValueError( + f'A database named "{database_name}" could not be found in the configuration' + ) + + hostname = connection_params['hostname'] or database.get( + 'restore_hostname', database.get('hostname') + ) + port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) + username = connection_params['username'] or database.get( + 'restore_username', database.get('username') + ) + password = connection_params['password'] or database.get( + 'restore_password', database.get('password') + ) + + restore_command = ( + ('mariadb', '--batch') + + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + + (('--host', hostname) if hostname else ()) + + (('--port', str(port)) if port else ()) + + (('--protocol', 'tcp') if hostname or port else ()) + + (('--user', username) if username else ()) + ) + extra_environment = {'MYSQL_PWD': password} if password else None + + logger.debug(f"{log_prefix}: Restoring MariaDB database {database['name']}{dry_run_label}") + if dry_run: + return + + # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning + # if the restore paths don't exist in the archive. + execute_command_with_processes( + restore_command, + [extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment=extra_environment, + ) diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index c94a084d..0f8cc2c6 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -59,26 +59,23 @@ def build_dump_command(database, dump_filename, dump_format): Return the mongodump command from a single database configuration. ''' all_databases = database['name'] == 'all' - command = ['mongodump'] - if dump_format == 'directory': - command.extend(('--out', dump_filename)) - if 'hostname' in database: - command.extend(('--host', database['hostname'])) - if 'port' in database: - command.extend(('--port', str(database['port']))) - if 'username' in database: - command.extend(('--username', database['username'])) - if 'password' in database: - command.extend(('--password', database['password'])) - if 'authentication_database' in database: - command.extend(('--authenticationDatabase', database['authentication_database'])) - if not all_databases: - command.extend(('--db', database['name'])) - if 'options' in database: - command.extend(database['options'].split(' ')) - if dump_format != 'directory': - command.extend(('--archive', '>', dump_filename)) - return command + + return ( + ('mongodump',) + + (('--out', dump_filename) if dump_format == 'directory' else ()) + + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--port', str(database['port'])) if 'port' in database else ()) + + (('--username', database['username']) if 'username' in database else ()) + + (('--password', database['password']) if 'password' in database else ()) + + ( + ('--authenticationDatabase', database['authentication_database']) + if 'authentication_database' in database + else () + ) + + (('--db', database['name']) if not all_databases else ()) + + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + (('--archive', '>', dump_filename) if dump_format != 'directory' else ()) + ) def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 25a664e4..1761311a 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -15,7 +15,7 @@ consistent snapshot that is more suited for backups. Fortunately, borgmatic includes built-in support for creating database dumps prior to running backups. For example, here is everything you need to dump and -backup a couple of local PostgreSQL databases and a MySQL/MariaDB database. +backup a couple of local PostgreSQL databases and a MySQL database. ```yaml postgresql_databases: @@ -46,6 +46,16 @@ sqlite_databases: path: /var/lib/sqlite3/mydb.sqlite ``` +New in version 1.8.2 If you're +using MariaDB, use the MariaDB database hook instead of `mysql_databases:` as +the MariaDB hook calls native MariaDB commands instead of the deprecated MySQL +ones. For instance: + +```yaml +mariadb_databases: + - name: comments +``` + As part of each backup, borgmatic streams a database dump for each configured database directly to Borg, so it's included in the backup without consuming additional disk space. (The exceptions are the PostgreSQL/MongoDB "directory" @@ -75,16 +85,23 @@ postgresql_databases: password: trustsome1 format: tar options: "--role=someone" +mariadb_databases: + - name: photos + hostname: database3.example.org + port: 3307 + username: root + password: trustsome1 + options: "--skip-comments" mysql_databases: - name: posts - hostname: database3.example.org + hostname: database4.example.org port: 3307 username: root password: trustsome1 options: "--skip-comments" mongodb_databases: - name: messages - hostname: database4.example.org + hostname: database5.example.org port: 27018 username: dbuser password: trustsome1 @@ -108,6 +125,8 @@ If you want to dump all databases on a host, use `all` for the database name: ```yaml postgresql_databases: - name: all +mariadb_databases: + - name: all mysql_databases: - name: all mongodb_databases: @@ -123,15 +142,18 @@ The SQLite hook in particular does not consider "all" a special database name. these options in the `hooks:` section of your configuration. New in version 1.7.6 With -PostgreSQL and MySQL, you can optionally dump "all" databases to separate -files instead of one combined dump file, allowing more convenient restores of -individual databases. Enable this by specifying your desired database dump -`format`: +PostgreSQL, MariaDB, and MySQL, you can optionally dump "all" databases to +separate files instead of one combined dump file, allowing more convenient +restores of individual databases. Enable this by specifying your desired +database dump `format`: ```yaml postgresql_databases: - name: all format: custom +mariadb_databases: + - name: all + format: sql mysql_databases: - name: all format: sql @@ -222,10 +244,16 @@ to prepare for this situation, it's a good idea to include borgmatic's own configuration files as part of your regular backups. That way, you can always bring back any missing configuration files in order to restore a database. +New in version 1.7.15 borgmatic +automatically includes configuration files in your backup. See [the +documentation on the `config bootstrap` +action](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive) +for more information. + ## Supported databases -As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, MongoDB, and SQLite +As of now, borgmatic supports PostgreSQL, MariaDB, MySQL, MongoDB, and SQLite databases directly. But see below about general-purpose preparation and cleanup hooks as a work-around with other database systems. Also, please [file a ticket](https://torsion.org/borgmatic/#issues) for additional database @@ -420,9 +448,9 @@ dumps with any database system. ## Troubleshooting -### PostgreSQL/MySQL authentication errors +### Authentication errors -With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors +With PostgreSQL, MariaDB, and MySQL, if you're getting authentication errors when borgmatic tries to connect to your database, a natural reaction is to increase your borgmatic verbosity with `--verbosity 2` and go looking in the logs. You'll notice though that your database password does not show up in the @@ -436,23 +464,24 @@ authenticated. For instance, with PostgreSQL, check your [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) file for that configuration. -Additionally, MySQL/MariaDB may be picking up some of your credentials from a -defaults file like `~/.my.cnf`. If that's the case, then it's possible -MySQL/MariaDB ends up using, say, a username from borgmatic's configuration -and a password from `~/.my.cnf`. This may result in authentication errors if -this combination of credentials is not what you intend. +Additionally, MariaDB or MySQL may be picking up some of your credentials from +a defaults file like `~/mariadb.cnf` or `~/.my.cnf`. If that's the case, then +it's possible MariaDB or MySQL end up using, say, a username from borgmatic's +configuration and a password from `~/mariadb.cnf` or `~/.my.cnf`. This may +result in authentication errors if this combination of credentials is not what +you intend. -### MySQL table lock errors +### MariaDB or MySQL table lock errors -If you encounter table lock errors during a database dump with MySQL/MariaDB, -you may need to [use a -transaction](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html#option_mysqldump_single-transaction). +If you encounter table lock errors during a database dump with MariaDB or +MySQL, you may need to [use a +transaction](https://mariadb.com/docs/skysql-dbaas/ref/mdb/cli/mariadb-dump/single-transaction/). You can add any additional flags to the `options:` in your database -configuration. Here's an example: +configuration. Here's an example with MariaDB: ```yaml -mysql_databases: +mariadb_databases: - name: posts options: "--single-transaction --quick" ``` diff --git a/setup.py b/setup.py index 26bff6be..622d2220 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.8.1' +VERSION = '1.8.2.dev0' setup( diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 8753ddd6..f87beb59 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -12,16 +12,16 @@ services: POSTGRES_DB: test POSTGRES_USER: postgres2 command: docker-entrypoint.sh -p 5433 - mysql: - image: docker.io/mariadb:10.5 + mariadb: + image: docker.io/mariadb:10.11.4 environment: - MYSQL_ROOT_PASSWORD: test - MYSQL_DATABASE: test - mysql2: - image: docker.io/mariadb:10.5 + MARIADB_ROOT_PASSWORD: test + MARIADB_DATABASE: test + mariadb2: + image: docker.io/mariadb:10.11.4 environment: - MYSQL_ROOT_PASSWORD: test2 - MYSQL_DATABASE: test + MARIADB_ROOT_PASSWORD: test2 + MARIADB_DATABASE: test command: docker-entrypoint.sh --port=3307 mongodb: image: docker.io/mongo:5.0.5 @@ -50,7 +50,7 @@ services: depends_on: - postgresql - postgresql2 - - mysql - - mysql2 + - mariadb + - mariadb2 - mongodb - mongodb2 diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index f9c36214..1ab02701 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -45,18 +45,32 @@ postgresql_databases: hostname: postgresql username: postgres password: test -mysql_databases: +mariadb_databases: - name: test - hostname: mysql + hostname: mariadb username: root password: test - name: all - hostname: mysql + hostname: mariadb username: root password: test - name: all format: sql - hostname: mysql + hostname: mariadb + username: root + password: test +mysql_databases: + - name: test + hostname: mariadb + username: root + password: test + - name: all + hostname: mariadb + username: root + password: test + - name: all + format: sql + hostname: mariadb username: root password: test mongodb_databases: @@ -111,12 +125,21 @@ postgresql_databases: restore_port: 5433 restore_username: postgres2 restore_password: test2 -mysql_databases: +mariadb_databases: - name: test - hostname: mysql + hostname: mariadb username: root password: test - restore_hostname: mysql2 + restore_hostname: mariadb2 + restore_port: 3307 + restore_username: root + restore_password: test2 +mysql_databases: + - name: test + hostname: mariadb + username: root + password: test + restore_hostname: mariadb2 restore_port: 3307 restore_username: root restore_password: test2 diff --git a/tests/unit/hooks/test_mariadb.py b/tests/unit/hooks/test_mariadb.py new file mode 100644 index 00000000..cc7f3fd6 --- /dev/null +++ b/tests/unit/hooks/test_mariadb.py @@ -0,0 +1,644 @@ +import logging + +import pytest +from flexmock import flexmock + +from borgmatic.hooks import mariadb as module + + +def test_database_names_to_dump_passes_through_name(): + extra_environment = flexmock() + log_prefix = '' + + names = module.database_names_to_dump( + {'name': 'foo'}, extra_environment, log_prefix, dry_run=False + ) + + assert names == ('foo',) + + +def test_database_names_to_dump_bails_for_dry_run(): + extra_environment = flexmock() + log_prefix = '' + flexmock(module).should_receive('execute_command_and_capture_output').never() + + names = module.database_names_to_dump( + {'name': 'all'}, extra_environment, log_prefix, dry_run=True + ) + + assert names == () + + +def test_database_names_to_dump_queries_mariadb_for_database_names(): + extra_environment = flexmock() + log_prefix = '' + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'), + extra_environment=extra_environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all'}, extra_environment, log_prefix, dry_run=False + ) + + assert names == ('foo', 'bar') + + +def test_dump_databases_dumps_each_database(): + databases = [{'name': 'foo'}, {'name': 'bar'}] + processes = [flexmock(), flexmock()] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( + ('bar',) + ) + + for name, process in zip(('foo', 'bar'), processes): + flexmock(module).should_receive('execute_dump_command').with_args( + database={'name': name}, + log_prefix=object, + dump_path=object, + database_names=(name,), + extra_environment=object, + dry_run=object, + dry_run_label=object, + ).and_return(process).once() + + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes + + +def test_dump_databases_dumps_with_password(): + database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} + process = flexmock() + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( + ('bar',) + ) + + flexmock(module).should_receive('execute_dump_command').with_args( + database=database, + log_prefix=object, + dump_path=object, + database_names=('foo',), + extra_environment={'MYSQL_PWD': 'trustsome1'}, + dry_run=object, + dry_run_label=object, + ).and_return(process).once() + + assert module.dump_databases([database], {}, 'test.yaml', dry_run=False) == [process] + + +def test_dump_databases_dumps_all_databases_at_once(): + databases = [{'name': 'all'}] + process = flexmock() + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) + flexmock(module).should_receive('execute_dump_command').with_args( + database={'name': 'all'}, + log_prefix=object, + dump_path=object, + database_names=('foo', 'bar'), + extra_environment=object, + dry_run=object, + dry_run_label=object, + ).and_return(process).once() + + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] + + +def test_dump_databases_dumps_all_databases_separately_when_format_configured(): + databases = [{'name': 'all', 'format': 'sql'}] + processes = [flexmock(), flexmock()] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) + + for name, process in zip(('foo', 'bar'), processes): + flexmock(module).should_receive('execute_dump_command').with_args( + database={'name': name, 'format': 'sql'}, + log_prefix=object, + dump_path=object, + database_names=(name,), + extra_environment=object, + dry_run=object, + dry_run_label=object, + ).and_return(process).once() + + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes + + +def test_database_names_to_dump_runs_mariadb_with_list_options(): + database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf'} + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mariadb', + '--defaults-extra-file=mariadb.cnf', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + extra_environment=None, + ).and_return(('foo\nbar')).once() + + assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') + + +def test_execute_dump_command_runs_mariadb_dump(): + process = flexmock() + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + extra_environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): + process = flexmock() + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + extra_environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'add_drop_database': False}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): + process = flexmock() + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--add-drop-database', + '--host', + 'database.example.org', + '--port', + '5433', + '--protocol', + 'tcp', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + extra_environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): + process = flexmock() + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--add-drop-database', + '--user', + 'root', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + extra_environment={'MYSQL_PWD': 'trustsome1'}, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment={'MYSQL_PWD': 'trustsome1'}, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mariadb_dump_with_options(): + process = flexmock() + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--stuff=such', + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + extra_environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'options': '--stuff=such'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() + flexmock(module).should_receive('execute_command').never() + + assert ( + module.execute_dump_command( + database={'name': 'foo'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=True, + dry_run_label='SO DRY', + ) + is None + ) + + +def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').never() + + assert ( + module.execute_dump_command( + database={'name': 'foo'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=True, + dry_run_label='SO DRY', + ) + is None + ) + + +def test_dump_databases_errors_for_missing_all_databases(): + databases = [{'name': 'all'}] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.dump).should_receive('make_database_dump_filename').and_return( + 'databases/localhost/all' + ) + flexmock(module).should_receive('database_names_to_dump').and_return(()) + + with pytest.raises(ValueError): + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) + + +def test_dump_databases_does_not_error_for_missing_all_databases_with_dry_run(): + databases = [{'name': 'all'}] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.dump).should_receive('make_database_dump_filename').and_return( + 'databases/localhost/all' + ) + flexmock(module).should_receive('database_names_to_dump').and_return(()) + + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=True) == [] + + +def test_restore_database_dump_runs_mariadb_to_restore(): + databases_config = [{'name': 'foo'}, {'name': 'bar'}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('mariadb', '--batch'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment=None, + ).once() + + module.restore_database_dump( + databases_config, + {}, + 'test.yaml', + database_name='foo', + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + +def test_restore_database_dump_errors_when_database_missing_from_configuration(): + databases_config = [{'name': 'foo'}, {'name': 'bar'}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').never() + + with pytest.raises(ValueError): + module.restore_database_dump( + databases_config, + {}, + 'test.yaml', + database_name='other', + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + +def test_restore_database_dump_runs_mariadb_with_options(): + databases_config = [{'name': 'foo', 'restore_options': '--harder'}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('mariadb', '--batch', '--harder'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment=None, + ).once() + + module.restore_database_dump( + databases_config, + {}, + 'test.yaml', + database_name='foo', + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + +def test_restore_database_dump_runs_mariadb_with_hostname_and_port(): + databases_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mariadb', + '--batch', + '--host', + 'database.example.org', + '--port', + '5433', + '--protocol', + 'tcp', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment=None, + ).once() + + module.restore_database_dump( + databases_config, + {}, + 'test.yaml', + database_name='foo', + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + +def test_restore_database_dump_runs_mariadb_with_username_and_password(): + databases_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('mariadb', '--batch', '--user', 'root'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'MYSQL_PWD': 'trustsome1'}, + ).once() + + module.restore_database_dump( + databases_config, + {}, + 'test.yaml', + database_name='foo', + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + +def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): + databases_config = [ + { + 'name': 'foo', + 'username': 'root', + 'password': 'trustsome1', + 'restore_hostname': 'restorehost', + 'restore_port': 'restoreport', + 'restore_username': 'restoreusername', + 'restore_password': 'restorepassword', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mariadb', + '--batch', + '--host', + 'clihost', + '--port', + 'cliport', + '--protocol', + 'tcp', + '--user', + 'cliusername', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'MYSQL_PWD': 'clipassword'}, + ).once() + + module.restore_database_dump( + databases_config, + {}, + 'test.yaml', + database_name='foo', + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': 'clihost', + 'port': 'cliport', + 'username': 'cliusername', + 'password': 'clipassword', + }, + ) + + +def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore(): + databases_config = [ + { + 'name': 'foo', + 'username': 'root', + 'password': 'trustsome1', + 'hostname': 'dbhost', + 'port': 'dbport', + 'restore_username': 'restoreuser', + 'restore_password': 'restorepass', + 'restore_hostname': 'restorehost', + 'restore_port': 'restoreport', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mariadb', + '--batch', + '--host', + 'restorehost', + '--port', + 'restoreport', + '--protocol', + 'tcp', + '--user', + 'restoreuser', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'MYSQL_PWD': 'restorepass'}, + ).once() + + module.restore_database_dump( + databases_config, + {}, + 'test.yaml', + database_name='foo', + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + +def test_restore_database_dump_with_dry_run_skips_restore(): + databases_config = [{'name': 'foo'}] + + flexmock(module).should_receive('execute_command_with_processes').never() + + module.restore_database_dump( + databases_config, + {}, + 'test.yaml', + database_name='foo', + dry_run=True, + extract_process=flexmock(), + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index a676b583..83d75d34 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -17,7 +17,7 @@ def test_dump_databases_runs_mongodump_for_each_database(): for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_command').with_args( - ['mongodump', '--db', name, '--archive', '>', f'databases/localhost/{name}'], + ('mongodump', '--db', name, '--archive', '>', f'databases/localhost/{name}'), shell=True, run_to_completion=False, ).and_return(process).once() @@ -47,7 +47,7 @@ def test_dump_databases_runs_mongodump_with_hostname_and_port(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - [ + ( 'mongodump', '--host', 'database.example.org', @@ -58,7 +58,7 @@ def test_dump_databases_runs_mongodump_with_hostname_and_port(): '--archive', '>', 'databases/database.example.org/foo', - ], + ), shell=True, run_to_completion=False, ).and_return(process).once() @@ -83,7 +83,7 @@ def test_dump_databases_runs_mongodump_with_username_and_password(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - [ + ( 'mongodump', '--username', 'mongo', @@ -96,7 +96,7 @@ def test_dump_databases_runs_mongodump_with_username_and_password(): '--archive', '>', 'databases/localhost/foo', - ], + ), shell=True, run_to_completion=False, ).and_return(process).once() @@ -114,7 +114,7 @@ def test_dump_databases_runs_mongodump_with_directory_format(): flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').with_args( - ['mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'], + ('mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'), shell=True, ).and_return(flexmock()).once() @@ -131,7 +131,7 @@ def test_dump_databases_runs_mongodump_with_options(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ['mongodump', '--db', 'foo', '--stuff=such', '--archive', '>', 'databases/localhost/foo'], + ('mongodump', '--db', 'foo', '--stuff=such', '--archive', '>', 'databases/localhost/foo'), shell=True, run_to_completion=False, ).and_return(process).once() @@ -149,7 +149,7 @@ def test_dump_databases_runs_mongodumpall_for_all_databases(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ['mongodump', '--archive', '>', 'databases/localhost/all'], + ('mongodump', '--archive', '>', 'databases/localhost/all'), shell=True, run_to_completion=False, ).and_return(process).once()