From 7c6ce9399cdaae40494781511f6eccb7cfca0dde Mon Sep 17 00:00:00 2001 From: Andrea Ghensi Date: Wed, 29 Dec 2021 22:18:50 +0100 Subject: [PATCH] fix integration tests and mongodb auth --- .drone.yml | 15 +++++++++++++++ borgmatic/config/schema.yaml | 18 ++++++++++++++---- borgmatic/hooks/mongodb.py | 6 +++++- scripts/run-full-tests | 2 +- tests/end-to-end/docker-compose.yaml | 5 +++++ tests/end-to-end/test_database.py | 19 +++++++++---------- tests/unit/hooks/test_mongodb.py | 10 ++++++++-- 7 files changed, 57 insertions(+), 18 deletions(-) diff --git a/.drone.yml b/.drone.yml index 0d4e4601..2a8dea1b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,6 +13,11 @@ services: environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test + - name: mongodb + image: mongo:5.0.5 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: test clone: skip_verify: true @@ -38,6 +43,11 @@ services: environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test + - name: mongodb + image: mongo:5.0.5 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: test clone: skip_verify: true @@ -63,6 +73,11 @@ services: environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test + - name: mongodb + image: mongo:5.0.5 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: test clone: skip_verify: true diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 64254a16..c211da47 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -811,15 +811,25 @@ properties: Password with which to connect to the database. Skip it if no authentication is needed. example: trustsome1 + auth_db: + type: string + description: | + Authentication database where the specified + username has been created. + If no authentication database is specified, + the databse provided in "name" will be used. + If "name" is "all", the "admin" database will + be used. + example: admin format: type: string enum: ['archive', 'directory'] description: | Database dump output format. One of "archive", - or "directory". Defaults to "archive" (unlike - raw pg_dump). See pg_dump documentation for - details. Note that format is ignored when the - database name is "all". + or "directory". Defaults to "archive". See + mongodump documentation for details. Note that + format is ignored when the database name is + "all". example: directory options: type: string diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index 88a367ab..e06fd026 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -35,7 +35,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dump_filename = dump.make_database_dump_filename( make_dump_path(location_config), name, database.get('hostname') ) - dump_format = database.get('format', 'custom') + dump_format = database.get('format', 'archive') logger.debug( '{}: Dumping MongoDB database {} to {}{}'.format( @@ -72,6 +72,8 @@ def build_dump_command(database, dump_filename, dump_format): command.extend(('--username', database['username'])) if 'password' in database: command.extend(('--password', database['password'])) + if 'auth_db' in database: + command.extend(('--authenticationDatabase', database['auth_db'])) if not all_databases: command.extend(('--db', database['name'])) if 'options' in database: @@ -155,4 +157,6 @@ def build_restore_command(extract_process, database, dump_filename): command.extend(('--username', database['username'])) if 'password' in database: command.extend(('--password', database['password'])) + if 'auth_db' in database: + command.extend(('--authenticationDatabase', database['auth_db'])) return command diff --git a/scripts/run-full-tests b/scripts/run-full-tests index 47e4f5f7..0e9d0233 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -10,7 +10,7 @@ set -e -apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client +apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true python3 -m pip install --upgrade pip==21.3.1 setuptools==58.2.0 diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 3c081342..094ac8d3 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -10,6 +10,11 @@ services: environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test + mongodb: + image: mongo:5.0.5 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: test tests: image: alpine:3.13 volumes: diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 208b9bbb..57e4219c 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -52,6 +52,7 @@ hooks: hostname: mongodb username: root password: test + auth_db: admin - name: all hostname: mongodb username: root @@ -77,15 +78,15 @@ def test_database_dump_and_restore(): write_configuration(config_path, repository_path, borgmatic_source_directory) subprocess.check_call( - 'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ') + ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] ) # Run borgmatic to generate a backup archive including a database dump. - subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' ')) + subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2']) # Get the created archive name. output = subprocess.check_output( - 'borgmatic --config {} list --json'.format(config_path).split(' ') + ['borgmatic', '--config', config_path, 'list', '--json'] ).decode(sys.stdout.encoding) parsed_output = json.loads(output) @@ -95,9 +96,7 @@ def test_database_dump_and_restore(): # Restore the database from the archive. subprocess.check_call( - 'borgmatic --config {} restore --archive {}'.format(config_path, archive_name).split( - ' ' - ) + ['borgmatic', '--config', config_path, 'restore', '--archive', archive_name] ) finally: os.chdir(original_working_directory) @@ -122,15 +121,15 @@ def test_database_dump_and_restore_with_directory_format(): ) subprocess.check_call( - 'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ') + ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] ) # Run borgmatic to generate a backup archive including a database dump. - subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' ')) + subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2']) # Restore the database from the archive. subprocess.check_call( - 'borgmatic --config {} restore --archive latest'.format(config_path).split(' ') + ['borgmatic', '--config', config_path, 'restore', '--archive', 'latest'] ) finally: os.chdir(original_working_directory) @@ -150,7 +149,7 @@ def test_database_dump_with_error_causes_borgmatic_to_exit(): write_configuration(config_path, repository_path, borgmatic_source_directory) subprocess.check_call( - 'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ') + ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] ) # Run borgmatic with a config override such that the database dump fails. diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index eec83bf7..377962e4 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -67,7 +67,7 @@ def test_dump_databases_runs_mongodump_with_hostname_and_port(): def test_dump_databases_runs_mongodump_with_username_and_password(): - databases = [{'name': 'foo', 'username': 'mongo', 'password': 'trustsome1'}] + databases = [{'name': 'foo', 'username': 'mongo', 'password': 'trustsome1', 'auth_db': "admin"}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_database_dump_filename').and_return( @@ -83,6 +83,8 @@ def test_dump_databases_runs_mongodump_with_username_and_password(): 'mongo', '--password', 'trustsome1', + '--authenticationDatabase', + 'admin', '--db', 'foo', '>', @@ -213,7 +215,9 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): def test_restore_database_dump_runs_pg_restore_with_username_and_password(): - database_config = [{'name': 'foo', 'username': 'mongo', 'password': 'trustsome1'}] + database_config = [ + {'name': 'foo', 'username': 'mongo', 'password': 'trustsome1', 'auth_db': 'admin'} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -229,6 +233,8 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): 'mongo', '--password', 'trustsome1', + '--authenticationDatabase', + 'admin', ], processes=[extract_process], output_log_level=logging.DEBUG,