diff --git a/.drone.yml b/.drone.yml
index 0d4e46015..a2e92ac95 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -1,54 +1,3 @@
----
-kind: pipeline
-name: python-3-6-alpine-3-9
-
-services:
- - name: postgresql
- image: postgres:11.9-alpine
- environment:
- POSTGRES_PASSWORD: test
- POSTGRES_DB: test
- - name: mysql
- image: mariadb:10.3
- environment:
- MYSQL_ROOT_PASSWORD: test
- MYSQL_DATABASE: test
-
-clone:
- skip_verify: true
-
-steps:
-- name: build
- image: alpine:3.9
- pull: always
- commands:
- - scripts/run-full-tests
----
-kind: pipeline
-name: python-3-7-alpine-3-10
-
-services:
- - name: postgresql
- image: postgres:11.9-alpine
- environment:
- POSTGRES_PASSWORD: test
- POSTGRES_DB: test
- - name: mysql
- image: mariadb:10.3
- environment:
- MYSQL_ROOT_PASSWORD: test
- MYSQL_DATABASE: test
-
-clone:
- skip_verify: true
-
-steps:
-- name: build
- image: alpine:3.10
- pull: always
- commands:
- - scripts/run-full-tests
----
kind: pipeline
name: python-3-8-alpine-3-13
@@ -63,6 +12,11 @@ services:
environment:
MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test
+ - name: mongodb
+ image: mongo:5.0.5
+ environment:
+ MONGO_INITDB_ROOT_USERNAME: root
+ MONGO_INITDB_ROOT_PASSWORD: test
clone:
skip_verify: true
diff --git a/NEWS b/NEWS
index 350af32e3..96663dae5 100644
--- a/NEWS
+++ b/NEWS
@@ -1,8 +1,11 @@
1.5.22.dev0
+ * #288: Database dump hooks for MongoDB.
* #470: Move mysqldump options to the beginning of the command due to MySQL bug 30994.
* #471: When command-line configuration override produces a parse error, error cleanly instead of
tracebacking.
* #476: Fix unicode error when restoring particular MySQL databases.
+ * Drop support for Python 3.6, which has been end-of-lifed.
+ * Add support for Python 3.10.
1.5.21
* #28: Optionally retry failing backups via "retries" and "retry_wait" configuration options.
diff --git a/README.md b/README.md
index 8fb2a1075..3d282dcfd 100644
--- a/README.md
+++ b/README.md
@@ -26,7 +26,6 @@ location:
repositories:
- 1234@usw-s001.rsync.net:backups.borg
- k8pDxu32@k8pDxu32.repo.borgbase.com:repo
- - user1@scp2.cdn.lima-labs.com:repo
- /var/lib/backups/local.borg
retention:
@@ -66,11 +65,11 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
+
-
@@ -93,7 +92,6 @@ referral links, but without any tracking scripts or cookies.)
- BorgBase: Borg hosting service with support for monitoring, 2FA, and append-only repos
- - Lima-Labs: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs
Additionally, [rsync.net](https://www.rsync.net/products/borg.html) and
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index d8cf10af8..64a2bdca9 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -781,6 +781,80 @@ properties:
mysqldump/mysql commands (from either MySQL or MariaDB). See
https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
https://mariadb.com/kb/en/library/mysqldump/ for details.
+ mongodb_databases:
+ type: array
+ items:
+ type: object
+ required: ['name']
+ additionalProperties: false
+ properties:
+ name:
+ type: string
+ description: |
+ Database name (required if using this hook). Or
+ "all" to dump all databases on the host. Note
+ that using this database hook implicitly enables
+ both read_special and one_file_system (see
+ above) to support dump and restore streaming.
+ example: users
+ hostname:
+ type: string
+ description: |
+ Database hostname to connect to. Defaults to
+ connecting to localhost.
+ example: database.example.org
+ port:
+ type: integer
+ description: Port to connect to. Defaults to 27017.
+ example: 27018
+ username:
+ type: string
+ description: |
+ Username with which to connect to the database.
+ Skip it if no authentication is needed.
+ example: dbuser
+ password:
+ type: string
+ description: |
+ Password with which to connect to the database.
+ Skip it if no authentication is needed.
+ example: trustsome1
+ authentication_database:
+ type: string
+ description: |
+ Authentication database where the specified
+ username exists. If no authentication database
+ is specified, the database provided in "name"
+ is used. If "name" is "all", the "admin"
+ database is used.
+ example: admin
+ format:
+ type: string
+ enum: ['archive', 'directory']
+ description: |
+ Database dump output format. One of "archive",
+ or "directory". Defaults to "archive". See
+ mongodump documentation for details. Note that
+ format is ignored when the database name is
+ "all".
+ example: directory
+ options:
+ type: string
+ description: |
+ Additional mongodump options to pass
+ directly to the dump command, without performing
+ any validation on them. See mongodump
+ documentation for details.
+ example: --role=someone
+ description: |
+ List of one or more MongoDB databases to dump before
+ creating a backup, run once per configuration file. The
+ database dumps are added to your source directories at
+ runtime, backed up, and removed afterwards. Requires
+ mongodump/mongorestore commands. See
+ https://docs.mongodb.com/database-tools/mongodump/ and
+ https://docs.mongodb.com/database-tools/mongorestore/ for
+ details.
healthchecks:
type: string
description: |
diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py
index 6fb2c8086..a689e7033 100644
--- a/borgmatic/hooks/dispatch.py
+++ b/borgmatic/hooks/dispatch.py
@@ -1,6 +1,6 @@
import logging
-from borgmatic.hooks import cronhub, cronitor, healthchecks, mysql, pagerduty, postgresql
+from borgmatic.hooks import cronhub, cronitor, healthchecks, mongodb, mysql, pagerduty, postgresql
logger = logging.getLogger(__name__)
@@ -11,6 +11,7 @@ HOOK_NAME_TO_MODULE = {
'pagerduty': pagerduty,
'postgresql_databases': postgresql,
'mysql_databases': mysql,
+ 'mongodb_databases': mongodb,
}
diff --git a/borgmatic/hooks/dump.py b/borgmatic/hooks/dump.py
index 8bc9fcb89..f905d4925 100644
--- a/borgmatic/hooks/dump.py
+++ b/borgmatic/hooks/dump.py
@@ -6,7 +6,7 @@ from borgmatic.borg.create import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
logger = logging.getLogger(__name__)
-DATABASE_HOOK_NAMES = ('postgresql_databases', 'mysql_databases')
+DATABASE_HOOK_NAMES = ('postgresql_databases', 'mysql_databases', 'mongodb_databases')
def make_database_dump_path(borgmatic_source_directory, database_hook_name):
diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py
new file mode 100644
index 000000000..feb4955d6
--- /dev/null
+++ b/borgmatic/hooks/mongodb.py
@@ -0,0 +1,162 @@
+import logging
+
+from borgmatic.execute import execute_command, execute_command_with_processes
+from borgmatic.hooks import dump
+
+logger = logging.getLogger(__name__)
+
+
+def make_dump_path(location_config): # pragma: no cover
+ '''
+ Make the dump path from the given location configuration and the name of this hook.
+ '''
+ return dump.make_database_dump_path(
+ location_config.get('borgmatic_source_directory'), 'mongodb_databases'
+ )
+
+
+def dump_databases(databases, log_prefix, location_config, dry_run):
+ '''
+ Dump the given MongoDB databases to a named pipe. The databases are supplied as a sequence of
+ dicts, one dict describing each database as per the configuration schema. Use the given log
+ prefix in any log entries. Use the given location configuration dict to construct the
+ destination path.
+
+ Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
+ pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
+ '''
+ dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+
+ logger.info('{}: Dumping MongoDB databases{}'.format(log_prefix, dry_run_label))
+
+ processes = []
+ for database in databases:
+ name = database['name']
+ dump_filename = dump.make_database_dump_filename(
+ make_dump_path(location_config), name, database.get('hostname')
+ )
+ dump_format = database.get('format', 'archive')
+
+ logger.debug(
+ '{}: Dumping MongoDB database {} to {}{}'.format(
+ log_prefix, name, dump_filename, dry_run_label
+ )
+ )
+ if dry_run:
+ continue
+
+ if dump_format == 'directory':
+ dump.create_parent_directory_for_dump(dump_filename)
+ else:
+ dump.create_named_pipe_for_dump(dump_filename)
+
+ command = build_dump_command(database, dump_filename, dump_format)
+ processes.append(execute_command(command, shell=True, run_to_completion=False))
+
+ return processes
+
+
+def build_dump_command(database, dump_filename, dump_format):
+ '''
+ Return the mongodump command from a single database configuration.
+ '''
+ all_databases = database['name'] == 'all'
+ command = ['mongodump', '--archive']
+ if dump_format == 'directory':
+ command.append(dump_filename)
+ if 'hostname' in database:
+ command.extend(('--host', database['hostname']))
+ if 'port' in database:
+ command.extend(('--port', str(database['port'])))
+ if 'username' in database:
+ command.extend(('--username', database['username']))
+ if 'password' in database:
+ command.extend(('--password', database['password']))
+ if 'authentication_database' in database:
+ command.extend(('--authenticationDatabase', database['authentication_database']))
+ if not all_databases:
+ command.extend(('--db', database['name']))
+ if 'options' in database:
+ command.extend(database['options'].split(' '))
+ if dump_format != 'directory':
+ command.extend(('>', dump_filename))
+ return command
+
+
+def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
+ '''
+ Remove all database dump files for this hook regardless of the given databases. Use the log
+ prefix in any log entries. Use the given location configuration dict to construct the
+ destination path. If this is a dry run, then don't actually remove anything.
+ '''
+ dump.remove_database_dumps(make_dump_path(location_config), 'MongoDB', log_prefix, dry_run)
+
+
+def make_database_dump_pattern(
+ databases, log_prefix, location_config, name=None
+): # pragma: no cover
+ '''
+ Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
+ and a database name to match, return the corresponding glob patterns to match the database dump
+ in an archive.
+ '''
+ return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
+
+
+def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+ '''
+ Restore the given MongoDB database from an extract stream. The database is supplied as a
+ one-element sequence containing a dict describing the database, as per the configuration schema.
+ Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
+ anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
+ output to consume.
+
+ If the extract process is None, then restore the dump from the filesystem rather than from an
+ extract stream.
+ '''
+ dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
+
+ if len(database_config) != 1:
+ raise ValueError('The database configuration value is invalid')
+
+ database = database_config[0]
+ dump_filename = dump.make_database_dump_filename(
+ make_dump_path(location_config), database['name'], database.get('hostname')
+ )
+ restore_command = build_restore_command(extract_process, database, dump_filename)
+
+ logger.debug(
+ '{}: Restoring MongoDB database {}{}'.format(log_prefix, database['name'], dry_run_label)
+ )
+ if dry_run:
+ return
+
+ execute_command_with_processes(
+ restore_command,
+ [extract_process] if extract_process else [],
+ output_log_level=logging.DEBUG,
+ input_file=extract_process.stdout if extract_process else None,
+ borg_local_path=location_config.get('local_path', 'borg'),
+ )
+
+
+def build_restore_command(extract_process, database, dump_filename):
+ '''
+ Return the mongorestore command from a single database configuration.
+ '''
+ command = ['mongorestore', '--archive']
+ if not extract_process:
+ command.append(dump_filename)
+ if database['name'] != 'all':
+ command.extend(('--drop', '--db', database['name']))
+ if 'hostname' in database:
+ command.extend(('--host', database['hostname']))
+ if 'port' in database:
+ command.extend(('--port', str(database['port'])))
+ if 'username' in database:
+ command.extend(('--username', database['username']))
+ if 'password' in database:
+ command.extend(('--password', database['password']))
+ if 'authentication_database' in database:
+ command.extend(('--authenticationDatabase', database['authentication_database']))
+ return command
diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md
index 19fa23d30..7eca0c626 100644
--- a/docs/how-to/backup-your-databases.md
+++ b/docs/how-to/backup-your-databases.md
@@ -15,7 +15,8 @@ consistent snapshot that is more suited for backups.
Fortunately, borgmatic includes built-in support for creating database dumps
prior to running backups. For example, here is everything you need to dump and
-backup a couple of local PostgreSQL databases and a MySQL/MariaDB database:
+backup a couple of local PostgreSQL databases, a MySQL/MariaDB database, and a
+MongoDB database:
```yaml
hooks:
@@ -24,12 +25,15 @@ hooks:
- name: orders
mysql_databases:
- name: posts
+ mongodb_databases:
+ - name: messages
```
As part of each backup, borgmatic streams a database dump for each configured
database directly to Borg, so it's included in the backup without consuming
-additional disk space. (The one exception is PostgreSQL's "directory" dump
-format, which can't stream and therefore does consume temporary disk space.)
+additional disk space. (The exceptions are the PostgreSQL/MongoDB "directory"
+dump formats, which can't stream and therefore do consume temporary disk
+space.)
To support this, borgmatic creates temporary named pipes in `~/.borgmatic` by
default. To customize this path, set the `borgmatic_source_directory` option
@@ -59,6 +63,14 @@ hooks:
username: root
password: trustsome1
options: "--skip-comments"
+ mongodb_databases:
+ - name: messages
+ hostname: database3.example.org
+ port: 27018
+ username: dbuser
+ password: trustsome1
+ authentication_database: mongousers
+ options: "--ssl"
```
If you want to dump all databases on a host, use `all` for the database name:
@@ -69,13 +81,15 @@ hooks:
- name: all
mysql_databases:
- name: all
+ mongodb_databases:
+ - name: all
```
Note that you may need to use a `username` of the `postgres` superuser for
this to work with PostgreSQL.
If you would like to backup databases only and not source directories, you can
-specify an empty `source_directories` value because it is a mandatory field:
+specify an empty `source_directories` value (as it is a mandatory field):
```yaml
location:
@@ -97,7 +111,7 @@ bring back any missing configuration files in order to restore a database.
## Supported databases
-As of now, borgmatic supports PostgreSQL and MySQL/MariaDB databases
+As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, and MongoDB databases
directly. But see below about general-purpose preparation and cleanup hooks as
a work-around with other database systems. Also, please [file a
ticket](https://torsion.org/borgmatic/#issues) for additional database systems
@@ -196,8 +210,8 @@ that may not be exhaustive.
If you prefer to restore a database without the help of borgmatic, first
[extract](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) an
archive containing a database dump, and then manually restore the dump file
-found within the extracted `~/.borgmatic/` path (e.g. with `pg_restore` or
-`mysql` commands).
+found within the extracted `~/.borgmatic/` path (e.g. with `pg_restore`,
+`mysql`, or `mongorestore`, commands).
## Preparation and cleanup hooks
diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md
index 1ef818737..8d643cfaa 100644
--- a/docs/how-to/make-backups-redundant.md
+++ b/docs/how-to/make-backups-redundant.md
@@ -22,7 +22,6 @@ location:
repositories:
- 1234@usw-s001.rsync.net:backups.borg
- k8pDxu32@k8pDxu32.repo.borgbase.com:repo
- - user1@scp2.cdn.lima-labs.com:repo
- /var/lib/backups/local.borg
```
@@ -35,8 +34,7 @@ Here's a way of visualizing what borgmatic does with the above configuration:
1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg`
2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo`
-3. Backup `/home` and `/etc` to `user1@scp2.cdn.lima-labs.com:repo`
-4. Backup `/home` and `/etc` to `/var/lib/backups/local.borg`
+3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg`
This gives you redundancy of your data across repositories and even
potentially across providers.
diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md
index e10b5b3ea..09576419a 100644
--- a/docs/how-to/set-up-backups.md
+++ b/docs/how-to/set-up-backups.md
@@ -101,7 +101,6 @@ referral links, but without any tracking scripts or cookies.)
- BorgBase: Borg hosting service with support for monitoring, 2FA, and append-only repos
- - Lima-Labs: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs
Additionally, [rsync.net](https://www.rsync.net/products/borg.html) and
diff --git a/docs/static/mongodb.png b/docs/static/mongodb.png
new file mode 100644
index 000000000..81118d9b1
Binary files /dev/null and b/docs/static/mongodb.png differ
diff --git a/docs/static/rsyncnet.png b/docs/static/rsyncnet.png
deleted file mode 100644
index 3c027be10..000000000
Binary files a/docs/static/rsyncnet.png and /dev/null differ
diff --git a/scripts/run-full-tests b/scripts/run-full-tests
index 47e4f5f7b..0e9d02330 100755
--- a/scripts/run-full-tests
+++ b/scripts/run-full-tests
@@ -10,7 +10,7 @@
set -e
-apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client
+apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools
# If certain dependencies of black are available in this version of Alpine, install them.
apk add --no-cache py3-typed-ast py3-regex || true
python3 -m pip install --upgrade pip==21.3.1 setuptools==58.2.0
diff --git a/test_requirements.txt b/test_requirements.txt
index 56482a7c2..6513bcaae 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -14,10 +14,10 @@ py==1.10.0
pycodestyle==2.6.0
pyflakes==2.2.0
jsonschema==3.2.0
-pytest==6.1.2
-pytest-cov==2.10.1
+pytest==6.2.5
+pytest-cov==3.0.0
regex; python_version >= '3.8'
requests==2.25.0
ruamel.yaml>0.15.0,<0.18.0
toml==0.10.2; python_version >= '3.8'
-typed-ast==1.4.2; python_version >= '3.8'
+typed-ast; python_version >= '3.8'
diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml
index 3c0813421..094ac8d37 100644
--- a/tests/end-to-end/docker-compose.yaml
+++ b/tests/end-to-end/docker-compose.yaml
@@ -10,6 +10,11 @@ services:
environment:
MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test
+ mongodb:
+ image: mongo:5.0.5
+ environment:
+ MONGO_INITDB_ROOT_USERNAME: root
+ MONGO_INITDB_ROOT_PASSWORD: test
tests:
image: alpine:3.13
volumes:
diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py
index 956918b29..c7f6366df 100644
--- a/tests/end-to-end/test_database.py
+++ b/tests/end-to-end/test_database.py
@@ -47,13 +47,22 @@ hooks:
hostname: mysql
username: root
password: test
+ mongodb_databases:
+ - name: test
+ hostname: mongodb
+ username: root
+ password: test
+ authentication_database: admin
+ - name: all
+ hostname: mongodb
+ username: root
+ password: test
'''.format(
config_path, repository_path, borgmatic_source_directory, postgresql_dump_format
)
- config_file = open(config_path, 'w')
- config_file.write(config)
- config_file.close()
+ with open(config_path, 'w') as config_file:
+ config_file.write(config)
def test_database_dump_and_restore():
@@ -69,15 +78,15 @@ def test_database_dump_and_restore():
write_configuration(config_path, repository_path, borgmatic_source_directory)
subprocess.check_call(
- 'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+ ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
)
# Run borgmatic to generate a backup archive including a database dump.
- subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' '))
+ subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
# Get the created archive name.
output = subprocess.check_output(
- 'borgmatic --config {} list --json'.format(config_path).split(' ')
+ ['borgmatic', '--config', config_path, 'list', '--json']
).decode(sys.stdout.encoding)
parsed_output = json.loads(output)
@@ -87,9 +96,7 @@ def test_database_dump_and_restore():
# Restore the database from the archive.
subprocess.check_call(
- 'borgmatic --config {} restore --archive {}'.format(config_path, archive_name).split(
- ' '
- )
+ ['borgmatic', '--config', config_path, 'restore', '--archive', archive_name]
)
finally:
os.chdir(original_working_directory)
@@ -114,15 +121,15 @@ def test_database_dump_and_restore_with_directory_format():
)
subprocess.check_call(
- 'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+ ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
)
# Run borgmatic to generate a backup archive including a database dump.
- subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' '))
+ subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
# Restore the database from the archive.
subprocess.check_call(
- 'borgmatic --config {} restore --archive latest'.format(config_path).split(' ')
+ ['borgmatic', '--config', config_path, 'restore', '--archive', 'latest']
)
finally:
os.chdir(original_working_directory)
@@ -142,7 +149,7 @@ def test_database_dump_with_error_causes_borgmatic_to_exit():
write_configuration(config_path, repository_path, borgmatic_source_directory)
subprocess.check_call(
- 'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+ ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
)
# Run borgmatic with a config override such that the database dump fails.
diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py
new file mode 100644
index 000000000..cc4ae4c4f
--- /dev/null
+++ b/tests/unit/hooks/test_mongodb.py
@@ -0,0 +1,308 @@
+import logging
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.hooks import mongodb as module
+
+
+def test_dump_databases_runs_mongodump_for_each_database():
+ databases = [{'name': 'foo'}, {'name': 'bar'}]
+ processes = [flexmock(), flexmock()]
+ flexmock(module).should_receive('make_dump_path').and_return('')
+ flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+ 'databases/localhost/foo'
+ ).and_return('databases/localhost/bar')
+ flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+ for name, process in zip(('foo', 'bar'), processes):
+ flexmock(module).should_receive('execute_command').with_args(
+ ['mongodump', '--archive', '--db', name, '>', 'databases/localhost/{}'.format(name)],
+ shell=True,
+ run_to_completion=False,
+ ).and_return(process).once()
+
+ assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
+
+
+def test_dump_databases_with_dry_run_skips_mongodump():
+ databases = [{'name': 'foo'}, {'name': 'bar'}]
+ flexmock(module).should_receive('make_dump_path').and_return('')
+ flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+ 'databases/localhost/foo'
+ ).and_return('databases/localhost/bar')
+ flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
+ flexmock(module).should_receive('execute_command').never()
+
+ assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == []
+
+
+def test_dump_databases_runs_mongodump_with_hostname_and_port():
+ databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+ process = flexmock()
+ flexmock(module).should_receive('make_dump_path').and_return('')
+ flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+ 'databases/database.example.org/foo'
+ )
+ flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+ flexmock(module).should_receive('execute_command').with_args(
+ [
+ 'mongodump',
+ '--archive',
+ '--host',
+ 'database.example.org',
+ '--port',
+ '5433',
+ '--db',
+ 'foo',
+ '>',
+ 'databases/database.example.org/foo',
+ ],
+ shell=True,
+ run_to_completion=False,
+ ).and_return(process).once()
+
+ assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
+def test_dump_databases_runs_mongodump_with_username_and_password():
+ databases = [
+ {
+ 'name': 'foo',
+ 'username': 'mongo',
+ 'password': 'trustsome1',
+ 'authentication_database': "admin",
+ }
+ ]
+ process = flexmock()
+ flexmock(module).should_receive('make_dump_path').and_return('')
+ flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+ 'databases/localhost/foo'
+ )
+ flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+ flexmock(module).should_receive('execute_command').with_args(
+ [
+ 'mongodump',
+ '--archive',
+ '--username',
+ 'mongo',
+ '--password',
+ 'trustsome1',
+ '--authenticationDatabase',
+ 'admin',
+ '--db',
+ 'foo',
+ '>',
+ 'databases/localhost/foo',
+ ],
+ shell=True,
+ run_to_completion=False,
+ ).and_return(process).once()
+
+ assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
+def test_dump_databases_runs_mongodump_with_directory_format():
+ databases = [{'name': 'foo', 'format': 'directory'}]
+ process = flexmock()
+ flexmock(module).should_receive('make_dump_path').and_return('')
+ flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+ 'databases/localhost/foo'
+ )
+ flexmock(module.dump).should_receive('create_parent_directory_for_dump')
+ flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
+
+ flexmock(module).should_receive('execute_command').with_args(
+ ['mongodump', '--archive', 'databases/localhost/foo', '--db', 'foo'],
+ shell=True,
+ run_to_completion=False,
+ ).and_return(process).once()
+
+ assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
+def test_dump_databases_runs_mongodump_with_options():
+ databases = [{'name': 'foo', 'options': '--stuff=such'}]
+ process = flexmock()
+ flexmock(module).should_receive('make_dump_path').and_return('')
+ flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+ 'databases/localhost/foo'
+ )
+ flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+ flexmock(module).should_receive('execute_command').with_args(
+ ['mongodump', '--archive', '--db', 'foo', '--stuff=such', '>', 'databases/localhost/foo'],
+ shell=True,
+ run_to_completion=False,
+ ).and_return(process).once()
+
+ assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
+def test_dump_databases_runs_mongodumpall_for_all_databases():
+ databases = [{'name': 'all'}]
+ process = flexmock()
+ flexmock(module).should_receive('make_dump_path').and_return('')
+ flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+ 'databases/localhost/all'
+ )
+ flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+ flexmock(module).should_receive('execute_command').with_args(
+ ['mongodump', '--archive', '>', 'databases/localhost/all'],
+ shell=True,
+ run_to_completion=False,
+ ).and_return(process).once()
+
+ assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
+def test_restore_database_dump_runs_pg_restore():
+ database_config = [{'name': 'foo'}]
+ extract_process = flexmock(stdout=flexmock())
+
+ flexmock(module).should_receive('make_dump_path')
+ flexmock(module.dump).should_receive('make_database_dump_filename')
+ flexmock(module).should_receive('execute_command_with_processes').with_args(
+ ['mongorestore', '--archive', '--drop', '--db', 'foo'],
+ processes=[extract_process],
+ output_log_level=logging.DEBUG,
+ input_file=extract_process.stdout,
+ borg_local_path='borg',
+ ).once()
+
+ module.restore_database_dump(
+ database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+ )
+
+
+def test_restore_database_dump_errors_on_multiple_database_config():
+ database_config = [{'name': 'foo'}, {'name': 'bar'}]
+
+ flexmock(module).should_receive('make_dump_path')
+ flexmock(module.dump).should_receive('make_database_dump_filename')
+ flexmock(module).should_receive('execute_command_with_processes').never()
+ flexmock(module).should_receive('execute_command').never()
+
+ with pytest.raises(ValueError):
+ module.restore_database_dump(
+ database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
+ )
+
+
+def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
+ database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+ extract_process = flexmock(stdout=flexmock())
+
+ flexmock(module).should_receive('make_dump_path')
+ flexmock(module.dump).should_receive('make_database_dump_filename')
+ flexmock(module).should_receive('execute_command_with_processes').with_args(
+ [
+ 'mongorestore',
+ '--archive',
+ '--drop',
+ '--db',
+ 'foo',
+ '--host',
+ 'database.example.org',
+ '--port',
+ '5433',
+ ],
+ processes=[extract_process],
+ output_log_level=logging.DEBUG,
+ input_file=extract_process.stdout,
+ borg_local_path='borg',
+ ).once()
+
+ module.restore_database_dump(
+ database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+ )
+
+
+def test_restore_database_dump_runs_pg_restore_with_username_and_password():
+ database_config = [
+ {
+ 'name': 'foo',
+ 'username': 'mongo',
+ 'password': 'trustsome1',
+ 'authentication_database': 'admin',
+ }
+ ]
+ extract_process = flexmock(stdout=flexmock())
+
+ flexmock(module).should_receive('make_dump_path')
+ flexmock(module.dump).should_receive('make_database_dump_filename')
+ flexmock(module).should_receive('execute_command_with_processes').with_args(
+ [
+ 'mongorestore',
+ '--archive',
+ '--drop',
+ '--db',
+ 'foo',
+ '--username',
+ 'mongo',
+ '--password',
+ 'trustsome1',
+ '--authenticationDatabase',
+ 'admin',
+ ],
+ processes=[extract_process],
+ output_log_level=logging.DEBUG,
+ input_file=extract_process.stdout,
+ borg_local_path='borg',
+ ).once()
+
+ module.restore_database_dump(
+ database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+ )
+
+
+def test_restore_database_dump_runs_psql_for_all_database_dump():
+ database_config = [{'name': 'all'}]
+ extract_process = flexmock(stdout=flexmock())
+
+ flexmock(module).should_receive('make_dump_path')
+ flexmock(module.dump).should_receive('make_database_dump_filename')
+ flexmock(module).should_receive('execute_command_with_processes').with_args(
+ ['mongorestore', '--archive'],
+ processes=[extract_process],
+ output_log_level=logging.DEBUG,
+ input_file=extract_process.stdout,
+ borg_local_path='borg',
+ ).once()
+
+ module.restore_database_dump(
+ database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+ )
+
+
+def test_restore_database_dump_with_dry_run_skips_restore():
+ database_config = [{'name': 'foo'}]
+
+ flexmock(module).should_receive('make_dump_path')
+ flexmock(module.dump).should_receive('make_database_dump_filename')
+ flexmock(module).should_receive('execute_command_with_processes').never()
+
+ module.restore_database_dump(
+ database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
+ )
+
+
+def test_restore_database_dump_without_extract_process_restores_from_disk():
+ database_config = [{'name': 'foo'}]
+
+ flexmock(module).should_receive('make_dump_path')
+ flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path')
+ flexmock(module).should_receive('execute_command_with_processes').with_args(
+ ['mongorestore', '--archive', '/dump/path', '--drop', '--db', 'foo'],
+ processes=[],
+ output_log_level=logging.DEBUG,
+ input_file=None,
+ borg_local_path='borg',
+ ).once()
+
+ module.restore_database_dump(
+ database_config, 'test.yaml', {}, dry_run=False, extract_process=None
+ )
diff --git a/tox.ini b/tox.ini
index 12a0f603f..7d8340c6f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = py36,py37,py38,py39
+envlist = py37,py38,py39,py310
skip_missing_interpreters = True
skipsdist = True
minversion = 3.14.1