feat: add dump-restore support for sqlite databases

This commit is contained in:
Divyansh Singh 2023-03-02 23:55:16 +05:30
parent 2a306bef12
commit af1cc27988
7 changed files with 244 additions and 5 deletions

View File

@ -931,6 +931,31 @@ properties:
mysqldump/mysql commands (from either MySQL or MariaDB). See
https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
https://mariadb.com/kb/en/library/mysqldump/ for details.
sqlite_databases:
type: array
items:
type: object
required: ['path','name']
additionalProperties: false
properties:
path:
type: string
description: |
Path to the SQLite database file to dump. If
relative, it is relative to the current working
directory. If absolute, it is relative to the
root of the filesystem. Note that using this
database hook implicitly enables both
read_special and one_file_system (see above) to
support dump and restore streaming.
example: /var/lib/sqlite/users.db
name:
type: string
description: |
This is used to tag the database dump file with
a name. It is not used to identify the database
file itself.
example: users
mongodb_databases:
type: array
items:

View File

@ -9,6 +9,7 @@ from borgmatic.hooks import (
ntfy,
pagerduty,
postgresql,
sqlite,
)
logger = logging.getLogger(__name__)
@ -22,6 +23,7 @@ HOOK_NAME_TO_MODULE = {
'ntfy': ntfy,
'pagerduty': pagerduty,
'postgresql_databases': postgresql,
'sqlite_databases': sqlite,
}

View File

@ -6,7 +6,12 @@ from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
logger = logging.getLogger(__name__)
DATABASE_HOOK_NAMES = ('postgresql_databases', 'mysql_databases', 'mongodb_databases')
DATABASE_HOOK_NAMES = (
'postgresql_databases',
'mysql_databases',
'mongodb_databases',
'sqlite_databases',
)
def make_database_dump_path(borgmatic_source_directory, database_hook_name):

122
borgmatic/hooks/sqlite.py Normal file
View File

@ -0,0 +1,122 @@
import logging
import os
from subprocess import CalledProcessError
from borgmatic.execute import execute_command, execute_command_with_processes
from borgmatic.hooks import dump
logger = logging.getLogger(__name__)
def make_dump_path(location_config): # pragma: no cover
'''
Make the dump path from the given location configuration and the name of this hook.
'''
return dump.make_database_dump_path(
location_config.get('borgmatic_source_directory'), 'sqlite_databases'
)
def dump_databases(databases, log_prefix, location_config, dry_run): # pragma: no cover
'''
Dump the given SQLite3 databases to a file. The databases are supplied as a sequence of
configuration dicts, as per the configuration schema. Use the given log prefix in any log
entries. Use the given location configuration dict to construct the destination path. If this
is a dry run, then don't actually dump anything.
'''
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
processes = []
logger.info('{}: Dumping SQLite databases{}'.format(log_prefix, dry_run_label))
for database in databases:
database_path = database['path']
database_filename = database['name']
dump_path = make_dump_path(location_config)
dump_filename = dump.make_database_dump_filename(dump_path, database_filename)
if os.path.exists(dump_filename):
logger.warning(
f'{log_prefix}: Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}'
)
continue
command = (
'sqlite3',
database_path,
'.dump',
'>',
dump_filename,
)
logger.debug(
f'{log_prefix}: Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}'
)
if dry_run:
continue
dump.create_parent_directory_for_dump(dump_filename)
processes.append(execute_command(command, shell=True, run_to_completion=False))
return processes
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
'''
Remove the given SQLite3 database dumps from the filesystem. The databases are supplied as a
sequence of configuration dicts, as per the configuration schema. Use the given log prefix in
any log entries. Use the given location configuration dict to construct the destination path.
If this is a dry run, then don't actually remove anything.
'''
dump.remove_database_dumps(make_dump_path(location_config), 'SQLite', log_prefix, dry_run)
def make_database_dump_pattern(
databases, log_prefix, location_config, name=None
): # pragma: no cover
'''
Make a pattern that matches the given SQLite3 databases. The databases are supplied as a
sequence of configuration dicts, as per the configuration schema.
'''
return dump.make_database_dump_filename(make_dump_path(location_config), name)
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
'''
Restore the given SQLite3 database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
output to consume.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
if len(database_config) != 1:
raise ValueError('The database configuration value is invalid')
database_path = database_config[0]['path']
logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
if dry_run:
return
remove_command = (
'rm',
database_path,
)
try:
execute_command(remove_command, shell=True)
except CalledProcessError:
logger.info(f'{log_prefix}: Database does not exist at {database_path}, skipping removal')
restore_command = (
'sqlite3',
database_path,
)
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
# if the restore paths don't exist in the archive.
execute_command_with_processes(
restore_command,
[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
)

View File

@ -15,8 +15,8 @@ consistent snapshot that is more suited for backups.
Fortunately, borgmatic includes built-in support for creating database dumps
prior to running backups. For example, here is everything you need to dump and
backup a couple of local PostgreSQL databases, a MySQL/MariaDB database, and a
MongoDB database:
backup a couple of local PostgreSQL databases, a MySQL/MariaDB database, a
MongoDB database and a SQLite database:
```yaml
hooks:
@ -27,6 +27,9 @@ hooks:
- name: posts
mongodb_databases:
- name: messages
sqlite_databases:
- path: /var/lib/sqlite3/mydb.sqlite
- name: mydb
```
As part of each backup, borgmatic streams a database dump for each configured
@ -74,6 +77,9 @@ hooks:
password: trustsome1
authentication_database: mongousers
options: "--ssl"
sqlite_databases:
- path: /var/lib/sqlite3/mydb.sqlite
- name: mydb
```
See your [borgmatic configuration
@ -97,7 +103,8 @@ hooks:
```
Note that you may need to use a `username` of the `postgres` superuser for
this to work with PostgreSQL.
this to work with PostgreSQL. Also, the `all` database name is not supported
for SQLite databases.
<span class="minilink minilink-addedin">New in version 1.7.6</span> With
PostgreSQL and MySQL, you can optionally dump "all" databases to separate
@ -154,7 +161,7 @@ bring back any missing configuration files in order to restore a database.
## Supported databases
As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, and MongoDB databases
As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, MongoDB databases and SQLite databases
directly. But see below about general-purpose preparation and cleanup hooks as
a work-around with other database systems. Also, please [file a
ticket](https://torsion.org/borgmatic/#issues) for additional database systems

View File

@ -73,6 +73,9 @@ hooks:
hostname: mongodb
username: root
password: test
sqlite_databases:
- path: /tmp/sqlite_test.db
- name: sqlite_test
'''
with open(config_path, 'w') as config_file:

View File

@ -0,0 +1,75 @@
import logging
import pytest
from flexmock import flexmock
from borgmatic.hooks import sqlite as module
def test_dump_databases_logs_and_skips_if_dump_already_exists():
databases = [{'path': '/path/to/database', 'name': 'database'}]
flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
'/path/to/dump/database'
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(logging).should_receive('info')
flexmock(logging).should_receive('warning')
assert module.dump_databases(databases, 'test.yaml', {}, dry_run = False) == []
def test_dump_databases_dumps_each_database():
databases = [
{'path': '/path/to/database1', 'name': 'database1'},
{'path': '/path/to/database2', 'name': 'database2'},
]
flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
'/path/to/dump/database'
)
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(logging).should_receive('info')
flexmock(logging).should_receive('warning')
flexmock(module).should_receive('dump.create_parent_directory_for_dump')
flexmock(module).should_receive('execute_command').and_return('process')
assert module.dump_databases(databases, 'test.yaml', {}, dry_run = False) == ['process', 'process']
def test_dump_databases_does_not_dump_if_dry_run():
databases = [{'path': '/path/to/database', 'name': 'database'}]
flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
'/path/to/dump/database'
)
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(logging).should_receive('info')
flexmock(logging).should_receive('warning')
flexmock(module).should_receive('dump.create_parent_directory_for_dump')
assert module.dump_databases(databases, 'test.yaml', {}, dry_run = True) == []
def test_restore_database_dump_restores_database():
database_config = [{'path': '/path/to/database', 'name': 'database'}]
extract_process = flexmock(stdout = flexmock())
flexmock(module).should_receive('execute_command_with_processes').and_return('process')
module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = False, extract_process = extract_process)
def test_restore_database_dump_does_not_restore_database_if_dry_run():
database_config = [{'path': '/path/to/database', 'name': 'database'}]
extract_process = flexmock(stdout = flexmock())
flexmock(module).should_receive('execute_command_with_processes').never()
module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = True, extract_process = extract_process)
def test_restore_database_dump_raises_error_if_database_config_is_invalid():
database_config = []
extract_process = flexmock(stdout = flexmock())
with pytest.raises(ValueError):
module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = False, extract_process = extract_process)