1213 lines
40 KiB
Python
1213 lines
40 KiB
Python
import logging
|
|
|
|
import pytest
|
|
from flexmock import flexmock
|
|
|
|
from borgmatic.hooks import postgresql as module
|
|
|
|
|
|
def test_make_extra_environment_maps_options_to_environment():
|
|
database = {
|
|
'name': 'foo',
|
|
'password': 'pass',
|
|
'ssl_mode': 'require',
|
|
'ssl_cert': 'cert.crt',
|
|
'ssl_key': 'key.key',
|
|
'ssl_root_cert': 'root.crt',
|
|
'ssl_crl': 'crl.crl',
|
|
}
|
|
expected = {
|
|
'PGPASSWORD': 'pass',
|
|
'PGSSLMODE': 'require',
|
|
'PGSSLCERT': 'cert.crt',
|
|
'PGSSLKEY': 'key.key',
|
|
'PGSSLROOTCERT': 'root.crt',
|
|
'PGSSLCRL': 'crl.crl',
|
|
}
|
|
|
|
extra_env = module.make_extra_environment(database)
|
|
|
|
assert extra_env == expected
|
|
|
|
|
|
def test_make_extra_environment_with_cli_password_sets_correct_password():
|
|
database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'}
|
|
|
|
extra = module.make_extra_environment(
|
|
database, restore_connection_params={'password': 'clipassword'}
|
|
)
|
|
|
|
assert extra['PGPASSWORD'] == 'clipassword'
|
|
|
|
|
|
def test_make_extra_environment_without_cli_password_or_configured_password_does_not_set_password():
|
|
database = {'name': 'foo'}
|
|
|
|
extra = module.make_extra_environment(
|
|
database, restore_connection_params={'username': 'someone'}
|
|
)
|
|
|
|
assert 'PGPASSWORD' not in extra
|
|
|
|
|
|
def test_make_extra_environment_without_ssl_mode_does_not_set_ssl_mode():
|
|
database = {'name': 'foo'}
|
|
|
|
extra = module.make_extra_environment(database)
|
|
|
|
assert 'PGSSLMODE' not in extra
|
|
|
|
|
|
def test_database_names_to_dump_passes_through_individual_database_name():
|
|
database = {'name': 'foo'}
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
|
|
'foo',
|
|
)
|
|
|
|
|
|
def test_database_names_to_dump_passes_through_individual_database_name_with_format():
|
|
database = {'name': 'foo', 'format': 'custom'}
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
|
|
'foo',
|
|
)
|
|
|
|
|
|
def test_database_names_to_dump_passes_through_all_without_format():
|
|
database = {'name': 'all'}
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
|
|
'all',
|
|
)
|
|
|
|
|
|
def test_database_names_to_dump_with_all_and_format_and_dry_run_bails():
|
|
database = {'name': 'all', 'format': 'custom'}
|
|
flexmock(module).should_receive('execute_command_and_capture_output').never()
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=True) == ()
|
|
|
|
|
|
def test_database_names_to_dump_with_all_and_format_lists_databases():
|
|
database = {'name': 'all', 'format': 'custom'}
|
|
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
|
|
'foo,test,\nbar,test,"stuff and such"'
|
|
)
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
|
|
'foo',
|
|
'bar',
|
|
)
|
|
|
|
|
|
def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
|
|
database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
|
|
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
|
(
|
|
'psql',
|
|
'--list',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--csv',
|
|
'--tuples-only',
|
|
'--host',
|
|
'localhost',
|
|
'--port',
|
|
'1234',
|
|
),
|
|
extra_environment=object,
|
|
).and_return('foo,test,\nbar,test,"stuff and such"')
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
|
|
'foo',
|
|
'bar',
|
|
)
|
|
|
|
|
|
def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
|
|
database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
|
|
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
|
(
|
|
'psql',
|
|
'--list',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--csv',
|
|
'--tuples-only',
|
|
'--username',
|
|
'postgres',
|
|
),
|
|
extra_environment=object,
|
|
).and_return('foo,test,\nbar,test,"stuff and such"')
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
|
|
'foo',
|
|
'bar',
|
|
)
|
|
|
|
|
|
def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
|
|
database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
|
|
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
|
('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'),
|
|
extra_environment=object,
|
|
).and_return('foo,test,\nbar,test,"stuff and such"')
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
|
|
'foo',
|
|
'bar',
|
|
)
|
|
|
|
|
|
def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
|
|
database = {'name': 'all', 'format': 'custom'}
|
|
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
|
|
'foo,test,\ntemplate0,test,blah'
|
|
)
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
|
|
'foo',
|
|
)
|
|
|
|
|
|
def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command():
|
|
database = {
|
|
'name': 'all',
|
|
'format': 'custom',
|
|
'psql_command': 'docker exec --workdir * mycontainer psql',
|
|
}
|
|
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
|
(
|
|
'docker',
|
|
'exec',
|
|
'--workdir',
|
|
"'*'", # Should get shell escaped to prevent injection attacks.
|
|
'mycontainer',
|
|
'psql',
|
|
'--list',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--csv',
|
|
'--tuples-only',
|
|
),
|
|
extra_environment=object,
|
|
).and_return('foo,text').once()
|
|
|
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
|
|
'foo',
|
|
)
|
|
|
|
|
|
def test_use_streaming_true_for_any_non_directory_format_databases():
|
|
assert module.use_streaming(
|
|
databases=[{'format': 'stuff'}, {'format': 'directory'}, {}],
|
|
config=flexmock(),
|
|
log_prefix=flexmock(),
|
|
)
|
|
|
|
|
|
def test_use_streaming_false_for_all_directory_format_databases():
|
|
assert not module.use_streaming(
|
|
databases=[{'format': 'directory'}, {'format': 'directory'}],
|
|
config=flexmock(),
|
|
log_prefix=flexmock(),
|
|
)
|
|
|
|
|
|
def test_use_streaming_false_for_no_databases():
|
|
assert not module.use_streaming(databases=[], config=flexmock(), log_prefix=flexmock())
|
|
|
|
|
|
def test_dump_data_sources_runs_pg_dump_for_each_database():
|
|
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
|
processes = [flexmock(), flexmock()]
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
|
|
('bar',)
|
|
)
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/localhost/foo'
|
|
).and_return('databases/localhost/bar')
|
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
|
|
|
for name, process in zip(('foo', 'bar'), processes):
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'pg_dump',
|
|
'--no-password',
|
|
'--clean',
|
|
'--if-exists',
|
|
'--format',
|
|
'custom',
|
|
name,
|
|
'>',
|
|
f'databases/localhost/{name}',
|
|
),
|
|
shell=True,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
run_to_completion=False,
|
|
).and_return(process).once()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False) == processes
|
|
|
|
|
|
def test_dump_data_sources_raises_when_no_database_names_to_dump():
|
|
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(())
|
|
|
|
with pytest.raises(ValueError):
|
|
module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False)
|
|
|
|
|
|
def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump():
|
|
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(())
|
|
|
|
module.dump_data_sources(databases, {}, 'test.yaml', dry_run=True) == []
|
|
|
|
|
|
def test_dump_data_sources_with_duplicate_dump_skips_pg_dump():
|
|
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
|
|
('bar',)
|
|
)
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/localhost/foo'
|
|
).and_return('databases/localhost/bar')
|
|
flexmock(module.os.path).should_receive('exists').and_return(True)
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
|
|
flexmock(module).should_receive('execute_command').never()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False) == []
|
|
|
|
|
|
def test_dump_data_sources_with_dry_run_skips_pg_dump():
|
|
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
|
|
('bar',)
|
|
)
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/localhost/foo'
|
|
).and_return('databases/localhost/bar')
|
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
|
|
flexmock(module).should_receive('execute_command').never()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=True) == []
|
|
|
|
|
|
def test_dump_data_sources_runs_pg_dump_with_hostname_and_port():
|
|
databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
|
process = flexmock()
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/database.example.org/foo'
|
|
)
|
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
|
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'pg_dump',
|
|
'--no-password',
|
|
'--clean',
|
|
'--if-exists',
|
|
'--host',
|
|
'database.example.org',
|
|
'--port',
|
|
'5433',
|
|
'--format',
|
|
'custom',
|
|
'foo',
|
|
'>',
|
|
'databases/database.example.org/foo',
|
|
),
|
|
shell=True,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
run_to_completion=False,
|
|
).and_return(process).once()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False) == [process]
|
|
|
|
|
|
def test_dump_data_sources_runs_pg_dump_with_username_and_password():
|
|
databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
|
|
process = flexmock()
|
|
flexmock(module).should_receive('make_extra_environment').and_return(
|
|
{'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}
|
|
)
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/localhost/foo'
|
|
)
|
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
|
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'pg_dump',
|
|
'--no-password',
|
|
'--clean',
|
|
'--if-exists',
|
|
'--username',
|
|
'postgres',
|
|
'--format',
|
|
'custom',
|
|
'foo',
|
|
'>',
|
|
'databases/localhost/foo',
|
|
),
|
|
shell=True,
|
|
extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
|
|
run_to_completion=False,
|
|
).and_return(process).once()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False) == [process]
|
|
|
|
|
|
def test_dump_data_sources_with_username_injection_attack_gets_escaped():
|
|
databases = [{'name': 'foo', 'username': 'postgres; naughty-command', 'password': 'trustsome1'}]
|
|
process = flexmock()
|
|
flexmock(module).should_receive('make_extra_environment').and_return(
|
|
{'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}
|
|
)
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/localhost/foo'
|
|
)
|
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
|
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'pg_dump',
|
|
'--no-password',
|
|
'--clean',
|
|
'--if-exists',
|
|
'--username',
|
|
"'postgres; naughty-command'",
|
|
'--format',
|
|
'custom',
|
|
'foo',
|
|
'>',
|
|
'databases/localhost/foo',
|
|
),
|
|
shell=True,
|
|
extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
|
|
run_to_completion=False,
|
|
).and_return(process).once()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False) == [process]
|
|
|
|
|
|
def test_dump_data_sources_runs_pg_dump_with_directory_format():
|
|
databases = [{'name': 'foo', 'format': 'directory'}]
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/localhost/foo'
|
|
)
|
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
|
flexmock(module.dump).should_receive('create_parent_directory_for_dump')
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
|
|
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'pg_dump',
|
|
'--no-password',
|
|
'--clean',
|
|
'--if-exists',
|
|
'--format',
|
|
'directory',
|
|
'--file',
|
|
'databases/localhost/foo',
|
|
'foo',
|
|
),
|
|
shell=True,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).and_return(flexmock()).once()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False) == []
|
|
|
|
|
|
def test_dump_data_sources_runs_pg_dump_with_options():
|
|
databases = [{'name': 'foo', 'options': '--stuff=such'}]
|
|
process = flexmock()
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/localhost/foo'
|
|
)
|
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
|
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'pg_dump',
|
|
'--no-password',
|
|
'--clean',
|
|
'--if-exists',
|
|
'--format',
|
|
'custom',
|
|
'--stuff=such',
|
|
'foo',
|
|
'>',
|
|
'databases/localhost/foo',
|
|
),
|
|
shell=True,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
run_to_completion=False,
|
|
).and_return(process).once()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False) == [process]
|
|
|
|
|
|
def test_dump_data_sources_runs_pg_dumpall_for_all_databases():
|
|
databases = [{'name': 'all'}]
|
|
process = flexmock()
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('all',))
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/localhost/all'
|
|
)
|
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
|
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'),
|
|
shell=True,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
run_to_completion=False,
|
|
).and_return(process).once()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False) == [process]
|
|
|
|
|
|
def test_dump_data_sources_runs_non_default_pg_dump():
|
|
databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump --compress *'}]
|
|
process = flexmock()
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
|
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
|
|
'databases/localhost/foo'
|
|
)
|
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
|
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'special_pg_dump',
|
|
'--compress',
|
|
"'*'", # Should get shell escaped to prevent injection attacks.
|
|
'--no-password',
|
|
'--clean',
|
|
'--if-exists',
|
|
'--format',
|
|
'custom',
|
|
'foo',
|
|
'>',
|
|
'databases/localhost/foo',
|
|
),
|
|
shell=True,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
run_to_completion=False,
|
|
).and_return(process).once()
|
|
|
|
assert module.dump_data_sources(databases, {}, 'test.yaml', dry_run=False) == [process]
|
|
|
|
|
|
def test_restore_data_source_dump_runs_pg_restore():
|
|
hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}]
|
|
extract_process = flexmock(stdout=flexmock())
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'pg_restore',
|
|
'--no-password',
|
|
'--if-exists',
|
|
'--exit-on-error',
|
|
'--clean',
|
|
'--dbname',
|
|
'foo',
|
|
),
|
|
processes=[extract_process],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=extract_process.stdout,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--dbname',
|
|
'foo',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source={'name': 'foo'},
|
|
dry_run=False,
|
|
extract_process=extract_process,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port():
|
|
hook_config = [
|
|
{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None}
|
|
]
|
|
extract_process = flexmock(stdout=flexmock())
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'pg_restore',
|
|
'--no-password',
|
|
'--if-exists',
|
|
'--exit-on-error',
|
|
'--clean',
|
|
'--dbname',
|
|
'foo',
|
|
'--host',
|
|
'database.example.org',
|
|
'--port',
|
|
'5433',
|
|
),
|
|
processes=[extract_process],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=extract_process.stdout,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--host',
|
|
'database.example.org',
|
|
'--port',
|
|
'5433',
|
|
'--dbname',
|
|
'foo',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source=hook_config[0],
|
|
dry_run=False,
|
|
extract_process=extract_process,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_runs_pg_restore_with_username_and_password():
|
|
hook_config = [
|
|
{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None}
|
|
]
|
|
extract_process = flexmock(stdout=flexmock())
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return(
|
|
{'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}
|
|
)
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'pg_restore',
|
|
'--no-password',
|
|
'--if-exists',
|
|
'--exit-on-error',
|
|
'--clean',
|
|
'--dbname',
|
|
'foo',
|
|
'--username',
|
|
'postgres',
|
|
),
|
|
processes=[extract_process],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=extract_process.stdout,
|
|
extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--username',
|
|
'postgres',
|
|
'--dbname',
|
|
'foo',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source=hook_config[0],
|
|
dry_run=False,
|
|
extract_process=extract_process,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore():
|
|
hook_config = [
|
|
{
|
|
'name': 'foo',
|
|
'hostname': 'database.example.org',
|
|
'port': 5433,
|
|
'username': 'postgres',
|
|
'password': 'trustsome1',
|
|
'restore_hostname': 'restorehost',
|
|
'restore_port': 'restoreport',
|
|
'restore_username': 'restoreusername',
|
|
'restore_password': 'restorepassword',
|
|
'schemas': None,
|
|
}
|
|
]
|
|
extract_process = flexmock(stdout=flexmock())
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return(
|
|
{'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}
|
|
)
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'pg_restore',
|
|
'--no-password',
|
|
'--if-exists',
|
|
'--exit-on-error',
|
|
'--clean',
|
|
'--dbname',
|
|
'foo',
|
|
'--host',
|
|
'clihost',
|
|
'--port',
|
|
'cliport',
|
|
'--username',
|
|
'cliusername',
|
|
),
|
|
processes=[extract_process],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=extract_process.stdout,
|
|
extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--host',
|
|
'clihost',
|
|
'--port',
|
|
'cliport',
|
|
'--username',
|
|
'cliusername',
|
|
'--dbname',
|
|
'foo',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source={'name': 'foo'},
|
|
dry_run=False,
|
|
extract_process=extract_process,
|
|
connection_params={
|
|
'hostname': 'clihost',
|
|
'port': 'cliport',
|
|
'username': 'cliusername',
|
|
'password': 'clipassword',
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_without_connection_params_uses_restore_params_in_config_for_restore():
|
|
hook_config = [
|
|
{
|
|
'name': 'foo',
|
|
'hostname': 'database.example.org',
|
|
'port': 5433,
|
|
'username': 'postgres',
|
|
'password': 'trustsome1',
|
|
'schemas': None,
|
|
'restore_hostname': 'restorehost',
|
|
'restore_port': 'restoreport',
|
|
'restore_username': 'restoreusername',
|
|
'restore_password': 'restorepassword',
|
|
}
|
|
]
|
|
extract_process = flexmock(stdout=flexmock())
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return(
|
|
{'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}
|
|
)
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'pg_restore',
|
|
'--no-password',
|
|
'--if-exists',
|
|
'--exit-on-error',
|
|
'--clean',
|
|
'--dbname',
|
|
'foo',
|
|
'--host',
|
|
'restorehost',
|
|
'--port',
|
|
'restoreport',
|
|
'--username',
|
|
'restoreusername',
|
|
),
|
|
processes=[extract_process],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=extract_process.stdout,
|
|
extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--host',
|
|
'restorehost',
|
|
'--port',
|
|
'restoreport',
|
|
'--username',
|
|
'restoreusername',
|
|
'--dbname',
|
|
'foo',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source=hook_config[0],
|
|
dry_run=False,
|
|
extract_process=extract_process,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_runs_pg_restore_with_options():
|
|
hook_config = [
|
|
{
|
|
'name': 'foo',
|
|
'restore_options': '--harder',
|
|
'analyze_options': '--smarter',
|
|
'schemas': None,
|
|
}
|
|
]
|
|
extract_process = flexmock(stdout=flexmock())
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'pg_restore',
|
|
'--no-password',
|
|
'--if-exists',
|
|
'--exit-on-error',
|
|
'--clean',
|
|
'--dbname',
|
|
'foo',
|
|
'--harder',
|
|
),
|
|
processes=[extract_process],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=extract_process.stdout,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--dbname',
|
|
'foo',
|
|
'--smarter',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source=hook_config[0],
|
|
dry_run=False,
|
|
extract_process=extract_process,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_runs_psql_for_all_database_dump():
|
|
hook_config = [{'name': 'all', 'schemas': None}]
|
|
extract_process = flexmock(stdout=flexmock())
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
),
|
|
processes=[extract_process],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=extract_process.stdout,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'),
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source={'name': 'all'},
|
|
dry_run=False,
|
|
extract_process=extract_process,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_runs_psql_for_plain_database_dump():
|
|
hook_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}]
|
|
extract_process = flexmock(stdout=flexmock())
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
('psql', '--no-password', '--no-psqlrc', '--dbname', 'foo'),
|
|
processes=[extract_process],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=extract_process.stdout,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--dbname',
|
|
'foo',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source=hook_config[0],
|
|
dry_run=False,
|
|
extract_process=extract_process,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql():
|
|
hook_config = [
|
|
{
|
|
'name': 'foo',
|
|
'pg_restore_command': 'docker exec --workdir * mycontainer pg_restore',
|
|
'psql_command': 'docker exec --workdir * mycontainer psql',
|
|
'schemas': None,
|
|
}
|
|
]
|
|
extract_process = flexmock(stdout=flexmock())
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'docker',
|
|
'exec',
|
|
'--workdir',
|
|
"'*'", # Should get shell escaped to prevent injection attacks.
|
|
'mycontainer',
|
|
'pg_restore',
|
|
'--no-password',
|
|
'--if-exists',
|
|
'--exit-on-error',
|
|
'--clean',
|
|
'--dbname',
|
|
'foo',
|
|
),
|
|
processes=[extract_process],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=extract_process.stdout,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'docker',
|
|
'exec',
|
|
'--workdir',
|
|
"'*'", # Should get shell escaped to prevent injection attacks.
|
|
'mycontainer',
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--dbname',
|
|
'foo',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source=hook_config[0],
|
|
dry_run=False,
|
|
extract_process=extract_process,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_with_dry_run_skips_restore():
|
|
hook_config = [{'name': 'foo', 'schemas': None}]
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename')
|
|
flexmock(module).should_receive('execute_command_with_processes').never()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source={'name': 'foo'},
|
|
dry_run=True,
|
|
extract_process=flexmock(),
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_without_extract_process_restores_from_disk():
|
|
hook_config = [{'name': 'foo', 'schemas': None}]
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'pg_restore',
|
|
'--no-password',
|
|
'--if-exists',
|
|
'--exit-on-error',
|
|
'--clean',
|
|
'--dbname',
|
|
'foo',
|
|
'/dump/path',
|
|
),
|
|
processes=[],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=None,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--dbname',
|
|
'foo',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source={'name': 'foo'},
|
|
dry_run=False,
|
|
extract_process=None,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|
|
|
|
|
|
def test_restore_data_source_dump_with_schemas_restores_schemas():
|
|
hook_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}]
|
|
|
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
|
flexmock(module).should_receive('make_dump_path')
|
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path')
|
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
|
(
|
|
'pg_restore',
|
|
'--no-password',
|
|
'--if-exists',
|
|
'--exit-on-error',
|
|
'--clean',
|
|
'--dbname',
|
|
'foo',
|
|
'/dump/path',
|
|
'--schema',
|
|
'bar',
|
|
'--schema',
|
|
'baz',
|
|
),
|
|
processes=[],
|
|
output_log_level=logging.DEBUG,
|
|
input_file=None,
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
flexmock(module).should_receive('execute_command').with_args(
|
|
(
|
|
'psql',
|
|
'--no-password',
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--dbname',
|
|
'foo',
|
|
'--command',
|
|
'ANALYZE',
|
|
),
|
|
extra_environment={'PGSSLMODE': 'disable'},
|
|
).once()
|
|
|
|
module.restore_data_source_dump(
|
|
hook_config,
|
|
{},
|
|
'test.yaml',
|
|
data_source=hook_config[0],
|
|
dry_run=False,
|
|
extract_process=None,
|
|
connection_params={
|
|
'hostname': None,
|
|
'port': None,
|
|
'username': None,
|
|
'password': None,
|
|
},
|
|
)
|