From 874fba76725ce76803b586525c57c59760404593 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 15:10:44 +0200 Subject: [PATCH 01/44] Fix PostgreSQL hook not using "psql_command" for list when dumping "all" --- borgmatic/hooks/postgresql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index d4799f5f..c0d4619b 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -59,8 +59,9 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): if dry_run: return () + psql_command = database.get('psql_command') or 'psql' list_command = ( - ('psql', '--list', '--no-password', '--csv', '--tuples-only') + (psql_command, '--list', '--no-password', '--csv', '--tuples-only') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) From 19a00371f5da3c501a8158b1dfb70ba40c09ac08 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 15:49:49 +0200 Subject: [PATCH 02/44] Run "psql" with "--no-psqlrc" Some settings in user's .psqlrc, e.g. "linestyle unicode", may break the CSV output. "--no-psqlrc" tells psql to not read startup file. This is not necessary for the analyze_command and restore_command (with all_databases), but it's generally recommended when running psql from a script. --- borgmatic/hooks/postgresql.py | 6 +++--- tests/unit/hooks/test_postgresql.py | 18 +++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index c0d4619b..1235d60e 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -61,7 +61,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): psql_command = database.get('psql_command') or 'psql' list_command = ( - (psql_command, '--list', '--no-password', '--csv', '--tuples-only') + (psql_command, '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -205,7 +205,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) psql_command = database.get('psql_command') or 'psql' analyze_command = ( - (psql_command, '--no-password', '--quiet') + (psql_command, '--no-password', '--no-psqlrc', '--quiet') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -219,7 +219,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ( ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) if not all_databases - else () + else ('--no-psqlrc',) ) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 349c04be..704d5a71 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -56,6 +56,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam 'psql', '--list', '--no-password', + '--no-psqlrc', '--csv', '--tuples-only', '--host', @@ -75,7 +76,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam def test_database_names_to_dump_with_all_and_format_lists_databases_with_username(): database = {'name': 'all', 'format': 'custom', 'username': 'postgres'} flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('psql', '--list', '--no-password', '--csv', '--tuples-only', '--username', 'postgres'), + ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--username', 'postgres'), extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') @@ -88,7 +89,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam def test_database_names_to_dump_with_all_and_format_lists_databases_with_options(): database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'} flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('psql', '--list', '--no-password', '--csv', '--tuples-only', '--harder'), + ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'), extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') @@ -433,7 +434,7 @@ def test_restore_database_dump_runs_pg_restore(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ('psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -487,6 +488,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): ( 'psql', '--no-password', + '--no-psqlrc', '--quiet', '--host', 'database.example.org', @@ -535,6 +537,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): ( 'psql', '--no-password', + '--no-psqlrc', '--quiet', '--username', 'postgres', @@ -580,6 +583,7 @@ def test_restore_database_dump_runs_pg_restore_with_options(): ( 'psql', '--no-password', + '--no-psqlrc', '--quiet', '--dbname', 'foo', @@ -603,14 +607,14 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( - ('psql', '--no-password'), + ('psql', '--no-password', '--no-psqlrc'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), + ('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -644,7 +648,7 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('special_psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ('special_psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -689,7 +693,7 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ('psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), extra_environment={'PGSSLMODE': 'disable'}, ).once() From 195024e505746ce9eb096422352762619b8fd429 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 16:11:42 +0200 Subject: [PATCH 03/44] Fix psql_command and pg_restore_command to accept command with arguments These commands are executed without `shell=True`, so the subprocess module treats e.g. "docker exec my_pg_container psql" as a single command (resulting in Errno 2 "No such file or directory") instead of a command with arguments. --- borgmatic/hooks/postgresql.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 1235d60e..fbe890a7 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -1,6 +1,7 @@ import csv import logging import os +import shlex from borgmatic.execute import ( execute_command, @@ -59,9 +60,10 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): if dry_run: return () - psql_command = database.get('psql_command') or 'psql' + psql_command = shlex.split(database.get('psql_command') or 'psql') list_command = ( - (psql_command, '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') + tuple(psql_command) + + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -203,9 +205,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, dump_filename = dump.make_database_dump_filename( make_dump_path(location_config), database['name'], database.get('hostname') ) - psql_command = database.get('psql_command') or 'psql' + psql_command = shlex.split(database.get('psql_command') or 'psql') analyze_command = ( - (psql_command, '--no-password', '--no-psqlrc', '--quiet') + tuple(psql_command) + + ('--no-password', '--no-psqlrc', '--quiet') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -213,9 +216,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + ('--command', 'ANALYZE') ) - pg_restore_command = database.get('pg_restore_command') or 'pg_restore' + pg_restore_command = shlex.split(database.get('pg_restore_command') or 'pg_restore') restore_command = ( - (psql_command if all_databases else pg_restore_command, '--no-password') + tuple(psql_command if all_databases else pg_restore_command) + + ('--no-password',) + ( ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) if not all_databases From dfccc1b94a57cf791bf4fe06b4353e20fa75c2ee Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 16:27:45 +0200 Subject: [PATCH 04/44] Exit on error when restoring all PostgreSQL databases "--set ON_ERROR_STOP=on" is equivalent to "--exit-on-error" in pg_restore. --- borgmatic/hooks/postgresql.py | 2 +- tests/unit/hooks/test_postgresql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index fbe890a7..b91b97b8 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -223,7 +223,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ( ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) if not all_databases - else ('--no-psqlrc',) + else ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') ) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 704d5a71..5f02978c 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -607,7 +607,7 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( - ('psql', '--no-password', '--no-psqlrc'), + ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, From f0f43174c6be21157d2d30955e14f04ddcead8a2 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 16:29:26 +0200 Subject: [PATCH 05/44] Swap if-else in restore_database_dump in postgresql hook for cleanliness --- borgmatic/hooks/postgresql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index b91b97b8..0f1a2df4 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -221,9 +221,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, tuple(psql_command if all_databases else pg_restore_command) + ('--no-password',) + ( - ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) - if not all_databases - else ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') + ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') + if all_databases + else ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) ) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) From 17f122bfe54a95e30cd186b552ae0efd295f7aa3 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 17:16:42 +0200 Subject: [PATCH 06/44] Use psql instead of pg_restore when format is "plain" pg_restore: error: input file appears to be a text format dump. Please use psql. --- borgmatic/hooks/postgresql.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 0f1a2df4..a64bb534 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -216,15 +216,17 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + ('--command', 'ANALYZE') ) + use_psql_command = all_databases or database.get('format') == 'plain' pg_restore_command = shlex.split(database.get('pg_restore_command') or 'pg_restore') restore_command = ( - tuple(psql_command if all_databases else pg_restore_command) + tuple(psql_command if use_psql_command else pg_restore_command) + ('--no-password',) + ( ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') - if all_databases - else ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) + if use_psql_command + else ('--if-exists', '--exit-on-error', '--clean') ) + + (('--dbname', database['name']) if not all_databases else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) From 991e08f16dc99693b32826c6cd21401c32b06633 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Apr 2023 09:13:13 -0700 Subject: [PATCH 07/44] Add Unraid borgmatic installation link to docs. --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 917eb437..ca703163 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -96,7 +96,7 @@ installing borgmatic: * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/) * [NixOS](https://search.nixos.org/packages?show=borgmatic&sort=relevance&type=packages&query=borgmatic) * [Ansible role](https://github.com/borgbase/ansible-role-borgbackup) - * [virtualenv](https://virtualenv.pypa.io/en/stable/) + * [Unraid](https://unraid.net/community/apps?q=borgmatic#r) ## Hosting providers From 9f5769f87be3e75a3b86ee1a6f31a6754ec2ed4e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 16 Apr 2023 15:41:17 -0700 Subject: [PATCH 08/44] Make docs/schema a little more container agnostic / less Docker specific. --- borgmatic/config/schema.yaml | 14 +++++++------- docs/how-to/backup-your-databases.md | 10 +++++----- docs/how-to/develop-on-borgmatic.md | 23 +++++++++++++---------- docs/how-to/set-up-backups.md | 4 ++-- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 26500855..c4fd3138 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -836,25 +836,25 @@ properties: Command to use instead of "pg_dump" or "pg_dumpall". This can be used to run a specific pg_dump version (e.g., one inside a running - docker container). Defaults to "pg_dump" for - single database dump or "pg_dumpall" to dump - all databases. + container). Defaults to "pg_dump" for single + database dump or "pg_dumpall" to dump all + databases. example: docker exec my_pg_container pg_dump pg_restore_command: type: string description: | Command to use instead of "pg_restore". This can be used to run a specific pg_restore - version (e.g., one inside a running docker - container). Defaults to "pg_restore". + version (e.g., one inside a running container). + Defaults to "pg_restore". example: docker exec my_pg_container pg_restore psql_command: type: string description: | Command to use instead of "psql". This can be used to run a specific psql version (e.g., - one inside a running docker container). - Defaults to "psql". + one inside a running container). Defaults to + "psql". example: docker exec my_pg_container psql options: type: string diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 91dba18f..db93c073 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -138,7 +138,7 @@ hooks: ### Containers -If your database is running within a Docker container and borgmatic is too, no +If your database is running within a container and borgmatic is too, no problem—simply configure borgmatic to connect to the container's name on its exposed port. For instance: @@ -154,10 +154,10 @@ hooks: But what if borgmatic is running on the host? You can still connect to a database container if its ports are properly exposed to the host. For -instance, when running the database container with Docker, you can specify -`--publish 127.0.0.1:5433:5432` so that it exposes the container's port 5432 -to port 5433 on the host (only reachable on localhost, in this case). Or the -same thing with Docker Compose: +instance, when running the database container, you can specify `--publish +127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433 +on the host (only reachable on localhost, in this case). Or the same thing +with Docker Compose: ```yaml services: diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 027a8145..19b4bd18 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -87,19 +87,20 @@ tox -e codespell borgmatic additionally includes some end-to-end tests that integration test with Borg and supported databases for a few representative scenarios. These tests don't run by default when running `tox`, because they're relatively slow -and depend on Docker containers for runtime dependencies. These tests tests do -run on the continuous integration (CI) server, and running them on your -developer machine is the closest thing to CI test parity. +and depend on containers for runtime dependencies. These tests do run on the +continuous integration (CI) server, and running them on your developer machine +is the closest thing to CI-test parity. -If you would like to run the full test suite, first install Docker and [Docker -Compose](https://docs.docker.com/compose/install/). Then run: +If you would like to run the full test suite, first install Docker (or Podman; +see below) and [Docker Compose](https://docs.docker.com/compose/install/). +Then run: ```bash scripts/run-end-to-end-dev-tests ``` -Note that this scripts assumes you have permission to run Docker. If you -don't, then you may need to run with `sudo`. +This script assumes you have permission to run `docker`. If you don't, then +you may need to run with `sudo`. #### Podman @@ -119,6 +120,7 @@ some key points to double-check: * Create a non-root Podman socket for that user: ```bash systemctl --user enable --now podman.socket + systemctl --user start --now podman.socket ``` Then you'll be able to run end-to-end tests as per normal, and the test script @@ -161,11 +163,12 @@ To build and view a copy of the documentation with your local changes, run the following from the root of borgmatic's source code: ```bash -sudo scripts/dev-docs +scripts/dev-docs ``` -This requires Docker to be installed on your system. You may not need to use -sudo if your non-root user has permissions to run Docker. +This requires Docker (or Podman; see below) to be installed on your system. +This script assumes you have permission to run `docker`. If you don't, then +you may need to run with `sudo`. After you run the script, you can point your web browser at http://localhost:8080 to view the documentation with your changes. diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index ca703163..93402828 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -82,8 +82,8 @@ on a relatively dedicated system, then a global install can work out fine. Besides the approaches described above, there are several other options for installing borgmatic: - * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files) - * [Docker image with multi-arch and Docker CLI support](https://hub.docker.com/r/modem7/borgmatic-docker/) + * [container image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files) + * [container image with multi-arch and Docker CLI support](https://hub.docker.com/r/modem7/borgmatic-docker/) * [Debian](https://tracker.debian.org/pkg/borgmatic) * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic) * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic) From 8bb7631f50998d9b0cd796f3c098162011bc1bc7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Apr 2023 21:22:51 -0700 Subject: [PATCH 09/44] Fix missing mock in unit test. --- tests/unit/borg/test_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 818ce276..d2ab397d 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -2550,7 +2550,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) - flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module).should_receive('check_all_source_directories_exist').and_raise(ValueError) with pytest.raises(ValueError): module.create_archive( From 3b21ce4ce8b8af640ccc690f2ec0b193a1adc9cb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Apr 2023 21:43:08 -0700 Subject: [PATCH 10/44] Rename "master" development branch to "main" to use more inclusive language (#684). --- .drone.yml | 2 +- NEWS | 4 +++- README.md | 2 +- docs/how-to/develop-on-borgmatic.md | 2 +- docs/how-to/set-up-backups.md | 6 +++--- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.drone.yml b/.drone.yml index 9a7630ac..406d7e09 100644 --- a/.drone.yml +++ b/.drone.yml @@ -53,6 +53,6 @@ trigger: repo: - borgmatic-collective/borgmatic branch: - - master + - main event: - push diff --git a/NEWS b/NEWS index 4a27d074..740cc6f8 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,8 @@ * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas + * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to + update your development checkouts accordingly. 1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. @@ -367,7 +369,7 @@ * #398: Clarify canonical home of borgmatic in documentation. * #406: Clarify that spaces in path names should not be backslashed in path names. * #423: Fix error handling to error loudly when Borg gets killed due to running out of memory! - * Fix build so as not to attempt to build and push documentation for a non-master branch. + * Fix build so as not to attempt to build and push documentation for a non-main branch. * "Fix" build failure with Alpine Edge by switching from Edge to Alpine 3.13. * Move #borgmatic IRC channel from Freenode to Libera Chat due to Freenode takeover drama. IRC connection info: https://torsion.org/borgmatic/#issues diff --git a/README.md b/README.md index eb827ae9..2af792ba 100644 --- a/README.md +++ b/README.md @@ -165,5 +165,5 @@ Also, please check out the [borgmatic development how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for info on cloning source code, running tests, etc. -![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/master) +![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/main) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 19b4bd18..6d2b13b5 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -7,7 +7,7 @@ eleventyNavigation: --- ## Source code -To get set up to hack on borgmatic, first clone master via HTTPS or SSH: +To get set up to hack on borgmatic, first clone it via HTTPS or SSH: ```bash git clone https://projects.torsion.org/borgmatic-collective/borgmatic.git diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 93402828..be229cb2 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -279,7 +279,7 @@ that, you can configure a separate job runner to invoke it periodically. ### cron If you're using cron, download the [sample cron -file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/master/sample/cron/borgmatic). +file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/main/sample/cron/borgmatic). Then, from the directory where you downloaded it: ```bash @@ -303,9 +303,9 @@ you may already have borgmatic systemd service and timer files. If so, you may be able to skip some of the steps below.) First, download the [sample systemd service -file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.service) +file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/main/sample/systemd/borgmatic.service) and the [sample systemd timer -file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer). +file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/main/sample/systemd/borgmatic.timer). Then, from the directory where you downloaded them: From 269fac074b1f9b6558f25dc3a33dc446e6bb653a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Apr 2023 23:14:51 -0700 Subject: [PATCH 11/44] Attempt to use Podman-in-Podman for building docs instead of Docker-in-Podman. --- .drone.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.drone.yml b/.drone.yml index 406d7e09..5a8bb56c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -38,16 +38,17 @@ clone: steps: - name: build - image: plugins/docker - settings: - username: + image: quay.io/stable/podman + environment: + USERNAME: from_secret: docker_username - password: + PASSWORD: from_secret: docker_password - registry: projects.torsion.org - repo: projects.torsion.org/borgmatic-collective/borgmatic - tags: docs - dockerfile: docs/Dockerfile + IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs + commands: + - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman push "$IMAGE_NAME" trigger: repo: From 0a6f5452f41e289c452e32cdd1aee93706e98d68 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Apr 2023 23:16:15 -0700 Subject: [PATCH 12/44] Fix broken Podman image name. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5a8bb56c..5137bacd 100644 --- a/.drone.yml +++ b/.drone.yml @@ -38,7 +38,7 @@ clone: steps: - name: build - image: quay.io/stable/podman + image: quay.io/podman/stable environment: USERNAME: from_secret: docker_username From 5f8c79dd164d144695f6385a0f4ed778f9e5daec Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 10:50:44 -0700 Subject: [PATCH 13/44] Attempt to get Podman-in-Podman builds working. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5137bacd..05f344fe 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt "unmask=/proc/*" --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 53ee0fcfada8d61ceefe652f109d63bf9f174b50 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:06:15 -0700 Subject: [PATCH 14/44] Another attempt at Podman-in-Podman incantations. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 05f344fe..1041e464 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt "unmask=/proc/*" --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt label=disable --security-opt unmask=ALL --device /dev/fuse --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From bb5028e4841c29834b63a19747c80bbb7df0a19c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:11:08 -0700 Subject: [PATCH 15/44] Sigh. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 1041e464..334962cb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt label=disable --security-opt unmask=ALL --device /dev/fuse --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt label=disable --device /dev/fuse --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From b85538c54c85bd21c326cb23255ad2f24397305f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:11:49 -0700 Subject: [PATCH 16/44] Double sigh. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 334962cb..5f4dbc94 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt label=disable --device /dev/fuse --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt label=disable --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 51bc53e5cade48f42c56ed8f707259e7c7b6d50e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:24:59 -0700 Subject: [PATCH 17/44] Whee. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5f4dbc94..3aa91923 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt label=disable --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 9e9a7c50e52a028adf9a20f1dc6ad123dcfc100f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:30:30 -0700 Subject: [PATCH 18/44] =?UTF-8?q?=F0=9F=98=8A=F0=9F=94=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 3aa91923..c4526039 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns-uid-map-group podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 4e78cf1b951a7abb6ec55fc341fe27f58f723e74 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:33:15 -0700 Subject: [PATCH 19/44] =?UTF-8?q?=E0=B2=A0=5F=E0=B2=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index c4526039..d95f0f00 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns-uid-map-group podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 02eeca1fc2845efc331212856dcd150a1bd9dbe7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:36:30 -0700 Subject: [PATCH 20/44] Hmm. --- .drone.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index d95f0f00..0e03945e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,8 +46,10 @@ steps: from_secret: docker_password IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: + - cat /etc/subuid + - cat /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From f82bf619ff6ba15fc54aee05f7bf69c00c8d7964 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:41:35 -0700 Subject: [PATCH 21/44] More. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 0e03945e..59fa1752 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,7 +49,7 @@ steps: - cat /etc/subuid - cat /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From d6e1cef356e29285b4d129f2b55bfc0daba75576 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:49:43 -0700 Subject: [PATCH 22/44] Throwing stuff at the wall. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 59fa1752..53de5f87 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,7 +49,7 @@ steps: - cat /etc/subuid - cat /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns host --userns-uid-map-user podman --userns-gid-map-group podman --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 8a31c270780f3c3cbe036c950c8b175fa4ded957 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:50:25 -0700 Subject: [PATCH 23/44] To see what sticks. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 53de5f87..c3e30adf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,7 +49,7 @@ steps: - cat /etc/subuid - cat /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns host --userns-uid-map-user podman --userns-gid-map-group podman --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns host --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 1721c05d2e2b20c275f002bcd00955afa20a33d7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:52:23 -0700 Subject: [PATCH 24/44] Yet more. --- .drone.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index c3e30adf..48216fcd 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,10 +46,10 @@ steps: from_secret: docker_password IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - - cat /etc/subuid - - cat /etc/subgid + - echo "podman:100000:65536" > /etc/subuid + - echo "podman:100000:65536" > /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns host --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 4302a07c9b37e8d9ca3ca37e01e32f35b9c70c22 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:53:52 -0700 Subject: [PATCH 25/44] WTF. --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 48216fcd..cb26aef0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,8 +46,8 @@ steps: from_secret: docker_password IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - - echo "podman:100000:65536" > /etc/subuid - - echo "podman:100000:65536" > /etc/subgid + - echo "podman:100000:100000" > /etc/subuid + - echo "podman:100000:100000" > /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" From 499e42df359aa5c7442a0a1d4b2d6e41e89d7d7a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:58:06 -0700 Subject: [PATCH 26/44] =?UTF-8?q?=F0=9F=98=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index cb26aef0..7808e29f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,7 +49,7 @@ steps: - echo "podman:100000:100000" > /etc/subuid - echo "podman:100000:100000" > /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt label=disable --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 7f7b89d79c733b1daa94b67ec4be3e17fb3514ec Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 12:03:51 -0700 Subject: [PATCH 27/44] Trying a different approach: Ditching Podman-in-Podman. --- .drone.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index 7808e29f..98b893e8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -32,13 +32,17 @@ steps: --- kind: pipeline name: documentation +type: exec + +platform: + os: linux + arch: amd64 clone: skip_verify: true steps: - name: build - image: quay.io/podman/stable environment: USERNAME: from_secret: docker_username @@ -46,10 +50,8 @@ steps: from_secret: docker_password IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - - echo "podman:100000:100000" > /etc/subuid - - echo "podman:100000:100000" > /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt label=disable --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From f947525fcafbd34676c0eef91f2fa207ea7b9e0d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:45:26 -0700 Subject: [PATCH 28/44] ? --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 98b893e8..25f57985 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume .:/app - podman push "$IMAGE_NAME" trigger: From c9bf52ee45628669f3b0c44ad741091ee0ef8382 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:46:49 -0700 Subject: [PATCH 29/44] Sigh again. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 25f57985..53a61c56 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume .:/app + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app - podman push "$IMAGE_NAME" trigger: From 1e03046d9a72155123679d9786e318a198ff0f35 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:50:26 -0700 Subject: [PATCH 30/44] *Seriously?* --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 53a61c56..441859bf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" - podman push "$IMAGE_NAME" trigger: From 08edecacae473a2cf979d58f88531ead0ad54472 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:55:37 -0700 Subject: [PATCH 31/44] WTF?! --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 441859bf..ddbb5d73 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" --storage-opt ignore_chown_errors - podman push "$IMAGE_NAME" trigger: From 7ff994a96467b06894a44cb2ac8a2d376d9e4a67 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:56:12 -0700 Subject: [PATCH 32/44] =?UTF-8?q?=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index ddbb5d73..2e0e0b17 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" --storage-opt ignore_chown_errors + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" --storage-opt ignore_chown_errors=true - podman push "$IMAGE_NAME" trigger: From f2f6fb537a9cfbd96aaf5ad416a74fbcfdaa8c84 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 14:19:34 -0700 Subject: [PATCH 33/44] !!! --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 2e0e0b17..11788468 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" --storage-opt ignore_chown_errors=true + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" . - podman push "$IMAGE_NAME" trigger: From 065be1d9d4feb4236712394f63a0c5d18ab624cb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 14:28:04 -0700 Subject: [PATCH 34/44] More inclusive language. --- SECURITY.md | 4 ++-- borgmatic/borg/rlist.py | 4 ++-- borgmatic/config/generate.py | 2 +- borgmatic/config/schema.yaml | 4 ++-- docs/how-to/backup-your-databases.md | 12 ++++++------ docs/how-to/inspect-your-backups.md | 2 +- docs/how-to/make-per-application-backups.md | 8 ++++---- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index d82b6f32..64c3d3b3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,8 +7,8 @@ permalink: security-policy/index.html While we want to hear about security vulnerabilities in all versions of borgmatic, security fixes are only made to the most recently released version. -It's simply not practical for our small volunteer effort to maintain multiple -release branches and put out separate security patches for each. +It's not practical for our small volunteer effort to maintain multiple release +branches and put out separate security patches for each. ## Reporting a vulnerability diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index c051a9ad..7f468705 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -17,8 +17,8 @@ def resolve_archive_name( ): ''' Given a local or remote repository path, an archive name, a storage config dict, a local Borg - path, and a remote Borg path, simply return the archive name. But if the archive name is - "latest", then instead introspect the repository for the latest archive and return its name. + path, and a remote Borg path, return the archive name. But if the archive name is "latest", + then instead introspect the repository for the latest archive and return its name. Raise ValueError if "latest" is given but there are no archives in the repository. ''' diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index d486f23c..081186e3 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -260,7 +260,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi ) continue - # This is some sort of scalar. Simply set it into the destination. + # This is some sort of scalar. Set it into the destination. destination_config[field_name] = source_config[field_name] return destination_config diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index c4fd3138..0cf02b25 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -60,7 +60,7 @@ properties: or port. If systemd service is used, then add local repository paths in the systemd service file to the ReadWritePaths list. Prior to borgmatic 1.7.10, repositories - was just a list of plain path strings. + was a list of plain path strings. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver @@ -1216,7 +1216,7 @@ properties: type: string description: | Healthchecks ping URL or UUID to notify when a - backup begins, ends, errors or just to send logs. + backup begins, ends, errors, or to send only logs. example: https://hc-ping.com/your-uuid-here verify_tls: type: boolean diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index db93c073..0a770d61 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -139,8 +139,8 @@ hooks: ### Containers If your database is running within a container and borgmatic is too, no -problem—simply configure borgmatic to connect to the container's name on its -exposed port. For instance: +problem—configure borgmatic to connect to the container's name on its exposed +port. For instance: ```yaml hooks: @@ -179,7 +179,7 @@ hooks: password: trustsome1 ``` -Of course, alter the ports in these examples to suit your particular database +You can alter the ports in these examples to suit your particular database system. @@ -397,9 +397,9 @@ dumps with any database system. With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors when borgmatic tries to connect to your database, a natural reaction is to increase your borgmatic verbosity with `--verbosity 2` and go looking in the -logs. You'll notice however that your database password does not show up in -the logs. This is likely not the cause of the authentication problem unless -you mistyped your password, however; borgmatic passes your password to the +logs. You'll notice though that your database password does not show up in the +logs. This is likely not the cause of the authentication problem unless you +mistyped your password, however; borgmatic passes your password to the database via an environment variable that does not appear in the logs. The cause of an authentication error is often on the database side—in the diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 73020eda..e89d23f6 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -169,7 +169,7 @@ brackets. For instance, the default log format is: `[{asctime}] {levelname}: {message}`. This means each log message is recorded as the log time (in square brackets), a logging level name, a colon, and the actual log message. -So if you just want each log message to get logged *without* a timestamp or a +So if you only want each log message to get logged *without* a timestamp or a logging level name: ```bash diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 9ee93f44..f2ddf012 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -86,8 +86,8 @@ uses the `archive_name_format` option to automatically limit which archives get used for actions operating on multiple archives. This prevents, for instance, duplicate archives from showing up in `rlist` or `info` results—even if the same repository appears in multiple borgmatic configuration files. To -take advantage of this feature, simply use a different `archive_name_format` -in each configuration file. +take advantage of this feature, use a different `archive_name_format` in each +configuration file. Under the hood, borgmatic accomplishes this by substituting globs for certain ephemeral data placeholders in your `archive_name_format`—and using the result @@ -108,8 +108,8 @@ archives used for actions like `rlist`, `info`, `prune`, `check`, etc. The end result is that when borgmatic runs the actions for a particular application-specific configuration file, it only operates on the archives -created for that application. Of course, this doesn't apply to actions like -`compact` that operate on an entire repository. +created for that application. But this doesn't apply to actions like `compact` +that operate on an entire repository. If this behavior isn't quite smart enough for your needs, you can use the `match_archives` option to override the pattern that borgmatic uses for From ee5c25f3bd9402b47738be1c7cfed5f282930ed7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 21:44:42 -0700 Subject: [PATCH 35/44] Add additional tests for PostgreSQL hook fixes (#678). --- NEWS | 5 ++ tests/unit/hooks/test_postgresql.py | 122 ++++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index 740cc6f8..69938b0f 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,11 @@ * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas + * #678: Fix error from PostgreSQL when dumping a database with a "format" of "plain". + * #678: Fix PostgreSQL hook to support "psql_command" and "pg_restore_command" options containing + commands with arguments. + * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break + database dumping. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 62b10d16..e3ecbdb6 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -76,7 +76,16 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam def test_database_names_to_dump_with_all_and_format_lists_databases_with_username(): database = {'name': 'all', 'format': 'custom', 'username': 'postgres'} flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--username', 'postgres'), + ( + 'psql', + '--list', + '--no-password', + '--no-psqlrc', + '--csv', + '--tuples-only', + '--username', + 'postgres', + ), extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') @@ -110,6 +119,28 @@ def test_database_names_to_dump_with_all_and_format_excludes_particular_database ) +def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): + database = {'name': 'all', 'format': 'custom', 'psql_command': 'docker exec mycontainer psql'} + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'docker', + 'exec', + 'mycontainer', + 'psql', + '--list', + '--no-password', + '--no-psqlrc', + '--csv', + '--tuples-only', + ), + extra_environment=object, + ).and_return('foo,text').once() + + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + 'foo', + ) + + def test_dump_databases_runs_pg_dump_for_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] @@ -434,7 +465,16 @@ def test_restore_database_dump_runs_pg_restore(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -632,12 +672,45 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): ) +def test_restore_database_dump_runs_psql_for_plain_database_dump(): + database_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on', '--dbname', 'foo'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + ) + + def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): database_config = [ { 'name': 'foo', - 'pg_restore_command': 'special_pg_restore', - 'psql_command': 'special_psql', + 'pg_restore_command': 'docker exec mycontainer pg_restore', + 'psql_command': 'docker exec mycontainer psql', 'schemas': None, } ] @@ -648,7 +721,10 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( ( - 'special_pg_restore', + 'docker', + 'exec', + 'mycontainer', + 'pg_restore', '--no-password', '--if-exists', '--exit-on-error', @@ -662,7 +738,19 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('special_psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ( + 'docker', + 'exec', + 'mycontainer', + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -707,7 +795,16 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -743,7 +840,16 @@ def test_restore_database_dump_with_schemas_restores_schemas(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), extra_environment={'PGSSLMODE': 'disable'}, ).once() From a14870ce4899a8aa38a2f38832bffe74b8490b78 Mon Sep 17 00:00:00 2001 From: Jesse Johnson Date: Wed, 19 Apr 2023 18:47:22 -0700 Subject: [PATCH 36/44] Expand source directories when checking for existence (#682). --- borgmatic/borg/create.py | 2 +- tests/unit/borg/test_create.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 8782dc6b..7413dfc5 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -314,7 +314,7 @@ def check_all_source_directories_exist(source_directories): missing_directories = [ source_directory for source_directory in source_directories - if not os.path.exists(source_directory) + if not all([os.path.exists(directory) for directory in expand_directory(source_directory)]) ] if missing_directories: raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}") diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index d2ab397d..5011f48b 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -2565,3 +2565,18 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ storage_config={}, local_borg_version='1.2.3', ) + + +def test_check_all_source_directories_exist_with_glob_and_tilde_directories(): + flexmock(module).should_receive('expand_directory').with_args('foo*').and_return( + ('foo', 'food') + ) + flexmock(module).should_receive('expand_directory').with_args('~/bar').and_return( + ('/root/bar',) + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.os.path).should_receive('exists').with_args('foo').and_return(True) + flexmock(module.os.path).should_receive('exists').with_args('food').and_return(True) + flexmock(module.os.path).should_receive('exists').with_args('/root/bar').and_return(True) + + module.check_all_source_directories_exist(['foo*', '~/bar']) From b555fcb95621ba46d1bc6d2208cbac79bd3e020c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 23:08:21 -0700 Subject: [PATCH 37/44] Add "source_directories_must_exist" expansion fix to NEWS (#682). --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 69938b0f..20214674 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,7 @@ commands with arguments. * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break database dumping. + * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. From 9ca31530a0d06196f2e3a64fb7c47ff05f89bf7d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 23:15:22 -0700 Subject: [PATCH 38/44] Add missing test for check_all_source_directories_exist() raising. --- tests/unit/borg/test_create.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 5011f48b..3728a0bf 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -2580,3 +2580,11 @@ def test_check_all_source_directories_exist_with_glob_and_tilde_directories(): flexmock(module.os.path).should_receive('exists').with_args('/root/bar').and_return(True) module.check_all_source_directories_exist(['foo*', '~/bar']) + + +def test_check_all_source_directories_exist_with_non_existent_directory_raises(): + flexmock(module).should_receive('expand_directory').with_args('foo').and_return(('foo',)) + flexmock(module.os.path).should_receive('exists').and_return(False) + + with pytest.raises(ValueError): + module.check_all_source_directories_exist(['foo']) From 71b75800cdf770fa6af519ae493fe5be8413b0a8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 23:32:57 -0700 Subject: [PATCH 39/44] Get more verbose in the end-to-end test restore. --- tests/end-to-end/test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 30aea4a8..e0c929f3 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -118,7 +118,7 @@ def test_database_dump_and_restore(): # Restore the database from the archive. subprocess.check_call( - ['borgmatic', '--config', config_path, 'restore', '--archive', archive_name] + ['borgmatic', '--config', '-v', '2', config_path, 'restore', '--archive', archive_name] ) finally: os.chdir(original_working_directory) From 3cefeaa229959c8e6bea0adf299f1fd513b6c6ee Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Apr 2023 09:30:08 -0700 Subject: [PATCH 40/44] Fix end-to-end test command-line syntax. --- tests/end-to-end/test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index e0c929f3..81e7c5e7 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -118,7 +118,7 @@ def test_database_dump_and_restore(): # Restore the database from the archive. subprocess.check_call( - ['borgmatic', '--config', '-v', '2', config_path, 'restore', '--archive', archive_name] + ['borgmatic', '-v, '2', '--config', config_path, 'restore', '--archive', archive_name] ) finally: os.chdir(original_working_directory) From ae12ccd8e6b0f0a009b2214f93b1181fb4d00842 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Apr 2023 09:31:37 -0700 Subject: [PATCH 41/44] And fixing again... --- tests/end-to-end/test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 81e7c5e7..5c4e22cc 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -118,7 +118,7 @@ def test_database_dump_and_restore(): # Restore the database from the archive. subprocess.check_call( - ['borgmatic', '-v, '2', '--config', config_path, 'restore', '--archive', archive_name] + ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name] ) finally: os.chdir(original_working_directory) From 7e64f415ba74a6a491171181196b2d5a80c876da Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Apr 2023 10:03:29 -0700 Subject: [PATCH 42/44] Attempt to fix failing end-to-end database test that only fails in CI. --- .drone.yml | 9 +++++---- NEWS | 1 + tests/end-to-end/docker-compose.yaml | 1 + tests/end-to-end/test_database.py | 6 +++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.drone.yml b/.drone.yml index 11788468..f2c5f7d7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3,17 +3,18 @@ name: python-3-8-alpine-3-13 services: - name: postgresql - image: postgres:13.1-alpine + image: docker.io/postgres:13.1-alpine environment: + POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: test - name: mysql - image: mariadb:10.5 + image: docker.io/mariadb:10.5 environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test - name: mongodb - image: mongo:5.0.5 + image: docker.io/mongo:5.0.5 environment: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: test @@ -23,7 +24,7 @@ clone: steps: - name: build - image: alpine:3.13 + image: docker.io/alpine:3.13 environment: TEST_CONTAINER: true pull: always diff --git a/NEWS b/NEWS index 20214674..530a447f 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,7 @@ commands with arguments. * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break database dumping. + * #678: Fix calls to psql in PostgreSQL hook to abort on error during a database restore. * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 0bbec8cc..f9895a8c 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -3,6 +3,7 @@ services: postgresql: image: docker.io/postgres:13.1-alpine environment: + POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: test mysql: diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 5c4e22cc..8b38e071 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -36,17 +36,17 @@ hooks: postgresql_databases: - name: test hostname: postgresql - username: postgres + username: test password: test format: {postgresql_dump_format} - name: all hostname: postgresql - username: postgres + username: test password: test - name: all format: custom hostname: postgresql - username: postgres + username: test password: test mysql_databases: - name: test From 5962fd473e54ada92de6a220db729da11c769bfe Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Apr 2023 10:34:50 -0700 Subject: [PATCH 43/44] Another try. Backing out psql error changes (#678). --- .drone.yml | 2 +- NEWS | 1 - borgmatic/hooks/postgresql.py | 6 +----- tests/end-to-end/docker-compose.yaml | 1 - tests/end-to-end/test_database.py | 6 +++--- tests/unit/hooks/test_postgresql.py | 8 ++++++-- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.drone.yml b/.drone.yml index f2c5f7d7..dcc19c67 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,3 +1,4 @@ +--- kind: pipeline name: python-3-8-alpine-3-13 @@ -5,7 +6,6 @@ services: - name: postgresql image: docker.io/postgres:13.1-alpine environment: - POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: test - name: mysql diff --git a/NEWS b/NEWS index 530a447f..20214674 100644 --- a/NEWS +++ b/NEWS @@ -7,7 +7,6 @@ commands with arguments. * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break database dumping. - * #678: Fix calls to psql in PostgreSQL hook to abort on error during a database restore. * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 859f7ece..3325391f 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -229,11 +229,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, restore_command = ( tuple(psql_command if use_psql_command else pg_restore_command) + ('--no-password',) - + ( - ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') - if use_psql_command - else ('--if-exists', '--exit-on-error', '--clean') - ) + + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean')) + (('--dbname', database['name']) if not all_databases else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index f9895a8c..0bbec8cc 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -3,7 +3,6 @@ services: postgresql: image: docker.io/postgres:13.1-alpine environment: - POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: test mysql: diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 8b38e071..5c4e22cc 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -36,17 +36,17 @@ hooks: postgresql_databases: - name: test hostname: postgresql - username: test + username: postgres password: test format: {postgresql_dump_format} - name: all hostname: postgresql - username: test + username: postgres password: test - name: all format: custom hostname: postgresql - username: test + username: postgres password: test mysql_databases: - name: test diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index e3ecbdb6..b3a55fa4 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -656,7 +656,11 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( - ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on'), + ( + 'psql', + '--no-password', + '--no-psqlrc', + ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -680,7 +684,7 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( - ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on', '--dbname', 'foo'), + ('psql', '--no-password', '--no-psqlrc', '--dbname', 'foo'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, From 22b84a2feaad01fe2a80e46e01c177e61bc80f8f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Apr 2023 10:07:40 -0700 Subject: [PATCH 44/44] Switch to Docker Compose for dev-docs script, so podman-docker is no longer needed for Podman users. --- docs/docker-compose.yaml | 22 ++++++++++++++++++++++ docs/how-to/develop-on-borgmatic.md | 5 ++--- scripts/dev-docs | 12 +++++++----- 3 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 docs/docker-compose.yaml diff --git a/docs/docker-compose.yaml b/docs/docker-compose.yaml new file mode 100644 index 00000000..e854c13a --- /dev/null +++ b/docs/docker-compose.yaml @@ -0,0 +1,22 @@ +version: '3' +services: + docs: + image: borgmatic-docs + container_name: docs + ports: + - 8080:80 + build: + dockerfile: docs/Dockerfile + context: .. + args: + ENVIRONMENT: dev + message: + image: alpine + container_name: message + command: + - sh + - -c + - | + echo "You can view dev docs at http://localhost:8080" + depends_on: + - docs diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 6d2b13b5..391d7950 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -113,8 +113,7 @@ borgmatic's end-to-end tests optionally support using Setting up Podman is outside the scope of this documentation, but here are some key points to double-check: - * Install Podman along with `podman-docker` and your desired networking - support. + * Install Podman and your desired networking support. * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the non-root user who will run tests. * Create a non-root Podman socket for that user: @@ -186,5 +185,5 @@ borgmatic's developer build for documentation optionally supports using [Podman](https://podman.io/) instead of Docker. Setting up Podman is outside the scope of this documentation. But once you -install `podman-docker`, then `scripts/dev-docs` should automatically use +install and configure Podman, then `scripts/dev-docs` should automatically use Podman instead of Docker. diff --git a/scripts/dev-docs b/scripts/dev-docs index 1ecc7e46..866ae0e9 100755 --- a/scripts/dev-docs +++ b/scripts/dev-docs @@ -2,8 +2,10 @@ set -e -docker build --tag borgmatic-docs --build-arg ENVIRONMENT=dev --file docs/Dockerfile . -echo -echo "You can view dev docs at http://localhost:8080" -echo -docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs +USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock + +if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then + export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH" +fi + +docker-compose --file docs/docker-compose.yaml up --build --force-recreate