Fix canonical home link in README.

Reviewed-on: borgmatic-collective/borgmatic#683

.drone.yml

@@ -1,19 +1,20 @@
+---
 kind: pipeline
 name: python-3-8-alpine-3-13
 
 services:
   - name: postgresql
-    image: postgres:13.1-alpine
+    image: docker.io/postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   - name: mysql
-    image: mariadb:10.5
+    image: docker.io/mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   - name: mongodb
-    image: mongo:5.0.5
+    image: docker.io/mongo:5.0.5
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
@@ -23,7 +24,7 @@ clone:
 
 steps:
 - name: build
-  image: alpine:3.13
+  image: docker.io/alpine:3.13
   environment:
     TEST_CONTAINER: true
   pull: always

NEWS

@@ -7,6 +7,7 @@
    commands with arguments.
  * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break
    database dumping.
+ * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories.
  * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to
    update your development checkouts accordingly.
 

README.md

@@ -11,7 +11,7 @@ borgmatic is simple, configuration-driven backup software for servers and
 workstations. Protect your files with client-side encryption. Backup your
 databases too. Monitor it all with integrated third-party services.
 
-The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic</a>.
+The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic/</a>
 
 Here's an example configuration file:
 

borgmatic/borg/create.py

@@ -314,7 +314,7 @@ def check_all_source_directories_exist(source_directories):
     missing_directories = [
         source_directory
         for source_directory in source_directories
-        if not os.path.exists(source_directory)
+        if not all([os.path.exists(directory) for directory in expand_directory(source_directory)])
     ]
     if missing_directories:
         raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")

borgmatic/hooks/postgresql.py

@@ -229,11 +229,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     restore_command = (
         tuple(psql_command if use_psql_command else pg_restore_command)
         + ('--no-password',)
-        + (
+        + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
-            ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on')
-            if use_psql_command
-            else ('--if-exists', '--exit-on-error', '--clean')
-        )
         + (('--dbname', database['name']) if not all_databases else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())

docs/docker-compose.yaml

@@ -0,0 +1,22 @@
+version: '3'
+services:
+  docs:
+    image: borgmatic-docs
+    container_name: docs
+    ports:
+      - 8080:80
+    build:
+      dockerfile: docs/Dockerfile
+      context: ..
+      args:
+        ENVIRONMENT: dev
+  message:
+    image: alpine
+    container_name: message
+    command:
+      - sh
+      - -c
+      - |
+        echo "You can view dev docs at http://localhost:8080"
+    depends_on:
+      - docs

docs/how-to/develop-on-borgmatic.md

@@ -113,8 +113,7 @@ borgmatic's end-to-end tests optionally support using
 Setting up Podman is outside the scope of this documentation, but here are
 some key points to double-check:
 
- * Install Podman along with `podman-docker` and your desired networking
+ * Install Podman and your desired networking support.
-   support.
  * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the
    non-root user who will run tests.
  * Create a non-root Podman socket for that user:
@@ -186,5 +185,5 @@ borgmatic's developer build for documentation optionally supports using
 [Podman](https://podman.io/) instead of Docker.
 
 Setting up Podman is outside the scope of this documentation. But once you
-install `podman-docker`, then `scripts/dev-docs` should automatically use
+install and configure Podman, then `scripts/dev-docs` should automatically use
 Podman instead of Docker.

scripts/dev-docs

@@ -2,8 +2,10 @@
 
 set -e
 
-docker build --tag borgmatic-docs --build-arg ENVIRONMENT=dev --file docs/Dockerfile .
+USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock
-echo
+
-echo "You can view dev docs at http://localhost:8080"
+if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then
-echo
+    export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH"
-docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs
+fi
+
+docker-compose --file docs/docker-compose.yaml up --build --force-recreate

tests/end-to-end/test_database.py

@@ -118,7 +118,7 @@ def test_database_dump_and_restore():
 
         # Restore the database from the archive.
         subprocess.check_call(
-            ['borgmatic', '--config', config_path, 'restore', '--archive', archive_name]
+            ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
         )
     finally:
         os.chdir(original_working_directory)

tests/unit/borg/test_create.py

@@ -2565,3 +2565,26 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_
             storage_config={},
             local_borg_version='1.2.3',
         )
+
+
+def test_check_all_source_directories_exist_with_glob_and_tilde_directories():
+    flexmock(module).should_receive('expand_directory').with_args('foo*').and_return(
+        ('foo', 'food')
+    )
+    flexmock(module).should_receive('expand_directory').with_args('~/bar').and_return(
+        ('/root/bar',)
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os.path).should_receive('exists').with_args('foo').and_return(True)
+    flexmock(module.os.path).should_receive('exists').with_args('food').and_return(True)
+    flexmock(module.os.path).should_receive('exists').with_args('/root/bar').and_return(True)
+
+    module.check_all_source_directories_exist(['foo*', '~/bar'])
+
+
+def test_check_all_source_directories_exist_with_non_existent_directory_raises():
+    flexmock(module).should_receive('expand_directory').with_args('foo').and_return(('foo',))
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+
+    with pytest.raises(ValueError):
+        module.check_all_source_directories_exist(['foo'])

tests/unit/hooks/test_postgresql.py

@@ -656,7 +656,11 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on'),
+        (
+            'psql',
+            '--no-password',
+            '--no-psqlrc',
+        ),
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
@@ -680,7 +684,7 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump():
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on', '--dbname', 'foo'),
+        ('psql', '--no-password', '--no-psqlrc', '--dbname', 'foo'),
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,