fix(restore): lock the app out via pg_hba during restore so it cannot race the import

Reviewed-on: https://git.coopcloud.tech/coop-cloud/immich/pulls/7

abra.sh

@@ -0,0 +1 @@
+export PG_BACKUP_VERSION=v1

compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 
 services:
   app:
-    image: ghcr.io/immich-app/immich-server:v2.5.6
+    image: ghcr.io/immich-app/immich-server:v2.7.5
     volumes:
       - uploads:/usr/src/app/upload
       - /etc/localtime:/etc/localtime:ro
@@ -30,7 +30,7 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "coop-cloud.${STACK_NAME}.version=1.4.0+v2.5.6"
+        - "coop-cloud.${STACK_NAME}.version=1.7.0+v2.7.5"
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
         - "backupbot.volumes.model-cache=false"
         - "backupbot.volumes.uploads=false"
@@ -38,7 +38,7 @@ services:
 
 
   immich-machine-learning: # TODO: this has to be that name, as the frontend tries to reach it at: http://immich-machine-learning:3003
-    image: ghcr.io/immich-app/immich-machine-learning:v2.5.6
+    image: ghcr.io/immich-app/immich-machine-learning:v2.7.5
     volumes:
       - model-cache:/cache
     networks:
@@ -47,7 +47,7 @@ services:
       disable: false
 
   redis:
-    image: docker.io/valkey/valkey:9@sha256:fb8d272e529ea567b9bf1302245796f21a2672b8368ca3fcb938ac334e613c8f
+    image: docker.io/valkey/valkey:9@sha256:3b55fbaa0cd93cf0d9d961f405e4dfcc70efe325e2d84da207a0a8e6d8fde4f9
     healthcheck:
       test: redis-cli ping || exit 1
     networks:
@@ -67,6 +67,21 @@ services:
       - postgres:/var/lib/postgresql/data
     networks:
       - backend
+    deploy:
+      labels:
+        backupbot.backup: "${ENABLE_BACKUPS:-true}"
+        backupbot.backup.pre-hook: "/pg_backup.sh backup"
+        backupbot.backup.volumes.postgres.path: "backup.sql"
+        backupbot.restore.post-hook: "/pg_backup.sh restore"
+    configs:
+      - source: pg_backup
+        target: /pg_backup.sh
+        mode: 0555
+
+configs:
+  pg_backup:
+    name: ${STACK_NAME}_pg_backup_${PG_BACKUP_VERSION}
+    file: pg_backup.sh
 
 secrets:
   db_password:

pg_backup.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Postgres backup/restore hook for the immich `database` service (VectorChord/pgvecto.rs image).
+# Invoked by backupbot-two via the deploy labels:
+#   backupbot.backup.pre-hook              = "/pg_backup.sh backup"
+#   backupbot.backup.volumes.postgres.path = "backup.sql"
+#   backupbot.restore.post-hook            = "/pg_backup.sh restore"
+# Backup dumps the immich DB to backup.sql (gzip) inside the postgres volume; backupbot archives that
+# file. Restore reads it back and reimports it.
+#
+# Two things make the immich DB tricky to restore, and BOTH are handled below:
+#
+# 1. VectorChord/pgvecto.rs search_path. A plain pg_dump emits
+#      SELECT pg_catalog.set_config('search_path', '', false);
+#    near the top; on reimport that empty search_path leaves the vector/vchord types + operator
+#    classes unresolvable, so the first statement that references them errors. immich's official
+#    restore (docs.immich.app/administration/backup-and-restore) rewrites that line to put
+#    `public, pg_catalog` back on the search_path BEFORE the import. We do the same.
+#
+# 2. The app races the restore. immich-server keeps a TCP connection pool to the DB and reconnects
+#    within milliseconds of being dropped, re-running its OWN startup migrations. If it does that
+#    while our single-transaction import is running, the two conflict (duplicate CREATE/relation
+#    errors) and ON_ERROR_STOP aborts our transaction — the import rolls back and nothing is
+#    restored, while the app finishes its migration and looks "healthy" on an empty DB. A one-shot
+#    pg_terminate_backend does NOT prevent the reconnect. So, like the (green) matrix-synapse hook,
+#    we replace pg_hba.conf with a LOCAL-trust-only policy and reload: postgres then REJECTS every
+#    TCP connection, locking the app out for the duration of the restore. We restore pg_hba (via an
+#    EXIT trap, so it always runs) once the import is done, and the app reconnects to the fully
+#    restored DB. Our own psql/createdb use the local socket (trust), so they are unaffected.
+
+set -e
+
+BACKUP_FILE='/var/lib/postgresql/data/backup.sql'
+HBA='/var/lib/postgresql/data/pg_hba.conf'
+export PGPASSWORD=$(cat "${POSTGRES_PASSWORD_FILE:-/run/secrets/db_password}")
+DB_USER="${POSTGRES_USER:-postgres}"
+DB_NAME="${POSTGRES_DB:-immich}"
+
+function backup {
+  pg_dump -U "$DB_USER" "$DB_NAME" | gzip > "$BACKUP_FILE"
+}
+
+function restore {
+  restore_hba() {
+    if [ -f "${HBA}.ccci.bak" ]; then
+      cat "${HBA}.ccci.bak" > "$HBA"
+      rm -f "${HBA}.ccci.bak"
+      psql -U "$DB_USER" -d postgres -c "SELECT pg_reload_conf();" >/dev/null 2>&1 || true
+    fi
+  }
+  trap restore_hba EXIT INT TERM
+
+  # Lock the networked immich-server OUT for the duration of the restore (see header note 2):
+  # local-trust-only pg_hba + reload makes postgres reject all TCP connections, then terminate the
+  # connections the app already holds so DROP DATABASE can proceed.
+  cp "$HBA" "${HBA}.ccci.bak"
+  printf 'local all all trust\n' > "$HBA"
+  psql -U "$DB_USER" -d postgres -c "SELECT pg_reload_conf();"
+  psql -U "$DB_USER" -d postgres -c \
+    "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='${DB_NAME}' AND pid<>pg_backend_pid();"
+
+  psql -U "$DB_USER" -d postgres -c "DROP DATABASE ${DB_NAME} WITH (FORCE);"
+  createdb -U "$DB_USER" "$DB_NAME"
+  # Rewrite the empty search_path the VectorChord dump sets (header note 1), then import alone.
+  gunzip -c "$BACKUP_FILE" \
+    | sed "s/SELECT pg_catalog.set_config('search_path', '', false);/SELECT pg_catalog.set_config('search_path', 'public, pg_catalog', true);/g" \
+    | psql -U "$DB_USER" -d "$DB_NAME" --single-transaction --set ON_ERROR_STOP=on
+
+  restore_hba
+  trap - EXIT INT TERM
+}
+
+$@

release/1.5.0+v2.6.3

@@ -0,0 +1 @@
+there might be some long running db migrations that cause the update to look like it's timed out, check `abra ps` to verify status

release/1.5.1+v2.6.3

@@ -0,0 +1 @@
+patch to fix my tagging screw up

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}