fix(db): make pg_upgrade 13->17 idempotent and use the cluster's real install user

Reviewed-on: https://git.coopcloud.tech/coop-cloud/discourse/pulls/14

.drone.yml

@@ -18,6 +18,23 @@ steps:
       STACK_NAME: discourse
       LETS_ENCRYPT_ENV: production
       SECRET_DB_PASSWORD_VERSION: v1
+      DB_ENTRYPOINT_VERSION: v1
 trigger:
   branch:
     - main
+---
+kind: pipeline
+name: generate recipe catalogue
+steps:
+  - name: release a new version
+    image: plugins/downstream
+    settings:
+      server: https://build.coopcloud.tech
+      token:
+        from_secret: drone_abra-bot_token
+      fork: true
+      repositories:
+        - toolshed/auto-recipes-catalogue-json
+
+trigger:
+  event: tag

README.md

@@ -7,7 +7,7 @@ A platform for community discussion
 <!-- metadata -->
 * **Category**: Apps
 * **Status**: 
-* **Image**: [`bitnami/discourse`](https://hub.docker.com/r/bitname/discourse)
+* **Image**: [`bitnami/discourse`](https://hub.docker.com/r/bitnami/discourse)
 * **Healthcheck**: yes
 * **Backups**: no
 * **Email**: yes

abra.sh

@@ -1 +1,2 @@
-export DB_ENTRYPOINT_VERSION=v1
+export DB_ENTRYPOINT_VERSION=v3
+export PG_BACKUP_VERSION=v2

compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 
 services:
   app:
-    image: bitnami/discourse:3.3.1
+    image: bitnamilegacy/discourse:3.5.0
     networks:
       - proxy
       - internal
@@ -43,16 +43,16 @@ services:
         #- "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}-redirect"
         #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLForceHost=true"
         #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLHost=${DOMAIN}"
-        - "coop-cloud.${STACK_NAME}.version=0.7.0+3.3.1"
+        - "coop-cloud.${STACK_NAME}.version=0.8.0+3.5.0"
     healthcheck:
       test: "ruby -e \"require 'uri'; require 'net/http'; uri = URI('http://localhost:3000/srv/status'); res = Net::HTTP.get_response(uri); if res.is_a?(Net::HTTPSuccess) then exit (0) else exit (1) end\""
       interval: 30s
       timeout: 10s
       retries: 6
-      start_period: 5m
+      start_period: 20m
 
   db:
-    image: postgres:13
+    image: pgvector/pgvector:pg17
     networks:
       - internal
     secrets:
@@ -63,6 +63,9 @@ services:
       - source: db_entrypoint
         target: /docker-entrypoint.sh
         mode: 0555
+      - source: pg_backup
+        target: /pg_backup.sh
+        mode: 0555
     entrypoint: /docker-entrypoint.sh
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
@@ -72,9 +75,9 @@ services:
     deploy:
       labels:
         backupbot.backup: "true"
-        backupbot.backup.pre-hook: "bash -c 'PGPASSWORD=$$(cat $${POSTGRES_PASSWORD_FILE}) pg_dump -U $${POSTGRES_USER} $${POSTGRES_DB} > /tmp/backup.sql'"
-        backupbot.backup.post-hook: "rm -rf /tmp/backup.sql"
-        backupbot.backup.path: "/tmp/backup.sql"
+        backupbot.backup.pre-hook: "/pg_backup.sh backup"
+        backupbot.backup.volumes.postgresql_data.path: "backup.sql"
+        backupbot.restore.post-hook: "/pg_backup.sh restore"
 
   redis:
     image: redis:7.4-alpine
@@ -84,7 +87,7 @@ services:
       - 'redis_data:/data'
 
   sidekiq:
-    image: bitnami/discourse:3.3.1
+    image: bitnamilegacy/discourse:3.5.0
     networks:
       - proxy
       - internal
@@ -132,3 +135,6 @@ configs:
     name: ${STACK_NAME}_db_entrypoint_${DB_ENTRYPOINT_VERSION}
     file: entrypoint.postgres.sh.tmpl
     template_driver: golang
+  pg_backup:
+    name: ${STACK_NAME}_pg_backup_${PG_BACKUP_VERSION}
+    file: pg_backup.sh

entrypoint.postgres.sh.tmpl

@@ -2,16 +2,23 @@
 
 set -e
 
-MIGRATION_MARKER=$PGDATA/migration_in_progress
 OLDDATA=$PGDATA/old_data
 NEWDATA=$PGDATA/new_data
 
 echo "Running as $(id)"
 
-if [ -e $MIGRATION_MARKER ]; then
-  echo "FATAL: migration was started but did not complete in a previous run. manual recovery necessary"
-  exit 1
-fi
+# The migration uses $OLDDATA/$NEWDATA as scratch and removes them when it
+# finishes; a leftover *empty* one means a run was interrupted before any data
+# moved (data still intact at $PGDATA) so we clear it and retry, while a
+# *non-empty* one means data may live only there, so we stop for manual recovery.
+for scratch in $OLDDATA $NEWDATA; do
+  if [ -d "$scratch" ] && [ -n "$(ls -A "$scratch")" ]; then
+    echo "FATAL: $scratch exists and is not empty - a previous migration did not"
+    echo "complete and the data may only exist there. manual recovery necessary."
+    exit 1
+  fi
+done
+rm -rf $OLDDATA $NEWDATA
 
 if [ -f $PGDATA/PG_VERSION ]; then
   DATA_VERSION=$(cat $PGDATA/PG_VERSION)
@@ -23,22 +30,33 @@ if [ -f $PGDATA/PG_VERSION ]; then
     apt-get update && apt-get install -y --no-install-recommends \
       postgresql-$DATA_VERSION \
       && rm -rf /var/lib/apt/lists/*
+    # pg_upgrade must run as the old cluster's bootstrap superuser (the "install
+    # user", oid 10), and the new cluster must be initialised with that same
+    # user. It is not necessarily $POSTGRES_USER (e.g. clusters created with the
+    # default "postgres" superuser and a separate app role), so read it from the
+    # old cluster: briefly start it and ask, connecting as the app role we know.
+    PGBIN=/usr/lib/postgresql/$DATA_VERSION/bin
+    gosu postgres $PGBIN/pg_ctl -D $PGDATA -w \
+      -o "-c listen_addresses= -c unix_socket_directories=/tmp" start
+    INSTALL_USER=$(gosu postgres psql -h /tmp -U "$POSTGRES_USER" -d postgres -tAc \
+      "select rolname from pg_roles where oid = 10")
+    gosu postgres $PGBIN/pg_ctl -D $PGDATA -w stop
+    echo "old cluster install user: $INSTALL_USER"
     echo "shuffling around"
     gosu postgres mkdir $OLDDATA $NEWDATA
     chmod 700 $OLDDATA $NEWDATA
     mv $PGDATA/* $OLDDATA/ || true
-    touch $MIGRATION_MARKER
     echo "running initdb"
-    # abuse entrypoint script for initdb by making server error out
-    gosu postgres bash -c "export PGDATA=$NEWDATA ; /usr/local/bin/docker-entrypoint.sh --invalid-arg || true"
+    # abuse entrypoint script for initdb by making server error out; initialise
+    # the new cluster with the same superuser as the old one so pg_upgrade matches
+    gosu postgres bash -c "export PGDATA=$NEWDATA POSTGRES_USER=$INSTALL_USER ; /usr/local/bin/docker-entrypoint.sh --invalid-arg || true"
     echo "running pg_upgrade"
     cd /tmp
-    gosu postgres pg_upgrade --link -b /usr/lib/postgresql/$DATA_VERSION/bin -d $OLDDATA -D $NEWDATA -U $POSTGRES_USER
+    gosu postgres pg_upgrade --link -b /usr/lib/postgresql/$DATA_VERSION/bin -d $OLDDATA -D $NEWDATA -U $INSTALL_USER
     cp $OLDDATA/pg_hba.conf $NEWDATA/
     mv $NEWDATA/* $PGDATA
     rm -rf $OLDDATA
     rmdir $NEWDATA
-    rm $MIGRATION_MARKER
     echo "migration complete"
   fi
 fi

pg_backup.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Postgres backup/restore hook for the discourse `db` service.
+
+set -e
+
+BACKUP_FILE='/var/lib/postgresql/data/backup.sql'
+export PGPASSWORD=$(cat "${POSTGRES_PASSWORD_FILE:-/run/secrets/db_password}")
+DB_USER="${POSTGRES_USER:-discourse}"
+DB_NAME="${POSTGRES_DB:-discourse}"
+
+function backup {
+  pg_dump -U "$DB_USER" "$DB_NAME" | gzip > "$BACKUP_FILE"
+}
+
+function restore {
+  cd /var/lib/postgresql/data/
+
+  # Block all non-local connections so the running discourse app + sidekiq cannot reconnect and
+  # interfere with the drop/recreate/reimport. Restored on exit.
+  restore_hba() {
+    cat pg_hba.conf.bak > pg_hba.conf
+    rm -f pg_hba.conf.bak
+    su postgres -c 'pg_ctl reload'
+  }
+  cp pg_hba.conf pg_hba.conf.bak
+  echo 'local all all trust' > pg_hba.conf
+  su postgres -c 'pg_ctl reload'
+  trap restore_hba EXIT INT TERM
+
+  # terminate any lingering local sessions before recreate
+  # see https://stackoverflow.com/questions/5108876/kill-a-postgresql-session-connection
+  psql -U "$DB_USER" -d postgres -c \
+    "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='${DB_NAME}' AND pid<>pg_backend_pid();"
+
+  # drop database and then recreate it
+  psql -U "$DB_USER" -d postgres -c "DROP DATABASE ${DB_NAME} WITH (FORCE);"
+  createdb -U "$DB_USER" "$DB_NAME"
+
+  # reimport data 
+  gunzip -c "$BACKUP_FILE" | psql -U "$DB_USER" -d "$DB_NAME" -1 -v ON_ERROR_STOP=1 -f -
+}
+
+$@