chore: regall sweep trigger

.env.sample

@@ -5,19 +5,7 @@ DOMAIN=plausible.example.com
 ## Domain aliases
 #EXTRA_DOMAINS=', `www.plausible.example.com`'
 LETS_ENCRYPT_ENV=production
-#LOG_LEVEL=debug
 
-DISABLE_REGISTRATION=true # true or false
+SECRET_KEY_BASE=replace-me
-
+DISABLE_AUTH=replace-me # true or false
-# Mail Configuration
+DISABLE_REGISTRATION=replace-me # true or false
-#SECRET_SMTP_PASSWORD_VERSION=v1 # generate=false
-#MAILER_ADAPTER=Bamboo.Mua
-#MAILER_EMAIL=noreply@example.com
-#MAILER_NAME=Noreply
-#SMTP_HOST_ADDR=mail.example.com
-#SMTP_HOST_PORT=587
-#SMTP_USER_NAME=foo
-#SMTP_HOST_SSL_ENABLED=false
-
-SECRET_TOTP_VAULT_KEY_VERSION=v1 # length=32 encoding=base64
-SECRET_KEY_BASE_VERSION=v1 # length=64

README.md

@@ -4,13 +4,13 @@
 
 <!-- metadata -->
 * **Category**: Apps
-* **Status**: 3
+* **Status**: 1, alpha
 * **Image**: [`plausible/analytics`](https://hub.docker.com/plausible/analytics), 4, upstream
 * **Healthcheck**:
 * **Backups**: Yes
-* **Email**: Yes
+* **Email**: No
-* **Tests**: No
+* **Tests**:
-* **SSO**: N/A
+* **SSO**: No
 <!-- endmetadata -->
 
 ## Basic usage
@@ -19,11 +19,11 @@
 2. Deploy [`coop-cloud/traefik`]
 3. `abra app new ${REPO_NAME} --secrets` (optionally with `--pass` if you'd like
    to save secrets in `pass`)
-4. `abra app secret insert YOURAPPDOMAIN smtp_password v1` and add your smtp password
+4. `abra app config YOURAPPDOMAIN` - be sure to change `$DOMAIN` to something that resolves to
-5. `abra app config YOURAPPDOMAIN` - be sure to change `$DOMAIN` to something that resolves to
    your Docker swarm box
-6. `abra app deploy YOURAPPDOMAIN`
+5. `abra app deploy YOURAPPDOMAIN`
-7. Open the configured domain in your browser to finish set-up and create an admin account
+6. Open the configured domain in your browser to finish set-up
 
 [`abra`]: https://git.coopcloud.tech/coop-cloud/abra
 [`coop-cloud/traefik`]: https://git.coopcloud.tech/coop-cloud/traefik
+p-cloud/traefik

abra.sh

@@ -1,6 +1,4 @@
-export CLICKHOUSE_CONFIG_VERSION=v2
+export CLICKHOUSE_CONF_VERSION=v2
-export CLICKHOUSE_USER_CONFIG_VERSION=v2
+export CLICKHOUSE_USER_CONF_VERSION=v2
-export CLICKHOUSE_ENTRYPOINT_VERSION=v7
+export DB_ENTRYPOINT_VERSION=v1
-export PG_BACKUP_VERSION=v1
+export CLICKHOUSE_ENTRYPOINT_VERSION=v2
-export CLICKHOUSE_BACKUP_VERSION=v1
-export PLAUSIBLE_ENTRYPOINT_VERSION=v2

clickhouse_backup.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# clickhouse-backup output lives inside the event-data volume (snapshotted via
-# backupbot.backup.volumes.event-data.path). Restoring the raw data files under a
-# running server is unsafe, so restore performs a logical restore instead.
-BACKUP_DIR=/var/lib/clickhouse/backup/events
-MIGRATIONS_TSV="$BACKUP_DIR/schema_migrations.tsv"
-
-backup() {
-  clickhouse-backup create events
-  # schema_migrations is a TinyLog table — clickhouse-backup only FREEZEs MergeTree
-  # data, so its rows aren't captured. Export them alongside the backup, else a restore
-  # leaves the ledger empty and the next boot re-runs every migration (DUPLICATE_COLUMN).
-  clickhouse-client --query "SELECT * FROM plausible_events_db.schema_migrations FORMAT TSV" > "$MIGRATIONS_TSV"
-}
-
-backup_cleanup() {
-  rm -rf "$BACKUP_DIR"
-}
-
-restore() {
-  clickhouse-backup restore --rm events
-  clickhouse-client --query "TRUNCATE TABLE plausible_events_db.schema_migrations"
-  clickhouse-client --query "INSERT INTO plausible_events_db.schema_migrations FORMAT TSV" < "$MIGRATIONS_TSV"
-  rm -rf "$BACKUP_DIR"
-}
-
-"$@"

compose.yml

@@ -3,117 +3,84 @@ version: "3.8"
 
 services:
   app:
-    image: ghcr.io/plausible/community-edition:v2.1.0
+    image: plausible/analytics:v2.0.0
-    entrypoint: /plausible_entrypoint.sh
+    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
     depends_on:
       - db
-      - events
+      - events_db
     environment:
       - BASE_URL=https://$DOMAIN
       - SECRET_KEY_BASE
       - DATABASE_URL=postgres://plausible:plausible@${STACK_NAME}_db:5432/plausible
-      - CLICKHOUSE_DATABASE_URL=http://${STACK_NAME}_events:8123/plausible_events_db
-      - MAILER_ADAPTER
       - SMTP_HOST_ADDR
       - MAILER_EMAIL
-      - MAILER_NAME
       - SMTP_HOST_PORT
       - SMTP_USER_NAME
       - SMTP_USER_PWD
       - SMTP_HOST_SSL_ENABLED
       - DISABLE_REGISTRATION
-      - LOG_LEVEL
+      - DISABLE_AUTH
-    secrets:
-      - totp_secret
-      - key_base
-      - smtp_password
-    configs:
-      - source: plausible_entrypoint
-        target: /plausible_entrypoint.sh
-        mode: 0555
     networks:
       - proxy
       - internal
-    healthcheck:
-          test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8000"]
-          interval: 15s
-          timeout: 10s
-          retries: 10
-          start_period: 15s
     deploy:
       restart_policy:
-        # `any`, not `on-failure`: a restore disrupts postgres under the app and Erlang then
+        condition: on-failure
-        # shuts down gracefully (exit 0), which on-failure treats as done and never restarts.
-        condition: any
       labels:
         - "traefik.enable=true"
         - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=8000"
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - coop-cloud.${STACK_NAME}.version=4.0.0+v2.1.0
+        - coop-cloud.${STACK_NAME}.version=3.0.1+v2.0.0
   db:
-    image: pgautoupgrade/pgautoupgrade:18-alpine
+    image: postgres:13.12
+    configs:
+      - source: db_entrypoint
+        target: /docker-entrypoint.sh
+        mode: 0555
+    # Custom docker entrypoint to handle major Postgres version upgrades
     volumes:
       - db-data:/var/lib/postgresql/data
+    entrypoint: /docker-entrypoint.sh
     environment:
-      # pin legacy PGDATA so the existing cluster on the volume is upgraded in place, not re-init'd
-      - PGDATA=/var/lib/postgresql/data
       - POSTGRES_USER=plausible
       - POSTGRES_PASSWORD=plausible
       - POSTGRES_DB=plausible
     networks:
       - internal
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U plausible -d plausible"]
-      interval: 5s
-      timeout: 5s
-      retries: 60
-    configs:
-      - source: pg_backup
-        target: /pg_backup.sh
-        mode: 0555
     deploy:
       labels:
         backupbot.backup: "true"
-        backupbot.backup.volumes.db-data.path: "postgres.dump.gz"
+        backupbot.backup.pre-hook: sh -c 'pg_dump -U "$$POSTGRES_USER" -Fc "$$POSTGRES_DB" | gzip > "/postgres.dump.gz"'
-        backupbot.backup.pre-hook: "/pg_backup.sh backup"
+        backupbot.backup.path: "/postgres.dump.gz"
-        backupbot.backup.post-hook: "/pg_backup.sh backup_cleanup"
+        backupbot.backup.post-hook: "rm -f /postgres.dump.gz"
         backupbot.restore: "true"
-        backupbot.restore.post-hook: "/pg_backup.sh restore"
+        backupbot.restore.post-hook: sh -c 'gzip -d /postgres.dump.gz && pg_restore --clean -U "$$POSTGRES_USER" --dbname="$$PLAUSIBLE_DB" < /postgres.dump && rm -f /postgres.dump'
 
-  events:
+  plausible_events_db:
-    image: clickhouse/clickhouse-server:24.3.3.102-alpine
+    image: clickhouse/clickhouse-server:23.4.2.11-alpine
     volumes:
       - event-data:/var/lib/clickhouse
     entrypoint: /custom-entrypoint.sh
     configs:
-      - source: clickhouse_config
+      - source: clickhouse-config
         target: /etc/clickhouse-server/config.d/logging.xml
-      - source: clickhouse_user_config
+      - source: clickhouse-user-config
         target: /etc/clickhouse-server/users.d/clickhouse-user-config.xml
       - source: clickhouse_entrypoint
         target: /custom-entrypoint.sh
         mode: 0555
-      - source: clickhouse_backup
-        target: /clickhouse_backup.sh
-        mode: 0555
     networks:
       - internal
-    healthcheck:
-          test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8123/ping || exit 1"]
-          interval: 15s
-          timeout: 10s
-          retries: 10
-          start_period: 15s
     deploy:
       labels:
         backupbot.backup: "true"
-        backupbot.backup.volumes.event-data.path: "backup/events"
+        backupbot.backup.pre-hook: clickhouse-backup create events
-        backupbot.backup.pre-hook: "/clickhouse_backup.sh backup"
+        backupbot.backup.path: "/var/lib/clickhouse/backup/events"
-        backupbot.backup.post-hook: "/clickhouse_backup.sh backup_cleanup"
+        backupbot.backup.post-hook: "rm -rf /var/lib/clickhouse/backup/events"
         backupbot.restore: "true"
-        backupbot.restore.post-hook: "/clickhouse_backup.sh restore"
+        backupbot.restore.post-hook: clickhouse-backup restore --rm events && rm -rf /var/lib/clickhouse/backup/events"
 
 volumes:
   db-data:
@@ -125,32 +92,16 @@ networks:
   internal:
 
 configs:
-  plausible_entrypoint:
+  clickhouse-config:
-    name: ${STACK_NAME}_plausible_entrypoint_${PLAUSIBLE_ENTRYPOINT_VERSION}
+    name: ${STACK_NAME}_clickhouse_config_${CLICKHOUSE_CONF_VERSION}
-    file: plausible_entrypoint.sh
-  clickhouse_config:
-    name: ${STACK_NAME}_clickhouse_config_${CLICKHOUSE_CONFIG_VERSION}
     file: clickhouse-config.xml
-  clickhouse_user_config:
+  clickhouse-user-config:
-    name: ${STACK_NAME}_clickhouse_user_config_${CLICKHOUSE_USER_CONFIG_VERSION}
+    name: ${STACK_NAME}_clickhouse_user_config_${CLICKHOUSE_USER_CONF_VERSION}
     file: clickhouse-user-config.xml
+  db_entrypoint:
+    name: ${STACK_NAME}_db_entrypoint_${DB_ENTRYPOINT_VERSION}
+    file: entrypoint.postgres.sh.tmpl
+    template_driver: golang
   clickhouse_entrypoint:
     name: ${STACK_NAME}_clickhouse_entrypoint_${CLICKHOUSE_ENTRYPOINT_VERSION}
     file: entrypoint.clickhouse.sh
-  pg_backup:
-    name: ${STACK_NAME}_pg_backup_${PG_BACKUP_VERSION}
-    file: pg_backup.sh
-  clickhouse_backup:
-    name: ${STACK_NAME}_clickhouse_backup_${CLICKHOUSE_BACKUP_VERSION}
-    file: clickhouse_backup.sh
-
-secrets:
-  totp_secret:
-    name: ${STACK_NAME}_totp_secret_${SECRET_TOTP_VAULT_KEY_VERSION}
-    external: true
-  key_base:
-    name: ${STACK_NAME}_key_base_${SECRET_KEY_BASE_VERSION}
-    external: true
-  smtp_password:
-    name: ${STACK_NAME}_smtp_password_${SECRET_SMTP_PASSWORD_VERSION}
-    external: true

entrypoint.clickhouse.sh

@@ -1,13 +1,8 @@
 #!/bin/bash
-# Install clickhouse-backup (powers this recipe's backup/restore hooks) before starting the
-# server. The binary is cached on the persistent volume keyed by version (downloaded at most
-# once per app) and fetched with bounded retries + a read timeout; the binary is verified before
-# being trusted or cached. If it truly cannot be installed the deploy fails loudly rather than
-# silently shipping broken backups.
 
-set -e
+set -ex
 
-CLICKHOUSE_BACKUP_VERSION=2.8.1
+CLICKHOUSE_BACKUP_VERSION=2.4.2
 
 ARCH=$(uname -m)
 if [[ $ARCH =~ "aarch64" ]]; then
@@ -22,38 +17,13 @@ elif [[ $ARCH =~ "x86_64" ]]; then
   ARCH="amd64"
 fi
 
-CACHE_DIR=/var/lib/clickhouse/.ccci-bin
+wget \
-CACHED="${CACHE_DIR}/clickhouse-backup-v${CLICKHOUSE_BACKUP_VERSION}"
+    --quiet \
-BIN=/usr/local/bin/clickhouse-backup
+    --continue \
-URL="https://github.com/Altinity/clickhouse-backup/releases/download/v${CLICKHOUSE_BACKUP_VERSION}/clickhouse-backup-linux-${ARCH}.tar.gz"
+    --no-clobber \
+    --output-document=/tmp/clickhouse-backup.tar.gz \
+    "https://github.com/AlexAkulov/clickhouse-backup/releases/download/v${CLICKHOUSE_BACKUP_VERSION}/clickhouse-backup-linux-${ARCH}.tar.gz" 2>/dev/null
 
-binary_ok() {
+tar -xf /tmp/clickhouse-backup.tar.gz --directory=/usr/local/bin --strip-components=3
-  "$1" --version >/dev/null 2>&1
-}
 
-install_clickhouse_backup() {
+/entrypoint.sh
-  mkdir -p "$CACHE_DIR"
-  if [ -x "$CACHED" ] && binary_ok "$CACHED"; then
-    cp -f "$CACHED" "$BIN"
-    echo "clickhouse-backup: using verified cached binary ($CACHED)"
-    return 0
-  fi
-  rm -f "$CACHED" # absent or fails to execute — re-fetch
-  for attempt in 1 2 3 4 5; do
-    if wget -T 30 --continue --output-document=/tmp/clickhouse-backup.tar.gz "$URL" \
-       && tar -xf /tmp/clickhouse-backup.tar.gz --directory=/usr/local/bin --strip-components=3 \
-       && binary_ok "$BIN"; then
-      cp -f "$BIN" "$CACHED" 2>/dev/null || true
-      echo "clickhouse-backup: downloaded, verified + cached (attempt ${attempt})"
-      return 0
-    fi
-    echo "clickhouse-backup: fetch attempt ${attempt}/5 failed" >&2
-    [ "$attempt" -lt 5 ] && sleep $((attempt * 10))
-  done
-  echo "clickhouse-backup: could not install after 5 attempts — failing the deploy (without it backup/restore would be silently broken)" >&2
-  return 1
-}
-
-install_clickhouse_backup
-
-exec /entrypoint.sh

entrypoint.postgres.sh.tmpl

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+
+MIGRATION_MARKER=$PGDATA/migration_in_progress
+OLDDATA=$PGDATA/old_data
+NEWDATA=$PGDATA/new_data
+
+if [ -e $MIGRATION_MARKER ]; then
+  echo "FATAL: migration was started but did not complete in a previous run. manual recovery necessary"
+  exit 1
+fi
+
+if [ -f $PGDATA/PG_VERSION ]; then
+  DATA_VERSION=$(cat $PGDATA/PG_VERSION)
+
+  if [ -n "$DATA_VERSION" -a "$PG_MAJOR" != "$DATA_VERSION" ]; then
+    echo "postgres data version $DATA_VERSION found, but need $PG_MAJOR. Starting migration"
+    echo "Installing postgres $DATA_VERSION"
+    sed -i "s/$/ $DATA_VERSION/" /etc/apt/sources.list.d/pgdg.list
+    apt-get update && apt-get install -y --no-install-recommends \
+      postgresql-$DATA_VERSION \
+      && rm -rf /var/lib/apt/lists/*
+    echo "shuffling around"
+    gosu postgres mkdir $OLDDATA $NEWDATA
+    chmod 700 $OLDDATA $NEWDATA
+    mv $PGDATA/* $OLDDATA/ || true
+    touch $MIGRATION_MARKER
+    echo "running initdb"
+    # abuse entrypoint script for initdb by making server error out
+    gosu postgres bash -c "export PGDATA=$NEWDATA ; /usr/local/bin/docker-entrypoint.sh --invalid-arg || true"
+    echo "running pg_upgrade"
+    cd /tmp
+    gosu postgres pg_upgrade --link -b /usr/lib/postgresql/$DATA_VERSION/bin -d $OLDDATA -D $NEWDATA -U $POSTGRES_USER
+    cp $OLDDATA/pg_hba.conf $NEWDATA/
+    mv $NEWDATA/* $PGDATA
+    rm -rf $OLDDATA
+    rmdir $NEWDATA
+    rm $MIGRATION_MARKER
+    echo "migration complete"
+  fi
+fi
+
+/usr/local/bin/docker-entrypoint.sh postgres

pg_backup.sh

@@ -1,29 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# The dump lives at the db-data volume root: backup-bot-two v2 snapshots paths inside
-# named volumes (backupbot.backup.volumes.db-data.path), not the container root fs.
-DUMP=/var/lib/postgresql/data/postgres.dump
-
-backup() {
-  pg_dump -U "$POSTGRES_USER" -Fc "$POSTGRES_DB" | gzip > "$DUMP.gz"
-}
-
-backup_cleanup() {
-  rm -f "$DUMP.gz"
-}
-
-restore() {
-  gzip -d "$DUMP.gz"
-  # --if-exists: otherwise DROPs on objects absent from the live db error out and
-  # pg_restore exits 1, killing the chain and leaving the dump behind.
-  pg_restore --clean --if-exists -U "$POSTGRES_USER" --dbname="$POSTGRES_DB" < "$DUMP"
-  rm -f "$DUMP"
-  # pg_restore --clean recreates objects under the live app, so its pooled connections
-  # keep stale type-OID caches ('cache lookup failed for type ...' crash loops, e.g.
-  # Oban). Terminate them so Ecto reconnects fresh.
-  psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid();"
-}
-
-"$@"

plausible_entrypoint.sh

@@ -1,9 +0,0 @@
-#!/bin/sh
-
-set -e
-
-[ -f /run/secrets/totp_secret ] && export TOTP_VAULT_KEY="$(cat /run/secrets/totp_secret)"
-[ -f /run/secrets/key_base ] && export SECRET_KEY_BASE="$(cat /run/secrets/key_base)"
-[ -f /run/secrets/smtp_password ] && export SMTP_USER_PWD="$(cat /run/secrets/smtp_password)"
-
-sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"

release/4.0.0+v2.1.0

@@ -1 +0,0 @@
-Switch to the new container of the Plausible Community Edition. Two new secrets need to be added. mail support is now available optionally. the clickhouse container was renamed, hopefully this does not lead to hickups.