From ecd770b9ca93a24a1481a175f6b1ed30f9053277 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Sat, 30 May 2026 00:08:10 +0100 Subject: [PATCH] feat(2): immich P3 2nd functional test (asset-processing: metadata extraction + library statistics) + PARITY/DECISIONS for immich postgres-backup recipe-PR Co-Authored-By: Claude Opus 4.8 (1M context) --- machine-docs/DECISIONS.md | 18 +++ tests/immich/PARITY.md | 28 ++-- .../functional/test_asset_processing.py | 123 ++++++++++++++++++ 3 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 tests/immich/functional/test_asset_processing.py diff --git a/machine-docs/DECISIONS.md b/machine-docs/DECISIONS.md index aac0c43..c590ab4 100644 --- a/machine-docs/DECISIONS.md +++ b/machine-docs/DECISIONS.md @@ -901,3 +901,21 @@ Two follow-on fixes from the first full mumble run: was the 1.0.0 release). So backup/restore are only meaningful AFTER the upgrade tier moves the app to head_ref (1.0.0+). With the upgrade fixed, backup/restore run against the backup-aware version and P4 (sqlite ci_marker survival) holds. The base (0.2.0) backup-unaware state is expected, not a defect. + +## immich postgres backup recipe-PR (Phase 2 Q3.5 P4) — 2026-05-30 +**Decision:** fix immich's P4 data-integrity gap with a recipe-PR (`recipe-maintainers/immich#1`), +not a §7.1 P4-N/A sign-off. The *published* immich recipe backs up NO database: `backupbot.backup` +sits only on the `app` service (whose sole data volume `uploads` is excluded), and the +`database`/postgres service had no backup label or pg_dump hook — so restoring a backup yields an +empty DB (total user-metadata loss). immich is the D10 large-volume/**data** category recipe; a P4-N/A +on its data path would be hollow (unlike mailu's mail-relay N/A). cc-ci exists to catch exactly this +class of bug, and the recipe mirror+PR flow (plan §0b/§4.1) is the sanctioned mechanism. +**Fix shape (matrix-synapse convention):** `database`-service `deploy.labels` +`backupbot.backup.pre-hook=/pg_backup.sh backup` + `backupbot.backup.volumes.postgres.path=backup.sql` ++ `backupbot.restore.post-hook=/pg_backup.sh restore`; a `configs:`-mounted `pg_backup.sh` +(top-level `configs.pg_backup`, versioned via `abra.sh` `PG_BACKUP_VERSION=v1`). The script: +backup = `pg_dump | gzip > /var/lib/postgresql/data/backup.sql`; restore = terminate immich-server +connections + `DROP DATABASE … WITH (FORCE)` + `createdb` + reimport (the matrix pg_hba "local trust" +trick does NOT cover immich-server's *networked* connections, so FORCE-drop is required). The +VectorChord/pgvecto.rs extensions (vchord, vector) + all tables round-trip cleanly — validated live, +then proven green end-to-end via `RECIPE=immich PR=1` (restore tier `test_restore_returns_state` PASS). diff --git a/tests/immich/PARITY.md b/tests/immich/PARITY.md index ac03869..b4e1546 100644 --- a/tests/immich/PARITY.md +++ b/tests/immich/PARITY.md @@ -7,18 +7,30 @@ Reference corpus: `references/recipe-maintainer/recipe-info/immich/tests/` (heal |---|---|---| | `health_check.py` | `tests/immich/functional/test_health_check.py::test_immich_returns_200` | HTTP 200/301/302 from `/` (immich web SPA served). | -## Recipe-specific functional tests (P3, ≥2 — characteristic behavior) +## Recipe-specific functional tests (P3, ≥2 separate tests — characteristic behavior) 1. **`test_asset_upload.py::test_immich_upload_asset_readback_and_thumbnail`** — the §4.3 - create-an-object + read-it-back + characteristic feature: admin-sign-up → login → **upload an - asset** (`POST /api/assets` multipart, 201 created with unique content) → **read it back** + create-an-object + read-it-back: admin-sign-up → login → **upload an asset** + (`POST /api/assets` multipart, 201 created with unique content) → **read it back** (`GET /api/assets/{id}` → 200, type IMAGE) → **immich generated a thumbnail/derivative** - (`GET /api/assets/{id}/thumbnail` → 200, polled for the async generation). Real assertions on - stored app state + the generated derivative — immich's whole purpose (photo ingest + derivatives). - This single test covers BOTH the create+read-back floor AND the distinctive derivative-generation - feature (≥2 characteristic behaviors). Verified against immich app version 2.7.5. + (`GET /api/assets/{id}/thumbnail` → 200, polled for the async generation). Asserts stored app + state + the generated derivative — immich's whole purpose (photo ingest + derivatives). +2. **`test_asset_processing.py::test_immich_processes_uploaded_asset_metadata_and_statistics`** — + immich's **asset-processing pipeline**, a distinct microservice path from thumbnailing: after + upload, poll `GET /api/assets/{id}` until the **metadata-extraction** job populated `exifInfo` + with the image dimensions (exifImageWidth/Height == 1 for the 1x1 PNG) → **proves the extraction + pipeline ran**; then `GET /api/assets/statistics` → the asset is **catalogued into the owner's + library** (images/total >= 1). Real assertions on processed state, not a 200 stand-in. + +Both verified against immich app version 2.7.5. Backup data-integrity (P4): Phase-1d/1e lifecycle overlays — `ops.py` seeds a postgres `ci_marker` -(postgres/immich DB, backupbot-labelled); `test_upgrade/backup/restore.py` assert it survives. +(postgres/immich DB); `test_upgrade/backup/restore.py` assert it survives. **P4 requires the +recipe-PR `recipe-maintainers/immich#1`** — the *published* recipe backed up NO database +(`backupbot.backup` sat only on the `app` service with all its volumes excluded; the +`database`/postgres service had no backup hook), so a restore yielded an empty DB. The PR adds a +`database`-service `/pg_backup.sh` config-mount + backupbot pre/restore hooks (matrix-synapse +convention); with it the ci_marker survives the recipe's real backup→restore. See DECISIONS.md +"immich postgres backup recipe-PR". ## Non-ports (documented — §7.1) - **`oidc_login.py`** — authentik-specific in the corpus. Per the operator SSO policy (DECISIONS diff --git a/tests/immich/functional/test_asset_processing.py b/tests/immich/functional/test_asset_processing.py new file mode 100644 index 0000000..b5df995 --- /dev/null +++ b/tests/immich/functional/test_asset_processing.py @@ -0,0 +1,123 @@ +"""immich — 2nd recipe-specific functional test (Phase 2 P3): asset PROCESSING pipeline. + +Distinct from `test_asset_upload.py` (which proves storage round-trip + thumbnail generation): +this exercises immich's **metadata-extraction microservice** and **library cataloging** — after an +upload, immich's background jobs read the image, populate `exifInfo` (image dimensions / file size), +and count the asset into the owner's library statistics. Characteristic immich behavior (it is a media +*management* server, not just a blob store), asserted on real stored app state — not a 200/health +stand-in. + + 1. admin-sign-up / login (first-run local admin; no SSO needed) + 2. POST /api/assets (upload a uniquely-keyed valid PNG) → 201 {id} + 3. poll GET /api/assets/{id} until immich's metadata-extraction job populated exifInfo with the + image dimensions (exifImageWidth/Height) — proves the extraction pipeline ran + 4. GET /api/assets/statistics → the asset is counted in the owner's library (images/total >= 1) + +Verified against immich app version 2.7.5. +""" + +from __future__ import annotations + +import os +import ssl +import sys +import time +import uuid + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "runner")) +from harness import http as harness_http # noqa: E402 + +# A valid 1x1 PNG (immich's decoder reads its dimensions → exifImageWidth/Height = 1). +_PNG = bytes.fromhex( + "89504e470d0a1a0a0000000d4948445200000001000000010806000000" + "1f15c4890000000a49444154789c6300010000050001" + "0d0a2db40000000049454e44ae426082" +) + +_ADMIN = {"email": "ci@ccci.local", "password": "ccciPass12345", "name": "ci"} + + +def _ctx(): + c = ssl.create_default_context() + c.check_hostname = False + c.verify_mode = ssl.CERT_NONE + return c + + +def _upload_asset(base: str, token: str, device_asset_id: str): + import json + import urllib.error + import urllib.request + + boundary = "----ccci" + uuid.uuid4().hex + parts: list[bytes] = [] + + def field(name: str, value: str) -> None: + parts.append( + f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n'.encode() + ) + + field("deviceAssetId", device_asset_id) + field("deviceId", "ccci") + field("fileCreatedAt", "2026-05-29T00:00:00.000Z") + field("fileModifiedAt", "2026-05-29T00:00:00.000Z") + # unique trailing bytes after IEND → unique content checksum so immich does not dedup (returns 201) + png = _PNG + device_asset_id.encode() + parts.append( + f'--{boundary}\r\nContent-Disposition: form-data; name="assetData"; filename="ccci.png"\r\n' + f"Content-Type: image/png\r\n\r\n".encode() + + png + + b"\r\n" + ) + parts.append(f"--{boundary}--\r\n".encode()) + body = b"".join(parts) + req = urllib.request.Request(f"{base}/api/assets", data=body, method="POST") + req.add_header("Content-Type", f"multipart/form-data; boundary={boundary}") + req.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(req, timeout=60, context=_ctx()) as r: + return r.status, json.loads(r.read() or b"null") + except urllib.error.HTTPError as e: + try: + return e.code, json.loads(e.read() or b"null") + except Exception: # noqa: BLE001 + return e.code, None + + +def test_immich_processes_uploaded_asset_metadata_and_statistics(live_app): + base = f"https://{live_app}" + + # admin + login + st, _ = harness_http.http_post(f"{base}/api/auth/admin-sign-up", data=_ADMIN) + assert st in (201, 400, 409), f"admin-sign-up unexpected HTTP {st}" + st, body = harness_http.http_post( + f"{base}/api/auth/login", data={"email": _ADMIN["email"], "password": _ADMIN["password"]} + ) + assert st in (200, 201) and isinstance(body, dict), f"login HTTP {st}: {body!r}" + token = body["accessToken"] + auth = {"Authorization": f"Bearer {token}"} + + # upload a uniquely-keyed asset + st, up = _upload_asset(base, token, f"ccci-meta-{uuid.uuid4().hex}") + assert st == 201 and isinstance(up, dict), f"asset upload HTTP {st}: {up!r}" + asset_id = up["id"] + + # immich's metadata-extraction job populates exifInfo (image dimensions) — poll (async job) + exif = None + for _ in range(24): + gst, got = harness_http.http_request("GET", f"{base}/api/assets/{asset_id}", headers=auth) + assert gst == 200 and isinstance(got, dict), f"asset read HTTP {gst}: {got!r}" + exif = got.get("exifInfo") + if exif and exif.get("exifImageWidth"): + break + time.sleep(5) + assert exif and exif.get("exifImageWidth") == 1 and exif.get("exifImageHeight") == 1, ( + f"immich metadata-extraction did not populate the 1x1 PNG dimensions in exifInfo: {exif!r}" + ) + + # the asset is catalogued into the owner's library statistics (list-back in aggregate) + sst, stats = harness_http.http_request("GET", f"{base}/api/assets/statistics", headers=auth) + assert sst == 200 and isinstance(stats, dict), f"statistics HTTP {sst}: {stats!r}" + assert stats.get("images", 0) >= 1 and stats.get("total", 0) >= 1, ( + f"uploaded asset not reflected in library statistics: {stats!r}" + )