fix: reclaim unused images during critical disk pressure

docs/warm.md

@@ -86,9 +86,10 @@ back cleanly to a full cold run (the PR is still tested).
   skips if a `run_recipe_ci.py` is active. At most MAX_TESTS apps are ever live at once.
 - **Warm keycloak shared safely** via per-run namespaced realms (above); orphan realms reaped.
 - **Disk** (warm is the budget, not RAM): the `ci-docker-prune` unit (`nix/modules/docker-prune.nix`,
-  Phase-2pc) prunes only **dangling** images/containers/build-cache (`until=24h`), and only under
-  genuine disk pressure (`/` ≥ 80%) with nothing in flight — **never `--all`** (keeps cached base/
-  in-use images warm; the local store IS the cache on this single host) and **never `--volumes`** (so
+  Phase-2pc) prunes dangling images/containers/build-cache (`until=24h`) under
+  disk pressure (`/` ≥ 80%) when no service is converging. At `/` ≥ 90%, it also removes unused
+  images older than seven days, even if a warm service is crash-looping. It skips while a run-app
+  stack is live and never uses `--volumes` (so
   data-warm canonical volumes survive). Each canonical = one data volume + one snapshot (small; the
   keycloak DB snapshot ~300M dominates). `canonical.prune_stale()` (run nightly) drops warm data for
   **de-enrolled** canonicals. Monitor with `df -h /` (the nightly logs it).

nix/modules/docker-prune.nix

@@ -5,24 +5,21 @@
 # between CI runs no test apps run, so it evicted the cached recipe base images and forced a cold
 # re-pull on the next run → the prune->re-pull->Docker-Hub-rate-limit churn documented in JOURNAL-2.
 #
-# On this SINGLE host, Docker's own local image store IS the cache (re-deploys reuse local layers,
-# no re-download; the daemon is PAT-authenticated). So we keep that store warm and only reclaim disk
-# under GENUINE pressure, and even then SURGICALLY:
-#   - dangling images + stopped containers + dangling build cache, age-gated (until=24h) — NEVER
-#     `--all` (would evict tagged base/in-use images), NEVER `--volumes` (warm canonical data — see
-#     swarm.nix's existing comment; warm volumes are reaped only by the warm reconcilers).
-# and only when nothing is in flight:
-#   - skip if any run-app stack is live (mid-pull layers can look prunable — "never prune mid-run");
-#   - skip if any swarm service has unmet replicas (a deploy/pull is converging, incl. warm redeploys).
+# On this single host, Docker's local image store is the CI cache. Keep tagged images during
+# routine pressure, but evict unused old images at 90% usage. This recovers from a warm service
+# crash loop: an unmet replica must not prevent the emergency cleanup that lets it restart.
+# Never prune volumes, and skip while a run-app stack is live.
 { pkgs, ... }:
 let
   # `/` usage % at/above which a surgical prune is permitted. Below this: keep the cache, no-op.
   threshold = 80;
+  emergencyThreshold = 90;
   prune = pkgs.writeShellApplication {
     name = "cc-ci-docker-prune";
     runtimeInputs = with pkgs; [ docker coreutils gnugrep gawk ];
     text = ''
       THRESH=${toString threshold}
+      EMERGENCY_THRESH=${toString emergencyThreshold}
       used="$(df --output=pcent / | tail -1 | tr -dc '0-9')"
       : "''${used:=0}"
       if [ "$used" -lt "$THRESH" ]; then
@@ -38,6 +35,14 @@ let
         echo "docker-prune: a run-app stack is live — skipping (never prune mid-run)"
         exit 0
       fi
+      if [ "$used" -ge "$EMERGENCY_THRESH" ]; then
+        echo "docker-prune: / at ''${used}% (>= ''${EMERGENCY_THRESH}%) — emergency prune of unused images older than 7d"
+        docker container prune -f --filter until=24h || true
+        docker image prune -af --filter until=168h || true
+        docker builder prune -f --filter until=24h || true
+        df -h /
+        exit 0
+      fi
       # NEVER prune while ANY swarm service is converging (unmet replicas => a pull/deploy in flight,
       # including infra warm redeploys). Replicas field is "running/desired" e.g. 1/1.
       converging="$(docker service ls --format '{{.Replicas}}' \
@@ -56,7 +61,7 @@ let
 in
 {
   systemd.services.ci-docker-prune = {
-    description = "Surgical disk-pressure-gated Docker prune (dangling+old only; never --all/--volumes; never mid-run)";
+    description = "Disk-pressure-gated Docker prune (old unused images at 90%; never volumes or mid-run)";
     after = [ "docker.service" ];
     requires = [ "docker.service" ];
     path = [ pkgs.docker ];