diff --git a/docs/warm.md b/docs/warm.md index 7e03608..566e5d1 100644 --- a/docs/warm.md +++ b/docs/warm.md @@ -86,9 +86,10 @@ back cleanly to a full cold run (the PR is still tested). skips if a `run_recipe_ci.py` is active. At most MAX_TESTS apps are ever live at once. - **Warm keycloak shared safely** via per-run namespaced realms (above); orphan realms reaped. - **Disk** (warm is the budget, not RAM): the `ci-docker-prune` unit (`nix/modules/docker-prune.nix`, - Phase-2pc) prunes only **dangling** images/containers/build-cache (`until=24h`), and only under - genuine disk pressure (`/` ≥ 80%) with nothing in flight — **never `--all`** (keeps cached base/ - in-use images warm; the local store IS the cache on this single host) and **never `--volumes`** (so + Phase-2pc) prunes dangling images/containers/build-cache (`until=24h`) under + disk pressure (`/` ≥ 80%) when no service is converging. At `/` ≥ 90%, it also removes unused + images older than seven days, even if a warm service is crash-looping. It skips while a run-app + stack is live and never uses `--volumes` (so data-warm canonical volumes survive). Each canonical = one data volume + one snapshot (small; the keycloak DB snapshot ~300M dominates). `canonical.prune_stale()` (run nightly) drops warm data for **de-enrolled** canonicals. Monitor with `df -h /` (the nightly logs it). diff --git a/nix/modules/docker-prune.nix b/nix/modules/docker-prune.nix index a809262..1242e84 100644 --- a/nix/modules/docker-prune.nix +++ b/nix/modules/docker-prune.nix @@ -5,24 +5,21 @@ # between CI runs no test apps run, so it evicted the cached recipe base images and forced a cold # re-pull on the next run → the prune->re-pull->Docker-Hub-rate-limit churn documented in JOURNAL-2. # -# On this SINGLE host, Docker's own local image store IS the cache (re-deploys reuse local layers, -# no re-download; the daemon is PAT-authenticated). So we keep that store warm and only reclaim disk -# under GENUINE pressure, and even then SURGICALLY: -# - dangling images + stopped containers + dangling build cache, age-gated (until=24h) — NEVER -# `--all` (would evict tagged base/in-use images), NEVER `--volumes` (warm canonical data — see -# swarm.nix's existing comment; warm volumes are reaped only by the warm reconcilers). -# and only when nothing is in flight: -# - skip if any run-app stack is live (mid-pull layers can look prunable — "never prune mid-run"); -# - skip if any swarm service has unmet replicas (a deploy/pull is converging, incl. warm redeploys). +# On this single host, Docker's local image store is the CI cache. Keep tagged images during +# routine pressure, but evict unused old images at 90% usage. This recovers from a warm service +# crash loop: an unmet replica must not prevent the emergency cleanup that lets it restart. +# Never prune volumes, and skip while a run-app stack is live. { pkgs, ... }: let # `/` usage % at/above which a surgical prune is permitted. Below this: keep the cache, no-op. threshold = 80; + emergencyThreshold = 90; prune = pkgs.writeShellApplication { name = "cc-ci-docker-prune"; runtimeInputs = with pkgs; [ docker coreutils gnugrep gawk ]; text = '' THRESH=${toString threshold} + EMERGENCY_THRESH=${toString emergencyThreshold} used="$(df --output=pcent / | tail -1 | tr -dc '0-9')" : "''${used:=0}" if [ "$used" -lt "$THRESH" ]; then @@ -38,6 +35,14 @@ let echo "docker-prune: a run-app stack is live — skipping (never prune mid-run)" exit 0 fi + if [ "$used" -ge "$EMERGENCY_THRESH" ]; then + echo "docker-prune: / at ''${used}% (>= ''${EMERGENCY_THRESH}%) — emergency prune of unused images older than 7d" + docker container prune -f --filter until=24h || true + docker image prune -af --filter until=168h || true + docker builder prune -f --filter until=24h || true + df -h / + exit 0 + fi # NEVER prune while ANY swarm service is converging (unmet replicas => a pull/deploy in flight, # including infra warm redeploys). Replicas field is "running/desired" e.g. 1/1. converging="$(docker service ls --format '{{.Replicas}}' \ @@ -56,7 +61,7 @@ let in { systemd.services.ci-docker-prune = { - description = "Surgical disk-pressure-gated Docker prune (dangling+old only; never --all/--volumes; never mid-run)"; + description = "Disk-pressure-gated Docker prune (old unused images at 90%; never volumes or mid-run)"; after = [ "docker.service" ]; requires = [ "docker.service" ]; path = [ pkgs.docker ];