fix: avoid stale OIDC secret during canonical promotion

Merge pull request enroll(rustdesk-server): test suite + bridge poll

rustdesk-server enrolled: status-key health overlay (sidecar serving the real hbbs public key), rendezvous ports + ws handshake + identity-key + punch-hole-amplification tests. Cold full-suite green on recipe main (punch-hole correctly skipped on 1.1.12) and on the upgrade head upgrade-2e65135 (passed actively on 1.1.16). Bridge polls recipe-maintainers/rustdesk-server.

docs/warm.md

@@ -86,9 +86,10 @@ back cleanly to a full cold run (the PR is still tested).
   skips if a `run_recipe_ci.py` is active. At most MAX_TESTS apps are ever live at once.
 - **Warm keycloak shared safely** via per-run namespaced realms (above); orphan realms reaped.
 - **Disk** (warm is the budget, not RAM): the `ci-docker-prune` unit (`nix/modules/docker-prune.nix`,
-  Phase-2pc) prunes only **dangling** images/containers/build-cache (`until=24h`), and only under
-  genuine disk pressure (`/` ≥ 80%) with nothing in flight — **never `--all`** (keeps cached base/
-  in-use images warm; the local store IS the cache on this single host) and **never `--volumes`** (so
+  Phase-2pc) prunes dangling images/containers/build-cache (`until=24h`) under
+  disk pressure (`/` ≥ 80%) when no service is converging. At `/` ≥ 90%, it also removes unused
+  images older than seven days, even if a warm service is crash-looping. It skips while a run-app
+  stack is live and never uses `--volumes` (so
   data-warm canonical volumes survive). Each canonical = one data volume + one snapshot (small; the
   keycloak DB snapshot ~300M dominates). `canonical.prune_stale()` (run nightly) drops warm data for
   **de-enrolled** canonicals. Monitor with `df -h /` (the nightly logs it).

nix/modules/docker-prune.nix

@@ -5,24 +5,21 @@
 # between CI runs no test apps run, so it evicted the cached recipe base images and forced a cold
 # re-pull on the next run → the prune->re-pull->Docker-Hub-rate-limit churn documented in JOURNAL-2.
 #
-# On this SINGLE host, Docker's own local image store IS the cache (re-deploys reuse local layers,
-# no re-download; the daemon is PAT-authenticated). So we keep that store warm and only reclaim disk
-# under GENUINE pressure, and even then SURGICALLY:
-#   - dangling images + stopped containers + dangling build cache, age-gated (until=24h) — NEVER
-#     `--all` (would evict tagged base/in-use images), NEVER `--volumes` (warm canonical data — see
-#     swarm.nix's existing comment; warm volumes are reaped only by the warm reconcilers).
-# and only when nothing is in flight:
-#   - skip if any run-app stack is live (mid-pull layers can look prunable — "never prune mid-run");
-#   - skip if any swarm service has unmet replicas (a deploy/pull is converging, incl. warm redeploys).
+# On this single host, Docker's local image store is the CI cache. Keep tagged images during
+# routine pressure, but evict unused old images at 90% usage. This recovers from a warm service
+# crash loop: an unmet replica must not prevent the emergency cleanup that lets it restart.
+# Never prune volumes, and skip while a run-app stack is live.
 { pkgs, ... }:
 let
   # `/` usage % at/above which a surgical prune is permitted. Below this: keep the cache, no-op.
   threshold = 80;
+  emergencyThreshold = 90;
   prune = pkgs.writeShellApplication {
     name = "cc-ci-docker-prune";
     runtimeInputs = with pkgs; [ docker coreutils gnugrep gawk ];
     text = ''
       THRESH=${toString threshold}
+      EMERGENCY_THRESH=${toString emergencyThreshold}
       used="$(df --output=pcent / | tail -1 | tr -dc '0-9')"
       : "''${used:=0}"
       if [ "$used" -lt "$THRESH" ]; then
@@ -38,6 +35,14 @@ let
         echo "docker-prune: a run-app stack is live — skipping (never prune mid-run)"
         exit 0
       fi
+      if [ "$used" -ge "$EMERGENCY_THRESH" ]; then
+        echo "docker-prune: / at ''${used}% (>= ''${EMERGENCY_THRESH}%) — emergency prune of unused images older than 7d"
+        docker container prune -f --filter until=24h || true
+        docker image prune -af --filter until=168h || true
+        docker builder prune -f --filter until=24h || true
+        df -h /
+        exit 0
+      fi
       # NEVER prune while ANY swarm service is converging (unmet replicas => a pull/deploy in flight,
       # including infra warm redeploys). Replicas field is "running/desired" e.g. 1/1.
       converging="$(docker service ls --format '{{.Replicas}}' \
@@ -56,7 +61,7 @@ let
 in
 {
   systemd.services.ci-docker-prune = {
-    description = "Surgical disk-pressure-gated Docker prune (dangling+old only; never --all/--volumes; never mid-run)";
+    description = "Disk-pressure-gated Docker prune (old unused images at 90%; never volumes or mid-run)";
     after = [ "docker.service" ];
     requires = [ "docker.service" ];
     path = [ pkgs.docker ];

runner/harness/abra.py

@@ -126,6 +126,25 @@ def recipe_checkout(recipe: str, version: str) -> None:
     subprocess.run(["git", "-C", path, "checkout", "-f", "--quiet", version], check=True)
 
 
+def ensure_secret_versions_from_recipe(recipe: str, domain: str) -> None:
+    """Fill missing secret versions after a pinned checkout.
+
+    `app new <recipe> <version>` creates its .env from whichever recipe tree was
+    checked out *before* it pins the version. A removed secret in the current
+    recipe can therefore be missing from the older base's .env. Keep existing
+    assignments, including versions intentionally changed by install hooks.
+    """
+    sample = os.path.join(recipe_dir(recipe), ".env.sample")
+    if not os.path.isfile(sample):
+        return
+    pattern = re.compile(r"^\s*(SECRET_[A-Z0-9_]+_VERSION)=(v[A-Za-z0-9._-]+)")
+    with open(sample) as fh:
+        for line in fh:
+            match = pattern.match(line)
+            if match and env_get(domain, match.group(1)) is None:
+                env_set(domain, match.group(1), match.group(2))
+
+
 def has_lightweight_version_tags(recipe: str) -> bool:
     """True if the recipe's local checkout has any LIGHTWEIGHT (non-annotated) version tag.
 

runner/harness/lifecycle.py

@@ -444,6 +444,7 @@ def deploy_app(
         )
     elif version:
         abra.recipe_checkout(recipe, version)
+        abra.ensure_secret_versions_from_recipe(recipe, domain)
         # A pinned (non-chaos) deploy runs `abra recipe lint`, which FATAs R014 ('only annotated
         # tags') if the upstream recipe ships a stray lightweight version tag (e.g. lasuite-meet's
         # 0.3.0+v1.16.0). In that case deploy the EXPLICITLY-checked-out pinned version with chaos:

tests/lasuite-docs/install_steps.sh

@@ -20,6 +20,24 @@ set -euo pipefail
 : "${CCCI_APP_DOMAIN:?missing}"
 ENV_PATH="${CCCI_APP_ENV:?missing}"
 
+# The v6 release requires two RSA PEM secrets with generate=false. CI must
+# supply those operator-managed keys before deploy, including when the base
+# is v5 and the upgrade head is v6. A PR head with its own key sidecar ignores
+# these secrets after upgrade.
+STACK_NAME="${CCCI_APP_DOMAIN//./_}"
+for SECRET_NAME in jwt_pk yhub_jwt_pk; do
+  ENV_NAME="SECRET_${SECRET_NAME^^}_VERSION"
+  VERSION=$(sed -n "s/^${ENV_NAME}=\([^ #]*\).*/\1/p" "$ENV_PATH" | tail -1)
+  if [ -n "$VERSION" ]; then
+    DOCKER_SECRET="${STACK_NAME}_${SECRET_NAME}_${VERSION}"
+    if ! docker secret inspect "$DOCKER_SECRET" >/dev/null 2>&1; then
+      openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 2>/dev/null \
+        | docker secret create "$DOCKER_SECRET" - >/dev/null
+      echo "  install_steps: supplied ${SECRET_NAME} for the published base recipe"
+    fi
+  fi
+done
+
 # No deps file / no keycloak entry → install-time provisioning failed or was skipped. NO-OP so the
 # recipe still boots; the @requires_deps OIDC custom test then SKIPs and F2-11 flips the run RED.
 if [ -z "${CCCI_DEPS_FILE:-}" ] || [ ! -s "${CCCI_DEPS_FILE}" ]; then
@@ -42,6 +60,13 @@ echo "  lasuite-docs install_steps: wiring OIDC at install against keycloak ${KC
 # can be created independently — so the single deploy below picks up v2.
 CUR_VER=$(grep -E '^\s*SECRET_OIDC_RPCS_VERSION=' "$ENV_PATH" | tail -1 | cut -d= -f2 | tr -d '"\r' || echo "v1")
 NEW_NUM=$((${CUR_VER#v} + 1))
+# A previous warm canonical may have left the next version behind. Its value
+# belongs to an old Keycloak realm, so select a fresh version instead of
+# reusing it or failing promotion with Swarm's AlreadyExists error.
+STACK_NAME="${CCCI_APP_DOMAIN//./_}"
+while docker secret inspect "${STACK_NAME}_oidc_rpcs_v${NEW_NUM}" >/dev/null 2>&1; do
+  NEW_NUM=$((NEW_NUM + 1))
+done
 NEW_VER="v${NEW_NUM}"
 INSERT_LOG=$(abra app secret insert "$CCCI_APP_DOMAIN" oidc_rpcs "$NEW_VER" "$KC_SECRET" --no-input -C -o 2>&1) ||
   INSERT_LOG=$(script -qec "abra app secret insert $CCCI_APP_DOMAIN oidc_rpcs $NEW_VER $KC_SECRET --no-input -C -o" /dev/null 2>&1) ||

tests/unit/test_abra_secret_versions.py

@@ -0,0 +1,30 @@
+"""Pinned base recipes need the secret versions declared by their own tag."""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
+from harness import abra  # noqa: E402
+
+
+def test_pinned_base_adds_removed_secret_without_overwriting_existing_version(
+    monkeypatch, tmp_path
+):
+    monkeypatch.setenv("HOME", str(tmp_path))
+    monkeypatch.setenv("ABRA_DIR", str(tmp_path / ".abra"))
+    recipe = tmp_path / ".abra" / "recipes" / "lasuite-docs"
+    recipe.mkdir(parents=True)
+    (recipe / ".env.sample").write_text(
+        "SECRET_COLLAB_SS_VERSION=v1\n"
+        "SECRET_JWT_PK_VERSION=v1 # generate=false\n"
+        "# SECRET_UNUSED_VERSION=v1\n"
+    )
+    env = tmp_path / ".abra" / "servers" / "default" / "test.example.env"
+    env.parent.mkdir(parents=True)
+    env.write_text("SECRET_JWT_PK_VERSION=v2\n")
+
+    abra.ensure_secret_versions_from_recipe("lasuite-docs", "test.example")
+
+    assert abra.env_get("test.example", "SECRET_COLLAB_SS_VERSION") == "v1"
+    assert abra.env_get("test.example", "SECRET_JWT_PK_VERSION") == "v2"
+    assert abra.env_get("test.example", "SECRET_UNUSED_VERSION") is None

tests/unit/test_previous.py

@@ -127,7 +127,13 @@ def test_previous_overlay_deploys_checked_out_base_without_clean_tree_gate(monke
         "prepull_images",
     ):
         monkeypatch.setattr(lifecycle, name, lambda *a, **kw: None)
-    for name in ("app_config_remove", "app_new", "env_set", "secret_generate"):
+    for name in (
+        "app_config_remove",
+        "app_new",
+        "env_set",
+        "secret_generate",
+        "ensure_secret_versions_from_recipe",
+    ):
         monkeypatch.setattr(lifecycle.abra, name, lambda *a, **kw: None)
     monkeypatch.setattr(lifecycle.meta_mod, "extra_env", lambda *a: {})
     monkeypatch.setattr(lifecycle.meta_mod, "hook_ctx", lambda *a: None)