feat(canon): M1.2 release-tag trigger + faithful mirror-sync in the weekly sweep (§2.C/§2.D)

- warm_reconcile.sweep_decision(latest_tag, canon_version): pure new-release-tag trigger keyed on version_key (NOT commit) — new tag>canon → run; ==/older → skip no-new-version (even with untagged main commits); no tag → skip never-released. Unit-tested. - scripts/recipe-mirror-sync.sh: faithful mirror sync (adapted from open-recipe-pr.sh --reconcile-only) — explicit coopcloud `upstream` remote (robust to inconsistent clone remotes), syncs main+TAGS, closes merged-upstream PRs, leaves unrelated PRs, bot-token auth. - nightly_sweep rewritten: per enrolled recipe → mirror_sync → fetch → sweep_decision → run_on_tag (checkout the release tag + CCCI_SKIP_FETCH=1 so head IS the tag → tagged-promote gate passes, REF empty → promote allowed). Skips logged; run-twice → skip-all determinism. - smoke-tested recipe-mirror-sync.sh live on custom-html: faithful no-op main/tags push, closed merged-upstream PR #2, left pending PR #5. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-17 06:45:37 +00:00
parent f089c30040
commit a20890a363
4 changed files with 234 additions and 29 deletions
--- a/runner/nightly_sweep.py
+++ b/runner/nightly_sweep.py
@ -1,18 +1,25 @@
 #!/usr/bin/env python3
-"""Nightly full-cold sweep (Phase 2w / WC6).
+"""Weekly canonical sweep (Phase 2w / WC6 + phase canon).

-Invoked by the `nightly-sweep` systemd timer (nix/modules/nightly-sweep.nix). Order (plan WC6):
+Invoked by the `nightly-sweep` systemd timer (nix/modules/nightly-sweep.nix), weekly. Order:
  1. Roll warm/infra to latest, HEALTH-GATED (WC1.1): re-run the keycloak + traefik reconcilers
     (warm_reconcile.py <app> — fetch latest recipe → deploy → health-gate → commit/rollback+alert).
     This is the health-gated "warm/infra → latest" step; a full operator `nixos-rebuild switch` is
-     the config-deploy path, not the autonomous nightly's job (DECISIONS Phase-2w WC6).
-  2. FULL-COLD sweep across enrolled (WARM_CANONICAL) recipes, SERIAL (MAX_TESTS honored — one at a
-     time), each `RECIPE=<r> run_recipe_ci.py` on LATEST (no REF) → a green run promotes/refreshes
-     that recipe's canonical (WC5). Serves as the daily authoritative regression.
+     the config-deploy path, not the autonomous sweep's job (DECISIONS Phase-2w WC6).
+  2. Per ENROLLED (WARM_CANONICAL) recipe, SERIAL (one at a time):
+     (C) faithfully mirror-sync the recipe to coopcloud upstream (main+tags, close merged-upstream
+         PRs) via scripts/recipe-mirror-sync.sh — so the sweep measures TRUE upstream tags/latest.
+     (D) NEW-RELEASE-TAG trigger (canon §2.D): compare the latest release tag to the recipe's
+         canonical version (NOT commit). No new tag → SKIP (even if `main` has new untagged commits).
+         New tag → cold-test that TAGGED version (run_on_tag) and, on green, promote the canonical to
+         it (run_recipe_ci.promote_canonical, gated on green+cold+latest+enrolled+TAGGED, canon §2.A).
+  Run-twice determinism (canon M2): a second immediate sweep finds latest tag == canonical for every
+  recipe → SKIPs all (clean no-op, no CI rerun).

 MUST NOT run while a test/Drone build is in flight: if a `run_recipe_ci.py` is already active, skip
-this nightly (defer to the next) rather than pile on the single node. Bounded + serial. Exit 0 even
-if some recipes fail (logs per-recipe results; a red recipe just doesn't advance its canonical).
+this sweep (defer) rather than pile on the single node. Bounded + serial. Exit 0 even if some recipes
+fail (logs per-recipe results; a red recipe just doesn't advance its canonical). NO AI at runtime —
+pure script + systemd timer.
 """

 from __future__ import annotations
@ -26,9 +33,11 @@ import sys
 # against $CCCI_REPO (default /root/cc-ci) — the same checkout run_recipe_ci already runs from.
 REPO = os.environ.get("CCCI_REPO", "/root/cc-ci")
 sys.path.insert(0, os.path.join(REPO, "runner"))
-from harness import canonical  # noqa: E402
+import warm_reconcile as wr  # noqa: E402
+from harness import abra, canonical  # noqa: E402

 WARM_APPS = ["keycloak", "traefik"]  # the live-warm/infra reconcilers to roll first (health-gated)
+MIRROR_SYNC = os.path.join(REPO, "scripts", "recipe-mirror-sync.sh")


 def _here() -> str:
@ -53,33 +62,71 @@ def roll_warm_infra() -> None:
        print(f"nightly: reconcile {app} rc={rc}", flush=True)


-def sweep() -> int:
-    recipes = canonical.enrolled_recipes()
-    print(f"\n===== nightly cold sweep: enrolled canonicals = {recipes} =====", flush=True)
-    results: dict[str, int] = {}
-    for r in recipes:
-        print(f"\n===== nightly: full-cold {r} (latest) =====", flush=True)
-        env = dict(os.environ, RECIPE=r)
-        env.pop("REF", None)  # latest, not a PR head
-        env.pop("CCCI_QUICK", None)
-        env.pop("MODE", None)
-        rc = subprocess.run(
-            [sys.executable, os.path.join(_here(), "run_recipe_ci.py")], env=env
-        ).returncode
-        results[r] = rc
+def mirror_sync(recipe: str) -> int:
+    """canon §2.C: faithfully reconcile the recipe MIRROR to coopcloud upstream (main+tags, close
+    merged-upstream PRs). Best-effort — a sync failure is logged but does NOT abort the recipe's run
+    (the trigger still reads upstream tags via the abra fetch below). Returns the script rc."""
+    if not os.path.isfile(MIRROR_SYNC):
        print(
-            f"nightly: {r} rc={rc} ({'green→canonical refreshed' if rc == 0 else 'red'})",
+            f"sweep: mirror-sync script missing ({MIRROR_SYNC}) — skipping sync for {recipe}",
            flush=True,
        )
+        return 0
+    rc = subprocess.run(["bash", MIRROR_SYNC, recipe]).returncode
+    if rc != 0:
+        print(f"sweep: mirror-sync {recipe} rc={rc} (non-fatal — continuing)", flush=True)
+    return rc
+
+
+def run_on_tag(recipe: str, tag: str) -> int:
+    """Run a full COLD CI on the recipe at the published RELEASE TAG `tag` (canon §2.D: the sweep
+    tests releases, not arbitrary `main` commits). Checks out the tag in the canonical recipe clone
+    and runs run_recipe_ci with CCCI_SKIP_FETCH=1 so the head under test IS the tag (head_version =
+    tag → the tagged-promote gate passes; REF stays empty → promote allowed). A green run promotes
+    the canonical to that tagged version (run_recipe_ci.should_promote_canonical)."""
+    abra.recipe_checkout(recipe, tag)
+    env = dict(os.environ, RECIPE=recipe, CCCI_SKIP_FETCH="1")
+    for k in ("REF", "CCCI_QUICK", "MODE", "VERSION"):
+        env.pop(k, None)  # cold (no PR head), full mode, head = the staged tag checkout
+    return subprocess.run(
+        [sys.executable, os.path.join(_here(), "run_recipe_ci.py")], env=env
+    ).returncode
+
+
+def sweep() -> int:
+    recipes = canonical.enrolled_recipes()
+    print(f"\n===== weekly canonical sweep: enrolled = {recipes} =====", flush=True)
+    results: dict[str, str] = {}
+    for r in recipes:
+        print(f"\n===== sweep: {r} =====", flush=True)
+        # C. faithful mirror-sync to upstream (best-effort) so we measure true upstream tags/latest.
+        mirror_sync(r)
+        # Ensure the local recipe clone reflects upstream tags for the trigger computation.
+        try:
+            wr.fetch_recipe(r)
+        except Exception as e:  # noqa: BLE001 — a fetch failure is logged; trigger uses what's local
+            print(f"sweep: {r} fetch_recipe failed (non-fatal): {e}", flush=True)
+        # D. new-release-tag trigger: latest release tag vs canonical version (NOT commit).
+        latest = wr.latest_version(wr.recipe_tags(r))
+        canon = (canonical.read_registry(r) or {}).get("version")
+        action, reason = wr.sweep_decision(latest, canon)
+        if action == "skip":
+            results[r] = f"SKIP ({reason})"
+            print(f"sweep: {r} SKIP — {reason}", flush=True)
+            continue
+        print(f"sweep: {r} RUN — {reason}; cold-testing tagged release {latest}", flush=True)
+        rc = run_on_tag(r, latest)
+        results[r] = "PASS (promoted)" if rc == 0 else "FAIL (canonical unchanged)"
+        print(f"sweep: {r} rc={rc} ({results[r]})", flush=True)
    # WC8 disk hygiene: drop warm data for de-enrolled canonicals; log the disk budget.
    pruned = canonical.prune_stale()
    if pruned:
-        print(f"nightly: pruned stale warm data for de-enrolled canonicals: {pruned}", flush=True)
+        print(f"sweep: pruned stale warm data for de-enrolled canonicals: {pruned}", flush=True)
    df = subprocess.run(["df", "-h", "/"], capture_output=True, text=True)
-    print(f"nightly: disk / →\n{df.stdout.strip()}", flush=True)
-    print("\n===== nightly sweep summary =====", flush=True)
-    for r, rc in results.items():
-        print(f"  {r}: {'PASS' if rc == 0 else 'FAIL'}", flush=True)
+    print(f"sweep: disk / →\n{df.stdout.strip()}", flush=True)
+    print("\n===== weekly sweep summary =====", flush=True)
+    for r, status in results.items():
+        print(f"  {r}: {status}", flush=True)
    return 0  # the sweep itself succeeds; per-recipe reds are reported, not fatal


--- a/runner/warm_reconcile.py
+++ b/runner/warm_reconcile.py
@ -185,6 +185,29 @@ def latest_version(tags) -> str | None:
    return s[-1] if s else None


+def sweep_decision(latest_tag: str | None, canon_version: str | None) -> tuple[str, str]:
+    """Pure new-release-tag TRIGGER for the weekly sweep (phase canon §2.D), keyed on the latest
+    RELEASE TAG vs the recipe's canonical version — NOT on commits. Returns (action, reason) where
+    action is "run" or "skip":
+      - no release tag at all              → skip ("never-released")  — recipe never cut a release
+      - no canonical yet                   → run  (seed at latest_tag)
+      - latest_tag <= canonical (by key)   → skip ("no-new-version")  — even if `main` has NEW
+                                             UNTAGGED commits: the sweep tests releases, not commits
+      - latest_tag  > canonical (by key)   → run  (cold-test the new tagged version, then promote)
+    This is the run-twice determinism property: after a green run promotes canonical→latest_tag, a
+    second immediate sweep finds latest_tag == canonical for every recipe → skips all (clean no-op)."""
+    if not latest_tag:
+        return ("skip", "never-released (no release tag)")
+    if not canon_version:
+        return ("run", f"no canonical yet → seed at {latest_tag}")
+    if version_key(latest_tag) <= version_key(canon_version):
+        return (
+            "skip",
+            f"no-new-version (latest release {latest_tag} <= canonical {canon_version})",
+        )
+    return ("run", f"new release {latest_tag} > canonical {canon_version}")
+
+
 def is_released_version(recipe: str, version: str | None) -> bool:
    """True iff `version` corresponds to a PUBLISHED RELEASE TAG of the recipe (phase canon §2.A:
    the canonical may only ever advance to a real release — never an arbitrary untagged `main`
--- a/scripts/recipe-mirror-sync.sh
+++ b/scripts/recipe-mirror-sync.sh
@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# recipe-mirror-sync :: faithfully reconcile a recipe MIRROR to its coopcloud UPSTREAM.
+# (phase canon §2.C — adapted from recipe-upgrade's `open-recipe-pr.sh --reconcile-only`.)
+# ---------------------------------------------------------------------------------------
+# Runs ON cc-ci (root) against the recipe checkout at ~/.abra/recipes/<recipe>. Invoked by the
+# weekly canonical sweep BEFORE the per-recipe CI so the sweep measures true upstream tags/latest.
+#
+# FAITHFUL MIRROR SYNC ONLY — never pushes our own changes:
+#   - upstream `main` + tags (git.coopcloud.tech/coop-cloud/<recipe>) are force-synced onto the
+#     mirror (git.autonomic.zone/recipe-maintainers/<recipe>): mirror main := upstream main, and all
+#     upstream tags pushed. This guarantees every open PR is measured against the real upstream.
+#   - any open mirror PR whose changes are ALREADY IN upstream main (merging it would be a no-op —
+#     i.e. it was merged upstream) is closed; UNRELATED PRs are left untouched.
+#   - NEVER merges anything; NEVER pushes a branch other than the faithful main/tags mirror.
+#
+# Why this is adapted rather than the verbatim open-recipe-pr.sh: that script assumes the clone's
+# `origin` IS coopcloud upstream, but abra recipe clones on cc-ci have inconsistent remotes (origin
+# is sometimes the mirror, sometimes coopcloud, sometimes there is no `upstream`). This version
+# pins an explicit `upstream` remote from the recipe name, and also syncs TAGS (canon needs the
+# release-tag trigger to see upstream tags). Auth: the bot gitea token (push via oauth2 URL, API via
+# Authorization header) so it is self-contained, not dependent on host .git-credentials state.
+set -o errexit -o nounset -o pipefail
+
+RECIPE="${1:?usage: recipe-mirror-sync.sh <recipe>}"
+export PATH="/run/current-system/sw/bin:${PATH}"
+
+GITEA_HOST="${GITEA_URL:-git.autonomic.zone}"
+NAMESPACE="${GITEA_NAMESPACE:-recipe-maintainers}"
+UPSTREAM_URL="https://git.coopcloud.tech/coop-cloud/${RECIPE}.git"
+TOKEN_FILE="${CCCI_GITEA_TOKEN_FILE:-/run/secrets/bridge_gitea_token}"
+RECIPE_DIR="${HOME}/.abra/recipes/${RECIPE}"
+
+[ -r "${TOKEN_FILE}" ] || {
+  echo "ERROR: gitea token not readable at ${TOKEN_FILE}"
+  exit 1
+}
+TOKEN="$(tr -d '[:space:]' <"${TOKEN_FILE}")"
+API="https://${GITEA_HOST}/api/v1"
+MIRROR_PUSH="https://oauth2:${TOKEN}@${GITEA_HOST}/${NAMESPACE}/${RECIPE}.git"
+auth=(-H "Authorization: token ${TOKEN}")
+
+# Ensure the recipe is cloned (abra recipe fetch clones from the catalogue → upstream).
+[ -d "${RECIPE_DIR}/.git" ] || abra recipe fetch "${RECIPE}" -n >/dev/null 2>&1 || true
+[ -d "${RECIPE_DIR}/.git" ] || {
+  echo "ERROR: ${RECIPE_DIR} is not a git repo"
+  exit 1
+}
+cd "${RECIPE_DIR}"
+
+# Pin an explicit upstream remote (coopcloud) regardless of how `origin` is configured.
+if git remote | grep -qx upstream; then
+  git remote set-url upstream "${UPSTREAM_URL}"
+else
+  git remote add upstream "${UPSTREAM_URL}"
+fi
+
+echo "→ Fetching upstream main + tags (${UPSTREAM_URL})..."
+git fetch --quiet --tags upstream main
+NEW_MAIN_SHA="$(git rev-parse refs/remotes/upstream/main)"
+MAIN_TREE="$(git rev-parse "${NEW_MAIN_SHA}^{tree}")"
+
+# Ensure the mirror repo exists (reconcile-only: if absent, nothing to do).
+STATUS="$(curl -s -o /dev/null -w "%{http_code}" "${auth[@]}" "${API}/repos/${NAMESPACE}/${RECIPE}")"
+if [ "${STATUS}" = "404" ]; then
+  echo "→ Mirror ${NAMESPACE}/${RECIPE} does not exist; nothing to reconcile."
+  exit 0
+elif [ "${STATUS}" != "200" ]; then
+  echo "ERROR: unexpected HTTP ${STATUS} checking mirror repo"
+  exit 1
+fi
+
+if git remote | grep -qx gitea; then
+  git remote set-url gitea "${MIRROR_PUSH}"
+else
+  git remote add gitea "${MIRROR_PUSH}"
+fi
+
+echo "→ Force-syncing mirror main := upstream main (${NEW_MAIN_SHA:0:8}) + tags..."
+git push --force gitea "${NEW_MAIN_SHA}:refs/heads/main"
+git push --force --tags gitea
+git fetch --quiet gitea '+refs/heads/*:refs/remotes/gitea/*' || true
+
+# Close open mirror PRs whose changes are already in upstream main (merged upstream); leave others.
+echo "→ Reconciling open PRs on ${NAMESPACE}/${RECIPE}..."
+close_pr() { # <index> <reason>
+  curl -s -o /dev/null "${auth[@]}" -H "Content-Type: application/json" -X POST \
+    "${API}/repos/${NAMESPACE}/${RECIPE}/issues/${1}/comments" \
+    -d "$(jq -n --arg b "Auto-closed by cc-ci canonical sweep: ${2}" '{body:$b}')" || true
+  curl -s -o /dev/null "${auth[@]}" -H "Content-Type: application/json" -X PATCH \
+    "${API}/repos/${NAMESPACE}/${RECIPE}/pulls/${1}" -d '{"state":"closed"}' || true
+  echo "  ✗ closed PR #${1} — ${2}"
+}
+OPEN_PRS="$(curl -s "${auth[@]}" "${API}/repos/${NAMESPACE}/${RECIPE}/pulls?state=open&base=main&limit=50")"
+while IFS=$'\t' read -r IDX HEAD_REF; do
+  [ -n "${IDX:-}" ] || continue
+  merged=0
+  if git rev-parse --verify --quiet "refs/remotes/gitea/${HEAD_REF}" >/dev/null; then
+    mt="$(git merge-tree --write-tree "${NEW_MAIN_SHA}" "refs/remotes/gitea/${HEAD_REF}" 2>/dev/null)" || mt=""
+    [ -n "${mt}" ] && [ "${mt}" = "${MAIN_TREE}" ] && merged=1
+  fi
+  if [ "${merged}" = "1" ]; then
+    close_pr "${IDX}" "its changes are already in upstream main (merged upstream); mirror main re-synced"
+  else
+    echo "  • PR #${IDX} (${HEAD_REF}) still open vs upstream main — left as-is"
+  fi
+done < <(echo "${OPEN_PRS}" | jq -r '.[] | [(.number | tostring), .head.ref] | @tsv' 2>/dev/null || true)
+
+echo "✓ mirror-sync done for ${NAMESPACE}/${RECIPE} (main+tags synced to upstream; merged-upstream PRs closed)."
--- a/tests/unit/test_warm_reconcile.py
+++ b/tests/unit/test_warm_reconcile.py
@ -45,6 +45,33 @@ def test_is_released_version(monkeypatch):
    assert wr.is_released_version("custom-html", "") is False


+def test_sweep_decision():
+    # canon §2.D new-release-tag trigger (pure), keyed on release tag vs canonical version.
+    # new release tag > canonical → run
+    assert wr.sweep_decision("1.13.0+1.31.1", "1.12.0+1.30.0")[0] == "run"
+    # latest tag == canonical → skip (no-new-version) — the run-twice determinism no-op
+    assert wr.sweep_decision("1.13.0+1.31.1", "1.13.0+1.31.1")[0] == "skip"
+    assert "no-new-version" in wr.sweep_decision("1.13.0+1.31.1", "1.13.0+1.31.1")[1]
+    # latest tag OLDER than canonical (shouldn't happen, but never downgrade) → skip
+    assert wr.sweep_decision("1.12.0+1.30.0", "1.13.0+1.31.1")[0] == "skip"
+    # no canonical yet → run (seed)
+    assert wr.sweep_decision("1.0.0+1.0.0", None)[0] == "run"
+    # no release tag at all → skip (never-released)
+    assert wr.sweep_decision(None, "1.0.0+1.0.0")[0] == "skip"
+    assert wr.sweep_decision(None, None)[0] == "skip"
+    assert "never-released" in wr.sweep_decision(None, None)[1]
+
+
+def test_sweep_decision_skips_untagged_ahead():
+    # The trigger compares the latest RELEASE TAG, not main HEAD: if main has new untagged commits
+    # but the latest tag still equals the canonical, the recipe is SKIPPED (no untagged promote).
+    # (Modelled here as latest_tag unchanged vs canonical — commits never enter the decision.)
+    assert wr.sweep_decision("2.0.0+9.9.9", "2.0.0+9.9.9") == (
+        "skip",
+        "no-new-version (latest release 2.0.0+9.9.9 <= canonical 2.0.0+9.9.9)",
+    )
+
+
 def test_is_released_version_no_tags(monkeypatch):
    # a recipe that has never cut a release → no version is a release
    monkeypatch.setattr(wr, "recipe_tags", lambda r: ["main"])