#!/usr/bin/env python3 """ADVERSARY check5 — WC1.1 MARQUEE health-gated rollback with data integrity (live, cold). Run via cc-ci-run from /root/cc-ci-adv-verify on cc-ci (PATH has abra/docker/git). Independent reproduce (does NOT trust the Builder's run): A. plant a MARKER realm on warm kc (the data whose survival proves integrity) B. stage fake tag 10.7.9+26.6.2 at the good commit -> reconcile -> expect HEALTHY upgrade, last_good advances to 10.7.9, marker preserved C. stage broken commit (KC_HOSTNAME=:::bad-host:::) tagged 10.7.10+26.6.2 -> reconcile -> expect ROLLBACK to 10.7.9, kc HEALTHY, marker INTACT, last_good NOT advanced, rollback alert D. cleanup: delete fake tags + broken commit, reconcile back to canonical 10.7.1+26.6.2, delete marker """ import json, os, subprocess, sys, time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from harness import sso import warm_reconcile as wr RECIPE, APP = "keycloak", "keycloak" D = "warm-keycloak.ci.commoninternet.net" RDIR = os.path.expanduser("~/.abra/recipes/keycloak") GOOD = "04400df" # HEAD = chore: upgrade to 10.7.1+26.6.2 CANON = "10.7.1+26.6.2" T_GOOD = "10.7.9+26.6.2" # fake, points at good commit T_BAD = "10.7.10+26.6.2" # fake, points at broken-KC_HOSTNAME commit MARKER = "advmarker-rollback" ALERTS = os.path.join(wr.warmsnap.DEFAULT_WARM_ROOT, "alerts") fails = [] def git(*a, check=True): return subprocess.run(["git", "-C", RDIR, "-c", "user.email=adv@cc-ci", "-c", "user.name=adv", *a], capture_output=True, text=True, check=check) def reconcile(): """Run the reconciler exactly as the unit would, but CCCI_SKIP_FETCH so my staged tags stand.""" env = {**os.environ, "CCCI_SKIP_FETCH": "1"} r = subprocess.run(["python3", os.path.join(os.path.dirname(__file__), "warm_reconcile.py"), APP], capture_output=True, text=True, env=env, timeout=1800) print(r.stdout[-1500:]); print(r.stderr[-600:], file=sys.stderr) for line in r.stdout.splitlines(): if line.startswith("RECONCILE RESULT:"): return line.split(":", 1)[1].strip() return f"" def health(): return wr.health_code(wr.SPECS[APP]) def realms(): return sorted(sso.list_realms(D)) def last_good(): return wr.read_last_good(RECIPE) def type_env(): return wr.current_version(D) print(f"=== START: TYPE={type_env()} last_good={last_good()} health={health()} realms={realms()}") # ---- A. plant marker realm (data) ---- sso.setup_keycloak_realm(D, MARKER, "marker-client", redirect_uris=["*"], web_origins=["*"]) assert MARKER in realms(), "marker realm not created" print(f"[A] marker realm planted: {MARKER in realms()}") # ---- B. healthy upgrade to fake 10.7.9 ---- git("tag", "-a", "-m", "adv", T_GOOD, GOOD + "^{commit}", check=False) wr.write_last_good(RECIPE, CANON) # baseline last_good = canonical print(f"[B] staged {T_GOOD}@good; reconcile #1 (expect upgrade->{T_GOOD})...") res1 = reconcile() print(f"[B] result={res1!r} last_good={last_good()} health={health()} markerIntact={MARKER in realms()}") if not res1.startswith("upgraded:"): fails.append(f"B not upgraded: {res1}") if last_good() != T_GOOD: fails.append(f"B last_good={last_good()} != {T_GOOD}") if health() != 200: fails.append(f"B health={health()}") if MARKER not in realms(): fails.append("B marker lost on healthy upgrade") # ---- C. broken latest 10.7.10 -> rollback ---- import shutil compose = os.path.join(RDIR, "compose.yml") bak = compose + ".advbak"; shutil.copy(compose, bak) txt = open(compose).read().replace("KC_HOSTNAME=https://${DOMAIN}", "KC_HOSTNAME=:::bad-host:::") open(compose, "w").write(txt) git("commit", "-am", "adv broken KC_HOSTNAME") broken_sha = git("rev-parse", "HEAD").stdout.strip() git("tag", "-a", "-m", "adv", T_BAD, broken_sha) git("reset", "--hard", GOOD) # branch back to good; tag keeps the broken commit alive shutil.copy(bak, compose); os.remove(bak) alerts_before = set(os.listdir(ALERTS)) if os.path.isdir(ALERTS) else set() print(f"[C] staged broken {T_BAD}@{broken_sha[:7]}; reconcile #2 (expect rollback->{T_GOOD})... (broken deploy may take minutes)") res2 = reconcile() alerts_after = set(os.listdir(ALERTS)) if os.path.isdir(ALERTS) else set() new_alerts = sorted(alerts_after - alerts_before) print(f"[C] result={res2!r} last_good={last_good()} health={health()} markerIntact={MARKER in realms()} newAlerts={new_alerts}") if not res2.startswith("rolled-back:"): fails.append(f"C not rolled-back: {res2}") if health() != 200: fails.append(f"C kc unhealthy after rollback: {health()}") if MARKER not in realms(): fails.append("C MARKER LOST — data integrity FAILED on rollback") if last_good() != T_GOOD: fails.append(f"C last_good advanced to {last_good()} (should stay {T_GOOD})") rb = [a for a in new_alerts if "rollback" in a] if not rb: fails.append("C no rollback alert written") else: rec = json.load(open(os.path.join(ALERTS, rb[0]))) print(f"[C] rollback alert: {rec}") if rec.get("attempted") != T_BAD: fails.append(f"alert attempted={rec.get('attempted')}") if rec.get("last_good") != T_GOOD: fails.append(f"alert last_good={rec.get('last_good')}") if rec.get("recovered") is not True: fails.append(f"alert recovered={rec.get('recovered')}") # ---- D. cleanup + restore canonical ---- print("[D] cleanup: delete fake tags, reconcile back to canonical, delete marker...") git("tag", "-d", T_GOOD, check=False); git("tag", "-d", T_BAD, check=False) git("reset", "--hard", GOOD) res3 = reconcile() # latest now = real CANON; current(env)=10.7.9 -> redeploys to CANON print(f"[D] result={res3!r} TYPE={type_env()} last_good={last_good()} health={health()}") sso.delete_keycloak_realm(D, MARKER) if type_env() != CANON: fails.append(f"D not restored to canonical TYPE: {type_env()}") if health() != 200: fails.append(f"D final health={health()}") fin = realms() if MARKER in fin: fails.append("D marker not cleaned") print(f"=== END: TYPE={type_env()} last_good={last_good()} health={health()} realms={fin}") print("\nRESULT:", "FAIL: " + "; ".join(fails) if fails else "PASS — WC1.1 marquee: healthy upgrade commits, broken latest rolls back with marker realm (data) INTACT, last_good not advanced, alert correct, canonical restored") sys.exit(1 if fails else 0)