119 lines
6.2 KiB
Python
119 lines
6.2 KiB
Python
#!/usr/bin/env python3
|
|
"""ADVERSARY check5 — WC1.1 MARQUEE health-gated rollback with data integrity (live, cold).
|
|
Run via cc-ci-run from /root/cc-ci-adv-verify on cc-ci (PATH has abra/docker/git).
|
|
|
|
Independent reproduce (does NOT trust the Builder's run):
|
|
A. plant a MARKER realm on warm kc (the data whose survival proves integrity)
|
|
B. stage fake tag 10.7.9+26.6.2 at the good commit -> reconcile -> expect HEALTHY upgrade,
|
|
last_good advances to 10.7.9, marker preserved
|
|
C. stage broken commit (KC_HOSTNAME=:::bad-host:::) tagged 10.7.10+26.6.2 -> reconcile ->
|
|
expect ROLLBACK to 10.7.9, kc HEALTHY, marker INTACT, last_good NOT advanced, rollback alert
|
|
D. cleanup: delete fake tags + broken commit, reconcile back to canonical 10.7.1+26.6.2, delete marker
|
|
"""
|
|
import json, os, subprocess, sys, time
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from harness import sso
|
|
import warm_reconcile as wr
|
|
|
|
RECIPE, APP = "keycloak", "keycloak"
|
|
D = "warm-keycloak.ci.commoninternet.net"
|
|
RDIR = os.path.expanduser("~/.abra/recipes/keycloak")
|
|
GOOD = "04400df" # HEAD = chore: upgrade to 10.7.1+26.6.2
|
|
CANON = "10.7.1+26.6.2"
|
|
T_GOOD = "10.7.9+26.6.2" # fake, points at good commit
|
|
T_BAD = "10.7.10+26.6.2" # fake, points at broken-KC_HOSTNAME commit
|
|
MARKER = "advmarker-rollback"
|
|
ALERTS = os.path.join(wr.warmsnap.DEFAULT_WARM_ROOT, "alerts")
|
|
fails = []
|
|
|
|
def git(*a, check=True):
|
|
return subprocess.run(["git", "-C", RDIR, "-c", "user.email=adv@cc-ci", "-c", "user.name=adv",
|
|
*a], capture_output=True, text=True, check=check)
|
|
|
|
def reconcile():
|
|
"""Run the reconciler exactly as the unit would, but CCCI_SKIP_FETCH so my staged tags stand."""
|
|
env = {**os.environ, "CCCI_SKIP_FETCH": "1"}
|
|
r = subprocess.run(["python3", os.path.join(os.path.dirname(__file__), "warm_reconcile.py"), APP],
|
|
capture_output=True, text=True, env=env, timeout=1800)
|
|
print(r.stdout[-1500:]); print(r.stderr[-600:], file=sys.stderr)
|
|
for line in r.stdout.splitlines():
|
|
if line.startswith("RECONCILE RESULT:"):
|
|
return line.split(":", 1)[1].strip()
|
|
return f"<no result rc={r.returncode}>"
|
|
|
|
def health():
|
|
return wr.health_code(wr.SPECS[APP])
|
|
|
|
def realms():
|
|
return sorted(sso.list_realms(D))
|
|
|
|
def last_good():
|
|
return wr.read_last_good(RECIPE)
|
|
|
|
def type_env():
|
|
return wr.current_version(D)
|
|
|
|
print(f"=== START: TYPE={type_env()} last_good={last_good()} health={health()} realms={realms()}")
|
|
|
|
# ---- A. plant marker realm (data) ----
|
|
sso.setup_keycloak_realm(D, MARKER, "marker-client", redirect_uris=["*"], web_origins=["*"])
|
|
assert MARKER in realms(), "marker realm not created"
|
|
print(f"[A] marker realm planted: {MARKER in realms()}")
|
|
|
|
# ---- B. healthy upgrade to fake 10.7.9 ----
|
|
git("tag", "-a", "-m", "adv", T_GOOD, GOOD + "^{commit}", check=False)
|
|
wr.write_last_good(RECIPE, CANON) # baseline last_good = canonical
|
|
print(f"[B] staged {T_GOOD}@good; reconcile #1 (expect upgrade->{T_GOOD})...")
|
|
res1 = reconcile()
|
|
print(f"[B] result={res1!r} last_good={last_good()} health={health()} markerIntact={MARKER in realms()}")
|
|
if not res1.startswith("upgraded:"): fails.append(f"B not upgraded: {res1}")
|
|
if last_good() != T_GOOD: fails.append(f"B last_good={last_good()} != {T_GOOD}")
|
|
if health() != 200: fails.append(f"B health={health()}")
|
|
if MARKER not in realms(): fails.append("B marker lost on healthy upgrade")
|
|
|
|
# ---- C. broken latest 10.7.10 -> rollback ----
|
|
import shutil
|
|
compose = os.path.join(RDIR, "compose.yml")
|
|
bak = compose + ".advbak"; shutil.copy(compose, bak)
|
|
txt = open(compose).read().replace("KC_HOSTNAME=https://${DOMAIN}", "KC_HOSTNAME=:::bad-host:::")
|
|
open(compose, "w").write(txt)
|
|
git("commit", "-am", "adv broken KC_HOSTNAME")
|
|
broken_sha = git("rev-parse", "HEAD").stdout.strip()
|
|
git("tag", "-a", "-m", "adv", T_BAD, broken_sha)
|
|
git("reset", "--hard", GOOD) # branch back to good; tag keeps the broken commit alive
|
|
shutil.copy(bak, compose); os.remove(bak)
|
|
alerts_before = set(os.listdir(ALERTS)) if os.path.isdir(ALERTS) else set()
|
|
print(f"[C] staged broken {T_BAD}@{broken_sha[:7]}; reconcile #2 (expect rollback->{T_GOOD})... (broken deploy may take minutes)")
|
|
res2 = reconcile()
|
|
alerts_after = set(os.listdir(ALERTS)) if os.path.isdir(ALERTS) else set()
|
|
new_alerts = sorted(alerts_after - alerts_before)
|
|
print(f"[C] result={res2!r} last_good={last_good()} health={health()} markerIntact={MARKER in realms()} newAlerts={new_alerts}")
|
|
if not res2.startswith("rolled-back:"): fails.append(f"C not rolled-back: {res2}")
|
|
if health() != 200: fails.append(f"C kc unhealthy after rollback: {health()}")
|
|
if MARKER not in realms(): fails.append("C MARKER LOST — data integrity FAILED on rollback")
|
|
if last_good() != T_GOOD: fails.append(f"C last_good advanced to {last_good()} (should stay {T_GOOD})")
|
|
rb = [a for a in new_alerts if "rollback" in a]
|
|
if not rb: fails.append("C no rollback alert written")
|
|
else:
|
|
rec = json.load(open(os.path.join(ALERTS, rb[0])))
|
|
print(f"[C] rollback alert: {rec}")
|
|
if rec.get("attempted") != T_BAD: fails.append(f"alert attempted={rec.get('attempted')}")
|
|
if rec.get("last_good") != T_GOOD: fails.append(f"alert last_good={rec.get('last_good')}")
|
|
if rec.get("recovered") is not True: fails.append(f"alert recovered={rec.get('recovered')}")
|
|
|
|
# ---- D. cleanup + restore canonical ----
|
|
print("[D] cleanup: delete fake tags, reconcile back to canonical, delete marker...")
|
|
git("tag", "-d", T_GOOD, check=False); git("tag", "-d", T_BAD, check=False)
|
|
git("reset", "--hard", GOOD)
|
|
res3 = reconcile() # latest now = real CANON; current(env)=10.7.9 -> redeploys to CANON
|
|
print(f"[D] result={res3!r} TYPE={type_env()} last_good={last_good()} health={health()}")
|
|
sso.delete_keycloak_realm(D, MARKER)
|
|
if type_env() != CANON: fails.append(f"D not restored to canonical TYPE: {type_env()}")
|
|
if health() != 200: fails.append(f"D final health={health()}")
|
|
fin = realms()
|
|
if MARKER in fin: fails.append("D marker not cleaned")
|
|
print(f"=== END: TYPE={type_env()} last_good={last_good()} health={health()} realms={fin}")
|
|
|
|
print("\nRESULT:", "FAIL: " + "; ".join(fails) if fails else "PASS — WC1.1 marquee: healthy upgrade commits, broken latest rolls back with marker realm (data) INTACT, last_good not advanced, alert correct, canonical restored")
|
|
sys.exit(1 if fails else 0)
|