cc-ci-orchestrator/cc-ci-plan/recipe-report.py

#!/usr/bin/env python3
"""recipe-report — data + HTML helper for the weekly "Recipe Report" (/recipe-report skill).

A newspaper-style front page: masthead, an editorial LEAD (overall recipe-fleet state + what to focus
on), a SECURITY BULLETIN of critical-CVE upgrades up top, then needs-attention / routine, and the
comprehensive table ("the full wire") at the end.

Subcommands (the /recipe-report agent runs them around its own review/classification):
  survey [DATE]            JSON of the run + every recipe's open PRs + CI verdict + per-recipe upgrade
                           notes (breaking-change/CVE analysis), and the /upgrade-all summary.
  render SPEC.json OUT.html   render the agent's report spec -> a self-contained newspaper HTML page
  publish OUT.html DATE    copy to cc-ci:/var/lib/cc-ci-reports/week-DATE.html and regen the archive index

SPEC SHAPE (the agent writes this JSON):
  {"date":"YYYY-MM-DD","subtitle":"Week of <human date>",
   "lead":"<editorial: overall fleet state + what to focus on; blank-line-separated paragraphs>",
   "security":[{"title":"recipe — CVE-… (critical)","body":"what it fixes","links":[{"text":"PR #","url":"…"}]}],
   "needs_attention":[{"title":"…","body":"…","links":[…]}],
   "routine":[ {same shape} ],
   "table":[{"recipe":"x","change":"a → b","status":"GREEN","ci":"build 154 ✓","ci_url":"…",
             "pr":"#4","pr_url":"…","notes":"…"}]}
PUBLIC PAGE — include only public-safe data (no secrets/tokens/raw logs).
"""
import base64, html, json, os, re, subprocess, sys, urllib.request
from datetime import datetime, timezone

LOGDIR = "/srv/cc-ci/.cc-ci-logs"
TESTENV = "/srv/cc-ci/.testenv"
INFRA = {"cc-ci", "cc-ci-orchestrator", "cc-ci-secrets"}
HOST_REPORTS = "/var/lib/cc-ci-reports"


def _env():
    e = {}
    try:
        for ln in open(TESTENV):
            ln = ln.strip()
            if "=" in ln and not ln.startswith("#"):
                k, v = ln.split("=", 1)
                e[k] = v.strip().strip('"').strip("'")
    except FileNotFoundError:
        pass
    return e


def _gitea_get(path, env):
    url = f"https://{env['GITEA_URL']}/api/v1{path}"
    auth = base64.b64encode(f"{env['GITEA_USERNAME']}:{env['GITEA_PASSWORD']}".encode()).decode()
    req = urllib.request.Request(url, headers={"Authorization": "Basic " + auth})
    try:
        with urllib.request.urlopen(req, timeout=20) as r:
            return json.load(r)
    except Exception:
        return None


def survey(date):
    env = _env()
    out = {"date": date, "generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")}
    sm = os.path.join(LOGDIR, "upgrades", f"upgrade-all-{date}.md")
    out["upgrade_summary_md"] = open(sm).read() if os.path.exists(sm) else None
    repos = _gitea_get("/orgs/recipe-maintainers/repos?limit=100", env) or []
    recipes = sorted(r["name"] for r in repos if r["name"] not in INFRA and not r["name"].startswith("archived-"))
    out["recipes"] = []
    for r in recipes:
        prs = _gitea_get(f"/repos/recipe-maintainers/{r}/pulls?state=open&limit=50", env) or []
        rec = {"recipe": r, "open_prs": []}
        for p in prs:
            sha = p["head"]["sha"]
            st = _gitea_get(f"/repos/recipe-maintainers/{r}/commits/{sha}/status", env) or {}
            ci = {"state": st.get("state")}
            for s in (st.get("statuses") or []):
                if s.get("context") == "cc-ci/testme":
                    ci = {"state": s.get("state"), "url": s.get("target_url"), "desc": s.get("description")}
            rec["open_prs"].append({"number": p["number"], "title": p["title"],
                                    "url": p["html_url"], "head": sha[:10], "ci": ci})
        # Per-recipe upgrade plan/summary (has the breaking-change + CVE/security analysis) — the agent
        # mines this to lead with critical-CVE upgrades.
        nf = os.path.join(LOGDIR, "upgrades", f"{r}-upgrade-{date}.md")
        rec["upgrade_notes_md"] = open(nf).read() if os.path.exists(nf) else None
        out["recipes"].append(rec)
    print(json.dumps(out, indent=2))


# ---------- newspaper HTML ----------
CSS = """
:root{--ink:#1a1a1a;--mut:#555;--rule:#111;--bg:#e8e4d9;--paper:#fbf9f3;--red:#9b1c1c;--accent:#6b4e00}
*{box-sizing:border-box}
body{font:18px/1.65 Georgia,'Times New Roman',serif;color:var(--ink);background:var(--bg);margin:0;padding:2rem 1rem 4rem}
.paper{max-width:62rem;margin:0 auto;background:var(--paper);padding:2.4rem 2.6rem 3rem;box-shadow:0 2px 10px rgba(0,0,0,.12)}
.mast{text-align:center;border-bottom:4px double var(--rule);padding-bottom:.4rem}
.mast .kicker{font-size:.78rem;font-weight:700;text-transform:uppercase;letter-spacing:.22em;color:var(--mut)}
.mast h1{font-size:3.7rem;line-height:1;margin:.25rem 0 .35rem;font-weight:900;letter-spacing:-.02em}
.dateline{display:flex;flex-wrap:wrap;gap:.5rem;justify-content:space-between;font-size:.76rem;text-transform:uppercase;letter-spacing:.09em;color:var(--mut);border-bottom:1px solid var(--rule);padding:.35rem 0;margin-bottom:1.5rem}
.lead{font-size:1.16rem;line-height:1.72}
.lead p{margin:.4rem 0}
.lead p:first-of-type::first-letter{float:left;font-size:3.6rem;line-height:.78;padding:.08rem .55rem .05rem 0;font-weight:900}
.kicker{font-size:.74rem;font-weight:700;text-transform:uppercase;letter-spacing:.14em;color:var(--red)}
h2{font-size:1.55rem;border-top:2px solid var(--rule);padding-top:.5rem;margin:2.1rem 0 .5rem;letter-spacing:-.01em}
.bulletin{border:2px solid var(--red);background:#fdf2f2;padding:1rem 1.2rem;margin:1.6rem 0}
.bulletin h2{border:0;margin:.1rem 0 .5rem;padding:0;color:var(--red);font-size:1.5rem}
.story{margin:.7rem 0;padding-bottom:.7rem;border-bottom:1px solid #d8d2c2}
.story .h{font-weight:700;font-size:1.1rem;line-height:1.3}
.story .b{margin:.2rem 0 .35rem;color:#2b2b2b}
.links a{font-size:.84rem;margin-right:.7rem}
a{color:#0b3d91;text-decoration:none}a:hover{text-decoration:underline}
.cols{column-count:2;column-gap:2.2rem}.cols .story{break-inside:avoid}
@media(max-width:640px){.cols{column-count:1}.mast h1{font-size:2.5rem}.paper{padding:1.4rem}}
table{border-collapse:collapse;width:100%;font:14px/1.45 Georgia,serif;margin-top:.5rem}
th,td{text-align:left;padding:.42rem .55rem;border-bottom:1px solid #ccc;vertical-align:top}
th{border-bottom:2px solid var(--rule);font-size:.72rem;text-transform:uppercase;letter-spacing:.05em;color:var(--mut)}
.s-GREEN{color:#1a7a3c;font-weight:700}.s-RED,.s-FAILED{color:var(--red);font-weight:700}
.s-STALE{color:var(--accent);font-weight:700}.s-SKIPPED,.s-UPTODATE{color:var(--mut)}
footer{margin-top:2.6rem;border-top:4px double var(--rule);padding-top:.8rem;font-size:.82rem;color:var(--mut);text-align:center}
.idx{list-style:none;padding:0}.idx li{padding:.55rem 0;border-bottom:1px solid #d8d2c2;font-size:1.1rem}
.idx .d{color:var(--mut);font-size:.85rem;float:right}
"""


def _esc(s):
    return html.escape(str(s or ""))


def _linkify_recipes(text, repo_url):
    """Linkify whole-word recipe-name mentions to their mirror repo. ONE pass over the (already
    HTML-escaped) text, longest names first so 'custom-html-tiny' wins over 'custom-html'; re.sub does
    not re-scan inserted hrefs, so URLs that end in a recipe name aren't double-linked."""
    if not repo_url:
        return text
    names = sorted(repo_url, key=len, reverse=True)
    pat = re.compile(r"(?<![\w-])(" + "|".join(re.escape(n) for n in names) + r")(?![\w-])")
    return pat.sub(lambda m: f'<a href="{repo_url[m.group(1)]}">{m.group(1)}</a>', text)


def _links(links):
    if not links:
        return ""
    return '<div class="links">' + " · ".join(
        f'<a href="{_esc(l["url"])}">{_esc(l["text"])}</a>' for l in links) + "</div>"


def _stories(items, repo_url=None):
    if not items:
        return "<p><em>Nothing this week.</em></p>"
    lk = (lambda x: _linkify_recipes(_esc(x), repo_url)) if repo_url else _esc
    return "\n".join(
        f'<div class="story"><div class="h">{lk(it.get("title"))}</div>'
        f'<div class="b">{lk(it.get("body"))}</div>{_links(it.get("links"))}</div>' for it in items)


def _table(rows):
    if not rows:
        return ""
    head = "<tr><th>Recipe</th><th>Change</th><th>Status</th><th>CI</th><th>PR</th><th>Notes</th></tr>"
    trs = []
    for r in rows:
        scls = "s-" + str(r.get("status", "")).upper().replace("-", "").replace(" ", "")
        ci = _esc(r.get("ci"))
        if r.get("ci_url"):
            ci = f'<a href="{_esc(r["ci_url"])}">{ci}</a>'
        pr = _esc(r.get("pr"))
        if r.get("pr_url"):
            pr = f'<a href="{_esc(r["pr_url"])}">{pr}</a>'
        trs.append(f"<tr><td>{_esc(r.get('recipe'))}</td><td>{_esc(r.get('change'))}</td>"
                   f'<td class="{scls}">{_esc(r.get("status"))}</td><td>{ci}</td><td>{pr}</td>'
                   f"<td>{_esc(r.get('notes'))}</td></tr>")
    return f"<table>{head}{''.join(trs)}</table>"


def _page(title, body):
    return (f'<!doctype html><html lang="en"><head><meta charset="utf-8">'
            f'<meta name="viewport" content="width=device-width,initial-scale=1">'
            f"<title>{_esc(title)}</title><style>{CSS}</style></head>"
            f'<body><div class="paper">{body}</div></body></html>')


def _mast():
    return ('<div class="mast"><div class="kicker">Co-op Cloud Recipe CI · Weekly Edition</div>'
            '<h1>The Recipe Report</h1></div>')


def render(spec_path, out_path):
    s = json.load(open(spec_path))
    gen = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
    sub = s.get("subtitle", "Week of " + s["date"])
    lead = s.get("lead", "") or ""
    # Auto-link recipe-name mentions in the lead to their mirror repos.
    gitea = _env().get("GITEA_URL", "git.autonomic.zone")
    repo_url = {r["recipe"]: f"https://{gitea}/recipe-maintainers/{r['recipe']}"
                for r in (s.get("table") or []) if r.get("recipe")}
    if lead and "<p>" not in lead:
        lead = "".join(f"<p>{_linkify_recipes(_esc(p.strip()), repo_url)}</p>"
                       for p in lead.split("\n\n") if p.strip())
    body = (_mast() +
            f'<div class="dateline"><span>{_esc(sub)}</span>'
            f'<span>report.ci.commoninternet.net</span><span>{gen}</span></div>'
            f'<div class="lead">{lead}</div>')
    if s.get("security"):
        body += ('<div class="bulletin"><div class="kicker">Security Bulletin</div>'
                 '<h2>🔒 Critical CVE upgrades — merge first</h2>' + _stories(s["security"], repo_url) + "</div>")
    body += f'<h2>⚑ Needs attention</h2>{_stories(s.get("needs_attention"), repo_url)}'
    body += f'<h2>Routine</h2><div class="cols">{_stories(s.get("routine"), repo_url)}</div>'
    body += f'<h2>The full wire — every recipe</h2>{_table(s.get("table"))}'
    body += (f'<footer>The Recipe Report · generated {gen} · '
             f'<a href="https://ci.commoninternet.net/">dashboard</a> · <a href="./">archive</a></footer>')
    open(out_path, "w").write(_page("The Recipe Report — " + s["date"], body))
    print("wrote", out_path)


def publish(html_path, date):
    page = f"week-{date}.html"
    subprocess.run(["ssh", "cc-ci", f"cat > {HOST_REPORTS}/{page}"], input=open(html_path, "rb").read(), check=True)
    listing = subprocess.run(["ssh", "cc-ci", f"ls -1 {HOST_REPORTS}/week-*.html 2>/dev/null"],
                             capture_output=True, text=True).stdout.split()
    dates = sorted({os.path.basename(p)[5:-5] for p in listing}, reverse=True)
    lis = "\n".join(f'<li><a href="week-{d}.html">Week of {d}</a><span class="d">{d}</span></li>' for d in dates)
    idx = _page("The Recipe Report — Archive", _mast() +
                '<div class="dateline"><span>Weekly review of Co-op Cloud recipe upgrades &amp; CI</span>'
                '<span>report.ci.commoninternet.net</span></div>'
                f'<ul class="idx">{lis or "<li><em>No reports yet.</em></li>"}</ul>')
    subprocess.run(["ssh", "cc-ci", f"cat > {HOST_REPORTS}/index.html"], input=idx.encode(), check=True)
    print(f"published https://report.ci.commoninternet.net/{page}  (+ index)")


def main():
    a = sys.argv[1:]
    cmd = a[0] if a else "survey"
    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
    if cmd == "survey":
        survey(a[1] if len(a) > 1 else today)
    elif cmd == "render":
        render(a[1], a[2])
    elif cmd == "publish":
        publish(a[1], a[2])
    else:
        print(__doc__); sys.exit(2)


if __name__ == "__main__":
    main()