f687174b53
Rename the table's Status column -> TESTS (the CI/test verdict, unchanged content). Add a new STATUS column showing the PR's LIVE state, fetched client-side: 'open' vs a ✓ for any not-open state (merged or closed). The cell is a JS hook (data-repo/data-pr) derived from existing recipe+pr fields; an inline, dependency-free, CSP-safe script GETs the same-origin /pr/<recipe>/<n> proxy (cc-ci nix/modules/reports.nix) on load and every 30s, and degrades to a muted '?' if the proxy/repo is unreachable. Blank cell when a row has no PR. Doc + SKILL updated. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
338 lines
18 KiB
Python
Executable File
338 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""recipe-report — data + HTML helper for the weekly "Recipe Report" (/recipe-report skill).
|
|
|
|
A newspaper-style front page: masthead, a SHORT editorial LEAD, then the comprehensive table ("the
|
|
full wire" — priority-sorted, CVEs column), an ADDENDUM of special issues, a SECURITY BULLETIN, and a
|
|
per-recipe "what changed" section at the end.
|
|
|
|
Subcommands (the /recipe-report agent runs them around its own review/classification):
|
|
survey [DATE] JSON of the run + every recipe's open PRs + CI verdict + per-recipe upgrade
|
|
notes (breaking-change/CVE analysis), and the /upgrade-all summary.
|
|
render SPEC.json OUT.html render the agent's report spec -> a self-contained newspaper HTML page
|
|
publish OUT.html DATE copy to cc-ci:/var/lib/cc-ci-reports/week-DATE.html and regen the archive index
|
|
|
|
Page order: short lead → the full wire table (priority-sorted, CVEs column) → Addendum → Security
|
|
Bulletin → per-recipe "What changed".
|
|
|
|
The full-wire table has two distinct status columns:
|
|
TESTS — the CI/test verdict for the PR (GREEN/FAILED/STALE/SKIPPED/UPTODATE; from the `status` field).
|
|
STATUS — the PR's LIVE state, fetched client-side and refreshed every ~30s: `open` vs a ✓ for any
|
|
not-open state (merged OR closed). Derived from `recipe` + `pr`; the inline script GETs the
|
|
same-origin `/pr/<recipe>/<n>` proxy served by the ccci-reports nginx stack (cc-ci repo,
|
|
nix/modules/reports.nix). Needs the recipe mirrors public; degrades to a muted "?" if the
|
|
proxy/repo is unreachable. No spec-shape change — STATUS uses existing row fields.
|
|
|
|
SPEC SHAPE (the agent writes this JSON):
|
|
{"date":"YYYY-MM-DD","subtitle":"Week of <human date>",
|
|
"lead":"<ONE short paragraph>",
|
|
"table":[{"recipe":"x","change":"a → b","status":"GREEN|FAILED|STALE|SKIPPED|UPTODATE", # -> TESTS col
|
|
"cve":16, # number of CVEs this PR fixes; 0/omit for none
|
|
"ci":"build 154 ✓","ci_url":"…","pr":"#4","pr_url":"…","notes":"…"}],
|
|
# ROWS SORTED by recommended priority to address — recipes WITH CVEs first, then failures,
|
|
# then stale-tests, then routine green, then up-to-date/skipped.
|
|
"addendum":["a special issue to look into (multiple open PRs, a CI-run oddity, a possible
|
|
improvement) …"], # only real issues — omit/empty if everything's clean
|
|
"security":[{"title":"recipe — CVE-… (high)","body":"what it fixes","links":[{"text":"PR #","url":"…"}]}],
|
|
"changes":[{"recipe":"x","body":"what changed in this recipe's PR","links":[{"text":"PR #4","url":"…"}]}]}
|
|
# one `changes` entry per recipe that has a PR this week.
|
|
PUBLIC PAGE — include only public-safe data (no secrets/tokens/raw logs).
|
|
"""
|
|
import base64, html, json, os, re, subprocess, sys, urllib.request
|
|
from datetime import datetime, timezone
|
|
|
|
LOGDIR = "/srv/cc-ci/.cc-ci-logs"
|
|
TESTENV = "/srv/cc-ci/.testenv"
|
|
INFRA = {"cc-ci", "cc-ci-orchestrator", "cc-ci-secrets"}
|
|
HOST_REPORTS = "/var/lib/cc-ci-reports"
|
|
|
|
|
|
def _env():
|
|
e = {}
|
|
try:
|
|
for ln in open(TESTENV):
|
|
ln = ln.strip()
|
|
if "=" in ln and not ln.startswith("#"):
|
|
k, v = ln.split("=", 1)
|
|
e[k] = v.strip().strip('"').strip("'")
|
|
except FileNotFoundError:
|
|
pass
|
|
return e
|
|
|
|
|
|
def _gitea_get(path, env):
|
|
url = f"https://{env['GITEA_URL']}/api/v1{path}"
|
|
auth = base64.b64encode(f"{env['GITEA_USERNAME']}:{env['GITEA_PASSWORD']}".encode()).decode()
|
|
req = urllib.request.Request(url, headers={"Authorization": "Basic " + auth})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=20) as r:
|
|
return json.load(r)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def survey(date):
|
|
env = _env()
|
|
out = {"date": date, "generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")}
|
|
sm = os.path.join(LOGDIR, "upgrades", f"upgrade-all-{date}.md")
|
|
out["upgrade_summary_md"] = open(sm).read() if os.path.exists(sm) else None
|
|
repos = _gitea_get("/orgs/recipe-maintainers/repos?limit=100", env) or []
|
|
recipes = sorted(r["name"] for r in repos if r["name"] not in INFRA and not r["name"].startswith("archived-"))
|
|
out["recipes"] = []
|
|
for r in recipes:
|
|
prs = _gitea_get(f"/repos/recipe-maintainers/{r}/pulls?state=open&limit=50", env) or []
|
|
rec = {"recipe": r, "open_prs": []}
|
|
for p in prs:
|
|
sha = p["head"]["sha"]
|
|
st = _gitea_get(f"/repos/recipe-maintainers/{r}/commits/{sha}/status", env) or {}
|
|
ci = {"state": st.get("state")}
|
|
for s in (st.get("statuses") or []):
|
|
if s.get("context") == "cc-ci/testme":
|
|
ci = {"state": s.get("state"), "url": s.get("target_url"), "desc": s.get("description")}
|
|
rec["open_prs"].append({"number": p["number"], "title": p["title"],
|
|
"url": p["html_url"], "head": sha[:10], "ci": ci})
|
|
# Per-recipe upgrade plan/summary (has the breaking-change + CVE/security analysis) — the agent
|
|
# mines this to lead with critical-CVE upgrades.
|
|
nf = os.path.join(LOGDIR, "upgrades", f"{r}-upgrade-{date}.md")
|
|
rec["upgrade_notes_md"] = open(nf).read() if os.path.exists(nf) else None
|
|
out["recipes"].append(rec)
|
|
print(json.dumps(out, indent=2))
|
|
|
|
|
|
# ---------- newspaper HTML ----------
|
|
CSS = """
|
|
:root{--ink:#1a1a1a;--mut:#555;--rule:#111;--bg:#e8e4d9;--paper:#fbf9f3;--red:#9b1c1c;--accent:#6b4e00}
|
|
*{box-sizing:border-box}
|
|
body{font:18px/1.65 Georgia,'Times New Roman',serif;color:var(--ink);background:var(--bg);margin:0;padding:2rem 1rem 4rem}
|
|
.paper{max-width:62rem;margin:0 auto;background:var(--paper);padding:2.4rem 2.6rem 3rem;box-shadow:0 2px 10px rgba(0,0,0,.12)}
|
|
.mast{text-align:center;border-bottom:4px double var(--rule);padding-bottom:.4rem}
|
|
.mast .kicker{font-size:.78rem;font-weight:700;text-transform:uppercase;letter-spacing:.22em;color:var(--mut)}
|
|
.mast h1{font-size:3.7rem;line-height:1;margin:.25rem 0 .35rem;font-weight:900;letter-spacing:-.02em}
|
|
.dateline{display:flex;flex-wrap:wrap;gap:.5rem;justify-content:space-between;font-size:.76rem;text-transform:uppercase;letter-spacing:.09em;color:var(--mut);border-bottom:1px solid var(--rule);padding:.35rem 0;margin-bottom:1.5rem}
|
|
.lead{font-size:1.16rem;line-height:1.72}
|
|
.lead p{margin:.4rem 0}
|
|
.lead p:first-of-type::first-letter{float:left;font-size:3.6rem;line-height:.78;padding:.08rem .55rem .05rem 0;font-weight:900}
|
|
.kicker{font-size:.74rem;font-weight:700;text-transform:uppercase;letter-spacing:.14em;color:var(--red)}
|
|
h2{font-size:1.55rem;border-top:2px solid var(--rule);padding-top:.5rem;margin:2.1rem 0 .5rem;letter-spacing:-.01em}
|
|
.bulletin{border:2px solid var(--red);background:#fdf2f2;padding:1rem 1.2rem;margin:1.6rem 0}
|
|
.bulletin h2{border:0;margin:.1rem 0 .5rem;padding:0;color:var(--red);font-size:1.5rem}
|
|
.story{margin:.7rem 0;padding-bottom:.7rem;border-bottom:1px solid #d8d2c2}
|
|
.story .h{font-weight:700;font-size:1.1rem;line-height:1.3}
|
|
.story .b{margin:.2rem 0 .35rem;color:#2b2b2b}
|
|
.links a{font-size:.84rem;margin-right:.7rem}
|
|
a{color:#0b3d91;text-decoration:none}a:hover{text-decoration:underline}
|
|
.cols{column-count:2;column-gap:2.2rem}.cols .story{break-inside:avoid}
|
|
@media(max-width:640px){.cols{column-count:1}.mast h1{font-size:2.5rem}.paper{padding:1.4rem}}
|
|
table{border-collapse:collapse;width:100%;font:14px/1.45 Georgia,serif;margin-top:.5rem}
|
|
th,td{text-align:left;padding:.42rem .55rem;border-bottom:1px solid #ccc;vertical-align:top}
|
|
th{border-bottom:2px solid var(--rule);font-size:.72rem;text-transform:uppercase;letter-spacing:.05em;color:var(--mut)}
|
|
.s-GREEN{color:#1a7a3c;font-weight:700}.s-RED,.s-FAILED{color:var(--red);font-weight:700}
|
|
.s-STALE{color:var(--accent);font-weight:700}.s-SKIPPED,.s-UPTODATE{color:var(--mut)}
|
|
footer{margin-top:2.6rem;border-top:4px double var(--rule);padding-top:.8rem;font-size:.82rem;color:var(--mut);text-align:center}
|
|
.idx{list-style:none;padding:0}.idx li{padding:.55rem 0;border-bottom:1px solid #d8d2c2;font-size:1.1rem}
|
|
.idx .d{color:var(--mut);font-size:.85rem;float:right}
|
|
td .cve{color:var(--red);font-weight:800}.muted{color:#999}
|
|
.pr-status{white-space:nowrap}
|
|
.pr-open{display:inline-block;font:700 .66rem/1 Georgia,serif;text-transform:uppercase;letter-spacing:.07em;color:#9b6b00;background:#fff3d6;border:1px solid #e3c98a;border-radius:.2rem;padding:.13rem .42rem}
|
|
.pr-done{color:#1a7a3c;font-weight:800;font-size:1.05rem}
|
|
.addendum-h{font-size:1rem;font-weight:700;text-transform:uppercase;letter-spacing:.08em;margin:1.7rem 0 .3rem;color:var(--ink)}
|
|
ul.addendum{margin:.2rem 0 1.2rem 1.2rem;padding:0}ul.addendum li{margin:.35rem 0;line-height:1.55}
|
|
.change{margin:.8rem 0;padding-bottom:.7rem;border-bottom:1px solid #d8d2c2}
|
|
.change .ch{font-weight:700;font-size:1.12rem}.change .cb{margin:.2rem 0 .35rem;color:#2b2b2b}
|
|
"""
|
|
|
|
|
|
# Inline (CSP-safe, dependency-free) script that lights up the live PR-STATUS column. For each
|
|
# `.pr-status[data-pr]` cell it GETs the same-origin /pr/<recipe>/<n> proxy (see reports.nix in the
|
|
# cc-ci repo) and renders BINARY state: `state==='open'` -> an "open" badge; any other state
|
|
# (closed/merged) -> a green ✓ (the title shows merged vs closed). It refreshes every 30s so an open
|
|
# tab tracks state in realtime, and is fully resilient — a network/parse error leaves a muted "?" and
|
|
# never overwrites a value already resolved, so the page renders fully even if the proxy is down.
|
|
STATUS_JS = """
|
|
(function(){
|
|
function refresh(){
|
|
document.querySelectorAll('.pr-status[data-pr]').forEach(function(cell){
|
|
var repo=cell.getAttribute('data-repo'), pr=cell.getAttribute('data-pr');
|
|
if(!repo||!pr) return;
|
|
fetch('/pr/'+encodeURIComponent(repo)+'/'+encodeURIComponent(pr),{headers:{'Accept':'application/json'}})
|
|
.then(function(r){ if(!r.ok) throw 0; return r.json(); })
|
|
.then(function(d){
|
|
var st=d&&d.state;
|
|
if(st==='open'){ cell.innerHTML='<span class="pr-open">open</span>'; cell.dataset.ok='1'; }
|
|
else if(st){ cell.innerHTML='<span class="pr-done" title="'+(d.merged?'merged':'closed')+'">\\u2713</span>'; cell.dataset.ok='1'; }
|
|
else throw 0;
|
|
})
|
|
.catch(function(){ if(!cell.dataset.ok){ cell.innerHTML='<span class="muted" title="live status unavailable">?</span>'; } });
|
|
});
|
|
}
|
|
if(document.readyState!=='loading'){ refresh(); } else { document.addEventListener('DOMContentLoaded', refresh); }
|
|
setInterval(refresh, 30000);
|
|
})();
|
|
"""
|
|
|
|
|
|
def _esc(s):
|
|
return html.escape(str(s or ""))
|
|
|
|
|
|
def _linkify_recipes(text, repo_url):
|
|
"""Linkify whole-word recipe-name mentions to their mirror repo. ONE pass over the (already
|
|
HTML-escaped) text, longest names first so 'custom-html-tiny' wins over 'custom-html'; re.sub does
|
|
not re-scan inserted hrefs, so URLs that end in a recipe name aren't double-linked."""
|
|
if not repo_url:
|
|
return text
|
|
names = sorted(repo_url, key=len, reverse=True)
|
|
pat = re.compile(r"(?<![\w-])(" + "|".join(re.escape(n) for n in names) + r")(?![\w-])")
|
|
return pat.sub(lambda m: f'<a href="{repo_url[m.group(1)]}">{m.group(1)}</a>', text)
|
|
|
|
|
|
def _links(links):
|
|
if not links:
|
|
return ""
|
|
return '<div class="links">' + " · ".join(
|
|
f'<a href="{_esc(l["url"])}">{_esc(l["text"])}</a>' for l in links) + "</div>"
|
|
|
|
|
|
def _stories(items, repo_url=None):
|
|
if not items:
|
|
return "<p><em>Nothing this week.</em></p>"
|
|
lk = (lambda x: _linkify_recipes(_esc(x), repo_url)) if repo_url else _esc
|
|
return "\n".join(
|
|
f'<div class="story"><div class="h">{lk(it.get("title"))}</div>'
|
|
f'<div class="b">{lk(it.get("body"))}</div>{_links(it.get("links"))}</div>' for it in items)
|
|
|
|
|
|
def _table(rows, repo_url=None):
|
|
if not rows:
|
|
return ""
|
|
# TESTS = the CI/test verdict (GREEN/FAILED/STALE/…); STATUS = the PR's LIVE state, fetched
|
|
# client-side (open vs ✓ for any not-open state) by the inline script in _page().
|
|
head = ("<tr><th>Recipe</th><th>Change</th><th>TESTS</th><th>CVEs</th>"
|
|
"<th>CI</th><th>PR</th><th>STATUS</th><th>Notes</th></tr>")
|
|
trs = []
|
|
for r in rows:
|
|
scls = "s-" + str(r.get("status", "")).upper().replace("-", "").replace(" ", "")
|
|
name = _esc(r.get("recipe"))
|
|
if repo_url and r.get("recipe") in repo_url:
|
|
name = f'<a href="{repo_url[r["recipe"]]}">{name}</a>'
|
|
cve = r.get("cve")
|
|
cve_cell = (f'<span class="cve">{int(cve)}</span>' if isinstance(cve, (int, float)) and cve
|
|
else '<span class="muted">none</span>')
|
|
ci = _esc(r.get("ci"))
|
|
if r.get("ci_url"):
|
|
ci = f'<a href="{_esc(r["ci_url"])}">{ci}</a>'
|
|
pr = _esc(r.get("pr"))
|
|
if r.get("pr_url"):
|
|
pr = f'<a href="{_esc(r["pr_url"])}">{pr}</a>'
|
|
# Live PR-STATUS cell: a JS hook carrying the repo + PR number derived from the existing
|
|
# `recipe` + `pr` fields. The inline script fills it from /pr/<recipe>/<n>. Blank when no PR.
|
|
recipe = r.get("recipe") or ""
|
|
m = re.search(r"\d+", str(r.get("pr") or ""))
|
|
if m and recipe:
|
|
status_cell = (f'<td class="pr-status" data-repo="{_esc(recipe)}" '
|
|
f'data-pr="{_esc(m.group(0))}"><span class="muted">…</span></td>')
|
|
else:
|
|
status_cell = "<td></td>"
|
|
trs.append(f"<tr><td>{name}</td><td>{_esc(r.get('change'))}</td>"
|
|
f'<td class="{scls}">{_esc(r.get("status"))}</td><td>{cve_cell}</td>'
|
|
f"<td>{ci}</td><td>{pr}</td>{status_cell}<td>{_esc(r.get('notes'))}</td></tr>")
|
|
return f"<table>{head}{''.join(trs)}</table>"
|
|
|
|
|
|
def _changes(items, repo_url=None):
|
|
"""Per-recipe 'what changed' sections (one per recipe that has a PR)."""
|
|
if not items:
|
|
return ""
|
|
lk = (lambda x: _linkify_recipes(_esc(x), repo_url)) if repo_url else _esc
|
|
out = []
|
|
for c in items:
|
|
name = c.get("recipe")
|
|
hdr = _esc(name)
|
|
if repo_url and name in repo_url:
|
|
hdr = f'<a href="{repo_url[name]}">{hdr}</a>'
|
|
out.append(f'<div class="change"><div class="ch">{hdr}</div>'
|
|
f'<div class="cb">{lk(c.get("body"))}</div>{_links(c.get("links"))}</div>')
|
|
return "\n".join(out)
|
|
|
|
|
|
def _page(title, body):
|
|
# The live-status script ships on every page (reports + archive index); harmless where there are
|
|
# no `.pr-status` cells. Self-contained: inline CSS + inline JS, no external requests.
|
|
return (f'<!doctype html><html lang="en"><head><meta charset="utf-8">'
|
|
f'<meta name="viewport" content="width=device-width,initial-scale=1">'
|
|
f"<title>{_esc(title)}</title><style>{CSS}</style></head>"
|
|
f'<body><div class="paper">{body}</div><script>{STATUS_JS}</script></body></html>')
|
|
|
|
|
|
def _mast():
|
|
return ('<div class="mast"><div class="kicker">Co-op Cloud Recipe CI · Weekly Edition</div>'
|
|
'<h1>The Recipe Report</h1></div>')
|
|
|
|
|
|
def render(spec_path, out_path):
|
|
s = json.load(open(spec_path))
|
|
gen = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
|
sub = s.get("subtitle", "Week of " + s["date"])
|
|
lead = s.get("lead", "") or ""
|
|
# Auto-link recipe-name mentions in the lead to their mirror repos.
|
|
gitea = _env().get("GITEA_URL", "git.autonomic.zone")
|
|
repo_url = {r["recipe"]: f"https://{gitea}/recipe-maintainers/{r['recipe']}"
|
|
for r in (s.get("table") or []) if r.get("recipe")}
|
|
if lead and "<p>" not in lead:
|
|
lead = "".join(f"<p>{_linkify_recipes(_esc(p.strip()), repo_url)}</p>"
|
|
for p in lead.split("\n\n") if p.strip())
|
|
body = (_mast() +
|
|
f'<div class="dateline"><span>{_esc(sub)}</span>'
|
|
f'<span>report.ci.commoninternet.net</span><span>{gen}</span></div>'
|
|
f'<div class="lead">{lead}</div>')
|
|
# 1) the full wire — every recipe, in the agent's recommended priority order (CVEs first); CVEs column.
|
|
body += f'<h2>The full wire — every recipe, in priority order</h2>{_table(s.get("table"), repo_url)}'
|
|
# 2) addendum — special issues to look into (normal-size header); omitted entirely if there are none.
|
|
add = [a for a in (s.get("addendum") or []) if str(a).strip()]
|
|
if add:
|
|
body += ('<p class="addendum-h">Addendum</p><ul class="addendum">'
|
|
+ "".join(f"<li>{_linkify_recipes(_esc(b), repo_url)}</li>" for b in add) + "</ul>")
|
|
# 3) security bulletin
|
|
if s.get("security"):
|
|
body += ('<div class="bulletin"><div class="kicker">Security Bulletin</div>'
|
|
'<h2>🔒 Critical CVE upgrades</h2>' + _stories(s["security"], repo_url) + "</div>")
|
|
# 4) what changed — a short section per recipe that has a PR
|
|
if s.get("changes"):
|
|
body += f'<h2>What changed</h2>{_changes(s.get("changes"), repo_url)}'
|
|
body += (f'<footer>The Recipe Report · generated {gen} · '
|
|
f'<a href="https://ci.commoninternet.net/">dashboard</a> · <a href="./">archive</a></footer>')
|
|
open(out_path, "w").write(_page("The Recipe Report — " + s["date"], body))
|
|
print("wrote", out_path)
|
|
|
|
|
|
def publish(html_path, date):
|
|
page = f"week-{date}.html"
|
|
subprocess.run(["ssh", "cc-ci", f"cat > {HOST_REPORTS}/{page}"], input=open(html_path, "rb").read(), check=True)
|
|
listing = subprocess.run(["ssh", "cc-ci", f"ls -1 {HOST_REPORTS}/week-*.html 2>/dev/null"],
|
|
capture_output=True, text=True).stdout.split()
|
|
dates = sorted({os.path.basename(p)[5:-5] for p in listing}, reverse=True)
|
|
lis = "\n".join(f'<li><a href="week-{d}.html">Week of {d}</a><span class="d">{d}</span></li>' for d in dates)
|
|
idx = _page("The Recipe Report — Archive", _mast() +
|
|
'<div class="dateline"><span>Weekly review of Co-op Cloud recipe upgrades & CI</span>'
|
|
'<span>report.ci.commoninternet.net</span></div>'
|
|
f'<ul class="idx">{lis or "<li><em>No reports yet.</em></li>"}</ul>')
|
|
subprocess.run(["ssh", "cc-ci", f"cat > {HOST_REPORTS}/index.html"], input=idx.encode(), check=True)
|
|
print(f"published https://report.ci.commoninternet.net/{page} (+ index)")
|
|
|
|
|
|
def main():
|
|
a = sys.argv[1:]
|
|
cmd = a[0] if a else "survey"
|
|
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
if cmd == "survey":
|
|
survey(a[1] if len(a) > 1 else today)
|
|
elif cmd == "render":
|
|
render(a[1], a[2])
|
|
elif cmd == "publish":
|
|
publish(a[1], a[2])
|
|
else:
|
|
print(__doc__); sys.exit(2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|