cc-ci-orchestrator/cc-ci-plan/pr-survey.py

#!/usr/bin/env python3
"""pr-survey — deterministic facts about every open recipe PR, for /cc-ci-cleanup to judge.

Open recipe PRs rot in specific, detectable ways. This gathers the evidence; it does NOT decide
anything — closing a PR is a judgement the skill makes, with these facts in hand.

RUN `reconcile-upstream.sh --all` FIRST. Every signal below is measured against the mirror's `main`,
and an unreconciled mirror makes all of them wrong: on 2026-08-11 three PRs (discourse #6 carrying
140 CVEs, keycloak #6 carrying 12, n8n #5) looked pending against a stale mirror while upstream had
already merged them. This tool refuses to guess about that — see `reconciled_recently`.

Per PR:
  behind_main      commits on main not in the branch — the "out of date" measure
  ahead            commits on the branch not on main
  mergeable        gitea's own verdict (false = conflicts, needs a rebase)
  diff_files       files the PR touches (0 = nothing left to merge)
  adds_images      the `+ image:` lines it introduces
  already_in_main  those `+ image:` lines ALREADY present in main -> the bump landed another way
  obsolete         true when every image it adds is already in main (nothing to contribute)
  ci               newest `!testme` verdict + build number parsed from the PR comments
  branch_kind      upgrade / fix / ci-artifact (`ci/*` sweep + probe branches) / other
  age_days, stale_days (since last update)

    pr-survey.py [recipe ...] [--json]
"""

from __future__ import annotations

import argparse
import base64
import json
import os
import re
import sys
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
USED_RECIPES = os.path.join(HERE, "used-recipes.md")
TESTENV = os.environ.get("CCCI_TESTENV", "/srv/cc-ci/.testenv")
NS = "recipe-maintainers"


def _env() -> dict:
    e = {}
    try:
        for ln in open(TESTENV):
            ln = ln.strip()
            if "=" in ln and not ln.startswith("#"):
                k, v = ln.split("=", 1)
                e[k] = v.strip().strip('"').strip("'")
    except OSError:
        pass
    return e


ENV = _env()
GITEA = os.environ.get("GITEA_URL") or ENV.get("GITEA_URL", "git.autonomic.zone")
_AUTH = base64.b64encode(
    f"{os.environ.get('GITEA_USERNAME') or ENV.get('GITEA_USERNAME','')}:"
    f"{os.environ.get('GITEA_PASSWORD') or ENV.get('GITEA_PASSWORD','')}".encode()
).decode()


def _get(path: str, raw: bool = False):
    req = urllib.request.Request(
        f"https://{GITEA}{path}",
        headers={"Authorization": f"Basic {_AUTH}", "User-Agent": "cc-ci-pr-survey"},
    )
    with urllib.request.urlopen(req, timeout=60) as r:
        body = r.read()
    return body.decode(errors="replace") if raw else json.loads(body)


def _days(iso: str | None) -> int | None:
    if not iso:
        return None
    try:
        d = datetime.fromisoformat(iso.replace("Z", "+00:00"))
    except ValueError:
        return None
    return (datetime.now(timezone.utc) - d).days


def _branch_kind(ref: str) -> str:
    if ref.startswith("ci/"):
        return "ci-artifact"          # regall/cfold sweeps + testme probes; never meant to merge
    if ref.startswith("upgrade"):
        return "upgrade"
    if re.match(r"^(fix|feat|chore|revert)", ref):
        return "fix"
    return "other"


def _main_images(recipe: str) -> set[str]:
    """Image refs pinned on the mirror's main — the baseline a PR is judged against."""
    out = set()
    for f in ("compose.yml",):
        try:
            txt = _get(f"/{NS}/{recipe}/raw/branch/main/{f}", raw=True)
        except Exception:  # noqa: BLE001
            continue
        for m in re.finditer(r"^\s*image:\s*[\"']?([^\"'\s]+)", txt, re.M):
            out.add(m.group(1))
    return out


def _ci_verdict(recipe: str, number: int) -> dict:
    """Newest cc-ci !testme outcome recorded on the PR."""
    try:
        cs = _get(f"/api/v1/repos/{NS}/{recipe}/issues/{number}/comments?limit=100")
    except Exception:  # noqa: BLE001
        return {"verdict": "unknown", "build": None}
    for c in reversed(cs):
        b = c.get("body") or ""
        if "cc-ci:testme" not in b:
            continue
        m = re.search(r"/cc-ci/(\d+)", b)
        if "✅" in b or "passed" in b:
            return {"verdict": "passed", "build": m.group(1) if m else None}
        if "❌" in b or "failure" in b:
            return {"verdict": "failed", "build": m.group(1) if m else None}
        if "⏳" in b or "in progress" in b:
            return {"verdict": "running", "build": m.group(1) if m else None}
    return {"verdict": "never-run", "build": None}


def survey_pr(recipe: str, pr: dict, main_images: set[str]) -> dict:
    n = pr["number"]
    head = pr["head"]["ref"]
    row = {
        "recipe": recipe, "number": n, "title": pr.get("title", ""), "head": head,
        "url": pr.get("html_url"), "branch_kind": _branch_kind(head),
        "age_days": _days(pr.get("created_at")), "stale_days": _days(pr.get("updated_at")),
        "mergeable": pr.get("mergeable"),
    }
    try:
        row["behind_main"] = _get(
            f"/api/v1/repos/{NS}/{recipe}/compare/{urllib.parse.quote(head, safe='')}...main"
        ).get("total_commits", 0)
        row["ahead"] = _get(
            f"/api/v1/repos/{NS}/{recipe}/compare/main...{urllib.parse.quote(head, safe='')}"
        ).get("total_commits", 0)
    except Exception:  # noqa: BLE001
        row["behind_main"], row["ahead"] = None, None
    # A FAILED diff fetch must never look like an empty diff: gitea#4 404s on .diff (force-pushed
    # branch) and would otherwise be flagged EMPTY-DIFF and closed — while being a verified, green,
    # needed fix. Unknown is its own state.
    diff = None
    try:
        body = _get(f"/{NS}/{recipe}/pulls/{n}.diff", raw=True)
        if body.lstrip().startswith(("diff --git", "From ")) or not body.strip():
            diff = body
    except Exception:  # noqa: BLE001
        diff = None
    row["diff_files"] = None if diff is None else len(re.findall(r"^diff --git ", diff, re.M))
    adds = re.findall(r"^\+\s*image:\s*[\"']?([^\"'\s]+)", diff or "", re.M)
    row["adds_images"] = sorted(set(adds))
    row["already_in_main"] = sorted({i for i in set(adds) if i in main_images})
    # Nothing left to contribute: it touches files but every image it introduces is already pinned.
    # Only claim obsolete when the diff was actually READ. No diff, no verdict.
    row["obsolete"] = diff is not None and bool(adds) and set(adds).issubset(main_images)
    row["ci"] = _ci_verdict(recipe, n)
    return row


def all_recipes() -> list[str]:
    out = []
    for ln in open(USED_RECIPES):
        p = ln.split()
        if len(p) >= 2 and not ln.startswith(("#", "`")) and p[1] in ("weekly", "external"):
            out.append(p[0])
    return out


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("recipes", nargs="*")
    ap.add_argument("--json", action="store_true")
    a = ap.parse_args()

    rows = []
    for r in (a.recipes or all_recipes()):
        try:
            prs = _get(f"/api/v1/repos/{NS}/{r}/pulls?state=open&limit=50")
        except urllib.error.HTTPError as e:
            rows.append({"recipe": r, "error": f"HTTP {e.code}"})
            continue
        if not prs:
            continue
        mi = _main_images(r)
        for pr in prs:
            rows.append(survey_pr(r, pr, mi))

    if a.json:
        print(json.dumps(rows, indent=2))
        return 0

    print(f"{len(rows)} open PR(s)\n")
    for x in sorted(rows, key=lambda z: (z.get("recipe", ""), z.get("number", 0))):
        if x.get("error"):
            print(f"  {x['recipe']}: {x['error']}")
            continue
        flags = []
        if x["obsolete"]:
            flags.append("OBSOLETE(images already in main)")
        if x["branch_kind"] == "ci-artifact":
            flags.append("CI-ARTIFACT")
        if x["diff_files"] == 0:
            flags.append("EMPTY-DIFF")
        if x["diff_files"] is None:
            flags.append("DIFF-UNREADABLE(do not close on this)")
        if x["mergeable"] is False:
            flags.append("CONFLICTS")
        if (x["behind_main"] or 0) > 0:
            flags.append(f"BEHIND-{x['behind_main']}")
        print(f"  {x['recipe']}#{x['number']:<3} {x['title'][:52]}")
        print(f"      {x['branch_kind']:12} age={x['age_days']}d idle={x['stale_days']}d "
              f"ci={x['ci']['verdict']}({x['ci']['build'] or '-'}) files={x['diff_files'] if x['diff_files'] is not None else '?'}")
        if x["adds_images"]:
            print(f"      adds: {', '.join(i.split('/')[-1] for i in x['adds_images'][:4])}")
        if flags:
            print(f"      >> {' | '.join(flags)}")
    return 0


if __name__ == "__main__":
    sys.exit(main())