advisory-scan: paginate, and count by advisory DATE when versions can't be ordered

Answers 'how can the weekly run produce counts like the hand count?' — by doing exactly what the hand count did, deterministically. Two changes: 1. PAGINATION. The scanner requested per_page=100 and stopped. This endpoint caps at 100 AND ignores ?page= (it re-returns the same rows — which is how a manual count first produced exact triplicates and a bogus 300). Busy projects were silently truncated: discourse has 286 advisories, so a single page could not see the window at all. Now follows the Link rel=next cursor to exhaustion. 2. DATE-BASED FALLBACK. Version strings cannot be ordered across a scheme change (discourse semver 3.5.3 -> calver 2026.7.1), which is why the scan first reported a false 133, then correctly refused. Release DATES always order. When the version path refuses, the scan now resolves both versions to their git tag dates on the primary repo and counts advisories PUBLISHED in that window, labelling the method in the output. The version path is still preferred when usable — it is exact rather than temporal. Verified: discourse 3.5.3 -> 2026.7.1 now reports 123, matching the hand count (1 critical, 16 high, 91 medium, 16 low; window 2025-12-30 -> 2026-07-31); gitea 1.27.0 -> 1.27.1 still reports 2 via the version path.
2026-08-10 21:02:39 +00:00
parent fc36d0e10f
commit 98a624a13a
1 changed files with 96 additions and 5 deletions
@@ -80,6 +80,57 @@ def _github_token() -> str | None:
        return None


+def _gh_paginate(url: str, hdrs: dict, max_pages: int = 20):
+    """Yield every row from a GitHub list endpoint, following Link rel=\"next\" cursors."""
+    seen_keys = set()
+    for _ in range(max_pages):
+        req = urllib.request.Request(url, headers={"User-Agent": UA, **hdrs})
+        with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
+            rows = json.load(r)
+            link = r.headers.get("Link", "") or ""
+        fresh = 0
+        for a in rows:
+            k = a.get("ghsa_id") or json.dumps(a, sort_keys=True)[:120]
+            if k not in seen_keys:
+                seen_keys.add(k); fresh += 1
+                yield a
+        nxt = None
+        for part in link.split(","):
+            if 'rel="next"' in part:
+                nxt = part.split(";")[0].strip().strip("<>")
+        if not nxt or fresh == 0:
+            return
+        url = nxt
+
+
+def _tag_date(owner: str, repo: str, version: str | None) -> str | None:
+    """Publish date of a release tag, for DATE-BASED classification (see classify_by_date).
+
+    Version strings cannot be ordered across a scheme change (semver → calver), but tag dates
+    always can. Tries the common tag spellings; returns an ISO timestamp or None."""
+    if not version:
+        return None
+    hdrs = {"Accept": "application/vnd.github+json"}
+    tok = _github_token()
+    if tok:
+        hdrs["Authorization"] = f"Bearer {tok}"
+    for tag in (f"v{version}", version):
+        try:
+            ref = json.loads(_fetch(f"https://api.github.com/repos/{owner}/{repo}/git/ref/tags/{tag}", hdrs))
+            obj = ref.get("object", {})
+            sha, typ = obj.get("sha"), obj.get("type")
+            if typ == "tag":
+                t = json.loads(_fetch(f"https://api.github.com/repos/{owner}/{repo}/git/tags/{sha}", hdrs))
+                if t.get("tagger", {}).get("date"):
+                    return t["tagger"]["date"]
+                sha = t.get("object", {}).get("sha")
+            c = json.loads(_fetch(f"https://api.github.com/repos/{owner}/{repo}/commits/{sha}", hdrs))
+            return c["commit"]["committer"]["date"]
+        except Exception:  # noqa: BLE001 — try the next spelling
+            continue
+    return None
+
+
 def _fetch(url: str, headers: dict | None = None) -> str:
    h = {"User-Agent": UA, "Accept-Encoding": "gzip"}
    h.update(headers or {})
@@ -142,7 +193,11 @@ def github_advisories(urls: list[str]) -> list[dict]:
            hdrs["Authorization"] = f"Bearer {tok}"
        entry = {"source": f"github-advisories:{owner}/{repo}", "status": "ok", "advisories": []}
        try:
-            for a in json.loads(_fetch(api, hdrs)):
+            # PAGINATE. This endpoint caps at 100 per response and IGNORES ?page= — it returns the
+            # same rows again, which silently truncates busy projects (discourse has 286; a hand
+            # count on 2026-08-10 found 123 CVEs in one upgrade window that a single page missed).
+            # Follow the Link rel="next" cursor to exhaustion instead.
+            for a in _gh_paginate(api, hdrs):
                # An advisory carries ONE ENTRY PER PATCHED RELEASE LINE. n8n patches three
                # (1.123.32, 2.17.4, 2.18.1); reading only vulnerabilities[0] silently dropped the
                # line our deployment is actually on, so CVE-2026-42231/42232 classified as
@@ -161,6 +216,7 @@ def github_advisories(urls: list[str]) -> list[dict]:
                            filter(None, (v.get("patched_versions") for v in vulns))
                        ) or None,
                        "url": a.get("html_url"),
+                        "published_at": a.get("published_at"),
                    }
                )
        except urllib.error.HTTPError as e:
@@ -243,7 +299,8 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) -

    def record(cve: str, src: str, **extra):
        e = report["cves"].setdefault(cve, {"sources": [], "severity": None, "ghsa": None,
-                                            "vulnerable_range": None, "patched": None, "context": None})
+                                            "vulnerable_range": None, "patched": None,
+                                            "context": None, "published_at": None})
        if src not in e["sources"]:
            e["sources"].append(src)
        for k, v in extra.items():
@@ -257,7 +314,7 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) -
            if a.get("cve"):
                record(a["cve"], entry["source"], severity=a.get("severity"), ghsa=a.get("ghsa"),
                       vulnerable_range=a.get("vulnerable_range"), patched=a.get("patched"),
-                       context=a.get("summary"))
+                       context=a.get("summary"), published_at=a.get("published_at"))

    for entry in vendor_pages(urls):
        report["sources"].append({"source": entry["source"], "status": entry["status"],
@@ -317,11 +374,41 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) -
            e["classification"] = "unclassified" if not (kf and kt and kp) else "outside-window"
            if e["classification"] == "unclassified":
                unknown.append(cve)
-    report["fixed_by_this_upgrade"] = sorted(fixed)
-    report["unclassified"] = sorted(unknown)
    # NEVER report 0 for something we could not determine. When classification was refused, the
    # count is UNKNOWN (null) — a 0 would be read as "no CVEs", which is an assertion this scan
    # cannot make. Consumers must distinguish "counted 0" from "could not count".
+    # DATE-BASED FALLBACK (phase datewin). Version strings cannot be ordered across a scheme change,
+    # but RELEASE DATES always can. Resolve both versions to their tag dates on the primary repo and
+    # count advisories PUBLISHED inside that window — the method a hand count used on 2026-08-10 to
+    # establish discourse 3.5.3 (2025-12-30) → 2026.7.1 (2026-07-31) = 123 CVEs, where version
+    # comparison had produced first a false 133 and then a refusal. Only used when the version path
+    # refuses; a successful version classification is always preferred (it is exact, not temporal).
+    date_window = None
+    if scheme_change and primary and primary.startswith("github-advisories:"):
+        owner_repo = primary.split(":", 1)[1]
+        owner, _, repo = owner_repo.partition("/")
+        d_from, d_to = _tag_date(owner, repo, v_from), _tag_date(owner, repo, v_to)
+        if d_from and d_to and d_from < d_to:
+            date_window = (d_from, d_to)
+            fixed, unknown = [], []
+            for cve, e in report["cves"].items():
+                pub = e.get("published_at")
+                if primary in e["sources"] and pub and d_from < pub <= d_to:
+                    e["classification"] = "fixed-by-this-upgrade (by advisory publish date)"
+                    fixed.append(cve)
+                else:
+                    e["classification"] = ("outside-window (by date)" if primary in e["sources"]
+                                           else "unclassified: different image than the given window")
+                    if primary not in e["sources"]:
+                        unknown.append(cve)
+            scheme_change = False  # resolved by date; a real count is available
+            report["classified_by"] = "advisory publish date (version scheme changed)"
+            report["date_window"] = {"from": d_from, "to": d_to}
+    if not date_window and not scheme_change:
+        report["classified_by"] = "patched version ranges"
+
+    report["fixed_by_this_upgrade"] = sorted(fixed)
+    report["unclassified"] = sorted(unknown)
    report["count_known"] = not scheme_change
    report["cve_count_fixed"] = len(fixed) if not scheme_change else None
    report["cve_count_total_seen"] = len(report["cves"])
@@ -361,6 +448,10 @@ def markdown(rep: dict) -> str:
        return "\n".join(L)
    if rep["fixed_by_this_upgrade"]:
        L.append(f"\n**CVEs fixed by this upgrade: {rep['cve_count_fixed']}**\n")
+        if rep.get("date_window"):
+            L.append(f"_Counted by advisory PUBLISH DATE ({rep['date_window']['from'][:10]} → "
+                     f"{rep['date_window']['to'][:10]}) because the version scheme changed across this "
+                     f"jump; version strings cannot be ordered across it._\n")
        L.append("| CVE | severity | fixed in | advisory | source |")
        L.append("|---|---|---|---|---|")
        for cve in rep["fixed_by_this_upgrade"]: