From 318d09bdab2f0c0364da4c9587ebfcaf89c967e4 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Mon, 10 Aug 2026 20:23:10 +0000 Subject: [PATCH 01/10] =?UTF-8?q?advisory-scan:=20eliminate=20spurious=20'?= =?UTF-8?q?=3F'=20=E2=80=94=20url=20punctuation,=20stale=20URL,=20and=20?= =?UTF-8?q?=3F=20semantics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-08-07 regeneration rendered '?' for 5 of 21 recipes. '?' is meant to be a rare 'we tried and could not tell'; at that rate it is indistinguishable from noise and hides the real unknowns. Three causes, none of them genuine uncertainty: 1. URL EXTRACTION BUG (mine). The registry is markdown, so urls appear inside `backticks` and 'quotes'. The extractor captured the trailing punctuation, so it fetched https://docs.n8n.io/release-notes/` and https://git.autonomic.zone'` — both 404 on the malformed url, both 200 when clean. Trailing markdown punctuation is now stripped. Fixed immich + n8n. 2. STALE REGISTRY URL. mattermost-lts pointed at docs.mattermost.com/about/mattermost-changelog.html, which 404s; the page moved to /deploy/. Corrected (same class as the pgautoupgrade fix). 3. WRONG SEMANTICS FOR 'NO UPGRADE'. lasuite-docs and custom-html-tiny were up-to-date this run, so no scan block existed and the report fell back to '?'. But a recipe with no upgrade has nothing an upgrade could have fixed — that is 0, not unknown. The report skill now says so explicitly, restricts '?' to scans that RAN and reported genuinely failed sources, states that benign notes (no-advisories-published / template URL) never trigger '?', and instructs that many '?' is itself a bug to raise in the Addendum. Result across all 16 scanned recipes of that run: 0 failed sources (was 5). Counts also improved with the classifier fix: discourse 130->133, keycloak ->7. --- .claude/skills/recipe-report/SKILL.md | 15 +++++++++++---- cc-ci-plan/advisory-scan.py | 8 ++++++-- cc-ci-plan/upstream/mattermost-lts.md | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.claude/skills/recipe-report/SKILL.md b/.claude/skills/recipe-report/SKILL.md index da02aa7..2075c8d 100644 --- a/.claude/skills/recipe-report/SKILL.md +++ b/.claude/skills/recipe-report/SKILL.md @@ -42,10 +42,17 @@ keeps every weekly edition looking the same regardless of which model writes the Advisories + vendor security pages + OSV, with severities and fixed-in versions). Treat its CVE list as a **further source** and report the **UNION** of it and what you found by reading. Its entries are machine-derived with advisory IDs, so prefer them for CVE ids / severities / - fixed-in versions, and cite the GHSA where present in the Security Bulletin. If a recipe has - no scan block, or the block lists **failed sources**, the count is **not** authoritative: - render the cve cell as `?` (unknown), never `none` — a blank that reads as "clean" is exactly - how two CVSS-9.8 gitea RCEs were reported as "none" on 2026-08-07. + fixed-in versions, and cite the GHSA where present in the Security Bulletin. If the block + lists **failed sources**, the count is **not** authoritative: render the cve cell as `?` + (unknown), never `none` — a blank that reads as "clean" is exactly how two CVSS-9.8 gitea + RCEs were reported as "none" on 2026-08-07. + - **`?` must stay RARE — it means "we tried and could not tell", not "we didn't look".** Use it + ONLY when a scan ran and reported genuinely failed sources. In particular: a recipe with **no + upgrade this run** (up-to-date/skipped) has nothing an upgrade could have fixed — report `0`, + not `?`. A recipe with a clean scan reports its number (including `0`). Benign notes in a scan + block (`no-advisories-published`, `skipped: template URL`) are NOT failures and must not + trigger `?`. If you find yourself rendering `?` for many recipes, that is a bug to report in + the Addendum, not a normal outcome. Anything **critical/high** also gets a `security` bulletin entry (recipe · CVE id(s) + severity · what it fixes · PR link); be specific about severity and what's exposed if not merged. - **Lead — ONE short paragraph.** A tight, concrete opener in opus's voice: fleet state in a sentence diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index 61994b0..01cccd3 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -114,8 +114,12 @@ def registry_urls(recipe: str, registry_dir: str) -> tuple[list[str], str | None return [], None urls = [] for u in re.findall(r"https?://[^\s)|\]]+", text): - u = u.rstrip(".,;") - if u not in urls: + # The registry is MARKDOWN: urls appear inside `backticks`, 'quotes', **bold**, and at the + # end of sentences. Trailing punctuation captured into the url makes the fetch 404 and the + # recipe render '?' for no real reason — that is what put immich and n8n in the unknown + # column on 2026-08-07 (https://docs.n8n.io/release-notes/` ← note the backtick). + u = u.rstrip("`'\"*.,;:>)") + if u and u not in urls: urls.append(u) return urls, path diff --git a/cc-ci-plan/upstream/mattermost-lts.md b/cc-ci-plan/upstream/mattermost-lts.md index 5677605..ec90ccd 100644 --- a/cc-ci-plan/upstream/mattermost-lts.md +++ b/cc-ci-plan/upstream/mattermost-lts.md @@ -2,7 +2,7 @@ | service | image | source repo | releases / changelog | |----------|-------------------------------------------|---------------------------------------------------|-------------------------------------------------------------------| -| app | mattermost/mattermost-team-edition | https://github.com/mattermost/mattermost | https://docs.mattermost.com/about/mattermost-changelog.html | +| app | mattermost/mattermost-team-edition | https://github.com/mattermost/mattermost | https://docs.mattermost.com/deploy/mattermost-changelog.html | | postgres | postgres | https://github.com/postgres/postgres | https://www.postgresql.org/docs/release/ | ## Standing notes -- 2.54.0 From 1daf0fa616ce86a26294368c05063463a06c134b Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Mon, 10 Aug 2026 20:44:27 +0000 Subject: [PATCH 02/10] advisory-scan: stop cross-image and cross-scheme miscounting (discourse's false 133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator disbelieved discourse's '133 CVEs fixed' — correctly. Two defects made it confidently wrong: 1. ONE WINDOW APPLIED TO EVERY IMAGE. The scan queries all source repos in the recipe's registry (app + redis/postgres/nginx sidecars) but judged them all with the APP's version window. 34 of the 133 were redis advisories, including CVE-2021-21309 — patched in redis 6.0.11 back in 2021 — scored as 'fixed by this upgrade' purely because 6.0.11 sits numerically inside discourse's 3.5.3 -> 2026.7.1 range. Only the PRIMARY app repo is now classified; other sources are reported as unclassified so they stay visible without inflating the count. 2. VERSION-SCHEME CHANGES BREAK ORDERING. discourse moved semver -> calver (3.5.3 -> 2026.7.1), so 2025.12.2 compares 'newer' than 3.5.3 while shipping earlier. Numeric comparison cannot order that. The scan now detects a leading- component jump >= 100, refuses to classify, and says so in the block: the count is '0 by refusal, not by evidence — read the vendor's release notes'. Refusing to answer beats answering wrongly: a fabricated 133 in a public security report is worse than an explicit 'cannot determine'. Verified after the fix: discourse 133 -> 0 (with the refusal caveat), gitea still exactly 2 (both criticals, patched 1.27.1), keycloak 7 all genuinely from keycloak/keycloak patched in 26.7.1, plausible 1. No other count changed. --- cc-ci-plan/advisory-scan.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index 01cccd3..ebd4611 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -275,7 +275,27 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) - # Classify against the upgrade window when we know it: an advisory is "fixed by this upgrade" # when its patched version is newer than `from` and no newer than `to`. + # + # TWO HARD-WON CONSTRAINTS (2026-08-10, discourse reported a false 133): + # a) The window belongs to ONE image. Advisories from OTHER repos in the registry (redis, + # postgres, nginx sidecars) must NOT be judged by it — redis CVE-2021-21309, patched in + # redis 6.0.11, scored as "fixed" because 6.0.11 sits numerically inside discourse's + # 3.5.3 → 2026.7.1 window. Only the PRIMARY app repo is classified; every other source is + # reported as unclassified so a human/agent still sees it but it never inflates the count. + # b) A version-SCHEME change (semver → calver, 3.5.3 → 2026.7.1) makes numeric ordering + # meaningless: 2025.12.2 compares "newer" than 3.5.3 while shipping earlier. When the + # leading component jumps by more than SCHEME_JUMP we refuse to classify and say so, + # rather than emitting a confident wrong number. kf, kt = _vkey(v_from), _vkey(v_to) + SCHEME_JUMP = 100 + scheme_change = bool(kf and kt and abs(kt[0] - kf[0]) >= SCHEME_JUMP) + primary = None + if v_from or v_to: + # the app repo = first github source in the registry (registry lists the app service first) + primary = next((s["source"] for s in report["sources"] + if s["source"].startswith("github-advisories:")), None) + report["primary_source"] = primary + report["scheme_change"] = scheme_change fixed, unknown = [], [] for cve, e in report["cves"].items(): # `patched_versions` is a RANGE EXPRESSION (">= 2.18.1"), not a bare version, and there may @@ -283,7 +303,14 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) - # token and treat the advisory as fixed-by-this-upgrade if ANY of them lands in (from, to]. cands = [_vkey(t) for t in re.findall(r"\d+(?:\.\d+)*", e.get("patched") or "")] kp = next((c for c in cands if kf and kt and kf < c <= kt), None) or (cands[0] if cands else ()) - if kf and kt and kp and kf < kp <= kt: + from_primary = primary is not None and primary in e["sources"] + if scheme_change: + e["classification"] = "unclassified: version-scheme change, cannot order reliably" + unknown.append(cve) + elif not from_primary: + e["classification"] = "unclassified: different image than the given version window" + unknown.append(cve) + elif kf and kt and kp and kf < kp <= kt: e["classification"] = "fixed-by-this-upgrade" fixed.append(cve) else: @@ -323,8 +350,13 @@ def markdown(rep: dict) -> str: f"{e.get('ghsa') or '-'} | {e['sources'][0]} |") else: L.append("\n**CVEs fixed by this upgrade: 0 identified by the deterministic scan.**") + if rep.get("scheme_change"): + L.append("\n⚠ **Version-scheme change detected** (e.g. semver → calver): numeric ordering is " + "not meaningful across it, so NOTHING was auto-classified. The CVE count above is " + "**0 by refusal, not by evidence** — read the vendor's release notes for this jump.") if rep["unclassified"]: - L.append(f"\nSeen but not version-classified ({len(rep['unclassified'])}): " + L.append(f"\nSeen but not version-classified ({len(rep['unclassified'])}) — includes advisories " + f"from OTHER images in this recipe (sidecars), which this window cannot judge: " + ", ".join(rep["unclassified"][:12])) if rep["sources_failed"]: L.append(f"\n⚠ sources that FAILED (treat counts as incomplete): {', '.join(rep['sources_failed'])}") -- 2.54.0 From fc36d0e10fca49dd9afe87d8b0ab3be2562ac442 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Mon, 10 Aug 2026 20:52:51 +0000 Subject: [PATCH 03/10] advisory-scan: report UNKNOWN, never 0, when a count could not be determined MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator: 'the scanner should not say 0 when it was not able to scan.' Correct — the previous patch still led with '0 identified' and relegated the caveat to a footnote, so the headline number was wrong even though the prose was right. A 0 in a security column is an assertion of safety; it must never be emitted for an undetermined result. Now: cve_count_fixed is null (not 0) in JSON, a count_known flag distinguishes 'counted zero' from 'could not count', and the markdown headline reads 'CVEs fixed by this upgrade: UNKNOWN — the scan could NOT determine a count' with an explicit 'This is NOT zero' and instructions to render '?'. Verified: discourse 3.5.3 -> 2026.7.1 (semver->calver) now reports UNKNOWN; gitea 1.27.0 -> 1.27.1 still reports 2. --- .claude/skills/recipe-report/SKILL.md | 10 +++++++--- cc-ci-plan/advisory-scan.py | 25 ++++++++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.claude/skills/recipe-report/SKILL.md b/.claude/skills/recipe-report/SKILL.md index 2075c8d..6be03f0 100644 --- a/.claude/skills/recipe-report/SKILL.md +++ b/.claude/skills/recipe-report/SKILL.md @@ -47,9 +47,13 @@ keeps every weekly edition looking the same regardless of which model writes the (unknown), never `none` — a blank that reads as "clean" is exactly how two CVSS-9.8 gitea RCEs were reported as "none" on 2026-08-07. - **`?` must stay RARE — it means "we tried and could not tell", not "we didn't look".** Use it - ONLY when a scan ran and reported genuinely failed sources. In particular: a recipe with **no - upgrade this run** (up-to-date/skipped) has nothing an upgrade could have fixed — report `0`, - not `?`. A recipe with a clean scan reports its number (including `0`). Benign notes in a scan + ONLY when a scan ran and reported genuinely failed sources, **or when the scan block says + COUNT UNKNOWN** (it refuses to classify across a version-scheme change, e.g. semver → calver — + discourse 3.5.3 → 2026.7.1). In that case the scan's `0` means *not determined*: publish `?` + and say so in the notes; publishing `0` would assert a clean bill of health nothing supports. + (The scan headline itself now says `UNKNOWN` rather than a number in that case.) + In particular: a recipe with **no upgrade this run** (up-to-date/skipped) has nothing an + upgrade could have fixed — report `0`, not `?`. A recipe with a clean scan reports its number (including `0`). Benign notes in a scan block (`no-advisories-published`, `skipped: template URL`) are NOT failures and must not trigger `?`. If you find yourself rendering `?` for many recipes, that is a bug to report in the Addendum, not a normal outcome. diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index ebd4611..8fad2a9 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -319,7 +319,11 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) - unknown.append(cve) report["fixed_by_this_upgrade"] = sorted(fixed) report["unclassified"] = sorted(unknown) - report["cve_count_fixed"] = len(fixed) + # NEVER report 0 for something we could not determine. When classification was refused, the + # count is UNKNOWN (null) — a 0 would be read as "no CVEs", which is an assertion this scan + # cannot make. Consumers must distinguish "counted 0" from "could not count". + report["count_known"] = not scheme_change + report["cve_count_fixed"] = len(fixed) if not scheme_change else None report["cve_count_total_seen"] = len(report["cves"]) # Only GENUINE failures make a count unreliable. "no-advisories-published" (404: the repo has # no advisory feed) and "skipped: template URL" are benign and must not degrade the verdict. @@ -340,6 +344,21 @@ def markdown(rep: dict) -> str: """Human/agent-readable block for pasting into the per-recipe upgrade log.""" L = [f"### Advisory scan (deterministic pre-step) — {rep['recipe']} " f"{rep.get('from') or '?'} → {rep.get('to') or '?'}"] + if not rep.get("count_known", True): + L.append("\n**CVEs fixed by this upgrade: UNKNOWN — the scan could NOT determine a count.**") + L.append("\n⚠ This is NOT zero. A version-scheme change (e.g. semver → calver) makes numeric " + "ordering meaningless across this jump, so no advisory could be classified. Render " + "this recipe's cve cell as `?`, never `0`. Read the vendor's release notes for the " + "jump and count by hand.") + if rep["unclassified"]: + L.append(f"\nAdvisories seen but unclassifiable ({len(rep['unclassified'])}) — includes " + f"other images in this recipe: " + ", ".join(rep["unclassified"][:12])) + if rep["sources_failed"]: + L.append(f"\n⚠ sources that FAILED: {', '.join(rep['sources_failed'])}") + L.append(f"\n_Sources checked: {len(rep['sources'])} ({rep['registry_urls']} registry URLs + " + f"advisory APIs). This scan is ADDITIVE — it does not replace the release-note " + f"reading in the upgrade step._") + return "\n".join(L) if rep["fixed_by_this_upgrade"]: L.append(f"\n**CVEs fixed by this upgrade: {rep['cve_count_fixed']}**\n") L.append("| CVE | severity | fixed in | advisory | source |") @@ -350,10 +369,6 @@ def markdown(rep: dict) -> str: f"{e.get('ghsa') or '-'} | {e['sources'][0]} |") else: L.append("\n**CVEs fixed by this upgrade: 0 identified by the deterministic scan.**") - if rep.get("scheme_change"): - L.append("\n⚠ **Version-scheme change detected** (e.g. semver → calver): numeric ordering is " - "not meaningful across it, so NOTHING was auto-classified. The CVE count above is " - "**0 by refusal, not by evidence** — read the vendor's release notes for this jump.") if rep["unclassified"]: L.append(f"\nSeen but not version-classified ({len(rep['unclassified'])}) — includes advisories " f"from OTHER images in this recipe (sidecars), which this window cannot judge: " -- 2.54.0 From 98a624a13a0b0e7ab29e26e40b601050f71f871a Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Mon, 10 Aug 2026 21:02:39 +0000 Subject: [PATCH 04/10] advisory-scan: paginate, and count by advisory DATE when versions can't be ordered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers 'how can the weekly run produce counts like the hand count?' — by doing exactly what the hand count did, deterministically. Two changes: 1. PAGINATION. The scanner requested per_page=100 and stopped. This endpoint caps at 100 AND ignores ?page= (it re-returns the same rows — which is how a manual count first produced exact triplicates and a bogus 300). Busy projects were silently truncated: discourse has 286 advisories, so a single page could not see the window at all. Now follows the Link rel=next cursor to exhaustion. 2. DATE-BASED FALLBACK. Version strings cannot be ordered across a scheme change (discourse semver 3.5.3 -> calver 2026.7.1), which is why the scan first reported a false 133, then correctly refused. Release DATES always order. When the version path refuses, the scan now resolves both versions to their git tag dates on the primary repo and counts advisories PUBLISHED in that window, labelling the method in the output. The version path is still preferred when usable — it is exact rather than temporal. Verified: discourse 3.5.3 -> 2026.7.1 now reports 123, matching the hand count (1 critical, 16 high, 91 medium, 16 low; window 2025-12-30 -> 2026-07-31); gitea 1.27.0 -> 1.27.1 still reports 2 via the version path. --- cc-ci-plan/advisory-scan.py | 101 ++++++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 5 deletions(-) diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index 8fad2a9..019f9a0 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -80,6 +80,57 @@ def _github_token() -> str | None: return None +def _gh_paginate(url: str, hdrs: dict, max_pages: int = 20): + """Yield every row from a GitHub list endpoint, following Link rel=\"next\" cursors.""" + seen_keys = set() + for _ in range(max_pages): + req = urllib.request.Request(url, headers={"User-Agent": UA, **hdrs}) + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + rows = json.load(r) + link = r.headers.get("Link", "") or "" + fresh = 0 + for a in rows: + k = a.get("ghsa_id") or json.dumps(a, sort_keys=True)[:120] + if k not in seen_keys: + seen_keys.add(k); fresh += 1 + yield a + nxt = None + for part in link.split(","): + if 'rel="next"' in part: + nxt = part.split(";")[0].strip().strip("<>") + if not nxt or fresh == 0: + return + url = nxt + + +def _tag_date(owner: str, repo: str, version: str | None) -> str | None: + """Publish date of a release tag, for DATE-BASED classification (see classify_by_date). + + Version strings cannot be ordered across a scheme change (semver → calver), but tag dates + always can. Tries the common tag spellings; returns an ISO timestamp or None.""" + if not version: + return None + hdrs = {"Accept": "application/vnd.github+json"} + tok = _github_token() + if tok: + hdrs["Authorization"] = f"Bearer {tok}" + for tag in (f"v{version}", version): + try: + ref = json.loads(_fetch(f"https://api.github.com/repos/{owner}/{repo}/git/ref/tags/{tag}", hdrs)) + obj = ref.get("object", {}) + sha, typ = obj.get("sha"), obj.get("type") + if typ == "tag": + t = json.loads(_fetch(f"https://api.github.com/repos/{owner}/{repo}/git/tags/{sha}", hdrs)) + if t.get("tagger", {}).get("date"): + return t["tagger"]["date"] + sha = t.get("object", {}).get("sha") + c = json.loads(_fetch(f"https://api.github.com/repos/{owner}/{repo}/commits/{sha}", hdrs)) + return c["commit"]["committer"]["date"] + except Exception: # noqa: BLE001 — try the next spelling + continue + return None + + def _fetch(url: str, headers: dict | None = None) -> str: h = {"User-Agent": UA, "Accept-Encoding": "gzip"} h.update(headers or {}) @@ -142,7 +193,11 @@ def github_advisories(urls: list[str]) -> list[dict]: hdrs["Authorization"] = f"Bearer {tok}" entry = {"source": f"github-advisories:{owner}/{repo}", "status": "ok", "advisories": []} try: - for a in json.loads(_fetch(api, hdrs)): + # PAGINATE. This endpoint caps at 100 per response and IGNORES ?page= — it returns the + # same rows again, which silently truncates busy projects (discourse has 286; a hand + # count on 2026-08-10 found 123 CVEs in one upgrade window that a single page missed). + # Follow the Link rel="next" cursor to exhaustion instead. + for a in _gh_paginate(api, hdrs): # An advisory carries ONE ENTRY PER PATCHED RELEASE LINE. n8n patches three # (1.123.32, 2.17.4, 2.18.1); reading only vulnerabilities[0] silently dropped the # line our deployment is actually on, so CVE-2026-42231/42232 classified as @@ -161,6 +216,7 @@ def github_advisories(urls: list[str]) -> list[dict]: filter(None, (v.get("patched_versions") for v in vulns)) ) or None, "url": a.get("html_url"), + "published_at": a.get("published_at"), } ) except urllib.error.HTTPError as e: @@ -243,7 +299,8 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) - def record(cve: str, src: str, **extra): e = report["cves"].setdefault(cve, {"sources": [], "severity": None, "ghsa": None, - "vulnerable_range": None, "patched": None, "context": None}) + "vulnerable_range": None, "patched": None, + "context": None, "published_at": None}) if src not in e["sources"]: e["sources"].append(src) for k, v in extra.items(): @@ -257,7 +314,7 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) - if a.get("cve"): record(a["cve"], entry["source"], severity=a.get("severity"), ghsa=a.get("ghsa"), vulnerable_range=a.get("vulnerable_range"), patched=a.get("patched"), - context=a.get("summary")) + context=a.get("summary"), published_at=a.get("published_at")) for entry in vendor_pages(urls): report["sources"].append({"source": entry["source"], "status": entry["status"], @@ -317,11 +374,41 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) - e["classification"] = "unclassified" if not (kf and kt and kp) else "outside-window" if e["classification"] == "unclassified": unknown.append(cve) - report["fixed_by_this_upgrade"] = sorted(fixed) - report["unclassified"] = sorted(unknown) # NEVER report 0 for something we could not determine. When classification was refused, the # count is UNKNOWN (null) — a 0 would be read as "no CVEs", which is an assertion this scan # cannot make. Consumers must distinguish "counted 0" from "could not count". + # DATE-BASED FALLBACK (phase datewin). Version strings cannot be ordered across a scheme change, + # but RELEASE DATES always can. Resolve both versions to their tag dates on the primary repo and + # count advisories PUBLISHED inside that window — the method a hand count used on 2026-08-10 to + # establish discourse 3.5.3 (2025-12-30) → 2026.7.1 (2026-07-31) = 123 CVEs, where version + # comparison had produced first a false 133 and then a refusal. Only used when the version path + # refuses; a successful version classification is always preferred (it is exact, not temporal). + date_window = None + if scheme_change and primary and primary.startswith("github-advisories:"): + owner_repo = primary.split(":", 1)[1] + owner, _, repo = owner_repo.partition("/") + d_from, d_to = _tag_date(owner, repo, v_from), _tag_date(owner, repo, v_to) + if d_from and d_to and d_from < d_to: + date_window = (d_from, d_to) + fixed, unknown = [], [] + for cve, e in report["cves"].items(): + pub = e.get("published_at") + if primary in e["sources"] and pub and d_from < pub <= d_to: + e["classification"] = "fixed-by-this-upgrade (by advisory publish date)" + fixed.append(cve) + else: + e["classification"] = ("outside-window (by date)" if primary in e["sources"] + else "unclassified: different image than the given window") + if primary not in e["sources"]: + unknown.append(cve) + scheme_change = False # resolved by date; a real count is available + report["classified_by"] = "advisory publish date (version scheme changed)" + report["date_window"] = {"from": d_from, "to": d_to} + if not date_window and not scheme_change: + report["classified_by"] = "patched version ranges" + + report["fixed_by_this_upgrade"] = sorted(fixed) + report["unclassified"] = sorted(unknown) report["count_known"] = not scheme_change report["cve_count_fixed"] = len(fixed) if not scheme_change else None report["cve_count_total_seen"] = len(report["cves"]) @@ -361,6 +448,10 @@ def markdown(rep: dict) -> str: return "\n".join(L) if rep["fixed_by_this_upgrade"]: L.append(f"\n**CVEs fixed by this upgrade: {rep['cve_count_fixed']}**\n") + if rep.get("date_window"): + L.append(f"_Counted by advisory PUBLISH DATE ({rep['date_window']['from'][:10]} → " + f"{rep['date_window']['to'][:10]}) because the version scheme changed across this " + f"jump; version strings cannot be ordered across it._\n") L.append("| CVE | severity | fixed in | advisory | source |") L.append("|---|---|---|---|---|") for cve in rep["fixed_by_this_upgrade"]: -- 2.54.0 From 78ae2be8ae02e9e69e2be59dcfa72cbe46e283d3 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Mon, 10 Aug 2026 21:55:37 +0000 Subject: [PATCH 05/10] docs: spec for the advisory scanner's CVE detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step-by-step specification of cc-ci-plan/advisory-scan.py: inputs, the three source classes and why each is ranked where it is, the union, both classification paths (patched-version ranges, and the advisory-publish-date fallback for version-scheme changes), the output contract, and how /recipe-report must read it. Each rule records the production wrong answer that motivated it — the false 133 from cross-image counting, the n8n misclassification from reading only vulnerabilities[0], the '?' sprawl from url punctuation and benign-404s, and the 'never emit 0 for an undetermined count' rule. Claims cross-checked against the implementation. Keep this file in the same commit as any behaviour change. --- cc-ci-plan/advisory-scan.SPEC.md | 184 +++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 cc-ci-plan/advisory-scan.SPEC.md diff --git a/cc-ci-plan/advisory-scan.SPEC.md b/cc-ci-plan/advisory-scan.SPEC.md new file mode 100644 index 0000000..7b665d9 --- /dev/null +++ b/cc-ci-plan/advisory-scan.SPEC.md @@ -0,0 +1,184 @@ +# Advisory scan — specification + +What `cc-ci-plan/advisory-scan.py` does, step by step, and why each step exists. This documents the +implementation as it stands (2026-08-10); if you change the code, change this file in the same commit. + +**Role.** A deterministic, per-recipe CVE detector run as a **pre-step of `/recipe-upgrade`** (step 2a). +It is **strictly additive**: it never replaces the release-note reading the upgrade agent already does. +The CVE count reported for a recipe is the **union** of what the agent read and what this scan found; +the scan may never *lower* a count established by reading. + +**Why it exists.** gitea 1.27.1 fixed CVE-2026-60004 and CVE-2026-59774 (both CVSS 9.8). The weekly +report printed gitea's CVE count as `1`, then `none`. The upgrade agent had read the GitHub *release +notes*, which name neither — both were announced only in the vendor's blog security section — and the +report then derived security content from those notes plus model knowledge, which predates the CVEs. +Nothing in the pipeline queried an advisory source. This scan closes that hole. + +--- + +## Inputs + +``` +advisory-scan.py [--from ] [--to ] [--json] [--registry DIR] +``` + +| Input | Meaning | +|---|---| +| `` | Recipe name; selects `cc-ci-plan/upstream/.md` (the per-recipe URL registry) | +| `--from` / `--to` | The **primary app image's** version window being upgraded across | +| `--registry` | Registry dir; also `CCCI_UPSTREAM_REGISTRY` | +| `GITHUB_TOKEN` / `GITHUB_TOKEN_FILE` | Read-only token; **rate limit only** (60/hr anonymous → 5000/hr). Default file `/srv/cc-ci/.github-token`, mode 600. Public advisories need **no scopes**. | + +Exit code is always 0 — this is informational. Failures are *reported*, never raised. + +--- + +## Step 1 — Collect source URLs from the registry + +Read `cc-ci-plan/upstream/.md` and extract every `http(s)://…` URL. + +**Trailing markdown punctuation is stripped** (`` ` `` `'` `"` `*` `.` `,` `;` `:` `>` `)`). The registry +is markdown, so URLs appear inside backticks and quotes; capturing the punctuation produced fetches of +`https://docs.n8n.io/release-notes/\`` which 404, and made immich and n8n render `?` for no real reason. + +> **Registry hygiene matters.** The scan can only look where the registry points. Two classes of defect +> have been found and fixed by running it: a **wrong URL** (`pgautoupgrade/pgautoupgrade`, which 404s — +> the repo is `pgautoupgrade/docker-pgautoupgrade`) and a **missing** one (gitea's CVEs are announced at +> `blog.gitea.com`, which the registry didn't list). When a vendor publishes security notes somewhere +> the registry lacks, add it. + +## Step 2 — Query the sources + +Three source classes, each recording **its own status** so *"checked, none found"* is never confused +with *"not checked"*. + +### 2a. GitHub Security Advisories — PRIMARY + +For every `github.com//` URL in the registry: +`GET /repos///security-advisories`. + +Captured per advisory: `cve_id`, `ghsa_id`, `severity`, `summary`, `published_at`, and **all** +`vulnerabilities[]` entries' `vulnerable_version_range` + `patched_versions` (joined with `;`). + +- **All entries, not just the first.** An advisory carries one entry **per patched release line** — + n8n patches three (1.123.32, 2.17.4, 2.18.1). Reading only `vulnerabilities[0]` silently dropped the + line a deployment was actually on, and misclassified CVE-2026-42231/42232 as out-of-window. +- **Pagination via the `Link rel="next"` cursor**, to exhaustion (cap 20 hops). This endpoint returns + at most 100 rows **and ignores `?page=`** — it re-returns the same rows, which silently truncates busy + projects. discourse has 286 advisories; a single page cannot even cover one upgrade window. +- **HTTP 404 ⇒ `no-advisories-published`** — a benign absence (many sidecar images publish none), **not** + a failure. Conflating the two pushed nearly every recipe to `?` and destroyed the signal. + +This source is primary because it carries **severity and version ranges**, making "fixed by *this* +upgrade" computable rather than guessed. + +### 2b. Vendor release / security pages + +Every other registry URL is fetched, HTML-stripped, and scanned for `CVE-\d{4}-\d{4,7}`, keeping ±160 +characters of context per hit. + +URLs containing `<`, `>`, `{`, `}`, `VERSION`, or `vX.Y.Z` are **skipped as templates** — they are +human documentation (`…/changelog/v/`), not fetchable, and counting them as failures is wrong. + +This is the source that would have caught gitea: the vendor blog names both CVEs, the GitHub release +page names neither. + +### 2c. OSV.dev — supplementary + +Only when the recipe has an entry in `OSV_PACKAGES` (ecosystem + package) and a version is given. + +> **Measured, not assumed.** For the two gitea CVEs, OSV **404'd on both** and returned only Go +> *dependency* advisories for the package; NVD's API had them by neither CPE, CVE id, nor keyword. +> **Advisory databases lag the vendor**, which is why 2a and 2b lead and this is supplementary. + +## Step 3 — Union + +All findings merge into one CVE map: id → `{sources[], severity, ghsa, vulnerable_range, patched, +published_at, context}`. A CVE seen by several sources keeps them all. + +## Step 4 — Classify against the upgrade window + +Two invariants govern this step, both learned from a wrong answer in production. + +> **A. The window belongs to ONE image.** Only advisories from the **primary** source (the first +> `github-advisories:` source, i.e. the app repo the registry lists first) are classified. Sidecar +> advisories are recorded as unclassified so they stay visible without inflating the count. +> *Why:* discourse once reported **133**, of which **34 were redis CVEs** — including +> `CVE-2021-21309`, patched in redis 6.0.11 in 2021 — counted purely because 6.0.11 sits numerically +> inside discourse's `3.5.3 → 2026.7.1` range. +> +> **B. Never emit a number you cannot justify.** If neither method below can order the window, the +> count is `null` / `UNKNOWN`, never `0`. A `0` in a security column asserts safety. + +### 4a. By patched version (preferred — exact) + +`patched_versions` is a **range expression** (`">= 2.18.1"`), possibly several joined by `;`. Extract +every version-looking token; the advisory is **fixed-by-this-upgrade** if **any** patched version `p` +satisfies `from < p <= to`, using a loose numeric key (leading integers per dot-part). + +### 4b. By advisory publish date (fallback — temporal) + +Used **only** when 4a cannot be trusted: a **version-scheme change**, detected as the leading version +component jumping by ≥ `SCHEME_JUMP` (100) — e.g. semver `3.5.3` → calver `2026.7.1`. + +Version strings are unorderable across such a jump (`2025.12.2` compares "newer" than `3.5.3` while +shipping earlier), but **release dates always order**. So: + +1. Resolve `--from` and `--to` to **git tag dates** on the primary repo (tries `v` then + ``; annotated tag → tagger date, else commit date). +2. An advisory counts as fixed when `date_from < published_at <= date_to`. + +Both the method and the resolved window appear in the output. This reproduces, automatically, the hand +count that established discourse `3.5.3` (2025-12-30) → `2026.7.1` (2026-07-31) = **123 CVEs**. + +*Assumption:* the vendor publishes advisories at fix time (true for discourse). The count includes +**first-party plugin advisories** where the vendor files them on the same repo — which is why a +plugin-rich project scores far higher than a monolith, not a statement about relative security. + +### 4c. Otherwise + +`count_known = false`, `cve_count_fixed = null`, and the markdown headline reads +**"CVEs fixed by this upgrade: UNKNOWN — the scan could NOT determine a count"** with an explicit +*"This is NOT zero"*. + +## Step 5 — Output + +Markdown (default) for pasting into the per-recipe upgrade log, or `--json`. + +| Field | Meaning | +|---|---| +| `cve_count_fixed` | Count, or **`null`** when undeterminable | +| `count_known` | Distinguishes "counted zero" from "could not count" | +| `classified_by` | `patched version ranges` or `advisory publish date (version scheme changed)` | +| `date_window` | The resolved date window, when 4b was used | +| `fixed_by_this_upgrade[]` | CVE ids, with severity / GHSA / fixed-in per id | +| `unclassified[]` | Seen but not attributable to this window (incl. other images) | +| `sources[]` | Every source with its own status | +| `sources_failed[]` | **Genuine** failures only | +| `sources_benign[]` | `no-advisories-published`, `skipped: template URL` | + +--- + +## How consumers must read it + +`/recipe-report` renders the `cve` column from the **union** of this scan and the agent's own reading: + +- a clean scan → its number, **including `0`**; +- **failed sources**, or `UNKNOWN` → **`?`**, never `none` — a blank reads as "clean", which is exactly + how two CVSS-9.8 gitea RCEs were published as `none`; +- **no upgrade this run** → `0`, not `?` — nothing an upgrade could have fixed; +- benign notes → never `?`. + +`?` must stay **rare**: it means *we tried and could not tell*, not *we did not look*. A rash of `?` is +a bug to raise in the report's Addendum, not a normal outcome — every instance so far traced to a defect +in this tool or stale registry data. + +## Known limits + +1. **One window per scan.** Sidecar bumps (redis, postgres) are not counted; they appear as + unclassified. Per-image windows would be the natural extension. +2. **Date-based counts are temporal**, not exact — they assume publish-at-fix-time. +3. **Registry-bound.** Unlisted vendor security pages are invisible; the scan cannot know what it was + never pointed at. +4. **Rate limit** without a token is 60/hr — a full weekly sweep will exhaust it and degrade to failed + sources (visibly, but degraded). -- 2.54.0 From b5f8543a9b8f6a29378ec244f7118f82fa7f384d Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Mon, 10 Aug 2026 22:01:28 +0000 Subject: [PATCH 06/10] advisory-scan: count sidecar CVEs via per-image windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A recipe upgrades several images, each through its own version range. The scan previously classified only the app repo, so sidecar bumps contributed nothing — the alternative to the earlier bug where sidecars were judged by the APP's window and produced a false 133. Now: --window KEY=FROM:TO (repeatable) gives any other source its own range; each window is classified independently (one may use patched-version ranges while another falls back to advisory dates) and the count is the UNION. An image with no window is still not counted — the scan will not guess a range it was not given. If ANY requested window cannot be ordered, the total is UNKNOWN rather than a partial number. /recipe-upgrade now instructs passing a --window per bumped sidecar. Verified on discourse app 3.5.3->2026.7.1 + redis 7.4->8.10: 128 = 123 (app, by publish date) + 5 (redis, by version range). The redis five are genuine for that bump (patched 7.4.1 / 7.4.6 / 8.2.3) and include CVE-2025-49844, CRITICAL — previously invisible. Regressions clean: gitea still 2, discourse without the sidecar window still 123. --- .claude/skills/recipe-upgrade/SKILL.md | 9 +- cc-ci-plan/advisory-scan.SPEC.md | 25 ++-- cc-ci-plan/advisory-scan.py | 180 +++++++++++++++---------- 3 files changed, 133 insertions(+), 81 deletions(-) diff --git a/.claude/skills/recipe-upgrade/SKILL.md b/.claude/skills/recipe-upgrade/SKILL.md index 4cd658b..5399985 100644 --- a/.claude/skills/recipe-upgrade/SKILL.md +++ b/.claude/skills/recipe-upgrade/SKILL.md @@ -163,9 +163,16 @@ Run the deterministic scanner for the exact upgrade window and **paste its markd into the per-recipe log**: ``` -python3 /srv/cc-ci/cc-ci-plan/advisory-scan.py --from --to +python3 /srv/cc-ci/cc-ci-plan/advisory-scan.py --from --to \ + [--window =: ...] ``` +**Pass a `--window` for EVERY sidecar you bumped** (redis, postgres, nginx …), not just the app — +each image is judged by its own version range, and an image without a window is not counted at all. +e.g. discourse bumping app 3.5.3→2026.7.1 *and* redis 7.4→8.10: +`--from 3.5.3 --to 2026.7.1 --window redis=7.4:8.10` → 128 CVEs (123 app + 5 redis), where the redis +five include a **critical** (CVE-2025-49844) that is invisible if the sidecar is left out. + It queries, per recipe: the **GitHub Security Advisories API** for every source repo in `cc-ci-plan/upstream/.md` (CVE + GHSA + severity + vulnerable/patched ranges, so "fixed by THIS upgrade" is computed, not guessed), every **vendor release/security URL** in that diff --git a/cc-ci-plan/advisory-scan.SPEC.md b/cc-ci-plan/advisory-scan.SPEC.md index 7b665d9..ff91e6b 100644 --- a/cc-ci-plan/advisory-scan.SPEC.md +++ b/cc-ci-plan/advisory-scan.SPEC.md @@ -26,6 +26,7 @@ advisory-scan.py [--from ] [--to ] [--json] [--regist |---|---| | `` | Recipe name; selects `cc-ci-plan/upstream/.md` (the per-recipe URL registry) | | `--from` / `--to` | The **primary app image's** version window being upgraded across | +| `--window KEY=FROM:TO` | A **sidecar's own** window (repeatable). `KEY` matches a source repo name, e.g. `--window redis=7.4:8.10`. Without it that image's advisories stay unclassified. | | `--registry` | Registry dir; also `CCCI_UPSTREAM_REGISTRY` | | `GITHUB_TOKEN` / `GITHUB_TOKEN_FILE` | Read-only token; **rate limit only** (60/hr anonymous → 5000/hr). Default file `/srv/cc-ci/.github-token`, mode 600. Public advisories need **no scopes**. | @@ -100,12 +101,17 @@ published_at, context}`. A CVE seen by several sources keeps them all. Two invariants govern this step, both learned from a wrong answer in production. -> **A. The window belongs to ONE image.** Only advisories from the **primary** source (the first -> `github-advisories:` source, i.e. the app repo the registry lists first) are classified. Sidecar -> advisories are recorded as unclassified so they stay visible without inflating the count. +> **A. Every image is judged by its OWN window.** The app repo uses `--from/--to`; each sidecar uses +> its own `--window KEY=FROM:TO`. An image with no window is **not** classified — its advisories are +> listed as unclassified so they stay visible without inflating the count. The reported count is the +> **union across windows**, and each window is classified independently (so one may use version +> ranges while another falls back to dates). > *Why:* discourse once reported **133**, of which **34 were redis CVEs** — including > `CVE-2021-21309`, patched in redis 6.0.11 in 2021 — counted purely because 6.0.11 sits numerically -> inside discourse's `3.5.3 → 2026.7.1` range. +> inside discourse's `3.5.3 → 2026.7.1` range. The fix is not to ignore sidecars but to give each one +> the window it actually moved through: with `--window redis=7.4:8.10`, discourse scores +> **128 = 123 (app, by date) + 5 (redis, by version range)** — and the redis five include +> `CVE-2025-49844`, **critical**, which was invisible while sidecars went uncounted. > > **B. Never emit a number you cannot justify.** If neither method below can order the window, the > count is `null` / `UNKNOWN`, never `0`. A `0` in a security column asserts safety. @@ -147,10 +153,11 @@ Markdown (default) for pasting into the per-recipe upgrade log, or `--json`. | Field | Meaning | |---|---| -| `cve_count_fixed` | Count, or **`null`** when undeterminable | +| `cve_count_fixed` | Union across all windows, or **`null`** if ANY requested window could not be ordered (a partial number would understate) | | `count_known` | Distinguishes "counted zero" from "could not count" | -| `classified_by` | `patched version ranges` or `advisory publish date (version scheme changed)` | -| `date_window` | The resolved date window, when 4b was used | +| `windows` | Every source classified, with its from/to | +| `classified_by` | **Per source**: `patched version ranges` or `advisory publish date (version scheme changed)` | +| `date_window` | **Per source**, when 4b was used | | `fixed_by_this_upgrade[]` | CVE ids, with severity / GHSA / fixed-in per id | | `unclassified[]` | Seen but not attributable to this window (incl. other images) | | `sources[]` | Every source with its own status | @@ -175,8 +182,8 @@ in this tool or stale registry data. ## Known limits -1. **One window per scan.** Sidecar bumps (redis, postgres) are not counted; they appear as - unclassified. Per-image windows would be the natural extension. +1. **Windows must be supplied.** An image with no `--window` is not counted — the scan will not + guess a version range it was not told. `/recipe-upgrade` passes one per image it bumped. 2. **Date-based counts are temporal**, not exact — they assume publish-at-fix-time. 3. **Registry-bound.** Unlisted vendor security pages are invisible; the scan cannot know what it was never pointed at. diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index 019f9a0..0a519c7 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -51,6 +51,8 @@ REGISTRY_DIR = os.environ.get("CCCI_UPSTREAM_REGISTRY", "/srv/cc-ci/cc-ci-plan/u UA = "cc-ci-advisory-scan (+https://git.autonomic.zone/recipe-maintainers/cc-ci)" TIMEOUT = int(os.environ.get("ADVISORY_SCAN_TIMEOUT", "45")) CVE_RE = re.compile(r"CVE-\d{4}-\d{4,7}") +# Leading-version-component jump that means the scheme changed (semver → calver). +SCHEME_JUMP = 100 # Optional OSV mappings: recipe -> (ecosystem, package). Supplementary only (see module docstring). OSV_PACKAGES: dict[str, tuple[str, str]] = { @@ -281,7 +283,8 @@ def osv(recipe: str, version: str | None) -> dict | None: return entry -def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) -> dict: +def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, + extra_windows: list[tuple[str, str, str]] | None = None) -> dict: urls, reg_path = registry_urls(recipe, registry_dir) report: dict = { "recipe": recipe, @@ -343,74 +346,92 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) - # meaningless: 2025.12.2 compares "newer" than 3.5.3 while shipping earlier. When the # leading component jumps by more than SCHEME_JUMP we refuse to classify and say so, # rather than emitting a confident wrong number. - kf, kt = _vkey(v_from), _vkey(v_to) - SCHEME_JUMP = 100 - scheme_change = bool(kf and kt and abs(kt[0] - kf[0]) >= SCHEME_JUMP) - primary = None - if v_from or v_to: - # the app repo = first github source in the registry (registry lists the app service first) - primary = next((s["source"] for s in report["sources"] - if s["source"].startswith("github-advisories:")), None) + # ── Classification ──────────────────────────────────────────────────────────────────────── + # A recipe upgrades SEVERAL images (app + redis/postgres/nginx sidecars), each with its OWN + # version window. Judging every advisory by the app's window is how discourse once reported a + # false 133 (34 of them redis CVEs, incl. one patched in redis 6.0.11 in 2021). So each source + # is classified against ITS OWN window, and the count is the union across windows. + # + # --from/--to → the PRIMARY app repo (first github source in the registry) + # --window K=F:T → any other source whose name contains K (repeatable), e.g. redis=7.4:8.10 + # + # A source with no window is not classified: its advisories are listed as unclassified so they + # stay visible without inflating the count. + gh_sources = [x["source"] for x in report["sources"] if x["source"].startswith("github-advisories:")] + primary = gh_sources[0] if (gh_sources and (v_from or v_to)) else None report["primary_source"] = primary - report["scheme_change"] = scheme_change - fixed, unknown = [], [] - for cve, e in report["cves"].items(): - # `patched_versions` is a RANGE EXPRESSION (">= 2.18.1"), not a bare version, and there may - # be several (one per patched release line, joined with ";"). Pull every version-looking - # token and treat the advisory as fixed-by-this-upgrade if ANY of them lands in (from, to]. - cands = [_vkey(t) for t in re.findall(r"\d+(?:\.\d+)*", e.get("patched") or "")] - kp = next((c for c in cands if kf and kt and kf < c <= kt), None) or (cands[0] if cands else ()) - from_primary = primary is not None and primary in e["sources"] - if scheme_change: - e["classification"] = "unclassified: version-scheme change, cannot order reliably" - unknown.append(cve) - elif not from_primary: - e["classification"] = "unclassified: different image than the given version window" - unknown.append(cve) - elif kf and kt and kp and kf < kp <= kt: - e["classification"] = "fixed-by-this-upgrade" - fixed.append(cve) - else: - e["classification"] = "unclassified" if not (kf and kt and kp) else "outside-window" - if e["classification"] == "unclassified": - unknown.append(cve) - # NEVER report 0 for something we could not determine. When classification was refused, the - # count is UNKNOWN (null) — a 0 would be read as "no CVEs", which is an assertion this scan - # cannot make. Consumers must distinguish "counted 0" from "could not count". - # DATE-BASED FALLBACK (phase datewin). Version strings cannot be ordered across a scheme change, - # but RELEASE DATES always can. Resolve both versions to their tag dates on the primary repo and - # count advisories PUBLISHED inside that window — the method a hand count used on 2026-08-10 to - # establish discourse 3.5.3 (2025-12-30) → 2026.7.1 (2026-07-31) = 123 CVEs, where version - # comparison had produced first a false 133 and then a refusal. Only used when the version path - # refuses; a successful version classification is always preferred (it is exact, not temporal). - date_window = None - if scheme_change and primary and primary.startswith("github-advisories:"): - owner_repo = primary.split(":", 1)[1] - owner, _, repo = owner_repo.partition("/") - d_from, d_to = _tag_date(owner, repo, v_from), _tag_date(owner, repo, v_to) - if d_from and d_to and d_from < d_to: - date_window = (d_from, d_to) - fixed, unknown = [], [] - for cve, e in report["cves"].items(): - pub = e.get("published_at") - if primary in e["sources"] and pub and d_from < pub <= d_to: - e["classification"] = "fixed-by-this-upgrade (by advisory publish date)" - fixed.append(cve) - else: - e["classification"] = ("outside-window (by date)" if primary in e["sources"] - else "unclassified: different image than the given window") - if primary not in e["sources"]: - unknown.append(cve) - scheme_change = False # resolved by date; a real count is available - report["classified_by"] = "advisory publish date (version scheme changed)" - report["date_window"] = {"from": d_from, "to": d_to} - if not date_window and not scheme_change: - report["classified_by"] = "patched version ranges" - report["fixed_by_this_upgrade"] = sorted(fixed) + windows = {} # source name -> (from, to) + if primary: + windows[primary] = (v_from, v_to) + for key, wf, wt in (extra_windows or []): + for src in gh_sources: + if key.lower() in src.lower() and src not in windows: + windows[src] = (wf, wt) + report["windows"] = {k: {"from": f, "to": t} for k, (f, t) in windows.items()} + + def _classify_window(src, wf, wt): + """Return (set_of_fixed_cves, method, date_window|None, unresolved:boolean) for one source.""" + kf, kt = _vkey(wf), _vkey(wt) + # A version-SCHEME change (semver 3.5.3 → calver 2026.7.1) makes numeric ordering + # meaningless: 2025.12.2 compares "newer" than 3.5.3 while shipping earlier. + scheme = bool(kf and kt and abs(kt[0] - kf[0]) >= SCHEME_JUMP) + if not scheme: + got = set() + for cve, e in report["cves"].items(): + if src not in e["sources"]: + continue + cands = [_vkey(t) for t in re.findall(r"\d+(?:\.\d+)*", e.get("patched") or "")] + if kf and kt and any(kf < c <= kt for c in cands): + got.add(cve) + return got, "patched version ranges", None, False + # DATE FALLBACK: release DATES always order, even across a scheme change. Resolve both + # versions to git tag dates and count advisories PUBLISHED in that window — the method a + # hand count used to establish discourse 3.5.3 (2025-12-30) → 2026.7.1 (2026-07-31) = 123. + owner, _, repo = src.split(":", 1)[1].partition("/") + d_from, d_to = _tag_date(owner, repo, wf), _tag_date(owner, repo, wt) + if d_from and d_to and d_from < d_to: + got = {cve for cve, e in report["cves"].items() + if src in e["sources"] and e.get("published_at") + and d_from < e["published_at"] <= d_to} + return got, "advisory publish date (version scheme changed)", (d_from, d_to), False + return set(), "unresolved", None, True + + fixed_set, methods, date_windows, unresolved_any = set(), {}, {}, False + for src, (wf, wt) in windows.items(): + got, method, dw, unresolved = _classify_window(src, wf, wt) + methods[src] = method + if dw: + date_windows[src] = {"from": dw[0], "to": dw[1]} + if unresolved: + unresolved_any = True + for cve in got: + report["cves"][cve]["classification"] = f"fixed-by-this-upgrade ({method}) via {src}" + fixed_set.add(cve) + + unknown = [] + for cve, e in report["cves"].items(): + if cve in fixed_set: + continue + if not any(src in e["sources"] for src in windows): + e["classification"] = "unclassified: no version window given for this image" + unknown.append(cve) + else: + e.setdefault("classification", "outside-window") + if e["classification"] == "outside-window": + pass + else: + unknown.append(cve) + + report["classified_by"] = methods + if date_windows: + report["date_window"] = date_windows + report["fixed_by_this_upgrade"] = sorted(fixed_set) report["unclassified"] = sorted(unknown) - report["count_known"] = not scheme_change - report["cve_count_fixed"] = len(fixed) if not scheme_change else None + # NEVER report 0 for something we could not determine — a 0 asserts safety. If ANY requested + # window could not be ordered at all, the total is UNKNOWN rather than a partial number. + report["count_known"] = not unresolved_any + report["cve_count_fixed"] = len(fixed_set) if not unresolved_any else None report["cve_count_total_seen"] = len(report["cves"]) # Only GENUINE failures make a count unreliable. "no-advisories-published" (404: the repo has # no advisory feed) and "skipped: template URL" are benign and must not degrade the verdict. @@ -448,10 +469,15 @@ def markdown(rep: dict) -> str: return "\n".join(L) if rep["fixed_by_this_upgrade"]: L.append(f"\n**CVEs fixed by this upgrade: {rep['cve_count_fixed']}**\n") - if rep.get("date_window"): - L.append(f"_Counted by advisory PUBLISH DATE ({rep['date_window']['from'][:10]} → " - f"{rep['date_window']['to'][:10]}) because the version scheme changed across this " - f"jump; version strings cannot be ordered across it._\n") + cb = rep.get("classified_by") or {} + if isinstance(cb, dict) and cb: + for src, method in cb.items(): + dw = (rep.get("date_window") or {}).get(src) + win = (rep.get("windows") or {}).get(src, {}) + span = f"{win.get('from')} → {win.get('to')}" + extra = (f" (dates {dw['from'][:10]} → {dw['to'][:10]})" if dw else "") + L.append(f"_{src.split(':',1)[-1]}: {span} — counted by {method}{extra}._") + L.append("") L.append("| CVE | severity | fixed in | advisory | source |") L.append("|---|---|---|---|---|") for cve in rep["fixed_by_this_upgrade"]: @@ -479,8 +505,20 @@ def main() -> int: ap.add_argument("--to", dest="v_to", default=None) ap.add_argument("--json", action="store_true", help="emit raw JSON instead of markdown") ap.add_argument("--registry", default=REGISTRY_DIR) + ap.add_argument("--window", action="append", default=[], metavar="KEY=FROM:TO", + help="extra image window, e.g. --window redis=7.4:8.10 (repeatable). " + "KEY matches a source repo name; its advisories are then counted " + "against ITS OWN bump instead of being left unclassified.") a = ap.parse_args() - rep = scan(a.recipe, a.v_from, a.v_to, a.registry) + wins = [] + for w in a.window: + key, _, rng = w.partition('=') + wf, _, wt = rng.partition(':') + if key and wf and wt: + wins.append((key, wf, wt)) + else: + print(f'ignoring malformed --window {w!r} (expected KEY=FROM:TO)', file=sys.stderr) + rep = scan(a.recipe, a.v_from, a.v_to, a.registry, wins) print(json.dumps(rep, indent=2) if a.json else markdown(rep)) return 0 -- 2.54.0 From 8d7320f32e107799f87d29dfc73404f194c0db83 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Mon, 10 Aug 2026 22:06:38 +0000 Subject: [PATCH 07/10] recipe-report: scheme changes and sidecars no longer mean '?' The skill still told the reporter to publish '?' whenever the scan hit a version-scheme change. The scan now resolves those by advisory publish date, so that instruction would have re-introduced a '?' for a count it can determine. Also documents that counts are a union across per-image windows, and that a sidecar-sourced critical must name its image in the bulletin. --- .claude/skills/recipe-report/SKILL.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.claude/skills/recipe-report/SKILL.md b/.claude/skills/recipe-report/SKILL.md index 6be03f0..1652645 100644 --- a/.claude/skills/recipe-report/SKILL.md +++ b/.claude/skills/recipe-report/SKILL.md @@ -48,9 +48,15 @@ keeps every weekly edition looking the same regardless of which model writes the RCEs were reported as "none" on 2026-08-07. - **`?` must stay RARE — it means "we tried and could not tell", not "we didn't look".** Use it ONLY when a scan ran and reported genuinely failed sources, **or when the scan block says - COUNT UNKNOWN** (it refuses to classify across a version-scheme change, e.g. semver → calver — - discourse 3.5.3 → 2026.7.1). In that case the scan's `0` means *not determined*: publish `?` - and say so in the notes; publishing `0` would assert a clean bill of health nothing supports. + COUNT UNKNOWN**. In that case the scan's `0` means *not determined*: publish `?` and say so in + the notes; publishing `0` would assert a clean bill of health nothing supports. Note a + **version-scheme change is no longer a reason for `?`** — the scan resolves semver→calver jumps + (discourse 3.5.3 → 2026.7.1) by falling back to advisory publish dates and reports a real number. + - **Counts span every image, each judged by its own window.** A scan block lists one line per + image with its version range and classification method; the headline is their union. So a + recipe's count legitimately includes **sidecar** CVEs (discourse's 128 = 123 app + 5 redis). + When a sidecar contributes a critical/high, name the image in the bulletin — CVE-2025-49844 is + a redis flaw, not a discourse one, and an operator reading "discourse" needs to know that. (The scan headline itself now says `UNKNOWN` rather than a number in that case.) In particular: a recipe with **no upgrade this run** (up-to-date/skipped) has nothing an upgrade could have fixed — report `0`, not `?`. A recipe with a clean scan reports its number (including `0`). Benign notes in a scan -- 2.54.0 From 65bf3c095b9e9d9e759469507e8438dd01070737 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Tue, 11 Aug 2026 00:51:12 +0000 Subject: [PATCH 08/10] advisory-scan: --image NAME FROM TO replaces --window KEY=FROM:TO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image name was packed into the value, so the flag needed a hand-rolled KEY=FROM:TO parser with its own malformed-input branch, and 'window' named the wrong thing — the tool has two kinds of window (version ranges and, on the date fallback, real date windows) and the flag meant only the first. Now each part is its own argument: --image redis 7.4 8.10, repeatable, all in one call. argparse enforces the arity, so the string parsing and its error path are deleted. 'windows' survives internally as the computed-range concept. Counts unchanged: discourse 128 with redis / 123 without, gitea 2. --- .claude/skills/recipe-upgrade/SKILL.md | 19 +++++++++++----- cc-ci-plan/advisory-scan.SPEC.md | 21 ++++++++++------- cc-ci-plan/advisory-scan.py | 31 +++++++++++--------------- 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/.claude/skills/recipe-upgrade/SKILL.md b/.claude/skills/recipe-upgrade/SKILL.md index 5399985..2e033e1 100644 --- a/.claude/skills/recipe-upgrade/SKILL.md +++ b/.claude/skills/recipe-upgrade/SKILL.md @@ -164,14 +164,21 @@ into the per-recipe log**: ``` python3 /srv/cc-ci/cc-ci-plan/advisory-scan.py --from --to \ - [--window =: ...] + [--image ]... ``` -**Pass a `--window` for EVERY sidecar you bumped** (redis, postgres, nginx …), not just the app — -each image is judged by its own version range, and an image without a window is not counted at all. -e.g. discourse bumping app 3.5.3→2026.7.1 *and* redis 7.4→8.10: -`--from 3.5.3 --to 2026.7.1 --window redis=7.4:8.10` → 128 CVEs (123 app + 5 redis), where the redis -five include a **critical** (CVE-2025-49844) that is invisible if the sidecar is left out. +**Pass an `--image` for EVERY sidecar you upgraded** (redis, postgres, nginx …), not just the app — +each image is judged by its own versions, and an image you don't name is not counted at all. Repeat +the flag for each one and pass them **all in a single call** (the count is a union across images). +e.g. discourse moving app 3.5.3→2026.7.1 *and* redis 7.4→8.10: + +``` +... --from 3.5.3 --to 2026.7.1 --image redis 7.4 8.10 +``` + +→ 128 CVEs (123 app + 5 redis), where the redis five include a **critical** (CVE-2025-49844) that is +invisible if the sidecar is left out. `` is substring-matched against source repo names, so make +it specific enough to hit exactly one. It queries, per recipe: the **GitHub Security Advisories API** for every source repo in `cc-ci-plan/upstream/.md` (CVE + GHSA + severity + vulnerable/patched ranges, so diff --git a/cc-ci-plan/advisory-scan.SPEC.md b/cc-ci-plan/advisory-scan.SPEC.md index ff91e6b..4884ee5 100644 --- a/cc-ci-plan/advisory-scan.SPEC.md +++ b/cc-ci-plan/advisory-scan.SPEC.md @@ -19,14 +19,15 @@ Nothing in the pipeline queried an advisory source. This scan closes that hole. ## Inputs ``` -advisory-scan.py [--from ] [--to ] [--json] [--registry DIR] +advisory-scan.py [--from ] [--to ] + [--image ]... [--json] [--registry DIR] ``` | Input | Meaning | |---|---| | `` | Recipe name; selects `cc-ci-plan/upstream/.md` (the per-recipe URL registry) | | `--from` / `--to` | The **primary app image's** version window being upgraded across | -| `--window KEY=FROM:TO` | A **sidecar's own** window (repeatable). `KEY` matches a source repo name, e.g. `--window redis=7.4:8.10`. Without it that image's advisories stay unclassified. | +| `--image NAME FROM TO` | A **sidecar image and the versions it moved between** (repeatable). `NAME` is matched as a substring against source repo names, e.g. `--image redis 7.4 8.10`. Without it that image's advisories stay unclassified. | | `--registry` | Registry dir; also `CCCI_UPSTREAM_REGISTRY` | | `GITHUB_TOKEN` / `GITHUB_TOKEN_FILE` | Read-only token; **rate limit only** (60/hr anonymous → 5000/hr). Default file `/srv/cc-ci/.github-token`, mode 600. Public advisories need **no scopes**. | @@ -101,15 +102,16 @@ published_at, context}`. A CVE seen by several sources keeps them all. Two invariants govern this step, both learned from a wrong answer in production. -> **A. Every image is judged by its OWN window.** The app repo uses `--from/--to`; each sidecar uses -> its own `--window KEY=FROM:TO`. An image with no window is **not** classified — its advisories are +> **A. Every image is judged by its OWN versions.** The app repo uses `--from/--to`; each sidecar uses +> its own `--image NAME FROM TO`. **Pass them all in ONE invocation** — the count is a union across +> images, and the UNKNOWN guarantee in B only holds when a single run sees every one. An image with no window is **not** classified — its advisories are > listed as unclassified so they stay visible without inflating the count. The reported count is the > **union across windows**, and each window is classified independently (so one may use version > ranges while another falls back to dates). > *Why:* discourse once reported **133**, of which **34 were redis CVEs** — including > `CVE-2021-21309`, patched in redis 6.0.11 in 2021 — counted purely because 6.0.11 sits numerically > inside discourse's `3.5.3 → 2026.7.1` range. The fix is not to ignore sidecars but to give each one -> the window it actually moved through: with `--window redis=7.4:8.10`, discourse scores +> the versions it actually moved through: with `--image redis 7.4 8.10`, discourse scores > **128 = 123 (app, by date) + 5 (redis, by version range)** — and the redis five include > `CVE-2025-49844`, **critical**, which was invisible while sidecars went uncounted. > @@ -155,7 +157,7 @@ Markdown (default) for pasting into the per-recipe upgrade log, or `--json`. |---|---| | `cve_count_fixed` | Union across all windows, or **`null`** if ANY requested window could not be ordered (a partial number would understate) | | `count_known` | Distinguishes "counted zero" from "could not count" | -| `windows` | Every source classified, with its from/to | +| `windows` | Every source classified, with its from/to (the internal computed ranges) | | `classified_by` | **Per source**: `patched version ranges` or `advisory publish date (version scheme changed)` | | `date_window` | **Per source**, when 4b was used | | `fixed_by_this_upgrade[]` | CVE ids, with severity / GHSA / fixed-in per id | @@ -182,10 +184,13 @@ in this tool or stale registry data. ## Known limits -1. **Windows must be supplied.** An image with no `--window` is not counted — the scan will not +1. **Versions must be supplied per image.** An image with no `--image` is not counted — the scan will not guess a version range it was not told. `/recipe-upgrade` passes one per image it bumped. 2. **Date-based counts are temporal**, not exact — they assume publish-at-fix-time. 3. **Registry-bound.** Unlisted vendor security pages are invisible; the scan cannot know what it was never pointed at. -4. **Rate limit** without a token is 60/hr — a full weekly sweep will exhaust it and degrade to failed +4. **`NAME` matching is substring-against-source-name**, so a short or generic name can attach to more + than one repo (`postgres` matches `discourse/discourse-postgres`). The primary source is claimed + first and cannot be stolen; be specific enough to hit one repo. +5. **Rate limit** without a token is 60/hr — a full weekly sweep will exhaust it and degrade to failed sources (visibly, but degraded). diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index 0a519c7..9718ea0 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -284,7 +284,7 @@ def osv(recipe: str, version: str | None) -> dict | None: def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, - extra_windows: list[tuple[str, str, str]] | None = None) -> dict: + images: list[tuple[str, str, str]] | None = None) -> dict: urls, reg_path = registry_urls(recipe, registry_dir) report: dict = { "recipe": recipe, @@ -352,8 +352,9 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, # false 133 (34 of them redis CVEs, incl. one patched in redis 6.0.11 in 2021). So each source # is classified against ITS OWN window, and the count is the union across windows. # - # --from/--to → the PRIMARY app repo (first github source in the registry) - # --window K=F:T → any other source whose name contains K (repeatable), e.g. redis=7.4:8.10 + # --from/--to → the PRIMARY app repo (first github source in the registry) + # --image NAME FROM TO → any other source whose name contains NAME (repeatable), + # e.g. --image redis 7.4 8.10 # # A source with no window is not classified: its advisories are listed as unclassified so they # stay visible without inflating the count. @@ -364,7 +365,7 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, windows = {} # source name -> (from, to) if primary: windows[primary] = (v_from, v_to) - for key, wf, wt in (extra_windows or []): + for key, wf, wt in (images or []): for src in gh_sources: if key.lower() in src.lower() and src not in windows: windows[src] = (wf, wt) @@ -414,7 +415,7 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, if cve in fixed_set: continue if not any(src in e["sources"] for src in windows): - e["classification"] = "unclassified: no version window given for this image" + e["classification"] = "unclassified: no versions given for this image" unknown.append(cve) else: e.setdefault("classification", "outside-window") @@ -505,20 +506,14 @@ def main() -> int: ap.add_argument("--to", dest="v_to", default=None) ap.add_argument("--json", action="store_true", help="emit raw JSON instead of markdown") ap.add_argument("--registry", default=REGISTRY_DIR) - ap.add_argument("--window", action="append", default=[], metavar="KEY=FROM:TO", - help="extra image window, e.g. --window redis=7.4:8.10 (repeatable). " - "KEY matches a source repo name; its advisories are then counted " - "against ITS OWN bump instead of being left unclassified.") + ap.add_argument("--image", action="append", default=[], nargs=3, + metavar=("NAME", "FROM", "TO"), + help="a sidecar image and the versions it moved between, e.g. " + "--image redis 7.4 8.10 (repeatable). NAME matches a source repo name; " + "its advisories are then counted against ITS OWN versions instead of " + "being left unclassified.") a = ap.parse_args() - wins = [] - for w in a.window: - key, _, rng = w.partition('=') - wf, _, wt = rng.partition(':') - if key and wf and wt: - wins.append((key, wf, wt)) - else: - print(f'ignoring malformed --window {w!r} (expected KEY=FROM:TO)', file=sys.stderr) - rep = scan(a.recipe, a.v_from, a.v_to, a.registry, wins) + rep = scan(a.recipe, a.v_from, a.v_to, a.registry, [tuple(x) for x in a.image]) print(json.dumps(rep, indent=2) if a.json else markdown(rep)) return 0 -- 2.54.0 From 46c4fff1a6d890f1c996d39aa9f50e5aac69635c Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Tue, 11 Aug 2026 00:52:40 +0000 Subject: [PATCH 09/10] advisory-scan: --image takes NAME=FROM:TO Restores the single-value form (operator preference) under the --image name. Repeat the flag per image, all in one call. Malformed values warn on stderr and are skipped rather than aborting the scan, since it is an additive pre-step. Counts unchanged: discourse 128 with redis / 123 without, gitea 2. --- .claude/skills/recipe-upgrade/SKILL.md | 4 ++-- cc-ci-plan/advisory-scan.SPEC.md | 8 ++++---- cc-ci-plan/advisory-scan.py | 19 +++++++++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.claude/skills/recipe-upgrade/SKILL.md b/.claude/skills/recipe-upgrade/SKILL.md index 2e033e1..7806050 100644 --- a/.claude/skills/recipe-upgrade/SKILL.md +++ b/.claude/skills/recipe-upgrade/SKILL.md @@ -164,7 +164,7 @@ into the per-recipe log**: ``` python3 /srv/cc-ci/cc-ci-plan/advisory-scan.py --from --to \ - [--image ]... + [--image =:]... ``` **Pass an `--image` for EVERY sidecar you upgraded** (redis, postgres, nginx …), not just the app — @@ -173,7 +173,7 @@ the flag for each one and pass them **all in a single call** (the count is a uni e.g. discourse moving app 3.5.3→2026.7.1 *and* redis 7.4→8.10: ``` -... --from 3.5.3 --to 2026.7.1 --image redis 7.4 8.10 +... --from 3.5.3 --to 2026.7.1 --image redis=7.4:8.10 ``` → 128 CVEs (123 app + 5 redis), where the redis five include a **critical** (CVE-2025-49844) that is diff --git a/cc-ci-plan/advisory-scan.SPEC.md b/cc-ci-plan/advisory-scan.SPEC.md index 4884ee5..bd0aad8 100644 --- a/cc-ci-plan/advisory-scan.SPEC.md +++ b/cc-ci-plan/advisory-scan.SPEC.md @@ -20,14 +20,14 @@ Nothing in the pipeline queried an advisory source. This scan closes that hole. ``` advisory-scan.py [--from ] [--to ] - [--image ]... [--json] [--registry DIR] + [--image =:]... [--json] [--registry DIR] ``` | Input | Meaning | |---|---| | `` | Recipe name; selects `cc-ci-plan/upstream/.md` (the per-recipe URL registry) | | `--from` / `--to` | The **primary app image's** version window being upgraded across | -| `--image NAME FROM TO` | A **sidecar image and the versions it moved between** (repeatable). `NAME` is matched as a substring against source repo names, e.g. `--image redis 7.4 8.10`. Without it that image's advisories stay unclassified. | +| `--image NAME=FROM:TO` | A **sidecar image and the versions it moved between** (repeatable). `NAME` is matched as a substring against source repo names, e.g. `--image redis=7.4:8.10`. Malformed values are warned about on stderr and skipped. Without it that image's advisories stay unclassified. | | `--registry` | Registry dir; also `CCCI_UPSTREAM_REGISTRY` | | `GITHUB_TOKEN` / `GITHUB_TOKEN_FILE` | Read-only token; **rate limit only** (60/hr anonymous → 5000/hr). Default file `/srv/cc-ci/.github-token`, mode 600. Public advisories need **no scopes**. | @@ -103,7 +103,7 @@ published_at, context}`. A CVE seen by several sources keeps them all. Two invariants govern this step, both learned from a wrong answer in production. > **A. Every image is judged by its OWN versions.** The app repo uses `--from/--to`; each sidecar uses -> its own `--image NAME FROM TO`. **Pass them all in ONE invocation** — the count is a union across +> its own `--image NAME=FROM:TO`. **Pass them all in ONE invocation** — the count is a union across > images, and the UNKNOWN guarantee in B only holds when a single run sees every one. An image with no window is **not** classified — its advisories are > listed as unclassified so they stay visible without inflating the count. The reported count is the > **union across windows**, and each window is classified independently (so one may use version @@ -111,7 +111,7 @@ Two invariants govern this step, both learned from a wrong answer in production. > *Why:* discourse once reported **133**, of which **34 were redis CVEs** — including > `CVE-2021-21309`, patched in redis 6.0.11 in 2021 — counted purely because 6.0.11 sits numerically > inside discourse's `3.5.3 → 2026.7.1` range. The fix is not to ignore sidecars but to give each one -> the versions it actually moved through: with `--image redis 7.4 8.10`, discourse scores +> the versions it actually moved through: with `--image redis=7.4:8.10`, discourse scores > **128 = 123 (app, by date) + 5 (redis, by version range)** — and the redis five include > `CVE-2025-49844`, **critical**, which was invisible while sidecars went uncounted. > diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index 9718ea0..bdb436a 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -353,8 +353,8 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, # is classified against ITS OWN window, and the count is the union across windows. # # --from/--to → the PRIMARY app repo (first github source in the registry) - # --image NAME FROM TO → any other source whose name contains NAME (repeatable), - # e.g. --image redis 7.4 8.10 + # --image NAME=FROM:TO → any other source whose name contains NAME (repeatable), + # e.g. --image redis=7.4:8.10 # # A source with no window is not classified: its advisories are listed as unclassified so they # stay visible without inflating the count. @@ -506,14 +506,21 @@ def main() -> int: ap.add_argument("--to", dest="v_to", default=None) ap.add_argument("--json", action="store_true", help="emit raw JSON instead of markdown") ap.add_argument("--registry", default=REGISTRY_DIR) - ap.add_argument("--image", action="append", default=[], nargs=3, - metavar=("NAME", "FROM", "TO"), + ap.add_argument("--image", action="append", default=[], metavar="NAME=FROM:TO", help="a sidecar image and the versions it moved between, e.g. " - "--image redis 7.4 8.10 (repeatable). NAME matches a source repo name; " + "--image redis=7.4:8.10 (repeatable). NAME matches a source repo name; " "its advisories are then counted against ITS OWN versions instead of " "being left unclassified.") a = ap.parse_args() - rep = scan(a.recipe, a.v_from, a.v_to, a.registry, [tuple(x) for x in a.image]) + images = [] + for spec in a.image: + name, _, rng = spec.partition('=') + vf, _, vt = rng.partition(':') + if name and vf and vt: + images.append((name, vf, vt)) + else: + print(f'ignoring malformed --image {spec!r} (expected NAME=FROM:TO)', file=sys.stderr) + rep = scan(a.recipe, a.v_from, a.v_to, a.registry, images) print(json.dumps(rep, indent=2) if a.json else markdown(rep)) return 0 -- 2.54.0 From 44cb9b6704e739b75ad56486e0ec3fa3c2b52668 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Tue, 11 Aug 2026 01:29:11 +0000 Subject: [PATCH 10/10] advisory-scan: tests, audit, and two real undercounts they found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds test-advisory-scan.py (58 offline tests on fixtures + 6 live regressions against the week-2026-08-07 report) and audit-advisory-scan.py, which re-derives every count with a SEPARATE semver implementation and its own release fetch and diffs against the scanner. Both found real defects: 1. Window membership was compared on ragged tuples, so (18,) < (18,0) — a CVE patched in 18.0 fell OUTSIDE a window ending at 18. Bare major tags are the norm for sidecars (postgres:18, redis:8-alpine). Now zero-padded, which also keeps the upper bound conservative (18.5 stays out of a window ending at 18). 2. Advisories with no knowable fix version were silently counted as 'not fixed'. Twelve redis advisories say patched_versions 'TBD' or '7.4.X' with an open-ended range — six of them high severity. They are now INDETERMINATE: not counted, not dismissed, and surfaced in the output. All twelve turned out to be genuinely fixed: redis names each in the release notes of every branch that got the fix (CVE-2025-32023 -> 6.2.19, 7.2.10, 7.4.5, 8.0.3, 8.2.0). So a third deterministic method resolves them from release notes, with the naming tags recorded as the citation. discourse's redis contribution goes 5 -> 17, and its total 128 -> 140. Pass 2 (--adjudicate) is the model-judged stage for what arithmetic cannot settle: it hands over each open case's full evidence, plus every verdict pass 1 reached, and takes FIXED/NOT-FIXED/STILL-UNKNOWN with a reason citing that evidence. It may only raise a count. Vendor-page-only CVEs — the shape of both gitea CVSS-9.8 RCEs — now reach it instead of being dropped. Tests cover pass 1 only, by design; pass 2's judgement is a model's. What is tested there is deterministic: which cases it selects, and that truncation is announced rather than silent. SPEC.md rewritten around the two passes. --- .claude/skills/recipe-report/SKILL.md | 3 + .claude/skills/recipe-upgrade/SKILL.md | 13 +- cc-ci-plan/advisory-scan.SPEC.md | 201 ++++++-- cc-ci-plan/advisory-scan.py | 354 +++++++++++++- cc-ci-plan/audit-advisory-scan.py | 151 ++++++ cc-ci-plan/test-advisory-scan.py | 623 +++++++++++++++++++++++++ 6 files changed, 1288 insertions(+), 57 deletions(-) create mode 100755 cc-ci-plan/audit-advisory-scan.py create mode 100755 cc-ci-plan/test-advisory-scan.py diff --git a/.claude/skills/recipe-report/SKILL.md b/.claude/skills/recipe-report/SKILL.md index 1652645..3f5ddd1 100644 --- a/.claude/skills/recipe-report/SKILL.md +++ b/.claude/skills/recipe-report/SKILL.md @@ -52,6 +52,9 @@ keeps every weekly edition looking the same regardless of which model writes the the notes; publishing `0` would assert a clean bill of health nothing supports. Note a **version-scheme change is no longer a reason for `?`** — the scan resolves semver→calver jumps (discourse 3.5.3 → 2026.7.1) by falling back to advisory publish dates and reports a real number. + - **A count with undetermined advisories is a FLOOR.** If the scan block says N advisories + "could NOT be judged", report the number but say in the notes that it is a floor — those + advisories are neither fixed nor safe, they are unmeasured. Never round them away. - **Counts span every image, each judged by its own window.** A scan block lists one line per image with its version range and classification method; the headline is their union. So a recipe's count legitimately includes **sidecar** CVEs (discourse's 128 = 123 app + 5 redis). diff --git a/.claude/skills/recipe-upgrade/SKILL.md b/.claude/skills/recipe-upgrade/SKILL.md index 7806050..9d3f0cf 100644 --- a/.claude/skills/recipe-upgrade/SKILL.md +++ b/.claude/skills/recipe-upgrade/SKILL.md @@ -176,9 +176,16 @@ e.g. discourse moving app 3.5.3→2026.7.1 *and* redis 7.4→8.10: ... --from 3.5.3 --to 2026.7.1 --image redis=7.4:8.10 ``` -→ 128 CVEs (123 app + 5 redis), where the redis five include a **critical** (CVE-2025-49844) that is -invisible if the sidecar is left out. `` is substring-matched against source repo names, so make -it specific enough to hit exactly one. +→ 140 CVEs (123 app + 17 redis), where the redis seventeen include a **critical** (CVE-2025-49844) +that is invisible if the sidecar is left out. `` is substring-matched against source repo names, +so make it specific enough to hit exactly one. + +**If the scan reports advisories it could NOT judge, re-run it with `--adjudicate`.** That is a second +pass: it collects each open case's full evidence (advisory prose, references, affected ranges, every +release naming the CVE) and asks YOU to decide FIXED / NOT-FIXED / STILL-UNKNOWN with a reason citing +that evidence. The deterministic number is a **floor** — add every FIXED to the count. Say +STILL-UNKNOWN rather than inferring from memory, and never record an undecided CVE as unaffected. +It also shows what pass 1 already decided; if a verdict looks wrong given its evidence, say so. It queries, per recipe: the **GitHub Security Advisories API** for every source repo in `cc-ci-plan/upstream/.md` (CVE + GHSA + severity + vulnerable/patched ranges, so diff --git a/cc-ci-plan/advisory-scan.SPEC.md b/cc-ci-plan/advisory-scan.SPEC.md index bd0aad8..4cb0fc7 100644 --- a/cc-ci-plan/advisory-scan.SPEC.md +++ b/cc-ci-plan/advisory-scan.SPEC.md @@ -1,12 +1,14 @@ # Advisory scan — specification What `cc-ci-plan/advisory-scan.py` does, step by step, and why each step exists. This documents the -implementation as it stands (2026-08-10); if you change the code, change this file in the same commit. +implementation as it stands (2026-08-11); if you change the code, change this file in the same commit. +`cc-ci-plan/test-advisory-scan.py` is the executable half of this spec — every rule below is asserted +there. -**Role.** A deterministic, per-recipe CVE detector run as a **pre-step of `/recipe-upgrade`** (step 2a). -It is **strictly additive**: it never replaces the release-note reading the upgrade agent already does. -The CVE count reported for a recipe is the **union** of what the agent read and what this scan found; -the scan may never *lower* a count established by reading. +**Role.** A per-recipe CVE detector run as a **pre-step of `/recipe-upgrade`** (step 2a). It is +**strictly additive**: it never replaces the release-note reading the upgrade agent already does. The +CVE count reported for a recipe is the **union** of what the agent read and what this scan found; the +scan may never *lower* a count established by reading. **Why it exists.** gitea 1.27.1 fixed CVE-2026-60004 and CVE-2026-59774 (both CVSS 9.8). The weekly report printed gitea's CVE count as `1`, then `none`. The upgrade agent had read the GitHub *release @@ -16,18 +18,35 @@ Nothing in the pipeline queried an advisory source. This scan closes that hole. --- +## Two passes + +| | Pass 1 — measure | Pass 2 — judge (`--adjudicate`) | +|---|---|---| +| Who | Pure Python, no model | The calling agent, a model | +| Does | Collects evidence and decides every case it can by arithmetic | Weighs the collected evidence on cases arithmetic cannot settle | +| Output | A count, or `UNKNOWN` | FIXED / NOT-FIXED / STILL-UNKNOWN per open case | +| Rule | Deterministic and reproducible | May only **raise** the count, never lower it | + +**Prefer pass 1.** Every case pass 1 decides is one that reproduces identically next week. Pass 2 exists +for evidence that is prose rather than data — a fallback, not a co-equal stage. When a class of case +keeps landing in pass 2, the fix is a new deterministic method in pass 1. §4c is exactly that: it moved +12 redis advisories out of pass 2 and into arithmetic. + +--- + ## Inputs ``` advisory-scan.py [--from ] [--to ] - [--image =:]... [--json] [--registry DIR] + [--image =:]... [--adjudicate] [--json] [--registry DIR] ``` | Input | Meaning | |---|---| | `` | Recipe name; selects `cc-ci-plan/upstream/.md` (the per-recipe URL registry) | | `--from` / `--to` | The **primary app image's** version window being upgraded across | -| `--image NAME=FROM:TO` | A **sidecar image and the versions it moved between** (repeatable). `NAME` is matched as a substring against source repo names, e.g. `--image redis=7.4:8.10`. Malformed values are warned about on stderr and skipped. Without it that image's advisories stay unclassified. | +| `--image NAME=FROM:TO` | A **sidecar image and the versions it moved between** (repeatable, all in ONE call). `NAME` is substring-matched against source repo names. Malformed values warn on stderr and are skipped. Without it that image's advisories stay unclassified. | +| `--adjudicate` | Run pass 2: append the evidence dossier for judgement | | `--registry` | Registry dir; also `CCCI_UPSTREAM_REGISTRY` | | `GITHUB_TOKEN` / `GITHUB_TOKEN_FILE` | Read-only token; **rate limit only** (60/hr anonymous → 5000/hr). Default file `/srv/cc-ci/.github-token`, mode 600. Public advisories need **no scopes**. | @@ -35,6 +54,8 @@ Exit code is always 0 — this is informational. Failures are *reported*, never --- +# Pass 1 — deterministic + ## Step 1 — Collect source URLs from the registry Read `cc-ci-plan/upstream/.md` and extract every `http(s)://…` URL. @@ -59,8 +80,8 @@ with *"not checked"*. For every `github.com//` URL in the registry: `GET /repos///security-advisories`. -Captured per advisory: `cve_id`, `ghsa_id`, `severity`, `summary`, `published_at`, and **all** -`vulnerabilities[]` entries' `vulnerable_version_range` + `patched_versions` (joined with `;`). +Captured per advisory: `cve_id`, `ghsa_id`, `severity`, `summary`, **`description`**, `published_at`, +and **all** `vulnerabilities[]` entries' `vulnerable_version_range` + `patched_versions` (joined `;`). - **All entries, not just the first.** An advisory carries one entry **per patched release line** — n8n patches three (1.123.32, 2.17.4, 2.18.1). Reading only `vulnerabilities[0]` silently dropped the @@ -68,12 +89,11 @@ Captured per advisory: `cve_id`, `ghsa_id`, `severity`, `summary`, `published_at - **Pagination via the `Link rel="next"` cursor**, to exhaustion (cap 20 hops). This endpoint returns at most 100 rows **and ignores `?page=`** — it re-returns the same rows, which silently truncates busy projects. discourse has 286 advisories; a single page cannot even cover one upgrade window. +- **The description is kept from this response.** It is already present here, and pass 2 needs the + prose; re-fetching it per advisory would cost one request each. - **HTTP 404 ⇒ `no-advisories-published`** — a benign absence (many sidecar images publish none), **not** a failure. Conflating the two pushed nearly every recipe to `?` and destroyed the signal. -This source is primary because it carries **severity and version ranges**, making "fixed by *this* -upgrade" computable rather than guessed. - ### 2b. Vendor release / security pages Every other registry URL is fetched, HTML-stripped, and scanned for `CVE-\d{4}-\d{4,7}`, keeping ±160 @@ -83,7 +103,8 @@ URLs containing `<`, `>`, `{`, `}`, `VERSION`, or `vX.Y.Z` are **skipped as temp human documentation (`…/changelog/v/`), not fetchable, and counting them as failures is wrong. This is the source that would have caught gitea: the vendor blog names both CVEs, the GitHub release -page names neither. +page names neither. A CVE found **only** here carries no version data, so pass 1 cannot place it — it +goes to pass 2 (§6). ### 2c. OSV.dev — supplementary @@ -96,7 +117,7 @@ Only when the recipe has an entry in `OSV_PACKAGES` (ecosystem + package) and a ## Step 3 — Union All findings merge into one CVE map: id → `{sources[], severity, ghsa, vulnerable_range, patched, -published_at, context}`. A CVE seen by several sources keeps them all. +description, published_at, url, cvss, context}`. A CVE seen by several sources keeps them all. ## Step 4 — Classify against the upgrade window @@ -104,25 +125,33 @@ Two invariants govern this step, both learned from a wrong answer in production. > **A. Every image is judged by its OWN versions.** The app repo uses `--from/--to`; each sidecar uses > its own `--image NAME=FROM:TO`. **Pass them all in ONE invocation** — the count is a union across -> images, and the UNKNOWN guarantee in B only holds when a single run sees every one. An image with no window is **not** classified — its advisories are -> listed as unclassified so they stay visible without inflating the count. The reported count is the -> **union across windows**, and each window is classified independently (so one may use version -> ranges while another falls back to dates). +> images, and the UNKNOWN guarantee in B only holds when a single run sees every one. An image with no +> window is **not** classified; its advisories are listed as unclassified so they stay visible without +> inflating the count. Each window is classified independently, so one may use version ranges while +> another falls back to dates. > *Why:* discourse once reported **133**, of which **34 were redis CVEs** — including > `CVE-2021-21309`, patched in redis 6.0.11 in 2021 — counted purely because 6.0.11 sits numerically > inside discourse's `3.5.3 → 2026.7.1` range. The fix is not to ignore sidecars but to give each one > the versions it actually moved through: with `--image redis=7.4:8.10`, discourse scores -> **128 = 123 (app, by date) + 5 (redis, by version range)** — and the redis five include -> `CVE-2025-49844`, **critical**, which was invisible while sidecars went uncounted. +> **140 = 123 (app, by date) + 17 (redis)**, and the redis seventeen include `CVE-2025-49844`, +> **critical**, invisible while sidecars went uncounted. > -> **B. Never emit a number you cannot justify.** If neither method below can order the window, the -> count is `null` / `UNKNOWN`, never `0`. A `0` in a security column asserts safety. +> **B. Never emit a number you cannot justify.** If no method can order a window, the count is +> `null` / `UNKNOWN`, never `0`. A `0` in a security column asserts safety. Equally, an advisory that +> cannot be judged is **indeterminate** (§4d) — never silently counted as "not fixed". ### 4a. By patched version (preferred — exact) `patched_versions` is a **range expression** (`">= 2.18.1"`), possibly several joined by `;`. Extract every version-looking token; the advisory is **fixed-by-this-upgrade** if **any** patched version `p` -satisfies `from < p <= to`, using a loose numeric key (leading integers per dot-part). +satisfies `from < p <= to` — exclusive lower (a fix already in the version you were on is not this +upgrade's doing), inclusive upper. + +**Comparison is zero-padded to equal length**, so `18` == `18.0` == `18.0.0` as semver means it. +Without padding, plain tuple order makes `(18,) < (18,0)`, i.e. a fix in 18.0 falls *outside* a window +ending at 18 — and bare major tags are the norm for sidecars (`postgres:18`, `redis:8-alpine`). Padding +is permissive at the lower bound and conservative at the upper: with `to = 18`, a fix in `18.5` is +**not** counted, because nothing proves which 18.x a floating tag resolved to. ### 4b. By advisory publish date (fallback — temporal) @@ -132,22 +161,50 @@ component jumping by ≥ `SCHEME_JUMP` (100) — e.g. semver `3.5.3` → calver Version strings are unorderable across such a jump (`2025.12.2` compares "newer" than `3.5.3` while shipping earlier), but **release dates always order**. So: -1. Resolve `--from` and `--to` to **git tag dates** on the primary repo (tries `v` then +1. Resolve `--from` and `--to` to **git tag dates** on that source (tries `v` then ``; annotated tag → tagger date, else commit date). -2. An advisory counts as fixed when `date_from < published_at <= date_to`. +2. An advisory counts as fixed when `date_from < published_at <= date_to` — the same exclusive/inclusive + boundaries as 4a, so the two methods agree at the edges. -Both the method and the resolved window appear in the output. This reproduces, automatically, the hand -count that established discourse `3.5.3` (2025-12-30) → `2026.7.1` (2026-07-31) = **123 CVEs**. +This reproduces, automatically, the hand count that established discourse `3.5.3` (2025-12-30) → +`2026.7.1` (2026-07-31) = **123 CVEs**. *Assumption:* the vendor publishes advisories at fix time (true for discourse). The count includes **first-party plugin advisories** where the vendor files them on the same repo — which is why a plugin-rich project scores far higher than a monolith, not a statement about relative security. -### 4c. Otherwise +### 4c. By release notes naming the CVE (rescue — for advisories with no fix version) -`count_known = false`, `cve_count_fixed = null`, and the markdown headline reads -**"CVEs fixed by this upgrade: UNKNOWN — the scan could NOT determine a count"** with an explicit -*"This is NOT zero"*. +Applied to advisories 4a/4b could not decide, **before** giving up on them. Fetch the source repo's +GitHub **releases** (cached per repo, 4 pages) and find every tag whose notes **name the CVE id**. If +any such tag falls inside the window by 4a's rule, the advisory is fixed by this upgrade, and the +naming tags are recorded in `fix_versions_from_release_notes` as the citation. + +> **Why this is not optional.** Vendors routinely publish an advisory with `patched_versions: "TBD"` +> and then name the CVE in the release notes of every branch that got the fix. **All 12** redis +> advisories crossed by discourse's redis bump are exactly this shape — `TBD`, or a placeholder like +> `7.4.X`, with an open-ended `vulnerable_version_range` (`All`, `>= 7.0.0`) — yet each is named in +> concrete releases (`CVE-2025-32023` → 6.2.19, 7.2.10, 7.4.5, 8.0.3, 8.2.0). Without this method +> discourse's redis contribution reads 5; with it, 17. Six of the twelve are high severity. + +### 4d. Otherwise — indeterminate, not excluded + +An advisory from a **windowed** source that none of 4a–4c could decide — no usable `patched_versions`, +an open-ended vulnerable range, and no release note naming it — is recorded as **indeterminate**. It is: + +- **not** added to the count (nothing justifies counting it), and +- **not** treated as unaffected (nothing justifies dismissing it either). + +It is listed prominently, the headline reads *"(at least — see undetermined below)"*, and it becomes an +input to pass 2. Silently excluding these is the same defect class as printing `0` for an unscanned +recipe, one level down. + +### 4e. Unorderable window + +If neither 4a nor 4b can order a window at all, `count_known = false`, `cve_count_fixed = null`, and the +headline reads **"CVEs fixed by this upgrade: UNKNOWN — the scan could NOT determine a count"** with an +explicit *"This is NOT zero"*. If **any** requested window is unorderable the whole count is suppressed; +a partial number would understate a security figure. ## Step 5 — Output @@ -155,16 +212,56 @@ Markdown (default) for pasting into the per-recipe upgrade log, or `--json`. | Field | Meaning | |---|---| -| `cve_count_fixed` | Union across all windows, or **`null`** if ANY requested window could not be ordered (a partial number would understate) | +| `cve_count_fixed` | Union across all windows, or **`null`** if any window was unorderable | | `count_known` | Distinguishes "counted zero" from "could not count" | -| `windows` | Every source classified, with its from/to (the internal computed ranges) | -| `classified_by` | **Per source**: `patched version ranges` or `advisory publish date (version scheme changed)` | +| `cve_count_indeterminate` / `indeterminate[]` | Judged by nothing; a **floor marker** on the count | +| `resolved_by_release_notes` | CVE → the tags that named it (4c citations) | +| `windows` | Every source classified, with its from/to | +| `classified_by` | **Per source**: which method decided it | | `date_window` | **Per source**, when 4b was used | | `fixed_by_this_upgrade[]` | CVE ids, with severity / GHSA / fixed-in per id | -| `unclassified[]` | Seen but not attributable to this window (incl. other images) | -| `sources[]` | Every source with its own status | -| `sources_failed[]` | **Genuine** failures only | -| `sources_benign[]` | `no-advisories-published`, `skipped: template URL` | +| `unclassified[]` | Seen but not attributable (other images, or vendor-page-only) | +| `sources[]` / `sources_failed[]` / `sources_benign[]` | Per-source status; only genuine failures in `failed` | + +--- + +# Pass 2 — adjudication (`--adjudicate`) + +## Step 6 — What gets judged + +Two kinds of open case, both real gaps rather than noise: + +1. **Indeterminate** (§4d) — from an image *with* a window, but no fix version is knowable anywhere. +2. **Vendor-page-only** — a CVE seen only on a vendor security page, with no structured advisory + behind it. **gitea's two CVSS-9.8 RCEs are this shape.** They carry no version data, so no + arithmetic can place them, but the page prose usually states the fixed release. + +## Step 7 — The evidence dossier + +Pass 1 collects; pass 2 judges. Nothing in the dossier interprets — it assembles what was *measured*, +so the judgement is made against evidence rather than recollection. That distinction is the whole +point: the original failure was a report leaning on model knowledge that predated the CVEs, with no +source queried at all. + +Per open case: severity, CVSS, sources, its window, why it is undecided, `patched_versions` and +`vulnerable_version_range` **as published**, summary, full description, references, affected ranges +with `first_patched_version`, and **every release tag naming the CVE** — whether or not in window, since +the model may reason about branch lines the arithmetic deliberately will not. + +**Pass 2 also sees every decision pass 1 made** — a compact table of counted and excluded advisories +with the evidence behind each verdict. A deterministic verdict can still be wrong (a mis-parsed range, +a release note that mentions a CVE without fixing it), and only a reader with the evidence in front of +it can catch that. Silence means agreement. + +## Step 8 — The verdict contract + +For each open case: **FIXED** / **NOT-FIXED** / **STILL-UNKNOWN**, each with a one-line reason +**citing the evidence shown**. Every FIXED is added to the recipe's count — pass 1's number is a floor, +not a total. If the evidence does not settle it, **STILL-UNKNOWN**: do not infer from memory of the +project, and never record an undecided CVE as unaffected. + +**No silent caps.** `MAX_ADJUDICATE` (25) and `MAX_REVIEW_ROWS` (400) bound the output; whenever either +truncates, the block says how many were dropped and that the unshown remain undetermined. --- @@ -176,21 +273,41 @@ Markdown (default) for pasting into the per-recipe upgrade log, or `--json`. - **failed sources**, or `UNKNOWN` → **`?`**, never `none` — a blank reads as "clean", which is exactly how two CVSS-9.8 gitea RCEs were published as `none`; - **no upgrade this run** → `0`, not `?` — nothing an upgrade could have fixed; +- a count with **indeterminate advisories** → the number is a floor; say so in the notes; - benign notes → never `?`. `?` must stay **rare**: it means *we tried and could not tell*, not *we did not look*. A rash of `?` is a bug to raise in the report's Addendum, not a normal outcome — every instance so far traced to a defect in this tool or stale registry data. +## Testing + +`test-advisory-scan.py` — 58 offline tests (fixtures, no network) plus 6 live regressions against the +counts published in week-2026-08-07. **The offline tier covers pass 1 only, by design**: pass 2's +judgement is a model's and cannot be asserted deterministically. What *is* tested about pass 2 is the +part that stays deterministic — which cases it selects, and that truncation is always announced. + +``` +python3 test-advisory-scan.py # offline +python3 test-advisory-scan.py --live # + historic report numbers +``` + +`audit-advisory-scan.py` re-derives the counts with a **separate** semver implementation and its own +release fetch, then diffs against the scanner. Run it after changing classification; it is what caught +the 12 undercounted redis CVEs. + ## Known limits -1. **Versions must be supplied per image.** An image with no `--image` is not counted — the scan will not - guess a version range it was not told. `/recipe-upgrade` passes one per image it bumped. +1. **Versions must be supplied per image.** An image with no `--image` is not counted — the scan will + not guess a version range it was not given. `/recipe-upgrade` passes one per image it bumped. 2. **Date-based counts are temporal**, not exact — they assume publish-at-fix-time. 3. **Registry-bound.** Unlisted vendor security pages are invisible; the scan cannot know what it was never pointed at. 4. **`NAME` matching is substring-against-source-name**, so a short or generic name can attach to more than one repo (`postgres` matches `discourse/discourse-postgres`). The primary source is claimed - first and cannot be stolen; be specific enough to hit one repo. -5. **Rate limit** without a token is 60/hr — a full weekly sweep will exhaust it and degrade to failed + first and cannot be stolen. A name matching **nothing** is silently ignored — a typo costs coverage + without warning. +5. **Release-note rescue (4c) trusts that naming implies fixing.** A release note that merely mentions a + CVE would be read as fixing it. Pass 2's review table exists partly to catch this. +6. **Rate limit** without a token is 60/hr — a full weekly sweep will exhaust it and degrade to failed sources (visibly, but degraded). diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index bdb436a..1c9697a 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -53,6 +53,12 @@ TIMEOUT = int(os.environ.get("ADVISORY_SCAN_TIMEOUT", "45")) CVE_RE = re.compile(r"CVE-\d{4}-\d{4,7}") # Leading-version-component jump that means the scheme changed (semver → calver). SCHEME_JUMP = 100 +# A patched_versions field that names no usable version. GitHub carries these verbatim from the +# vendor: redis publishes "TBD" for 11 advisories and "6.2.X, 7.2.X, 7.4.X" for another, and their +# vulnerable_version_range is open-ended ("All", ">= 7.0.0"), so the fix version is NOT recoverable. +# Such an advisory must be reported as INDETERMINATE, never silently counted as "not fixed" — that +# would assert an upgrade did not fix something we simply cannot judge. +PLACEHOLDER_RE = re.compile(r"\bTBD\b|\bunknown\b|\bnone\b|\d+\.[Xx]\b|\?", re.I) # Optional OSV mappings: recipe -> (ecosystem, package). Supplementary only (see module docstring). OSV_PACKAGES: dict[str, tuple[str, str]] = { @@ -158,6 +164,21 @@ def _vkey(v: str | None) -> tuple: return tuple(out) +def _within(kf: tuple, kt: tuple, c: tuple) -> bool: + """Is patched-version `c` inside the window (kf, kt] — exclusive lower, inclusive upper? + + Compares ZERO-PADDED to equal length, so "18" == "18.0" == "18.0.0" the way semver means it. + Without the padding, plain tuple order says (18,) < (18,0), i.e. a CVE patched in 18.0 falls + OUTSIDE a window ending at 18 — and bare major tags are the norm for sidecars (postgres:18, + redis:8-alpine), so that silently dropped real fixes. Padding also keeps the upper bound + conservative: a fix in 18.5 is still outside a window ending at "18", because nothing proves + which 18.x a floating tag resolved to. + """ + n = max(len(kf), len(kt), len(c)) + pad = lambda t: t + (0,) * (n - len(t)) + return pad(kf) < pad(c) <= pad(kt) + + def registry_urls(recipe: str, registry_dir: str) -> tuple[list[str], str | None]: path = os.path.join(registry_dir, f"{recipe}.md") try: @@ -219,6 +240,10 @@ def github_advisories(urls: list[str]) -> list[dict]: ) or None, "url": a.get("html_url"), "published_at": a.get("published_at"), + # The list response ALREADY carries the prose. Keep it: the adjudication + # pass needs it, and re-fetching per advisory costs a request each. + "description": (a.get("description") or "")[:4000], + "cvss": ((a.get("cvss") or {}).get("vector_string")), } ) except urllib.error.HTTPError as e: @@ -283,6 +308,244 @@ def osv(recipe: str, version: str | None) -> dict | None: return entry +_RELEASE_CACHE: dict[str, list[tuple[str, str]]] = {} + + +def _releases(owner: str, repo: str, max_pages: int = 4) -> list[tuple[str, str]]: + """[(tag, body)] for a repo's GitHub releases, cached per repo for the process.""" + key = f"{owner}/{repo}" + if key in _RELEASE_CACHE: + return _RELEASE_CACHE[key] + hdrs = {"Accept": "application/vnd.github+json"} + tok = _github_token() + if tok: + hdrs["Authorization"] = f"Bearer {tok}" + out: list[tuple[str, str]] = [] + try: + for rel in _gh_paginate( + f"https://api.github.com/repos/{owner}/{repo}/releases?per_page=100", hdrs, max_pages + ): + out.append((rel.get("tag_name") or "", + f"{rel.get('name') or ''}\n{rel.get('body') or ''}")) + except Exception: # noqa: BLE001 — best effort; absence just leaves advisories undetermined + pass + _RELEASE_CACHE[key] = out + return out + + +def release_fix_versions(source: str, cve: str) -> list[str]: + """Release tags whose notes NAME this CVE — a deterministic fix version when the advisory has none. + + Vendors routinely publish an advisory with `patched_versions: "TBD"` and then name the CVE in the + release notes of every branch that got the fix. redis does exactly this: all 12 of its advisories + that discourse's redis bump crosses carry TBD, yet each is named in concrete releases + (CVE-2025-32023 → 6.2.19, 7.2.10, 7.4.5, 8.0.3, 8.2.0). Ignoring that evidence undercounted + discourse by 12 CVEs, so this is checked BEFORE giving up on an advisory. + """ + if not source.startswith("github-advisories:"): + return [] + owner, _, repo = source.split(":", 1)[1].partition("/") + return [tag for tag, body in _releases(owner, repo) if cve in body] + + +def advisory_text(ghsa: str, source: str | None = None) -> dict: + """Full text of one advisory, for the ADJUDICATION pass (see adjudication_block). + + The structured `patched_versions` field is often "TBD" while the prose description and the + linked references DO state where the fix landed. That prose is not machine-parseable in general + — which is the point: it is collected here for a MODEL to judge, not for a regex.""" + hdrs = {"Accept": "application/vnd.github+json"} + tok = _github_token() + if tok: + hdrs["Authorization"] = f"Bearer {tok}" + out = {"ghsa": ghsa, "status": "ok"} + # Repo-scoped FIRST. Many repository advisories are never mirrored into the global GitHub + # Advisory Database, so /advisories/ 404s for them (all 12 redis ones, for instance) + # while /repos///security-advisories/ returns the full record. + cands = [] + if source and source.startswith("github-advisories:"): + cands.append(f"https://api.github.com/repos/{source.split(':',1)[1]}/security-advisories/{ghsa}") + cands.append(f"https://api.github.com/advisories/{ghsa}") + try: + a, last = None, None + for u in cands: + try: + a = json.loads(_fetch(u, hdrs)); break + except Exception as ex: # noqa: BLE001 — try the next endpoint + last = ex + if a is None: + raise last or RuntimeError("no advisory endpoint responded") + out.update({ + "summary": a.get("summary"), + "description": (a.get("description") or "")[:4000], + "severity": a.get("severity"), + "published_at": a.get("published_at"), + "references": [r for r in (a.get("references") or [])][:12] or ( + [a.get("html_url")] if a.get("html_url") else []), + "cvss": (a.get("cvss") or {}).get("vector_string"), + "vulnerabilities": [ + {"package": (v.get("package") or {}).get("name"), + "vulnerable_version_range": v.get("vulnerable_version_range"), + "first_patched_version": v.get("first_patched_version")} + for v in (a.get("vulnerabilities") or []) + ], + }) + except Exception as e: # noqa: BLE001 + out["status"] = f"error: {type(e).__name__}: {e}" + return out + + +MAX_ADJUDICATE = int(os.environ.get("ADVISORY_SCAN_MAX_ADJUDICATE", "25")) +# Compact review rows for advisories pass 1 DID decide. Pass 2 sees these too, so a wrong +# deterministic verdict can be caught rather than inherited. +MAX_REVIEW_ROWS = int(os.environ.get("ADVISORY_SCAN_MAX_REVIEW", "400")) + + +def needs_judgement(rep: dict) -> list[str]: + """CVEs the deterministic pass could not decide — the input set for the adjudication pass. + + Two kinds, both real gaps rather than noise: + 1. INDETERMINATE — from an image WITH a window, but no fix version is knowable (advisory says + `TBD`/`7.4.X`, range is open-ended, and no release note names it). + 2. VENDOR-PAGE-ONLY — a CVE seen only on a vendor security page, with no structured advisory + behind it at all. gitea's two CVSS-9.8 RCEs are this shape. They carry no version data, so + no arithmetic can place them, but the page's prose usually states the fixed release. + """ + windows = rep.get("windows") or {} + out = list(rep.get("indeterminate") or []) + for cve in rep.get("unclassified") or []: + srcs = rep["cves"][cve]["sources"] + if not any(s.startswith("github-advisories:") for s in srcs) and cve not in out: + out.append(cve) + return sorted(out) + + +def evidence_bundle(rep: dict, cve: str) -> dict: + """EVERY deterministic signal held about one CVE, gathered for a model to weigh. + + Pass 1 collects; pass 2 judges. Nothing here interprets — it assembles what was measured, so the + judgement is made against evidence rather than recollection (the exact failure that let two + CVSS-9.8 gitea RCEs be published as "none": the report leaned on model knowledge that predated + them, and no source had been queried at all). + """ + e = rep["cves"][cve] + src = e["sources"][0] + windows = rep.get("windows") or {} + win = next((windows[s] for s in e["sources"] if s in windows), None) + ev = { + "cve": cve, + "severity": e.get("severity"), + "cvss": e.get("cvss"), + "sources": e["sources"], + "window": win, + "why_undecided": ("no fix version published and no release note names it" + if cve in (rep.get("indeterminate") or []) + else "seen only on a vendor page — no structured advisory, no version data"), + "patched_as_published": e.get("patched"), + "vulnerable_range_as_published": e.get("vulnerable_range"), + "summary": e.get("context"), + "description": e.get("description"), + "advisory_url": e.get("url"), + # Release tags NAMING this CVE, whether or not they fall in the window — the model may + # reason about branch lines the arithmetic deliberately would not. + "releases_naming_it": release_fix_versions(src, cve) if src.startswith("github-advisories:") else [], + "references": [], + } + if e.get("ghsa"): + t = advisory_text(e["ghsa"], src) + if t.get("status") == "ok": + ev["references"] = t.get("references") or [] + ev["description"] = t.get("description") or ev["description"] + ev["summary"] = t.get("summary") or ev["summary"] + ev["affected"] = t.get("vulnerabilities") or [] + else: + ev["detail_fetch"] = t.get("status") + return ev + + +def adjudication_block(rep: dict) -> str: + """SECOND PASS: present the collected evidence and ask for a judgement on each open case. + + This block decides nothing. The deterministic count stands as a FLOOR; a verdict here may only + ADD to it, matching the rule that this scan raises a count on evidence but never lowers one. + """ + todo = needs_judgement(rep) + if not todo: + return "" + shown, dropped = todo[:MAX_ADJUDICATE], max(0, len(todo) - MAX_ADJUDICATE) + L = ["", "---", "", + f"## Adjudication pass — {len(todo)} advisory/advisories need judgement", "", + "Pass 1 collected the evidence below deterministically and could NOT decide these cases. " + "Weigh the evidence and decide each one.", "", + "**For each: did the version move in its window fix it?** Answer **FIXED** / **NOT-FIXED** / " + "**STILL-UNKNOWN**, each with a one-line reason **citing the evidence shown** — a fixed " + "release named in the text, a branch line, an affected range. Add every FIXED to the " + "recipe's CVE count; the deterministic number is a floor, not a total. If the evidence does " + "not settle it, say STILL-UNKNOWN: do NOT infer from memory of the project, and never " + "record an undecided CVE as unaffected.", ""] + for src, win in (rep.get("windows") or {}).items(): + L.append(f"- window: `{src.split(':',1)[-1]}` {win['from']} → {win['to']}") + if dropped: + L += ["", f"⚠ Showing the first {MAX_ADJUDICATE} of {len(todo)}; **{dropped} not shown** " + f"(raise ADVISORY_SCAN_MAX_ADJUDICATE). The unshown remain undetermined — do not " + f"treat them as absent."] + L.append("") + for cve in shown: + ev = evidence_bundle(rep, cve) + L.append(f"### {cve} — {ev['severity'] or '?'}") + L.append(f"- undecided because: {ev['why_undecided']}") + L.append(f"- source: `{ev['sources'][0]}`" + + (f" · window {ev['window']['from']} → {ev['window']['to']}" if ev["window"] else + " · **no version window** for this image")) + L.append(f"- patched_versions as published: `{ev['patched_as_published']}`") + L.append(f"- vulnerable_range as published: `{ev['vulnerable_range_as_published']}`") + if ev["releases_naming_it"]: + L.append(f"- **releases naming this CVE**: {', '.join(ev['releases_naming_it'][:14])}") + for v in ev.get("affected") or []: + L.append(f"- affects `{v.get('package')}` {v.get('vulnerable_version_range')} — " + f"first_patched_version: {v.get('first_patched_version')}") + if ev["references"]: + L.append(f"- references: {', '.join(r.strip() for r in ev['references'][:6])}") + if ev.get("detail_fetch"): + L.append(f"- ⚠ detail fetch failed: {ev['detail_fetch']} (evidence below is from pass 1)") + if ev["summary"]: + L += ["", f"> {ev['summary']}"] + if ev["description"]: + L += ["", "```", (ev["description"] or "").strip()[:2000], "```"] + L.append("") + + # ── everything pass 1 DID decide, with the evidence behind each verdict ────────────────────── + # Pass 2 must see the whole picture, not only the leftovers: a deterministic verdict can still + # be wrong (a mis-parsed range, a release note that names a CVE without fixing it), and only a + # reader with the evidence in front of it can catch that. + decided = [] + for cve in rep.get("fixed_by_this_upgrade") or []: + e = rep["cves"][cve] + decided.append((cve, "COUNTED", e)) + for cve, e in sorted(rep.get("cves", {}).items()): + if e.get("classification") == "outside-window": + decided.append((cve, "excluded (outside window)", e)) + if decided: + shown_rows, dropped_rows = decided[:MAX_REVIEW_ROWS], max(0, len(decided) - MAX_REVIEW_ROWS) + L += ["---", "", + f"## Pass 1 decisions — {len(decided)} already judged deterministically", "", + "Review these too. If any verdict looks wrong given its evidence, say so and explain; " + "a correction here changes the count. Silence means you agree.", ""] + if dropped_rows: + L.append(f"⚠ Showing {MAX_REVIEW_ROWS} of {len(decided)}; **{dropped_rows} not shown** " + f"(raise ADVISORY_SCAN_MAX_REVIEW).") + L.append("") + L += ["| CVE | verdict | severity | patched as published | releases naming it | source |", + "|---|---|---|---|---|---|"] + for cve, verdict, e in shown_rows: + rel = e.get("fix_versions_from_release_notes") or [] + L.append(f"| {cve} | {verdict} | {e.get('severity') or '?'} | " + f"{(e.get('patched') or '—')[:60]} | {', '.join(rel[:5]) or '—'} | " + f"{e['sources'][0].split(':',1)[-1]} |") + L.append("") + return "\n".join(L) + + def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, images: list[tuple[str, str, str]] | None = None) -> dict: urls, reg_path = registry_urls(recipe, registry_dir) @@ -303,7 +566,8 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, def record(cve: str, src: str, **extra): e = report["cves"].setdefault(cve, {"sources": [], "severity": None, "ghsa": None, "vulnerable_range": None, "patched": None, - "context": None, "published_at": None}) + "context": None, "published_at": None, + "description": None, "url": None, "cvss": None}) if src not in e["sources"]: e["sources"].append(src) for k, v in extra.items(): @@ -317,7 +581,8 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, if a.get("cve"): record(a["cve"], entry["source"], severity=a.get("severity"), ghsa=a.get("ghsa"), vulnerable_range=a.get("vulnerable_range"), patched=a.get("patched"), - context=a.get("summary"), published_at=a.get("published_at")) + context=a.get("summary"), published_at=a.get("published_at"), + description=a.get("description"), url=a.get("url"), cvss=a.get("cvss")) for entry in vendor_pages(urls): report["sources"].append({"source": entry["source"], "status": entry["status"], @@ -372,20 +637,28 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, report["windows"] = {k: {"from": f, "to": t} for k, (f, t) in windows.items()} def _classify_window(src, wf, wt): - """Return (set_of_fixed_cves, method, date_window|None, unresolved:boolean) for one source.""" + """Return (fixed, method, date_window|None, unresolved, indeterminate) for one source. + + `indeterminate` = advisories from this source that the method COULD NOT JUDGE (no usable + patched version, or no publish date). They are neither counted nor dismissed.""" kf, kt = _vkey(wf), _vkey(wt) # A version-SCHEME change (semver 3.5.3 → calver 2026.7.1) makes numeric ordering # meaningless: 2025.12.2 compares "newer" than 3.5.3 while shipping earlier. scheme = bool(kf and kt and abs(kt[0] - kf[0]) >= SCHEME_JUMP) if not scheme: - got = set() + got, undecidable = set(), set() for cve, e in report["cves"].items(): if src not in e["sources"]: continue - cands = [_vkey(t) for t in re.findall(r"\d+(?:\.\d+)*", e.get("patched") or "")] - if kf and kt and any(kf < c <= kt for c in cands): + patched = e.get("patched") or "" + cands = [_vkey(t) for t in re.findall(r"\d+(?:\.\d+)*", patched)] + if kf and kt and any(_within(kf, kt, c) for c in cands): got.add(cve) - return got, "patched version ranges", None, False + elif not patched or PLACEHOLDER_RE.search(patched): + # No fix version published ("TBD") or only a placeholder ("7.4.X" — which could + # be 7.4.1, inside the window). We cannot say either way, so say so. + undecidable.add(cve) + return got, "patched version ranges", None, False, undecidable # DATE FALLBACK: release DATES always order, even across a scheme change. Resolve both # versions to git tag dates and count advisories PUBLISHED in that window — the method a # hand count used to establish discourse 3.5.3 (2025-12-30) → 2026.7.1 (2026-07-31) = 123. @@ -395,12 +668,16 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, got = {cve for cve, e in report["cves"].items() if src in e["sources"] and e.get("published_at") and d_from < e["published_at"] <= d_to} - return got, "advisory publish date (version scheme changed)", (d_from, d_to), False - return set(), "unresolved", None, True + undecidable = {cve for cve, e in report["cves"].items() + if src in e["sources"] and not e.get("published_at")} + return got, "advisory publish date (version scheme changed)", (d_from, d_to), False, undecidable + return set(), "unresolved", None, True, set() fixed_set, methods, date_windows, unresolved_any = set(), {}, {}, False + indeterminate: set = set() for src, (wf, wt) in windows.items(): - got, method, dw, unresolved = _classify_window(src, wf, wt) + got, method, dw, unresolved, undecidable = _classify_window(src, wf, wt) + indeterminate |= undecidable methods[src] = method if dw: date_windows[src] = {"from": dw[0], "to": dw[1]} @@ -427,7 +704,37 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, report["classified_by"] = methods if date_windows: report["date_window"] = date_windows + # THIRD METHOD: before declaring an advisory undecidable, look for the CVE id in the project's + # own release notes. A tag that names it, inside the window, IS the fix version the advisory + # failed to publish. Deterministic and citable — not a judgement call. + resolved_by_release = {} + for cve in sorted(indeterminate - fixed_set): + e = report["cves"][cve] + for src in e["sources"]: + if src not in windows: + continue + wf, wt = windows[src] + kf, kt = _vkey(wf), _vkey(wt) + if not (kf and kt): + continue + hits = [t for t in release_fix_versions(src, cve) if _within(kf, kt, _vkey(t))] + if hits: + fixed_set.add(cve) + resolved_by_release[cve] = sorted(hits) + e["classification"] = (f"fixed-by-this-upgrade (named in release notes " + f"{', '.join(sorted(hits))}) via {src}") + e["fix_versions_from_release_notes"] = sorted(hits) + break + if resolved_by_release: + report["resolved_by_release_notes"] = resolved_by_release + + indeterminate -= fixed_set + for cve in indeterminate: + report["cves"][cve]["classification"] = "indeterminate: no fix version published" + unknown = [c for c in unknown if c not in fixed_set] report["fixed_by_this_upgrade"] = sorted(fixed_set) + report["indeterminate"] = sorted(indeterminate) + report["cve_count_indeterminate"] = len(indeterminate) report["unclassified"] = sorted(unknown) # NEVER report 0 for something we could not determine — a 0 asserts safety. If ANY requested # window could not be ordered at all, the total is UNKNOWN rather than a partial number. @@ -468,8 +775,10 @@ def markdown(rep: dict) -> str: f"advisory APIs). This scan is ADDITIVE — it does not replace the release-note " f"reading in the upgrade step._") return "\n".join(L) + ind = rep.get("cve_count_indeterminate") or 0 if rep["fixed_by_this_upgrade"]: - L.append(f"\n**CVEs fixed by this upgrade: {rep['cve_count_fixed']}**\n") + floor = " (at least — see undetermined below)" if ind else "" + L.append(f"\n**CVEs fixed by this upgrade: {rep['cve_count_fixed']}**{floor}\n") cb = rep.get("classified_by") or {} if isinstance(cb, dict) and cb: for src, method in cb.items(): @@ -487,6 +796,15 @@ def markdown(rep: dict) -> str: f"{e.get('ghsa') or '-'} | {e['sources'][0]} |") else: L.append("\n**CVEs fixed by this upgrade: 0 identified by the deterministic scan.**") + if ind: + L.append(f"\n⚠ **{ind} advisory/advisories could NOT be judged** — the vendor published no fix " + f"version (GitHub carries `TBD` or a placeholder like `7.4.X`) and the vulnerable " + f"range is open-ended, so neither method can tell whether this upgrade fixed them. " + f"They are NOT included in the count above and must NOT be read as unaffected: " + + ", ".join(f"{c} ({rep['cves'][c].get('severity') or '?'})" + for c in rep["indeterminate"][:15]) + + (" …" if len(rep["indeterminate"]) > 15 else "") + + "\n\nRe-run with `--adjudicate` for the collected evidence on each, to judge.") if rep["unclassified"]: L.append(f"\nSeen but not version-classified ({len(rep['unclassified'])}) — includes advisories " f"from OTHER images in this recipe (sidecars), which this window cannot judge: " @@ -505,6 +823,10 @@ def main() -> int: ap.add_argument("--from", dest="v_from", default=None) ap.add_argument("--to", dest="v_to", default=None) ap.add_argument("--json", action="store_true", help="emit raw JSON instead of markdown") + ap.add_argument("--adjudicate", action="store_true", + help="SECOND PASS: for advisories the deterministic pass could not judge (no " + "fix version published), fetch their full text + references and append a " + "block for the agent to judge. Additive: it never changes the count above.") ap.add_argument("--registry", default=REGISTRY_DIR) ap.add_argument("--image", action="append", default=[], metavar="NAME=FROM:TO", help="a sidecar image and the versions it moved between, e.g. " @@ -521,7 +843,15 @@ def main() -> int: else: print(f'ignoring malformed --image {spec!r} (expected NAME=FROM:TO)', file=sys.stderr) rep = scan(a.recipe, a.v_from, a.v_to, a.registry, images) - print(json.dumps(rep, indent=2) if a.json else markdown(rep)) + if a.adjudicate: + rep["adjudication"] = [evidence_bundle(rep, c) + for c in needs_judgement(rep)[:MAX_ADJUDICATE]] + if a.json: + print(json.dumps(rep, indent=2)) + else: + print(markdown(rep)) + if a.adjudicate: + print(adjudication_block(rep)) return 0 diff --git a/cc-ci-plan/audit-advisory-scan.py b/cc-ci-plan/audit-advisory-scan.py new file mode 100755 index 0000000..10c76ed --- /dev/null +++ b/cc-ci-plan/audit-advisory-scan.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Independent audit of advisory-scan's counts. + +Deliberately does NOT reuse the scanner's classifier. Re-parses patched versions with a separate +semver implementation and re-derives membership, then diffs against what the scanner concluded. +Anything the two disagree on is a miscategorization in one of them. +""" +import importlib.util, json, re, sys + +spec = importlib.util.spec_from_file_location("A", "/srv/cc-ci-orch/cc-ci-plan/advisory-scan.py") +A = importlib.util.module_from_spec(spec); spec.loader.exec_module(A) +REG = "/srv/cc-ci-orch/cc-ci-plan/upstream" + + +def sv(s): + """Independent semver parse: strict 3-tuple, missing parts are 0.""" + m = re.match(r"^\s*v?(\d+)(?:\.(\d+))?(?:\.(\d+))?", s or "") + if not m: + return None + return tuple(int(x) if x else 0 for x in m.groups()) + + +def in_window(f, t, patched_expr): + """Independent membership: any patched token strictly above f and at most t.""" + kf, kt = sv(f), sv(t) + for tok in re.findall(r"\d+(?:\.\d+)*", patched_expr or ""): + c = sv(tok) + if c and kf and kt and kf < c <= kt: + return True + return False + + +_RELS = {} + + +def fetch_releases(owner_repo): + """Independent releases fetch — deliberately NOT the scanner's cache or pagination helper.""" + if owner_repo in _RELS: + return _RELS[owner_repo] + import urllib.request + tok = None + try: + tok = open("/srv/cc-ci/.github-token").read().strip() + except OSError: + pass + h = {"User-Agent": "audit", "Accept": "application/vnd.github+json"} + if tok: + h["Authorization"] = f"Bearer {tok}" + out, url, pages = [], f"https://api.github.com/repos/{owner_repo}/releases?per_page=100", 0 + while url and pages < 4: + req = urllib.request.Request(url, headers=h) + with urllib.request.urlopen(req, timeout=45) as r: + rows = json.load(r) + link = r.headers.get("Link", "") or "" + out += [(x.get("tag_name") or "", (x.get("body") or "") + " " + (x.get("name") or "")) + for x in rows] + url = None + for part in link.split(","): + if 'rel="next"' in part: + url = part.split(";")[0].strip().strip("<>") + pages += 1 + _RELS[owner_repo] = out + return out + + +def audit(recipe, vf, vt, images=None, label=""): + rep = A.scan(recipe, vf, vt, REG, images) + print(f"\n{'='*78}\n{recipe} {vf} → {vt} {label}\n{'='*78}") + print(f"scanner count = {rep['cve_count_fixed']} known={rep['count_known']} " + f"failed_sources={rep['sources_failed']}") + counted = set(rep["fixed_by_this_upgrade"]) + + by_src = {} + for cve, e in rep["cves"].items(): + by_src.setdefault(e["sources"][0], []).append((cve, e)) + + windows = rep["windows"] + total_mismatch = 0 + for src, win in windows.items(): + f, t = win["from"], win["to"] + method = rep["classified_by"][src] + rows = by_src.get(src, []) + scanner_here = {c for c, e in rows if c in counted} + print(f"\n ── {src} ({f} → {t}) method={method} advisories={len(rows)}") + if "publish date" in method: + dw = rep["date_window"][src] + indep = {c for c, e in rows + if e.get("published_at") and dw["from"] < e["published_at"] <= dw["to"]} + print(f" date window {dw['from'][:10]} → {dw['to'][:10]}") + else: + indep = {c for c, e in rows if in_window(f, t, e.get("patched"))} + # Independently redo the release-note method: fetch the repo's releases ourselves and + # confirm a tag NAMING the CVE really does fall inside (f, t]. + kf, kt = sv(f), sv(t) + owner_repo = src.split(":", 1)[1] + rels = fetch_releases(owner_repo) + for c, e in rows: + if c in indep: + continue + naming = [tag for tag, body in rels if c in body] + if any(kf < sv(tag) <= kt for tag in naming if sv(tag)): + indep.add(c) + missed = indep - scanner_here + extra = scanner_here - indep + print(f" scanner counted {len(scanner_here)} | independent {len(indep)}" + f" | missed_by_scanner {len(missed)} | over_counted {len(extra)}") + if missed: + print(f" !! MISSED: {sorted(missed)}") + for c in sorted(missed): + print(f" {c} patched={dict(rows)[c].get('patched')!r}") + if extra: + print(f" !! OVER-COUNTED: {sorted(extra)}") + for c in sorted(extra): + print(f" {c} patched={dict(rows)[c].get('patched')!r}") + total_mismatch += len(missed) + len(extra) + + # Anything counted that belongs to NO window would be a leak. + leaked = {c for c in counted if not any(s in rep["cves"][c]["sources"] for s in windows)} + if leaked: + print(f"\n !! COUNTED BUT OUTSIDE EVERY WINDOW: {sorted(leaked)}") + total_mismatch += len(leaked) + + # Unclassified entries that belong to a WINDOWED source would mean a judged CVE was dropped. + dropped = [c for c in rep["unclassified"] + if any(s in rep["cves"][c]["sources"] for s in windows)] + if dropped: + print(f"\n !! UNCLASSIFIED DESPITE HAVING A WINDOW: {sorted(dropped)[:10]}") + total_mismatch += len(dropped) + + unwindowed = {} + for cve in rep["unclassified"]: + unwindowed.setdefault(rep["cves"][cve]["sources"][0], []).append(cve) + if unwindowed: + print("\n unclassified by source (expected: images with no --image given):") + for s, cs in sorted(unwindowed.items()): + print(f" {len(cs):4d} {s}") + + print(f"\n VERDICT: {'CLEAN' if total_mismatch == 0 else f'{total_mismatch} DISAGREEMENTS'}") + return total_mismatch, rep + + +if __name__ == "__main__": + bad = 0 + bad += audit("gitea", "1.27.0", "1.27.1")[0] + bad += audit("discourse", "3.5.3", "2026.7.1")[0] + bad += audit("discourse", "3.5.3", "2026.7.1", [("redis", "7.4", "8.10")], "+redis sidecar")[0] + bad += audit("keycloak", "26.7.0", "26.7.1")[0] + bad += audit("mailu", "2024.06.55", "2024.06.57", [("redis", "8.8.0", "8.10.0")], "+redis")[0] + bad += audit("n8n", "1.123.0", "2.18.1")[0] + print(f"\n\n{'#'*78}\nOVERALL: {'CLEAN — no disagreements' if bad == 0 else f'{bad} DISAGREEMENTS'}") + sys.exit(1 if bad else 0) diff --git a/cc-ci-plan/test-advisory-scan.py b/cc-ci-plan/test-advisory-scan.py new file mode 100755 index 0000000..65b1fc6 --- /dev/null +++ b/cc-ci-plan/test-advisory-scan.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 +"""Tests for advisory-scan.py. + +Two tiers: + + OFFLINE (default) — pure logic, fixtures injected in place of the network. Fast, deterministic, + no token, no rate limit. These encode every classification rule and every guarantee the CVE count + makes, including the specific production defects that motivated them. + + LIVE (--live) — re-derives the CVE counts published in the week-2026-08-07 report against the real + advisory APIs. Slow, needs network + ideally a GitHub token. Run before changing classification. + +Usage: + python3 test-advisory-scan.py # offline only + python3 test-advisory-scan.py --live # offline + historic report regressions +""" + +from __future__ import annotations + +import importlib.util +import io +import json +import os +import pathlib +import sys +import unittest +import unittest.mock + +HERE = pathlib.Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location("advisory_scan", HERE / "advisory-scan.py") +A = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(A) + + +# ── fixture helpers ─────────────────────────────────────────────────────────────────────────────── + +def adv(cve, patched=None, published=None, severity="high", ghsa=None): + """One GitHub advisory row as github_advisories() would emit it.""" + return {"cve": cve, "ghsa": ghsa or f"GHSA-fake-{cve[-4:]}", "severity": severity, + "summary": f"summary for {cve}", "vulnerable_range": None, "patched": patched, + "url": None, "published_at": published} + + +def gh(owner_repo, advisories, status="ok"): + return {"source": f"github-advisories:{owner_repo}", "status": status, "advisories": advisories} + + +def vendor(url, cves=(), status="ok"): + return {"source": url, "status": status, "cves": list(cves), + "context": {c: f"...{c}..." for c in cves}} + + +def run_scan(gh_entries=(), vendor_entries=(), tag_dates=None, *, v_from=None, v_to=None, + images=None, recipe="fixture", urls=None, releases=None): + """scan() with every network call replaced by fixtures. + + `releases` maps CVE id -> tags whose release notes name it (the third, release-note method).""" + tag_dates = tag_dates or {} + releases = releases or {} + urls = urls if urls is not None else ["https://github.com/app/app"] + with unittest.mock.patch.object(A, "registry_urls", lambda r, d: (list(urls), "/fake/reg.md")), \ + unittest.mock.patch.object(A, "github_advisories", lambda u: list(gh_entries)), \ + unittest.mock.patch.object(A, "vendor_pages", lambda u: list(vendor_entries)), \ + unittest.mock.patch.object(A, "osv", lambda r, v: None), \ + unittest.mock.patch.object(A, "_tag_date", lambda o, r, v: tag_dates.get(v)), \ + unittest.mock.patch.object(A, "release_fix_versions", lambda src, cve: list(releases.get(cve, []))): + return A.scan(recipe, v_from, v_to, "/fake", images) + + +def parse_image_args(argv): + """Drive main()'s --image parsing exactly as the CLI does, returning the tuples scan() receives.""" + captured = {} + + def fake_scan(recipe, vf, vt, reg, images): + captured["images"] = images + return {"recipe": recipe, "from": vf, "to": vt, "registry": reg, "registry_urls": 0, + "sources": [], "cves": {}, "fixed_by_this_upgrade": [], "unclassified": [], + "sources_failed": [], "sources_benign": [], "count_known": True, + "cve_count_fixed": 0, "windows": {}, "classified_by": {}} + + err = io.StringIO() + with unittest.mock.patch.object(A, "scan", fake_scan), \ + unittest.mock.patch.object(sys, "argv", ["advisory-scan.py", *argv]), \ + unittest.mock.patch.object(sys, "stdout", io.StringIO()), \ + unittest.mock.patch.object(sys, "stderr", err): + A.main() + return captured["images"], err.getvalue() + + +# ── A. version ordering ─────────────────────────────────────────────────────────────────────────── + +class TestVersionKey(unittest.TestCase): + def test_strips_prefix_and_suffix(self): + self.assertEqual(A._vkey("v1.27.1"), (1, 27, 1)) + self.assertEqual(A._vkey("1.27.1-rootless"), (1, 27, 1)) + self.assertEqual(A._vkey("2024.06.55"), (2024, 6, 55)) + + def test_empty_and_none(self): + self.assertEqual(A._vkey(None), ()) + self.assertEqual(A._vkey(""), ()) + + def test_dotted_minor_is_numeric_not_lexical(self): + # The bug this guards: "8.10" must be NEWER than "8.2.3". String compare says otherwise. + self.assertGreater(A._vkey("8.10"), A._vkey("8.2.3")) + self.assertGreater(A._vkey("1.27.10"), A._vkey("1.27.9")) + + def test_shorter_prefix_orders_below_its_own_patch(self): + # 7.4 < 7.4.1, so a CVE patched in 7.4.1 IS fixed by moving off a bare 7.4 pin. + self.assertLess(A._vkey("7.4"), A._vkey("7.4.1")) + + +class TestWindowMembership(unittest.TestCase): + """(from, to] membership — exclusive lower, inclusive upper, compared zero-padded.""" + + def _in(self, f, t, c): + return A._within(A._vkey(f), A._vkey(t), A._vkey(c)) + + def test_bounds(self): + self.assertTrue(self._in("1.27.0", "1.27.1", "1.27.1")) # upper inclusive + self.assertFalse(self._in("1.27.0", "1.27.1", "1.27.0")) # lower exclusive + self.assertFalse(self._in("1.27.0", "1.27.1", "1.26.9")) + self.assertFalse(self._in("1.27.0", "1.27.1", "1.28.0")) + + def test_bare_major_upper_bound_includes_its_dot_zero(self): + # Regression: plain tuple order makes (18,) < (18,0), so a fix in 18.0 fell OUTSIDE a + # window ending at 18. Bare major tags are the norm for sidecars (postgres:18, redis:8). + self.assertTrue(self._in("17", "18", "18.0")) + self.assertTrue(self._in("7", "8", "8.0")) + self.assertTrue(self._in("7.4", "8.10", "8.0.4")) + + def test_bare_major_upper_bound_excludes_later_patches(self): + # Conservative on the other side: nothing proves which 18.x a floating tag resolved to. + self.assertFalse(self._in("17", "18", "18.5")) + + def test_bare_version_is_read_literally_as_dot_zero(self): + # from="8" means 8.0, so a fix in 8.0.4 is inside a window that ends at 9. + self.assertTrue(self._in("8", "9", "8.0.4")) + self.assertFalse(self._in("8", "9", "8.0")) # == the stated lower bound + + def test_prefix_lower_bound_still_counts_its_patches(self): + self.assertTrue(self._in("7.4", "8.10", "7.4.1")) + self.assertTrue(self._in("7.4", "8.10", "7.4.6")) + + def test_the_false_133_cve_stays_out(self): + self.assertFalse(self._in("7.4", "8.10", "6.0.11")) + + +# ── B. registry URL extraction ──────────────────────────────────────────────────────────────────── + +class TestRegistryUrls(unittest.TestCase): + def _write(self, tmp, text): + p = pathlib.Path(tmp) / "r.md" + p.write_text(text) + return A.registry_urls("r", tmp) + + def test_strips_trailing_markdown_punctuation(self): + # Production defect: a captured backtick 404'd the fetch and rendered n8n/immich as '?'. + import tempfile + with tempfile.TemporaryDirectory() as tmp: + urls, _ = self._write(tmp, "see `https://docs.n8n.io/release-notes/` and " + "**https://example.com/sec.html**, plus https://a.test/x.") + self.assertIn("https://docs.n8n.io/release-notes/", urls) + self.assertIn("https://example.com/sec.html", urls) + self.assertIn("https://a.test/x", urls) + self.assertFalse([u for u in urls if u.endswith(("`", "*", ".", ","))]) + + def test_dedupes_and_reports_missing_registry(self): + import tempfile + with tempfile.TemporaryDirectory() as tmp: + urls, path = self._write(tmp, "https://a.test/x https://a.test/x") + self.assertEqual(urls.count("https://a.test/x"), 1) + self.assertTrue(path.endswith("r.md")) + urls, path = A.registry_urls("does-not-exist", "/nonexistent-dir") + self.assertEqual((urls, path), ([], None)) + + +# ── C. --image argument parsing ─────────────────────────────────────────────────────────────────── + +class TestImageArgParsing(unittest.TestCase): + def test_single_and_repeated(self): + imgs, _ = parse_image_args(["r", "--image", "redis=7.4:8.10"]) + self.assertEqual(imgs, [("redis", "7.4", "8.10")]) + imgs, _ = parse_image_args(["r", "--image", "redis=7.4:8.10", "--image", "postgres=17:18"]) + self.assertEqual(imgs, [("redis", "7.4", "8.10"), ("postgres", "17", "18")]) + + def test_malformed_is_skipped_with_a_warning_not_a_crash(self): + # It is an ADDITIVE pre-step: one typo must not abort the upgrade's scan step. + for bad in ("redis=7.4", "redis", "=7.4:8.10", "redis=:8.10", "redis=7.4:"): + imgs, err = parse_image_args(["r", "--image", bad]) + self.assertEqual(imgs, [], f"{bad!r} should be rejected") + self.assertIn("malformed", err) + + def test_good_and_bad_mixed_keeps_the_good(self): + imgs, err = parse_image_args(["r", "--image", "redis=7.4:8.10", "--image", "nope"]) + self.assertEqual(imgs, [("redis", "7.4", "8.10")]) + self.assertIn("malformed", err) + + +# ── D. classification ───────────────────────────────────────────────────────────────────────────── + +class TestClassificationBoundaries(unittest.TestCase): + def _one(self, patched, v_from="1.27.0", v_to="1.27.1"): + rep = run_scan([gh("app/app", [adv("CVE-2026-0001", patched=patched)])], + v_from=v_from, v_to=v_to) + return rep + + def test_patched_at_upper_bound_counts(self): + self.assertEqual(self._one("1.27.1")["fixed_by_this_upgrade"], ["CVE-2026-0001"]) + + def test_patched_at_lower_bound_does_not_count(self): + # Already fixed in the version we were ON — this upgrade did not fix it. + rep = self._one("1.27.0") + self.assertEqual(rep["fixed_by_this_upgrade"], []) + self.assertEqual(rep["cve_count_fixed"], 0) + + def test_patched_below_and_above_window_do_not_count(self): + self.assertEqual(self._one("1.26.0")["fixed_by_this_upgrade"], []) + self.assertEqual(self._one("1.28.0")["fixed_by_this_upgrade"], []) + + def test_any_of_several_patched_lines_counts(self): + # n8n regression: one advisory patches several release lines; reading only the first + # dropped the line the deployment was on (CVE-2026-42231/42232 misclassified). + rep = self._one("1.123.32; 2.17.4; 2.18.1", v_from="2.17.0", v_to="2.17.4") + self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2026-0001"]) + + def test_no_patched_data_is_not_counted(self): + self.assertEqual(self._one(None)["fixed_by_this_upgrade"], []) + + +class TestPerImageWindows(unittest.TestCase): + """The false-133 family of defects: an image must only ever be judged by its OWN versions.""" + + APP = gh("discourse/discourse", [adv("CVE-APP-0001", patched="3.5.4", published="2026-03-01T00:00:00Z")]) + REDIS = gh("redis/redis", [ + adv("CVE-2021-21309", patched="6.0.11", published="2021-02-01T00:00:00Z"), + adv("CVE-2025-49844", patched="7.4.6; 8.0.4; 8.2.2", published="2025-10-01T00:00:00Z", + severity="critical"), + ]) + URLS = ["https://github.com/discourse/discourse", "https://github.com/redis/redis"] + + def test_sidecar_cve_is_not_judged_by_the_app_window(self): + # redis 6.0.11 sits numerically inside discourse 3.5.3 -> 2026.7.1. It must NOT count. + rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="2026.7.1", + tag_dates={"3.5.3": "2025-12-30T00:00:00Z", "2026.7.1": "2026-07-31T00:00:00Z"}, + urls=self.URLS) + self.assertNotIn("CVE-2021-21309", rep["fixed_by_this_upgrade"]) + self.assertIn("CVE-2021-21309", rep["unclassified"]) + + def test_unwindowed_image_is_unclassified_never_counted(self): + rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="3.5.4", urls=self.URLS) + self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-APP-0001"]) + for cve in ("CVE-2021-21309", "CVE-2025-49844"): + self.assertIn(cve, rep["unclassified"]) + + def test_sidecar_window_counts_only_what_that_bump_fixed(self): + rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="3.5.4", + images=[("redis", "7.4", "8.10")], urls=self.URLS) + self.assertIn("CVE-2025-49844", rep["fixed_by_this_upgrade"]) # patched 7.4.6, in window + self.assertNotIn("CVE-2021-21309", rep["fixed_by_this_upgrade"]) # patched 6.0.11, below it + self.assertEqual(rep["cve_count_fixed"], 2) # app 1 + redis 1 + + def test_count_is_the_union_across_images(self): + rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="3.5.4", + images=[("redis", "7.4", "8.10")], urls=self.URLS) + self.assertEqual(sorted(rep["fixed_by_this_upgrade"]), ["CVE-2025-49844", "CVE-APP-0001"]) + + def test_each_image_classified_independently(self): + # App crosses a scheme change (date method); redis does not (version method). Both resolve. + rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="2026.7.1", + images=[("redis", "7.4", "8.10")], urls=self.URLS, + tag_dates={"3.5.3": "2025-12-30T00:00:00Z", "2026.7.1": "2026-07-31T00:00:00Z"}) + methods = rep["classified_by"] + self.assertIn("publish date", methods["github-advisories:discourse/discourse"]) + self.assertEqual(methods["github-advisories:redis/redis"], "patched version ranges") + self.assertTrue(rep["count_known"]) + + def test_image_name_matches_as_substring(self): + rep = run_scan([self.APP, gh("discourse/discourse-postgres", [adv("CVE-PG-1", patched="18.0")])], + v_from="3.5.3", v_to="3.5.4", images=[("postgres", "17", "18")], + urls=["https://github.com/discourse/discourse", + "https://github.com/discourse/discourse-postgres"]) + self.assertIn("github-advisories:discourse/discourse-postgres", rep["windows"]) + self.assertIn("CVE-PG-1", rep["fixed_by_this_upgrade"]) + + def test_primary_cannot_be_stolen_by_a_loose_image_name(self): + rep = run_scan([self.APP, gh("discourse/discourse-postgres", [adv("CVE-PG-1", patched="18.0")])], + v_from="3.5.3", v_to="3.5.4", images=[("discourse", "1", "2")], + urls=["https://github.com/discourse/discourse", + "https://github.com/discourse/discourse-postgres"]) + self.assertEqual(rep["windows"]["github-advisories:discourse/discourse"], + {"from": "3.5.3", "to": "3.5.4"}) + + def test_unmatched_image_name_is_silently_ignored(self): + # Documents CURRENT behaviour: a typo'd name costs coverage without warning. + rep = run_scan([self.APP], v_from="3.5.3", v_to="3.5.4", + images=[("nosuchimage", "1", "2")], urls=self.URLS[:1]) + self.assertEqual(list(rep["windows"]), ["github-advisories:discourse/discourse"]) + self.assertTrue(rep["count_known"]) + + +class TestSchemeChangeDateFallback(unittest.TestCase): + DATES = {"3.5.3": "2025-12-30T00:00:00Z", "2026.7.1": "2026-07-31T00:00:00Z"} + + def _rep(self, advisories, dates=None): + return run_scan([gh("discourse/discourse", advisories)], v_from="3.5.3", v_to="2026.7.1", + tag_dates=self.DATES if dates is None else dates) + + def test_counts_advisories_published_inside_the_date_window(self): + rep = self._rep([adv("CVE-IN-1", published="2026-03-01T00:00:00Z"), + adv("CVE-OUT-1", published="2025-06-01T00:00:00Z"), + adv("CVE-OUT-2", published="2026-09-01T00:00:00Z")]) + self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-IN-1"]) + + def test_date_boundaries_match_the_version_rule(self): + # Exclusive lower, inclusive upper — same as 4a, so the two methods agree at the edges. + rep = self._rep([adv("CVE-LOWER", published=self.DATES["3.5.3"]), + adv("CVE-UPPER", published=self.DATES["2026.7.1"])]) + self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-UPPER"]) + + def test_advisory_without_a_publish_date_is_not_counted(self): + self.assertEqual(self._rep([adv("CVE-NODATE", published=None)])["fixed_by_this_upgrade"], []) + + def test_scheme_change_is_detected_not_version_compared(self): + rep = self._rep([adv("CVE-IN-1", patched="2026.1.0", published="2026-03-01T00:00:00Z")]) + self.assertIn("publish date", rep["classified_by"]["github-advisories:discourse/discourse"]) + self.assertIn("github-advisories:discourse/discourse", rep["date_window"]) + + def test_small_major_bump_still_uses_version_ranges(self): + rep = run_scan([gh("app/app", [adv("CVE-X", patched="3.0.0")])], v_from="2.9.0", v_to="3.0.0") + self.assertEqual(rep["classified_by"]["github-advisories:app/app"], "patched version ranges") + + +# ── E. count guarantees ─────────────────────────────────────────────────────────────────────────── + +class TestCountGuarantees(unittest.TestCase): + def test_unresolvable_window_yields_unknown_never_zero(self): + # Scheme change AND tag dates unresolvable -> must refuse to emit a number. + rep = run_scan([gh("app/app", [adv("CVE-1", published="2026-01-01T00:00:00Z")])], + v_from="3.5.3", v_to="2026.7.1", tag_dates={}) + self.assertIs(rep["cve_count_fixed"], None) + self.assertFalse(rep["count_known"]) + md = A.markdown(rep) + self.assertIn("UNKNOWN", md) + self.assertIn("NOT zero", md) + + def test_one_unresolvable_image_makes_the_whole_count_unknown(self): + # A partial number would understate a security figure, so it is suppressed entirely. + rep = run_scan([gh("app/app", [adv("CVE-APP", patched="1.1")]), + gh("redis/redis", [adv("CVE-REDIS", patched="8.0")])], + v_from="1.0", v_to="1.1", images=[("redis", "7.4", "9999.1")], + tag_dates={}, urls=["https://github.com/app/app", "https://github.com/redis/redis"]) + self.assertIs(rep["cve_count_fixed"], None) + self.assertFalse(rep["count_known"]) + + def test_genuine_zero_is_reported_as_zero(self): + rep = run_scan([gh("app/app", [adv("CVE-1", patched="9.9.9")])], v_from="1.0", v_to="1.1") + self.assertEqual(rep["cve_count_fixed"], 0) + self.assertTrue(rep["count_known"]) + self.assertIn("0 identified", A.markdown(rep)) + + def test_404_advisory_feed_is_benign_not_a_failure(self): + rep = run_scan([gh("app/app", [adv("CVE-1", patched="1.1")]), + gh("side/car", [], status="no-advisories-published")], + v_from="1.0", v_to="1.1") + self.assertEqual(rep["sources_failed"], []) + self.assertIn("github-advisories:side/car", rep["sources_benign"]) + self.assertEqual(rep["cve_count_fixed"], 1) + + def test_template_url_is_benign_not_a_failure(self): + rep = run_scan([gh("app/app", [adv("CVE-1", patched="1.1")])], + [vendor("https://x.test/changelog/v/", status="skipped: template URL")], + v_from="1.0", v_to="1.1") + self.assertEqual(rep["sources_failed"], []) + + def test_real_source_failure_is_surfaced(self): + rep = run_scan([gh("app/app", [adv("CVE-1", patched="1.1")], status="error: HTTP 500")], + v_from="1.0", v_to="1.1") + self.assertIn("github-advisories:app/app", rep["sources_failed"]) + self.assertIn("FAILED", A.markdown(rep)) + + def test_no_window_given_classifies_nothing(self): + rep = run_scan([gh("app/app", [adv("CVE-1", patched="1.1")])]) + self.assertEqual(rep["fixed_by_this_upgrade"], []) + self.assertIn("CVE-1", rep["unclassified"]) + + +class TestVendorOnlyCves(unittest.TestCase): + """The gitea case: CVEs named ONLY on a vendor page, absent from the GitHub advisory feed.""" + + def test_vendor_only_cve_is_recorded_and_surfaced(self): + rep = run_scan([gh("go-gitea/gitea", [])], + [vendor("https://blog.gitea.com/release-1.27.1/", ["CVE-2026-60004"])], + v_from="1.27.0", v_to="1.27.1") + self.assertIn("CVE-2026-60004", rep["cves"]) + self.assertIn("CVE-2026-60004", rep["unclassified"]) + + def test_vendor_only_cve_is_NOT_counted_but_IS_sent_for_judgement(self): + # It carries no version data, so no arithmetic can place it — the deterministic count must + # not include it. It must not be silently dropped either: pass 2 gets it with its evidence. + rep = run_scan([gh("go-gitea/gitea", [])], + [vendor("https://blog.gitea.com/release-1.27.1/", ["CVE-2026-60004"])], + v_from="1.27.0", v_to="1.27.1") + self.assertEqual(rep["fixed_by_this_upgrade"], []) + self.assertEqual(rep["cve_count_fixed"], 0) + self.assertIn("CVE-2026-60004", A.needs_judgement(rep)) + + def test_cve_in_both_vendor_and_advisory_feed_is_counted_once(self): + rep = run_scan([gh("go-gitea/gitea", [adv("CVE-2026-60004", patched="1.27.1")])], + [vendor("https://blog.gitea.com/x/", ["CVE-2026-60004"])], + v_from="1.27.0", v_to="1.27.1") + self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2026-60004"]) + self.assertEqual(rep["cve_count_fixed"], 1) + self.assertEqual(len(rep["cves"]["CVE-2026-60004"]["sources"]), 2) + + +class TestIndeterminateBucket(unittest.TestCase): + """An advisory with no knowable fix version is neither counted nor dismissed.""" + + def test_tbd_patched_is_indeterminate_not_excluded(self): + rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + self.assertEqual(rep["fixed_by_this_upgrade"], []) + self.assertIn("CVE-TBD", rep["indeterminate"]) + self.assertEqual(rep["cve_count_indeterminate"], 1) + + def test_placeholder_patched_is_indeterminate(self): + # "7.4.X" could be 7.4.1 — inside the window. Extracting a bare 7.4 and excluding it was + # how CVE-2024-46981 (high) went missing. + rep = run_scan([gh("redis/redis", [adv("CVE-X", patched="6.2.X, 7.2.X, 7.4.X")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + self.assertIn("CVE-X", rep["indeterminate"]) + + def test_real_versions_outside_the_window_are_decided_not_indeterminate(self): + rep = run_scan([gh("redis/redis", [adv("CVE-OLD", patched="6.0.11")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + self.assertEqual(rep["indeterminate"], []) + self.assertEqual(rep["cve_count_fixed"], 0) + + def test_indeterminate_is_surfaced_in_the_markdown_and_not_read_as_zero(self): + rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD", severity="critical")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + md = A.markdown(rep) + self.assertIn("could NOT be judged", md) + self.assertIn("must NOT be read as unaffected", md) + + +class TestReleaseNoteResolution(unittest.TestCase): + """Third method: a release whose notes NAME the CVE supplies the fix version the advisory lacks.""" + + def test_release_naming_the_cve_inside_the_window_counts_it(self): + rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"], + releases={"CVE-TBD": ["6.2.19", "7.2.10", "7.4.5", "8.0.3"]}) + self.assertIn("CVE-TBD", rep["fixed_by_this_upgrade"]) + self.assertEqual(rep["indeterminate"], []) + self.assertEqual(rep["resolved_by_release_notes"]["CVE-TBD"], ["7.4.5", "8.0.3"]) + + def test_release_naming_it_only_outside_the_window_stays_indeterminate(self): + rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"], + releases={"CVE-TBD": ["6.2.19"]}) + self.assertEqual(rep["fixed_by_this_upgrade"], []) + self.assertIn("CVE-TBD", rep["indeterminate"]) + + def test_release_evidence_is_recorded_for_audit(self): + rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"], + releases={"CVE-TBD": ["7.4.5"]}) + e = rep["cves"]["CVE-TBD"] + self.assertEqual(e["fix_versions_from_release_notes"], ["7.4.5"]) + self.assertIn("named in release notes", e["classification"]) + + def test_it_does_not_override_a_version_range_decision(self): + # A CVE already counted by patched ranges is untouched; the method only rescues undecided. + rep = run_scan([gh("redis/redis", [adv("CVE-OK", patched="7.4.1")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"], + releases={"CVE-OK": ["7.4.1"]}) + self.assertNotIn("CVE-OK", rep.get("resolved_by_release_notes") or {}) + + +class TestAdjudicationEvidenceAssembly(unittest.TestCase): + """Pass 2's JUDGEMENT is a model's and not testable; what IS testable is what it gets shown.""" + + def test_selects_indeterminate_and_vendor_only_cases(self): + rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])], + [vendor("https://blog.test/sec", ["CVE-VENDOR"])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + todo = A.needs_judgement(rep) + self.assertIn("CVE-TBD", todo) + self.assertIn("CVE-VENDOR", todo) + + def test_does_not_re_ask_about_cases_pass_1_settled(self): + rep = run_scan([gh("redis/redis", [adv("CVE-OK", patched="7.4.1")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + self.assertNotIn("CVE-OK", A.needs_judgement(rep)) + + def test_evidence_bundle_carries_the_window_and_published_fields(self): + rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + with unittest.mock.patch.object(A, "advisory_text", lambda g, s=None: {"status": "skipped"}), \ + unittest.mock.patch.object(A, "release_fix_versions", lambda s, c: []): + ev = A.evidence_bundle(rep, "CVE-TBD") + self.assertEqual(ev["window"], {"from": "7.4", "to": "8.10"}) + self.assertEqual(ev["patched_as_published"], "TBD") + self.assertIn("no fix version", ev["why_undecided"]) + + def test_pass_1_decisions_are_included_for_review(self): + rep = run_scan([gh("redis/redis", [adv("CVE-OK", patched="7.4.1"), + adv("CVE-OLD", patched="6.0.11"), + adv("CVE-TBD", patched="TBD")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + with unittest.mock.patch.object(A, "advisory_text", lambda g, s=None: {"status": "skipped"}), \ + unittest.mock.patch.object(A, "release_fix_versions", lambda s, c: []): + block = A.adjudication_block(rep) + self.assertIn("Pass 1 decisions", block) + self.assertIn("CVE-OK", block) # counted + self.assertIn("CVE-OLD", block) # excluded as outside-window + self.assertIn("CVE-TBD", block) # needs judgement + + def test_truncation_is_announced_never_silent(self): + advs = [adv(f"CVE-2026-{1000+i}", patched="TBD") for i in range(30)] + rep = run_scan([gh("redis/redis", advs)], v_from="7.4", v_to="8.10", + urls=["https://github.com/redis/redis"]) + with unittest.mock.patch.object(A, "advisory_text", lambda g, s=None: {"status": "skipped"}), \ + unittest.mock.patch.object(A, "release_fix_versions", lambda s, c: []), \ + unittest.mock.patch.object(A, "MAX_ADJUDICATE", 5): + block = A.adjudication_block(rep) + self.assertIn("not shown", block) + self.assertIn("do not", block.lower()) + + +class TestMarkdownOutput(unittest.TestCase): + def test_lists_every_window_with_its_method(self): + rep = run_scan([gh("discourse/discourse", [adv("CVE-A", patched="3.5.4")]), + gh("redis/redis", [adv("CVE-B", patched="8.0")])], + v_from="3.5.3", v_to="3.5.4", images=[("redis", "7.4", "8.10")], + urls=["https://github.com/discourse/discourse", "https://github.com/redis/redis"]) + md = A.markdown(rep) + self.assertIn("discourse/discourse: 3.5.3 → 3.5.4", md) + self.assertIn("redis/redis: 7.4 → 8.10", md) + self.assertIn("**CVEs fixed by this upgrade: 2**", md) + + def test_severity_and_fixed_in_are_rendered(self): + rep = run_scan([gh("redis/redis", [adv("CVE-2025-49844", patched="7.4.6; 8.2.2", + severity="critical")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + md = A.markdown(rep) + self.assertIn("critical", md) + self.assertIn("7.4.6", md) + + +# ── F. live regressions against published historic reports ──────────────────────────────────────── + +class TestHistoricReportNumbers(unittest.TestCase): + """Re-derive counts published in week-2026-08-07. Network + GitHub token; opt in with --live.""" + + REGISTRY = str(HERE / "upstream") + + @classmethod + def setUpClass(cls): + if not os.environ.get("ADVISORY_SCAN_LIVE"): + raise unittest.SkipTest("live tests: re-run with --live") + + def _count(self, recipe, v_from, v_to, images=None): + rep = A.scan(recipe, v_from, v_to, self.REGISTRY, images) + self.assertEqual(rep["sources_failed"], [], f"{recipe}: source failures make the count unsafe") + self.assertTrue(rep["count_known"], f"{recipe}: count came back UNKNOWN") + return rep + + def test_gitea_1_27_0_to_1_27_1_is_2(self): + rep = self._count("gitea", "1.27.0", "1.27.1") + self.assertEqual(rep["cve_count_fixed"], 2) + # Both CVSS-9.8 RCEs — the pair whose omission is why this tool exists. + self.assertEqual(set(rep["fixed_by_this_upgrade"]), {"CVE-2026-59774", "CVE-2026-60004"}) + + def test_discourse_app_only_is_123(self): + rep = self._count("discourse", "3.5.3", "2026.7.1") + self.assertEqual(rep["cve_count_fixed"], 123) + self.assertIn("publish date", rep["classified_by"]["github-advisories:discourse/discourse"]) + + def test_discourse_with_redis_sidecar_is_140(self): + # 123 app + 17 redis. Five redis advisories carry a usable patched_versions; the other + # twelve say "TBD" and are resolved from the release notes that name them. + rep = self._count("discourse", "3.5.3", "2026.7.1", [("redis", "7.4", "8.10")]) + self.assertEqual(rep["cve_count_fixed"], 140) + self.assertEqual(len(rep.get("resolved_by_release_notes") or {}), 12) + self.assertEqual(rep["cve_count_indeterminate"], 0) + # The five redis advisories that a sidecar-blind scan missed, incl. one critical. + for cve in ("CVE-2024-31227", "CVE-2024-31228", "CVE-2024-31449", + "CVE-2025-49844", "CVE-2025-62507"): + self.assertIn(cve, rep["fixed_by_this_upgrade"], f"{cve} missing from discourse+redis") + self.assertEqual(rep["cves"]["CVE-2025-49844"]["severity"], "critical") + + def test_discourse_redis_delta_is_exactly_seventeen(self): + app = self._count("discourse", "3.5.3", "2026.7.1") + both = self._count("discourse", "3.5.3", "2026.7.1", [("redis", "7.4", "8.10")]) + delta = set(both["fixed_by_this_upgrade"]) - set(app["fixed_by_this_upgrade"]) + self.assertEqual(len(delta), 17) + for cve in delta: + self.assertIn("redis", both["cves"][cve]["sources"][0]) + + def test_mailu_scan_finds_zero_and_says_so_knowingly(self): + # Published as 2 via the UNION with release-note reading; the scan's own contribution is 0, + # and 0 here must mean "checked, none", not "could not check". + rep = self._count("mailu", "2024.06.55", "2024.06.57", [("redis", "8.8.0", "8.10.0")]) + self.assertEqual(rep["cve_count_fixed"], 0) + self.assertTrue(rep["count_known"]) + + def test_keycloak_26_7_0_to_26_7_1_is_7(self): + rep = self._count("keycloak", "26.7.0", "26.7.1") + self.assertEqual(rep["cve_count_fixed"], 7) + + +def _main(): + live = "--live" in sys.argv + if live: + sys.argv.remove("--live") + os.environ["ADVISORY_SCAN_LIVE"] = "1" + unittest.main(verbosity=2) + + +if __name__ == "__main__": + _main() -- 2.54.0