From fc36d0e10fca49dd9afe87d8b0ab3be2562ac442 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Mon, 10 Aug 2026 20:52:51 +0000 Subject: [PATCH] advisory-scan: report UNKNOWN, never 0, when a count could not be determined MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator: 'the scanner should not say 0 when it was not able to scan.' Correct — the previous patch still led with '0 identified' and relegated the caveat to a footnote, so the headline number was wrong even though the prose was right. A 0 in a security column is an assertion of safety; it must never be emitted for an undetermined result. Now: cve_count_fixed is null (not 0) in JSON, a count_known flag distinguishes 'counted zero' from 'could not count', and the markdown headline reads 'CVEs fixed by this upgrade: UNKNOWN — the scan could NOT determine a count' with an explicit 'This is NOT zero' and instructions to render '?'. Verified: discourse 3.5.3 -> 2026.7.1 (semver->calver) now reports UNKNOWN; gitea 1.27.0 -> 1.27.1 still reports 2. --- .claude/skills/recipe-report/SKILL.md | 10 +++++++--- cc-ci-plan/advisory-scan.py | 25 ++++++++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.claude/skills/recipe-report/SKILL.md b/.claude/skills/recipe-report/SKILL.md index 2075c8d..6be03f0 100644 --- a/.claude/skills/recipe-report/SKILL.md +++ b/.claude/skills/recipe-report/SKILL.md @@ -47,9 +47,13 @@ keeps every weekly edition looking the same regardless of which model writes the (unknown), never `none` — a blank that reads as "clean" is exactly how two CVSS-9.8 gitea RCEs were reported as "none" on 2026-08-07. - **`?` must stay RARE — it means "we tried and could not tell", not "we didn't look".** Use it - ONLY when a scan ran and reported genuinely failed sources. In particular: a recipe with **no - upgrade this run** (up-to-date/skipped) has nothing an upgrade could have fixed — report `0`, - not `?`. A recipe with a clean scan reports its number (including `0`). Benign notes in a scan + ONLY when a scan ran and reported genuinely failed sources, **or when the scan block says + COUNT UNKNOWN** (it refuses to classify across a version-scheme change, e.g. semver → calver — + discourse 3.5.3 → 2026.7.1). In that case the scan's `0` means *not determined*: publish `?` + and say so in the notes; publishing `0` would assert a clean bill of health nothing supports. + (The scan headline itself now says `UNKNOWN` rather than a number in that case.) + In particular: a recipe with **no upgrade this run** (up-to-date/skipped) has nothing an + upgrade could have fixed — report `0`, not `?`. A recipe with a clean scan reports its number (including `0`). Benign notes in a scan block (`no-advisories-published`, `skipped: template URL`) are NOT failures and must not trigger `?`. If you find yourself rendering `?` for many recipes, that is a bug to report in the Addendum, not a normal outcome. diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index ebd4611..8fad2a9 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -319,7 +319,11 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str) - unknown.append(cve) report["fixed_by_this_upgrade"] = sorted(fixed) report["unclassified"] = sorted(unknown) - report["cve_count_fixed"] = len(fixed) + # NEVER report 0 for something we could not determine. When classification was refused, the + # count is UNKNOWN (null) — a 0 would be read as "no CVEs", which is an assertion this scan + # cannot make. Consumers must distinguish "counted 0" from "could not count". + report["count_known"] = not scheme_change + report["cve_count_fixed"] = len(fixed) if not scheme_change else None report["cve_count_total_seen"] = len(report["cves"]) # Only GENUINE failures make a count unreliable. "no-advisories-published" (404: the repo has # no advisory feed) and "skipped: template URL" are benign and must not degrade the verdict. @@ -340,6 +344,21 @@ def markdown(rep: dict) -> str: """Human/agent-readable block for pasting into the per-recipe upgrade log.""" L = [f"### Advisory scan (deterministic pre-step) — {rep['recipe']} " f"{rep.get('from') or '?'} → {rep.get('to') or '?'}"] + if not rep.get("count_known", True): + L.append("\n**CVEs fixed by this upgrade: UNKNOWN — the scan could NOT determine a count.**") + L.append("\n⚠ This is NOT zero. A version-scheme change (e.g. semver → calver) makes numeric " + "ordering meaningless across this jump, so no advisory could be classified. Render " + "this recipe's cve cell as `?`, never `0`. Read the vendor's release notes for the " + "jump and count by hand.") + if rep["unclassified"]: + L.append(f"\nAdvisories seen but unclassifiable ({len(rep['unclassified'])}) — includes " + f"other images in this recipe: " + ", ".join(rep["unclassified"][:12])) + if rep["sources_failed"]: + L.append(f"\n⚠ sources that FAILED: {', '.join(rep['sources_failed'])}") + L.append(f"\n_Sources checked: {len(rep['sources'])} ({rep['registry_urls']} registry URLs + " + f"advisory APIs). This scan is ADDITIVE — it does not replace the release-note " + f"reading in the upgrade step._") + return "\n".join(L) if rep["fixed_by_this_upgrade"]: L.append(f"\n**CVEs fixed by this upgrade: {rep['cve_count_fixed']}**\n") L.append("| CVE | severity | fixed in | advisory | source |") @@ -350,10 +369,6 @@ def markdown(rep: dict) -> str: f"{e.get('ghsa') or '-'} | {e['sources'][0]} |") else: L.append("\n**CVEs fixed by this upgrade: 0 identified by the deterministic scan.**") - if rep.get("scheme_change"): - L.append("\n⚠ **Version-scheme change detected** (e.g. semver → calver): numeric ordering is " - "not meaningful across it, so NOTHING was auto-classified. The CVE count above is " - "**0 by refusal, not by evidence** — read the vendor's release notes for this jump.") if rep["unclassified"]: L.append(f"\nSeen but not version-classified ({len(rep['unclassified'])}) — includes advisories " f"from OTHER images in this recipe (sidecars), which this window cannot judge: "