recipe-maintainers/cc-ci-orchestrator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

advisory-scan: attribute vendor-changelog CVEs to the release that fixed them

Browse Source 
nginx publishes NO GitHub security advisories. Every nginx CVE we can see comes
from nginx.org/en/CHANGES, and the scan scraped ids out of it without attributing
them to a release - so they had no patched version, could never be classified, and
every nginx bump in the fleet reported 0 CVEs. nginx is a sidecar in most recipes,
so this was a fleet-wide blind spot.

Measured on the two PRs that prompted the question:
  lasuite-docs#7  nginx 1.31.1 -> 1.31.3   0 -> 6 CVEs
  lasuite-drive#6 nginx 1.31.2 -> 1.31.3   0 -> 3 CVEs
matching a hand count of the changelog exactly (three fixed in 1.31.2, three in
1.31.3; the narrower window correctly counts only the latter).

How: when a vendor page is organised by release, each CVE is attributed to the
nearest preceding release heading ('Changes with nginx 1.31.3', '## v1.31.3'),
and that becomes its fixed-in version. The CVE is tied to a window by the image
name appearing in the page URL (window 'nginx' <-> nginx.org/...). A changelog
lists the project's whole history, so only releases the window actually crosses
count - asserted by a test that the 2013 entries stay out.

76 tests. discourse 140 / gitea 2 / mailu 2 / keycloak 12 unchanged.
This commit is contained in:
autonomic-bot
2026-08-11 19:49:43 +00:00
parent 4bad1ea6db
commit db37f1618b
3 changed files with 134 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cc-ci-plan/advisory-scan.py
+66 -3
View File
@@ -272,6 +272,37 @@ def github_advisories(urls: list[str]) -> list[dict]:
return results
# Release headings in a vendor changelog. nginx's CHANGES uses "Changes with nginx 1.31.3", most
# markdown changelogs use "## 1.31.3" / "## v1.31.3".
_HEADING_RE = re.compile(
r"^\s*(?:#{1,4}\s*)?(?:Changes with\s+\S+\s+|Version\s+|Release\s+)?v?(\d+\.\d+(?:\.\d+)*)\s*$"
r"|^\s*Changes with\s+\S+\s+(\d+\.\d+(?:\.\d+)*)", re.I)
def _changelog_versions(text: str) -> dict:
"""{cve: version} for a changelog that is ORGANISED BY RELEASE.
Why this exists: nginx publishes NO GitHub security advisories. Every nginx CVE we can see comes
from nginx.org/en/CHANGES, and scraping ids out of it without attributing them to a release
leaves them with no patched version — so they can never be classified, and an nginx bump reports
0 CVEs forever. nginx 1.31.1 -> 1.31.3 in fact fixes SIX (three in .2, three in .3), and nginx is
a sidecar in most of the fleet, so that was a fleet-wide blind spot.
Attributes each CVE to the nearest PRECEDING release heading — the release that fixed it.
"""
plain = re.sub(r"<[^>]+>", " ", text)
out, cur = {}, None
for line in plain.splitlines():
m = _HEADING_RE.match(line)
if m:
cur = m.group(1) or m.group(2)
continue
if cur:
for cve in CVE_RE.findall(line):
out.setdefault(cve, cur)
return out
def vendor_pages(urls: list[str]) -> list[dict]:
"""Fetch each registry URL and regex out CVE ids, with a little surrounding context."""
out = []
@@ -284,7 +315,7 @@ def vendor_pages(urls: list[str]) -> list[dict]:
# correct; counting them as failures would wrongly mark the recipe's count unreliable.
out.append({"source": u, "status": "skipped: template URL (not fetchable)", "cves": [], "context": {}})
continue
entry = {"source": u, "status": "ok", "cves": [], "context": {}}
entry = {"source": u, "status": "ok", "cves": [], "context": {}, "fixed_in": {}}
try:
text = _fetch(u)
plain = re.sub(r"<[^>]+>", " ", text)
@@ -292,6 +323,7 @@ def vendor_pages(urls: list[str]) -> list[dict]:
entry["cves"].append(cve)
i = plain.find(cve)
entry["context"][cve] = re.sub(r"\s+", " ", plain[max(0, i - 160) : i + 200]).strip()
entry["fixed_in"] = _changelog_versions(text)
except Exception as e: # noqa: BLE001
entry["status"] = f"error: {type(e).__name__}: {e}"
out.append(entry)
@@ -595,7 +627,8 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str,
e = report["cves"].setdefault(cve, {"sources": [], "severity": None, "ghsa": None,
"vulnerable_range": None, "patched": None,
"context": None, "published_at": None,
"description": None, "url": None, "cvss": None})
"description": None, "url": None, "cvss": None,
"changelog_fixed_in": None})
if src not in e["sources"]:
e["sources"].append(src)
for k, v in extra.items():
@@ -616,7 +649,8 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str,
report["sources"].append({"source": entry["source"], "status": entry["status"],
"found": len(entry.get("cves", []))})
for cve in entry.get("cves", []):
record(cve, entry["source"], context=entry["context"].get(cve))
record(cve, entry["source"], context=entry["context"].get(cve),
changelog_fixed_in=(entry.get("fixed_in") or {}).get(cve))
for version in filter(None, (v_from, v_to)):
o = osv(recipe, version)
@@ -656,8 +690,10 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str,
report["primary_source"] = primary
windows = {} # source name -> (from, to)
window_key = {} # source name -> the image name it covers
if primary:
windows[primary] = (v_from, v_to)
window_key[primary] = primary.split("/")[-1]
for key, wf, wt in (images or []):
for src in gh_sources:
if src in windows:
@@ -669,6 +705,7 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str,
k, name = key.lower(), src.split("/")[-1].lower()
if k in src.lower() or name in k:
windows[src] = (wf, wt)
window_key[src] = key
report["windows"] = {k: {"from": f, "to": t} for k, (f, t) in windows.items()}
def _classify_window(src, wf, wt):
@@ -726,6 +763,32 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str,
report["cves"][cve]["classification"] = f"fixed-by-this-upgrade ({method}) via {src}"
fixed_set.add(cve)
# A CVE seen only in a vendor CHANGELOG has no advisory feed behind it, but the changelog says
# which release fixed it (see _changelog_versions). Tie it to a window by the image name
# appearing in the page URL — nginx's window is `nginx`, and its changelog is nginx.org/... .
# Without this, projects that publish no GitHub advisories (nginx being the big one) can never
# contribute a CVE, and every nginx bump in the fleet silently reports 0.
from_changelog = {}
for cve, e in report["cves"].items():
if cve in fixed_set or not e.get("changelog_fixed_in"):
continue
for src, (wf, wt) in windows.items():
key = (window_key.get(src) or "").lower()
if not key:
continue
if not any(key in s_.lower() for s_ in e["sources"] if s_.startswith("http")):
continue
kf, kt = _vkey(wf), _vkey(wt)
cand = _vkey(e["changelog_fixed_in"])
if kf and kt and cand and _within(kf, kt, cand):
e["classification"] = (f"fixed-by-this-upgrade (named under {e['changelog_fixed_in']} "
f"in the vendor changelog) via {src}")
fixed_set.add(cve)
from_changelog[cve] = e["changelog_fixed_in"]
break
if from_changelog:
report["resolved_by_changelog"] = from_changelog
unknown = []
for cve, e in report["cves"].items():
if cve in fixed_set: