autonomic-bot
1db85a7e77
immich pins two images with BOTH a tag and a digest, which makes abra FATA and
abandon the WHOLE recipe. It therefore contributed no version data at all and
silently dropped out of every survey — indistinguishable from 'up to date'. The
standing answer was prose in three skills telling an agent to check registries by
hand. This replaces it with a tool.
resolve-images.py reads the compose files and queries registries itself:
- Docker Hub, ghcr, and any OCI registry via its own auth challenge (lscr.io
and dock.mau.dev advertise different realms; assuming ghcr's shape 401'd).
- tag SHAPES (digits -> '#') so -alpine stays on -alpine and 'latest' is never
proposed as an upgrade.
- reports newest_within_major AND newest_same_shape, and refuses to choose:
immich's postgres tag encodes the pg major plus the vectorchord/pgvectors
build immich-server expects, so taking the newest breaks the deploy.
- integrity check: if the CURRENT pin is absent from the listing, the listing
was truncated and any 'newest' is a guess. ghcr caps out past 40k tags, so
that falls back to the project's GitHub releases.
- per-repo cache + backoff + Docker Hub auth: a fleet sweep re-reads nginx,
redis and postgres many times and was getting 429s reported as 'unresolved'.
21/21 recipes now resolve. It found upgrades abra missed entirely in five:
mumble (abra said 'no new versions'; four patches behind), plausible's
clickhouse, lasuite-drive's collabora, gitea's mariadb, immich's postgres.
plausible's carried four CVEs, three high.
Also fixes a real over-count found while validating that: a fix inside the
numeric window is not a fix on the branch you land on. ClickHouse patched
CVE-2023-48704 in 23.9.6.20 AND 23.10.5.20 — landing on 23.10.4.25 crosses the
23.9 fix but sits below its own line's, so it does NOT have it. A fix named on
the target's own line and above the target is now proof of absence.
70 tests (64 offline + 6 live). keycloak's live expectation moves 7 -> 12 and
mailu's 0 -> 2: both are the release-note source finding real fixes that were
never filed as advisories.
689 lines
38 KiB
Python
Executable File
689 lines
38 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Tests for advisory-scan.py.
|
|
|
|
Two tiers:
|
|
|
|
OFFLINE (default) — pure logic, fixtures injected in place of the network. Fast, deterministic,
|
|
no token, no rate limit. These encode every classification rule and every guarantee the CVE count
|
|
makes, including the specific production defects that motivated them.
|
|
|
|
LIVE (--live) — re-derives the CVE counts published in the week-2026-08-07 report against the real
|
|
advisory APIs. Slow, needs network + ideally a GitHub token. Run before changing classification.
|
|
|
|
Usage:
|
|
python3 test-advisory-scan.py # offline only
|
|
python3 test-advisory-scan.py --live # offline + historic report regressions
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
import unittest.mock
|
|
|
|
HERE = pathlib.Path(__file__).resolve().parent
|
|
_spec = importlib.util.spec_from_file_location("advisory_scan", HERE / "advisory-scan.py")
|
|
A = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(A)
|
|
|
|
|
|
# ── fixture helpers ───────────────────────────────────────────────────────────────────────────────
|
|
|
|
def adv(cve, patched=None, published=None, severity="high", ghsa=None):
|
|
"""One GitHub advisory row as github_advisories() would emit it."""
|
|
return {"cve": cve, "ghsa": ghsa or f"GHSA-fake-{cve[-4:]}", "severity": severity,
|
|
"summary": f"summary for {cve}", "vulnerable_range": None, "patched": patched,
|
|
"url": None, "published_at": published}
|
|
|
|
|
|
def gh(owner_repo, advisories, status="ok"):
|
|
return {"source": f"github-advisories:{owner_repo}", "status": status, "advisories": advisories}
|
|
|
|
|
|
def vendor(url, cves=(), status="ok"):
|
|
return {"source": url, "status": status, "cves": list(cves),
|
|
"context": {c: f"...{c}..." for c in cves}}
|
|
|
|
|
|
def run_scan(gh_entries=(), vendor_entries=(), tag_dates=None, *, v_from=None, v_to=None,
|
|
images=None, recipe="fixture", urls=None, releases=None):
|
|
"""scan() with every network call replaced by fixtures.
|
|
|
|
`releases` maps CVE id -> tags whose release notes name it (the third, release-note method)."""
|
|
tag_dates = tag_dates or {}
|
|
releases = releases or {}
|
|
urls = urls if urls is not None else ["https://github.com/app/app"]
|
|
with unittest.mock.patch.object(A, "registry_urls", lambda r, d: (list(urls), "/fake/reg.md")), \
|
|
unittest.mock.patch.object(A, "github_advisories", lambda u: list(gh_entries)), \
|
|
unittest.mock.patch.object(A, "vendor_pages", lambda u: list(vendor_entries)), \
|
|
unittest.mock.patch.object(A, "osv", lambda r, v: None), \
|
|
unittest.mock.patch.object(A, "_tag_date", lambda o, r, v: tag_dates.get(v)), \
|
|
unittest.mock.patch.object(A, "release_fix_versions", lambda src, cve: list(releases.get(cve, []))):
|
|
return A.scan(recipe, v_from, v_to, "/fake", images)
|
|
|
|
|
|
def parse_image_args(argv):
|
|
"""Drive main()'s --image parsing exactly as the CLI does, returning the tuples scan() receives."""
|
|
captured = {}
|
|
|
|
def fake_scan(recipe, vf, vt, reg, images):
|
|
captured["images"] = images
|
|
return {"recipe": recipe, "from": vf, "to": vt, "registry": reg, "registry_urls": 0,
|
|
"sources": [], "cves": {}, "fixed_by_this_upgrade": [], "unclassified": [],
|
|
"sources_failed": [], "sources_benign": [], "count_known": True,
|
|
"cve_count_fixed": 0, "windows": {}, "classified_by": {}}
|
|
|
|
err = io.StringIO()
|
|
with unittest.mock.patch.object(A, "scan", fake_scan), \
|
|
unittest.mock.patch.object(sys, "argv", ["advisory-scan.py", *argv]), \
|
|
unittest.mock.patch.object(sys, "stdout", io.StringIO()), \
|
|
unittest.mock.patch.object(sys, "stderr", err):
|
|
A.main()
|
|
return captured["images"], err.getvalue()
|
|
|
|
|
|
# ── A. version ordering ───────────────────────────────────────────────────────────────────────────
|
|
|
|
class TestVersionKey(unittest.TestCase):
|
|
def test_strips_prefix_and_suffix(self):
|
|
self.assertEqual(A._vkey("v1.27.1"), (1, 27, 1))
|
|
self.assertEqual(A._vkey("1.27.1-rootless"), (1, 27, 1))
|
|
self.assertEqual(A._vkey("2024.06.55"), (2024, 6, 55))
|
|
|
|
def test_empty_and_none(self):
|
|
self.assertEqual(A._vkey(None), ())
|
|
self.assertEqual(A._vkey(""), ())
|
|
|
|
def test_dotted_minor_is_numeric_not_lexical(self):
|
|
# The bug this guards: "8.10" must be NEWER than "8.2.3". String compare says otherwise.
|
|
self.assertGreater(A._vkey("8.10"), A._vkey("8.2.3"))
|
|
self.assertGreater(A._vkey("1.27.10"), A._vkey("1.27.9"))
|
|
|
|
def test_shorter_prefix_orders_below_its_own_patch(self):
|
|
# 7.4 < 7.4.1, so a CVE patched in 7.4.1 IS fixed by moving off a bare 7.4 pin.
|
|
self.assertLess(A._vkey("7.4"), A._vkey("7.4.1"))
|
|
|
|
|
|
class TestWindowMembership(unittest.TestCase):
|
|
"""(from, to] membership — exclusive lower, inclusive upper, compared zero-padded."""
|
|
|
|
def _in(self, f, t, c):
|
|
return A._within(A._vkey(f), A._vkey(t), A._vkey(c))
|
|
|
|
def test_bounds(self):
|
|
self.assertTrue(self._in("1.27.0", "1.27.1", "1.27.1")) # upper inclusive
|
|
self.assertFalse(self._in("1.27.0", "1.27.1", "1.27.0")) # lower exclusive
|
|
self.assertFalse(self._in("1.27.0", "1.27.1", "1.26.9"))
|
|
self.assertFalse(self._in("1.27.0", "1.27.1", "1.28.0"))
|
|
|
|
def test_bare_major_upper_bound_includes_its_dot_zero(self):
|
|
# Regression: plain tuple order makes (18,) < (18,0), so a fix in 18.0 fell OUTSIDE a
|
|
# window ending at 18. Bare major tags are the norm for sidecars (postgres:18, redis:8).
|
|
self.assertTrue(self._in("17", "18", "18.0"))
|
|
self.assertTrue(self._in("7", "8", "8.0"))
|
|
self.assertTrue(self._in("7.4", "8.10", "8.0.4"))
|
|
|
|
def test_bare_major_upper_bound_excludes_later_patches(self):
|
|
# Conservative on the other side: nothing proves which 18.x a floating tag resolved to.
|
|
self.assertFalse(self._in("17", "18", "18.5"))
|
|
|
|
def test_bare_version_is_read_literally_as_dot_zero(self):
|
|
# from="8" means 8.0, so a fix in 8.0.4 is inside a window that ends at 9.
|
|
self.assertTrue(self._in("8", "9", "8.0.4"))
|
|
self.assertFalse(self._in("8", "9", "8.0")) # == the stated lower bound
|
|
|
|
def test_prefix_lower_bound_still_counts_its_patches(self):
|
|
self.assertTrue(self._in("7.4", "8.10", "7.4.1"))
|
|
self.assertTrue(self._in("7.4", "8.10", "7.4.6"))
|
|
|
|
def test_the_false_133_cve_stays_out(self):
|
|
self.assertFalse(self._in("7.4", "8.10", "6.0.11"))
|
|
|
|
|
|
# ── B. registry URL extraction ────────────────────────────────────────────────────────────────────
|
|
|
|
class TestRegistryUrls(unittest.TestCase):
|
|
def _write(self, tmp, text):
|
|
p = pathlib.Path(tmp) / "r.md"
|
|
p.write_text(text)
|
|
return A.registry_urls("r", tmp)
|
|
|
|
def test_strips_trailing_markdown_punctuation(self):
|
|
# Production defect: a captured backtick 404'd the fetch and rendered n8n/immich as '?'.
|
|
import tempfile
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
urls, _ = self._write(tmp, "see `https://docs.n8n.io/release-notes/` and "
|
|
"**https://example.com/sec.html**, plus https://a.test/x.")
|
|
self.assertIn("https://docs.n8n.io/release-notes/", urls)
|
|
self.assertIn("https://example.com/sec.html", urls)
|
|
self.assertIn("https://a.test/x", urls)
|
|
self.assertFalse([u for u in urls if u.endswith(("`", "*", ".", ","))])
|
|
|
|
def test_dedupes_and_reports_missing_registry(self):
|
|
import tempfile
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
urls, path = self._write(tmp, "https://a.test/x https://a.test/x")
|
|
self.assertEqual(urls.count("https://a.test/x"), 1)
|
|
self.assertTrue(path.endswith("r.md"))
|
|
urls, path = A.registry_urls("does-not-exist", "/nonexistent-dir")
|
|
self.assertEqual((urls, path), ([], None))
|
|
|
|
|
|
# ── C. --image argument parsing ───────────────────────────────────────────────────────────────────
|
|
|
|
class TestImageArgParsing(unittest.TestCase):
|
|
def test_single_and_repeated(self):
|
|
imgs, _ = parse_image_args(["r", "--image", "redis=7.4:8.10"])
|
|
self.assertEqual(imgs, [("redis", "7.4", "8.10")])
|
|
imgs, _ = parse_image_args(["r", "--image", "redis=7.4:8.10", "--image", "postgres=17:18"])
|
|
self.assertEqual(imgs, [("redis", "7.4", "8.10"), ("postgres", "17", "18")])
|
|
|
|
def test_malformed_is_skipped_with_a_warning_not_a_crash(self):
|
|
# It is an ADDITIVE pre-step: one typo must not abort the upgrade's scan step.
|
|
for bad in ("redis=7.4", "redis", "=7.4:8.10", "redis=:8.10", "redis=7.4:"):
|
|
imgs, err = parse_image_args(["r", "--image", bad])
|
|
self.assertEqual(imgs, [], f"{bad!r} should be rejected")
|
|
self.assertIn("malformed", err)
|
|
|
|
def test_good_and_bad_mixed_keeps_the_good(self):
|
|
imgs, err = parse_image_args(["r", "--image", "redis=7.4:8.10", "--image", "nope"])
|
|
self.assertEqual(imgs, [("redis", "7.4", "8.10")])
|
|
self.assertIn("malformed", err)
|
|
|
|
|
|
# ── D. classification ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
class TestClassificationBoundaries(unittest.TestCase):
|
|
def _one(self, patched, v_from="1.27.0", v_to="1.27.1"):
|
|
rep = run_scan([gh("app/app", [adv("CVE-2026-0001", patched=patched)])],
|
|
v_from=v_from, v_to=v_to)
|
|
return rep
|
|
|
|
def test_patched_at_upper_bound_counts(self):
|
|
self.assertEqual(self._one("1.27.1")["fixed_by_this_upgrade"], ["CVE-2026-0001"])
|
|
|
|
def test_patched_at_lower_bound_does_not_count(self):
|
|
# Already fixed in the version we were ON — this upgrade did not fix it.
|
|
rep = self._one("1.27.0")
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], [])
|
|
self.assertEqual(rep["cve_count_fixed"], 0)
|
|
|
|
def test_patched_below_and_above_window_do_not_count(self):
|
|
self.assertEqual(self._one("1.26.0")["fixed_by_this_upgrade"], [])
|
|
self.assertEqual(self._one("1.28.0")["fixed_by_this_upgrade"], [])
|
|
|
|
def test_any_of_several_patched_lines_counts(self):
|
|
# n8n regression: one advisory patches several release lines; reading only the first
|
|
# dropped the line the deployment was on (CVE-2026-42231/42232 misclassified).
|
|
rep = self._one("1.123.32; 2.17.4; 2.18.1", v_from="2.17.0", v_to="2.17.4")
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2026-0001"])
|
|
|
|
def test_no_patched_data_is_not_counted(self):
|
|
self.assertEqual(self._one(None)["fixed_by_this_upgrade"], [])
|
|
|
|
|
|
class TestPerImageWindows(unittest.TestCase):
|
|
"""The false-133 family of defects: an image must only ever be judged by its OWN versions."""
|
|
|
|
APP = gh("discourse/discourse", [adv("CVE-APP-0001", patched="3.5.4", published="2026-03-01T00:00:00Z")])
|
|
REDIS = gh("redis/redis", [
|
|
adv("CVE-2021-21309", patched="6.0.11", published="2021-02-01T00:00:00Z"),
|
|
adv("CVE-2025-49844", patched="7.4.6; 8.0.4; 8.2.2", published="2025-10-01T00:00:00Z",
|
|
severity="critical"),
|
|
])
|
|
URLS = ["https://github.com/discourse/discourse", "https://github.com/redis/redis"]
|
|
|
|
def test_sidecar_cve_is_not_judged_by_the_app_window(self):
|
|
# redis 6.0.11 sits numerically inside discourse 3.5.3 -> 2026.7.1. It must NOT count.
|
|
rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="2026.7.1",
|
|
tag_dates={"3.5.3": "2025-12-30T00:00:00Z", "2026.7.1": "2026-07-31T00:00:00Z"},
|
|
urls=self.URLS)
|
|
self.assertNotIn("CVE-2021-21309", rep["fixed_by_this_upgrade"])
|
|
self.assertIn("CVE-2021-21309", rep["unclassified"])
|
|
|
|
def test_unwindowed_image_is_unclassified_never_counted(self):
|
|
rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="3.5.4", urls=self.URLS)
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-APP-0001"])
|
|
for cve in ("CVE-2021-21309", "CVE-2025-49844"):
|
|
self.assertIn(cve, rep["unclassified"])
|
|
|
|
def test_sidecar_window_counts_only_what_that_bump_fixed(self):
|
|
rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="3.5.4",
|
|
images=[("redis", "7.4", "8.10")], urls=self.URLS)
|
|
self.assertIn("CVE-2025-49844", rep["fixed_by_this_upgrade"]) # patched 7.4.6, in window
|
|
self.assertNotIn("CVE-2021-21309", rep["fixed_by_this_upgrade"]) # patched 6.0.11, below it
|
|
self.assertEqual(rep["cve_count_fixed"], 2) # app 1 + redis 1
|
|
|
|
def test_count_is_the_union_across_images(self):
|
|
rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="3.5.4",
|
|
images=[("redis", "7.4", "8.10")], urls=self.URLS)
|
|
self.assertEqual(sorted(rep["fixed_by_this_upgrade"]), ["CVE-2025-49844", "CVE-APP-0001"])
|
|
|
|
def test_each_image_classified_independently(self):
|
|
# App crosses a scheme change (date method); redis does not (version method). Both resolve.
|
|
rep = run_scan([self.APP, self.REDIS], v_from="3.5.3", v_to="2026.7.1",
|
|
images=[("redis", "7.4", "8.10")], urls=self.URLS,
|
|
tag_dates={"3.5.3": "2025-12-30T00:00:00Z", "2026.7.1": "2026-07-31T00:00:00Z"})
|
|
methods = rep["classified_by"]
|
|
self.assertIn("publish date", methods["github-advisories:discourse/discourse"])
|
|
self.assertEqual(methods["github-advisories:redis/redis"], "patched version ranges")
|
|
self.assertTrue(rep["count_known"])
|
|
|
|
def test_image_name_matches_as_substring(self):
|
|
rep = run_scan([self.APP, gh("discourse/discourse-postgres", [adv("CVE-PG-1", patched="18.0")])],
|
|
v_from="3.5.3", v_to="3.5.4", images=[("postgres", "17", "18")],
|
|
urls=["https://github.com/discourse/discourse",
|
|
"https://github.com/discourse/discourse-postgres"])
|
|
self.assertIn("github-advisories:discourse/discourse-postgres", rep["windows"])
|
|
self.assertIn("CVE-PG-1", rep["fixed_by_this_upgrade"])
|
|
|
|
def test_primary_cannot_be_stolen_by_a_loose_image_name(self):
|
|
rep = run_scan([self.APP, gh("discourse/discourse-postgres", [adv("CVE-PG-1", patched="18.0")])],
|
|
v_from="3.5.3", v_to="3.5.4", images=[("discourse", "1", "2")],
|
|
urls=["https://github.com/discourse/discourse",
|
|
"https://github.com/discourse/discourse-postgres"])
|
|
self.assertEqual(rep["windows"]["github-advisories:discourse/discourse"],
|
|
{"from": "3.5.3", "to": "3.5.4"})
|
|
|
|
def test_unmatched_image_name_is_silently_ignored(self):
|
|
# Documents CURRENT behaviour: a typo'd name costs coverage without warning.
|
|
rep = run_scan([self.APP], v_from="3.5.3", v_to="3.5.4",
|
|
images=[("nosuchimage", "1", "2")], urls=self.URLS[:1])
|
|
self.assertEqual(list(rep["windows"]), ["github-advisories:discourse/discourse"])
|
|
self.assertTrue(rep["count_known"])
|
|
|
|
|
|
class TestSchemeChangeDateFallback(unittest.TestCase):
|
|
DATES = {"3.5.3": "2025-12-30T00:00:00Z", "2026.7.1": "2026-07-31T00:00:00Z"}
|
|
|
|
def _rep(self, advisories, dates=None):
|
|
return run_scan([gh("discourse/discourse", advisories)], v_from="3.5.3", v_to="2026.7.1",
|
|
tag_dates=self.DATES if dates is None else dates)
|
|
|
|
def test_counts_advisories_published_inside_the_date_window(self):
|
|
rep = self._rep([adv("CVE-IN-1", published="2026-03-01T00:00:00Z"),
|
|
adv("CVE-OUT-1", published="2025-06-01T00:00:00Z"),
|
|
adv("CVE-OUT-2", published="2026-09-01T00:00:00Z")])
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-IN-1"])
|
|
|
|
def test_date_boundaries_match_the_version_rule(self):
|
|
# Exclusive lower, inclusive upper — same as 4a, so the two methods agree at the edges.
|
|
rep = self._rep([adv("CVE-LOWER", published=self.DATES["3.5.3"]),
|
|
adv("CVE-UPPER", published=self.DATES["2026.7.1"])])
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-UPPER"])
|
|
|
|
def test_advisory_without_a_publish_date_is_not_counted(self):
|
|
self.assertEqual(self._rep([adv("CVE-NODATE", published=None)])["fixed_by_this_upgrade"], [])
|
|
|
|
def test_scheme_change_is_detected_not_version_compared(self):
|
|
rep = self._rep([adv("CVE-IN-1", patched="2026.1.0", published="2026-03-01T00:00:00Z")])
|
|
self.assertIn("publish date", rep["classified_by"]["github-advisories:discourse/discourse"])
|
|
self.assertIn("github-advisories:discourse/discourse", rep["date_window"])
|
|
|
|
def test_small_major_bump_still_uses_version_ranges(self):
|
|
rep = run_scan([gh("app/app", [adv("CVE-X", patched="3.0.0")])], v_from="2.9.0", v_to="3.0.0")
|
|
self.assertEqual(rep["classified_by"]["github-advisories:app/app"], "patched version ranges")
|
|
|
|
|
|
# ── E. count guarantees ───────────────────────────────────────────────────────────────────────────
|
|
|
|
class TestCountGuarantees(unittest.TestCase):
|
|
def test_unresolvable_window_yields_unknown_never_zero(self):
|
|
# Scheme change AND tag dates unresolvable -> must refuse to emit a number.
|
|
rep = run_scan([gh("app/app", [adv("CVE-1", published="2026-01-01T00:00:00Z")])],
|
|
v_from="3.5.3", v_to="2026.7.1", tag_dates={})
|
|
self.assertIs(rep["cve_count_fixed"], None)
|
|
self.assertFalse(rep["count_known"])
|
|
md = A.markdown(rep)
|
|
self.assertIn("UNKNOWN", md)
|
|
self.assertIn("NOT zero", md)
|
|
|
|
def test_one_unresolvable_image_makes_the_whole_count_unknown(self):
|
|
# A partial number would understate a security figure, so it is suppressed entirely.
|
|
rep = run_scan([gh("app/app", [adv("CVE-APP", patched="1.1")]),
|
|
gh("redis/redis", [adv("CVE-REDIS", patched="8.0")])],
|
|
v_from="1.0", v_to="1.1", images=[("redis", "7.4", "9999.1")],
|
|
tag_dates={}, urls=["https://github.com/app/app", "https://github.com/redis/redis"])
|
|
self.assertIs(rep["cve_count_fixed"], None)
|
|
self.assertFalse(rep["count_known"])
|
|
|
|
def test_genuine_zero_is_reported_as_zero(self):
|
|
rep = run_scan([gh("app/app", [adv("CVE-1", patched="9.9.9")])], v_from="1.0", v_to="1.1")
|
|
self.assertEqual(rep["cve_count_fixed"], 0)
|
|
self.assertTrue(rep["count_known"])
|
|
self.assertIn("0 identified", A.markdown(rep))
|
|
|
|
def test_404_advisory_feed_is_benign_not_a_failure(self):
|
|
rep = run_scan([gh("app/app", [adv("CVE-1", patched="1.1")]),
|
|
gh("side/car", [], status="no-advisories-published")],
|
|
v_from="1.0", v_to="1.1")
|
|
self.assertEqual(rep["sources_failed"], [])
|
|
self.assertIn("github-advisories:side/car", rep["sources_benign"])
|
|
self.assertEqual(rep["cve_count_fixed"], 1)
|
|
|
|
def test_template_url_is_benign_not_a_failure(self):
|
|
rep = run_scan([gh("app/app", [adv("CVE-1", patched="1.1")])],
|
|
[vendor("https://x.test/changelog/v<VERSION>/", status="skipped: template URL")],
|
|
v_from="1.0", v_to="1.1")
|
|
self.assertEqual(rep["sources_failed"], [])
|
|
|
|
def test_real_source_failure_is_surfaced(self):
|
|
rep = run_scan([gh("app/app", [adv("CVE-1", patched="1.1")], status="error: HTTP 500")],
|
|
v_from="1.0", v_to="1.1")
|
|
self.assertIn("github-advisories:app/app", rep["sources_failed"])
|
|
self.assertIn("FAILED", A.markdown(rep))
|
|
|
|
def test_no_window_given_classifies_nothing(self):
|
|
rep = run_scan([gh("app/app", [adv("CVE-1", patched="1.1")])])
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], [])
|
|
self.assertIn("CVE-1", rep["unclassified"])
|
|
|
|
|
|
class TestVendorOnlyCves(unittest.TestCase):
|
|
"""The gitea case: CVEs named ONLY on a vendor page, absent from the GitHub advisory feed."""
|
|
|
|
def test_vendor_only_cve_is_recorded_and_surfaced(self):
|
|
rep = run_scan([gh("go-gitea/gitea", [])],
|
|
[vendor("https://blog.gitea.com/release-1.27.1/", ["CVE-2026-60004"])],
|
|
v_from="1.27.0", v_to="1.27.1")
|
|
self.assertIn("CVE-2026-60004", rep["cves"])
|
|
self.assertIn("CVE-2026-60004", rep["unclassified"])
|
|
|
|
def test_vendor_only_cve_is_NOT_counted_but_IS_sent_for_judgement(self):
|
|
# It carries no version data, so no arithmetic can place it — the deterministic count must
|
|
# not include it. It must not be silently dropped either: pass 2 gets it with its evidence.
|
|
rep = run_scan([gh("go-gitea/gitea", [])],
|
|
[vendor("https://blog.gitea.com/release-1.27.1/", ["CVE-2026-60004"])],
|
|
v_from="1.27.0", v_to="1.27.1")
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], [])
|
|
self.assertEqual(rep["cve_count_fixed"], 0)
|
|
self.assertIn("CVE-2026-60004", A.needs_judgement(rep))
|
|
|
|
def test_cve_in_both_vendor_and_advisory_feed_is_counted_once(self):
|
|
rep = run_scan([gh("go-gitea/gitea", [adv("CVE-2026-60004", patched="1.27.1")])],
|
|
[vendor("https://blog.gitea.com/x/", ["CVE-2026-60004"])],
|
|
v_from="1.27.0", v_to="1.27.1")
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2026-60004"])
|
|
self.assertEqual(rep["cve_count_fixed"], 1)
|
|
self.assertEqual(len(rep["cves"]["CVE-2026-60004"]["sources"]), 2)
|
|
|
|
|
|
class TestIndeterminateBucket(unittest.TestCase):
|
|
"""An advisory with no knowable fix version is neither counted nor dismissed."""
|
|
|
|
def test_tbd_patched_is_indeterminate_not_excluded(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], [])
|
|
self.assertIn("CVE-TBD", rep["indeterminate"])
|
|
self.assertEqual(rep["cve_count_indeterminate"], 1)
|
|
|
|
def test_placeholder_patched_is_indeterminate(self):
|
|
# "7.4.X" could be 7.4.1 — inside the window. Extracting a bare 7.4 and excluding it was
|
|
# how CVE-2024-46981 (high) went missing.
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-X", patched="6.2.X, 7.2.X, 7.4.X")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
self.assertIn("CVE-X", rep["indeterminate"])
|
|
|
|
def test_real_versions_outside_the_window_are_decided_not_indeterminate(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-OLD", patched="6.0.11")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
self.assertEqual(rep["indeterminate"], [])
|
|
self.assertEqual(rep["cve_count_fixed"], 0)
|
|
|
|
def test_indeterminate_is_surfaced_in_the_markdown_and_not_read_as_zero(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD", severity="critical")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
md = A.markdown(rep)
|
|
self.assertIn("could NOT be judged", md)
|
|
self.assertIn("must NOT be read as unaffected", md)
|
|
|
|
|
|
class TestReleaseNoteResolution(unittest.TestCase):
|
|
"""Third method: a release whose notes NAME the CVE supplies the fix version the advisory lacks."""
|
|
|
|
def test_release_naming_the_cve_inside_the_window_counts_it(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"],
|
|
releases={"CVE-TBD": ["6.2.19", "7.2.10", "7.4.5", "8.0.3"]})
|
|
self.assertIn("CVE-TBD", rep["fixed_by_this_upgrade"])
|
|
self.assertEqual(rep["indeterminate"], [])
|
|
self.assertEqual(rep["resolved_by_release_notes"]["CVE-TBD"], ["7.4.5", "8.0.3"])
|
|
|
|
def test_naming_releases_all_below_the_window_means_ALREADY_fixed(self):
|
|
# Every known fix predates the version we were already on, so this upgrade did not deliver
|
|
# it. That is a DECISION, not an unknown — mailu's redis 8.8.0 → 8.10.0 crosses 12 such
|
|
# advisories, and calling them "could not judge" overstates the uncertainty.
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"],
|
|
releases={"CVE-TBD": ["6.2.19"]})
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], [])
|
|
self.assertEqual(rep["indeterminate"], [])
|
|
self.assertIn("CVE-TBD", rep["already_fixed_before_upgrade"])
|
|
self.assertIn("outside-window", rep["cves"]["CVE-TBD"]["classification"])
|
|
|
|
def test_naming_releases_only_ABOVE_the_window_stays_indeterminate(self):
|
|
# The fix landed after our target, so we are still exposed. Deliberately NOT decided as a
|
|
# tidy "not fixed": it is an open vulnerability and must stay visible to the operator.
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"],
|
|
releases={"CVE-TBD": ["9.0.0"]})
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], [])
|
|
self.assertIn("CVE-TBD", rep["indeterminate"])
|
|
|
|
def test_vendor_page_cve_on_the_same_repo_uses_release_notes(self):
|
|
# mailu announces its Roundcube CVEs only on github.com/Mailu/Mailu/releases. Requiring an
|
|
# advisory feed sent a deterministic case to pass 2; it is now decided in pass 1.
|
|
rep = run_scan([gh("Mailu/Mailu", [])],
|
|
[vendor("https://github.com/Mailu/Mailu/releases", ["CVE-2026-54432"])],
|
|
v_from="2024.06.55", v_to="2024.06.57",
|
|
urls=["https://github.com/Mailu/Mailu"],
|
|
releases={"CVE-2026-54432": ["2024.06.56"]})
|
|
self.assertIn("CVE-2026-54432", rep["fixed_by_this_upgrade"])
|
|
self.assertEqual(rep["cve_count_fixed"], 1)
|
|
|
|
def test_release_evidence_is_recorded_for_audit(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"],
|
|
releases={"CVE-TBD": ["7.4.5"]})
|
|
e = rep["cves"]["CVE-TBD"]
|
|
self.assertEqual(e["fix_versions_from_release_notes"], ["7.4.5"])
|
|
self.assertIn("named in release notes", e["classification"])
|
|
|
|
def test_it_does_not_override_a_version_range_decision(self):
|
|
# A CVE already counted by patched ranges is untouched; the method only rescues undecided.
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-OK", patched="7.4.1")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"],
|
|
releases={"CVE-OK": ["7.4.1"]})
|
|
self.assertNotIn("CVE-OK", rep.get("resolved_by_release_notes") or {})
|
|
|
|
|
|
class TestReleaseLineSemantics(unittest.TestCase):
|
|
"""A fix inside the numeric window is not a fix on the branch you actually land on."""
|
|
|
|
def test_fix_later_on_the_targets_own_line_is_not_counted(self):
|
|
# ClickHouse fixed CVE-2023-48704 in 23.9.6.20 AND 23.10.5.20. Landing on 23.10.4.25 crosses
|
|
# the 23.9 fix numerically but is BELOW its own line's fix, so it does not have it.
|
|
rep = run_scan([gh("ClickHouse/ClickHouse",
|
|
[adv("CVE-2023-48704", patched="v23.10.5.20; v23.9.6.20; v23.8.8.20")])],
|
|
v_from="23.4.2.11", v_to="23.10.4.25",
|
|
urls=["https://github.com/ClickHouse/ClickHouse"])
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], [])
|
|
|
|
def test_fix_earlier_on_the_targets_own_line_is_counted(self):
|
|
rep = run_scan([gh("ClickHouse/ClickHouse",
|
|
[adv("CVE-2023-47118", patched="v23.10.2.13; v23.8.6.16")])],
|
|
v_from="23.4.2.11", v_to="23.10.4.25",
|
|
urls=["https://github.com/ClickHouse/ClickHouse"])
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2023-47118"])
|
|
|
|
def test_fix_exactly_at_the_target_is_counted(self):
|
|
rep = run_scan([gh("ClickHouse/ClickHouse",
|
|
[adv("CVE-2023-48298", patched="v23.10.4.25; v23.9.5.29")])],
|
|
v_from="23.4.2.11", v_to="23.10.4.25",
|
|
urls=["https://github.com/ClickHouse/ClickHouse"])
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2023-48298"])
|
|
|
|
def test_no_fix_on_the_target_line_falls_back_to_the_window(self):
|
|
# redis fixes 7.4.6/8.0.4/8.2.2 with no 8.10.x entry; landing on 8.10 still has them,
|
|
# because nothing on the 8.10 line is named as a LATER fix.
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-2025-49844", patched="7.4.6; 8.0.4; 8.2.2")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2025-49844"])
|
|
|
|
|
|
class TestAdjudicationEvidenceAssembly(unittest.TestCase):
|
|
"""Pass 2's JUDGEMENT is a model's and not testable; what IS testable is what it gets shown."""
|
|
|
|
def test_selects_indeterminate_and_vendor_only_cases(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])],
|
|
[vendor("https://blog.test/sec", ["CVE-VENDOR"])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
todo = A.needs_judgement(rep)
|
|
self.assertIn("CVE-TBD", todo)
|
|
self.assertIn("CVE-VENDOR", todo)
|
|
|
|
def test_does_not_re_ask_about_cases_pass_1_settled(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-OK", patched="7.4.1")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
self.assertNotIn("CVE-OK", A.needs_judgement(rep))
|
|
|
|
def test_evidence_bundle_carries_the_window_and_published_fields(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-TBD", patched="TBD")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
with unittest.mock.patch.object(A, "advisory_text", lambda g, s=None: {"status": "skipped"}), \
|
|
unittest.mock.patch.object(A, "release_fix_versions", lambda s, c: []):
|
|
ev = A.evidence_bundle(rep, "CVE-TBD")
|
|
self.assertEqual(ev["window"], {"from": "7.4", "to": "8.10"})
|
|
self.assertEqual(ev["patched_as_published"], "TBD")
|
|
self.assertIn("no fix version", ev["why_undecided"])
|
|
|
|
def test_pass_1_decisions_are_included_for_review(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-OK", patched="7.4.1"),
|
|
adv("CVE-OLD", patched="6.0.11"),
|
|
adv("CVE-TBD", patched="TBD")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
with unittest.mock.patch.object(A, "advisory_text", lambda g, s=None: {"status": "skipped"}), \
|
|
unittest.mock.patch.object(A, "release_fix_versions", lambda s, c: []):
|
|
block = A.adjudication_block(rep)
|
|
self.assertIn("Pass 1 decisions", block)
|
|
self.assertIn("CVE-OK", block) # counted
|
|
self.assertIn("CVE-OLD", block) # excluded as outside-window
|
|
self.assertIn("CVE-TBD", block) # needs judgement
|
|
|
|
def test_truncation_is_announced_never_silent(self):
|
|
advs = [adv(f"CVE-2026-{1000+i}", patched="TBD") for i in range(30)]
|
|
rep = run_scan([gh("redis/redis", advs)], v_from="7.4", v_to="8.10",
|
|
urls=["https://github.com/redis/redis"])
|
|
with unittest.mock.patch.object(A, "advisory_text", lambda g, s=None: {"status": "skipped"}), \
|
|
unittest.mock.patch.object(A, "release_fix_versions", lambda s, c: []), \
|
|
unittest.mock.patch.object(A, "MAX_ADJUDICATE", 5):
|
|
block = A.adjudication_block(rep)
|
|
self.assertIn("not shown", block)
|
|
self.assertIn("do not", block.lower())
|
|
|
|
|
|
class TestMarkdownOutput(unittest.TestCase):
|
|
def test_lists_every_window_with_its_method(self):
|
|
rep = run_scan([gh("discourse/discourse", [adv("CVE-A", patched="3.5.4")]),
|
|
gh("redis/redis", [adv("CVE-B", patched="8.0")])],
|
|
v_from="3.5.3", v_to="3.5.4", images=[("redis", "7.4", "8.10")],
|
|
urls=["https://github.com/discourse/discourse", "https://github.com/redis/redis"])
|
|
md = A.markdown(rep)
|
|
self.assertIn("discourse/discourse: 3.5.3 → 3.5.4", md)
|
|
self.assertIn("redis/redis: 7.4 → 8.10", md)
|
|
self.assertIn("**CVEs fixed by this upgrade: 2**", md)
|
|
|
|
def test_severity_and_fixed_in_are_rendered(self):
|
|
rep = run_scan([gh("redis/redis", [adv("CVE-2025-49844", patched="7.4.6; 8.2.2",
|
|
severity="critical")])],
|
|
v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"])
|
|
md = A.markdown(rep)
|
|
self.assertIn("critical", md)
|
|
self.assertIn("7.4.6", md)
|
|
|
|
|
|
# ── F. live regressions against published historic reports ────────────────────────────────────────
|
|
|
|
class TestHistoricReportNumbers(unittest.TestCase):
|
|
"""Re-derive counts published in week-2026-08-07. Network + GitHub token; opt in with --live."""
|
|
|
|
REGISTRY = str(HERE / "upstream")
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
if not os.environ.get("ADVISORY_SCAN_LIVE"):
|
|
raise unittest.SkipTest("live tests: re-run with --live")
|
|
|
|
def _count(self, recipe, v_from, v_to, images=None):
|
|
rep = A.scan(recipe, v_from, v_to, self.REGISTRY, images)
|
|
self.assertEqual(rep["sources_failed"], [], f"{recipe}: source failures make the count unsafe")
|
|
self.assertTrue(rep["count_known"], f"{recipe}: count came back UNKNOWN")
|
|
return rep
|
|
|
|
def test_gitea_1_27_0_to_1_27_1_is_2(self):
|
|
rep = self._count("gitea", "1.27.0", "1.27.1")
|
|
self.assertEqual(rep["cve_count_fixed"], 2)
|
|
# Both CVSS-9.8 RCEs — the pair whose omission is why this tool exists.
|
|
self.assertEqual(set(rep["fixed_by_this_upgrade"]), {"CVE-2026-59774", "CVE-2026-60004"})
|
|
|
|
def test_discourse_app_only_is_123(self):
|
|
rep = self._count("discourse", "3.5.3", "2026.7.1")
|
|
self.assertEqual(rep["cve_count_fixed"], 123)
|
|
self.assertIn("publish date", rep["classified_by"]["github-advisories:discourse/discourse"])
|
|
|
|
def test_discourse_with_redis_sidecar_is_140(self):
|
|
# 123 app + 17 redis. Five redis advisories carry a usable patched_versions; the other
|
|
# twelve say "TBD" and are resolved from the release notes that name them.
|
|
rep = self._count("discourse", "3.5.3", "2026.7.1", [("redis", "7.4", "8.10")])
|
|
self.assertEqual(rep["cve_count_fixed"], 140)
|
|
self.assertEqual(len(rep.get("resolved_by_release_notes") or {}), 12)
|
|
self.assertEqual(rep["cve_count_indeterminate"], 0)
|
|
# The five redis advisories that a sidecar-blind scan missed, incl. one critical.
|
|
for cve in ("CVE-2024-31227", "CVE-2024-31228", "CVE-2024-31449",
|
|
"CVE-2025-49844", "CVE-2025-62507"):
|
|
self.assertIn(cve, rep["fixed_by_this_upgrade"], f"{cve} missing from discourse+redis")
|
|
self.assertEqual(rep["cves"]["CVE-2025-49844"]["severity"], "critical")
|
|
|
|
def test_discourse_redis_delta_is_exactly_seventeen(self):
|
|
app = self._count("discourse", "3.5.3", "2026.7.1")
|
|
both = self._count("discourse", "3.5.3", "2026.7.1", [("redis", "7.4", "8.10")])
|
|
delta = set(both["fixed_by_this_upgrade"]) - set(app["fixed_by_this_upgrade"])
|
|
self.assertEqual(len(delta), 17)
|
|
for cve in delta:
|
|
self.assertIn("redis", both["cves"][cve]["sources"][0])
|
|
|
|
def test_mailu_finds_the_roundcube_pair_without_an_agent(self):
|
|
# Published as 2 on 2026-08-07, but only because an agent read the release notes; the scan
|
|
# itself contributed 0. It now reaches 2 deterministically: the CVEs appear only on
|
|
# github.com/Mailu/Mailu/releases, and release 2024.06.56 (inside the window) names them.
|
|
rep = self._count("mailu", "2024.06.55", "2024.06.57", [("redis", "8.8.0", "8.10.0")])
|
|
self.assertEqual(rep["cve_count_fixed"], 2)
|
|
self.assertEqual(set(rep["fixed_by_this_upgrade"]), {"CVE-2026-54432", "CVE-2026-54433"})
|
|
# The redis bump fixes nothing new — every advisory it crosses was fixed at or before 8.6.3.
|
|
self.assertEqual(rep["cve_count_indeterminate"], 0)
|
|
|
|
def test_keycloak_26_7_0_to_26_7_1_is_12(self):
|
|
# Was 7 while only the GHSA feed was consulted. keycloak lists five more CVEs in the 26.7.1
|
|
# release notes' fixed-issues section that it never filed as advisories — the gitea pattern.
|
|
rep = self._count("keycloak", "26.7.0", "26.7.1")
|
|
self.assertEqual(rep["cve_count_fixed"], 12)
|
|
self.assertEqual(len(rep.get("resolved_by_release_notes") or {}), 5)
|
|
|
|
|
|
def _main():
|
|
live = "--live" in sys.argv
|
|
if live:
|
|
sys.argv.remove("--live")
|
|
os.environ["ADVISORY_SCAN_LIVE"] = "1"
|
|
unittest.main(verbosity=2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
_main()
|