From 1db85a7e773d055da206c43a43ab00530dcabf6b Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Tue, 11 Aug 2026 04:56:39 +0000 Subject: [PATCH] resolve-images: abra-independent version resolver; fix release-line over-count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit immich pins two images with BOTH a tag and a digest, which makes abra FATA and abandon the WHOLE recipe. It therefore contributed no version data at all and silently dropped out of every survey — indistinguishable from 'up to date'. The standing answer was prose in three skills telling an agent to check registries by hand. This replaces it with a tool. resolve-images.py reads the compose files and queries registries itself: - Docker Hub, ghcr, and any OCI registry via its own auth challenge (lscr.io and dock.mau.dev advertise different realms; assuming ghcr's shape 401'd). - tag SHAPES (digits -> '#') so -alpine stays on -alpine and 'latest' is never proposed as an upgrade. - reports newest_within_major AND newest_same_shape, and refuses to choose: immich's postgres tag encodes the pg major plus the vectorchord/pgvectors build immich-server expects, so taking the newest breaks the deploy. - integrity check: if the CURRENT pin is absent from the listing, the listing was truncated and any 'newest' is a guess. ghcr caps out past 40k tags, so that falls back to the project's GitHub releases. - per-repo cache + backoff + Docker Hub auth: a fleet sweep re-reads nginx, redis and postgres many times and was getting 429s reported as 'unresolved'. 21/21 recipes now resolve. It found upgrades abra missed entirely in five: mumble (abra said 'no new versions'; four patches behind), plausible's clickhouse, lasuite-drive's collabora, gitea's mariadb, immich's postgres. plausible's carried four CVEs, three high. Also fixes a real over-count found while validating that: a fix inside the numeric window is not a fix on the branch you land on. ClickHouse patched CVE-2023-48704 in 23.9.6.20 AND 23.10.5.20 — landing on 23.10.4.25 crosses the 23.9 fix but sits below its own line's, so it does NOT have it. A fix named on the target's own line and above the target is now proof of absence. 70 tests (64 offline + 6 live). keycloak's live expectation moves 7 -> 12 and mailu's 0 -> 2: both are the release-note source finding real fixes that were never filed as advisories. --- .claude/skills/cve-check/SKILL.md | 20 +- cc-ci-plan/advisory-scan.py | 19 ++ cc-ci-plan/recipe-report.py | 4 +- cc-ci-plan/resolve-images.py | 459 ++++++++++++++++++++++++++++++ cc-ci-plan/test-advisory-scan.py | 54 +++- 5 files changed, 543 insertions(+), 13 deletions(-) create mode 100644 cc-ci-plan/resolve-images.py diff --git a/.claude/skills/cve-check/SKILL.md b/.claude/skills/cve-check/SKILL.md index 8f2d683..1a64867 100644 --- a/.claude/skills/cve-check/SKILL.md +++ b/.claude/skills/cve-check/SKILL.md @@ -46,10 +46,16 @@ This is `/recipe-upgrade` step 1's research, stopping before it implements anyth > `git -C ~/.abra/recipes/ remote set-url origin "https://$GITEA_USERNAME:$GITEA_PASSWORD@git.autonomic.zone/recipe-maintainers/.git"` > 3. **dirty worktree** — usually just the untracked cc-ci overlay; `git stash -u` before, `stash pop` > after. Only a genuinely dirty TRACKED tree is a skip. -> 4. **tag+digest pins abra cannot parse** — abra FATAs and aborts the WHOLE recipe (immich). This is -> **not** "not fetchable": enumerate the compose's `image:` refs yourself and check the upstream -> registry directly for the ones abra could not read, picking the newest tag the app version -> supports rather than the numerically highest. +> 4. **tag+digest pins abra cannot parse** — abra FATAs and aborts the WHOLE recipe (immich). Do not +> hand-check the registry; run the resolver, which is abra-independent and covers every image: +> ``` +> python3 /srv/cc-ci/cc-ci-plan/resolve-images.py --ssh cc-ci --table +> ``` +> It reports, per image, `newest_within_major` (the compatibility-safe pick) and +> `newest_same_shape` (the newest of that tag's form). **Use `newest_within_major` unless you have +> checked the app supports the major jump** — immich's postgres tag encodes the pg major plus the +> vectorchord/pgvectors versions immich-server is built against, so taking the newest would break +> the deploy. `all_resolved: false` means an image could NOT be resolved — that is a `?`, never a 0. **Reconcile the mirror from true upstream FIRST — ALWAYS, no exceptions.** This is the same reconcile `/upgrade-all` does. Do not skip it in the name of keeping the sweep read-only: skipping it makes you @@ -66,6 +72,12 @@ nothing; it also auto-closes mirror PRs whose changes upstream has already merge > API (`/api/v1/repos/coop-cloud/` → `default_branch`) before reading any file, and never > `git reset --hard origin/main` on a checkout that tracks `master`. +**Cross-check abra with the resolver.** abra is the primary source, but it silently contributes +nothing for images it cannot parse, and it reported "no new versions" for images that did have them +(mumble v1.6.870-0 → -4). Run `resolve-images.py` for every recipe and take the UNION of the two: on +the first real sweep the resolver found upgrades abra missed entirely in five recipes, one of which +(plausible's clickhouse) carried four CVEs. + Then read versions: ``` set -a; . /srv/cc-ci/.testenv; set +a diff --git a/cc-ci-plan/advisory-scan.py b/cc-ci-plan/advisory-scan.py index 622f741..93751e5 100755 --- a/cc-ci-plan/advisory-scan.py +++ b/cc-ci-plan/advisory-scan.py @@ -164,6 +164,21 @@ def _vkey(v: str | None) -> tuple: return tuple(out) +def _superseded_on_target_line(kt: tuple, cands: list[tuple]) -> bool: + """Does a patched version on the TARGET's own release line sit ABOVE the target? + + Projects maintain several branches at once and backport per branch, so "some patched version is + inside the numeric window" is not the same as "the version we land on has the fix". ClickHouse + fixed CVE-2023-48704 in 23.9.6.20 AND 23.10.5.20; an upgrade landing on 23.10.4.25 crosses the + 23.9 fix numerically but is still BELOW its own line's fix, so it does NOT have it. When the + advisory names a fix on the target's own line and the target is older than it, that is proof of + absence and outranks any other candidate.""" + if len(kt) < 2: + return False + line = kt[:2] + return any(c[:2] == line and c > kt for c in cands if len(c) >= 2) + + def _within(kf: tuple, kt: tuple, c: tuple) -> bool: """Is patched-version `c` inside the window (kf, kt] — exclusive lower, inclusive upper? @@ -665,6 +680,10 @@ def scan(recipe: str, v_from: str | None, v_to: str | None, registry_dir: str, continue patched = e.get("patched") or "" cands = [_vkey(t) for t in re.findall(r"\d+(?:\.\d+)*", patched)] + if kf and kt and _superseded_on_target_line(kt, cands): + # The target's own line got the fix LATER than the target: not fixed here. + e.setdefault("classification", "outside-window") + continue if kf and kt and any(_within(kf, kt, c) for c in cands): got.add(cve) elif not patched or PLACEHOLDER_RE.search(patched): diff --git a/cc-ci-plan/recipe-report.py b/cc-ci-plan/recipe-report.py index b124996..f73cf78 100755 --- a/cc-ci-plan/recipe-report.py +++ b/cc-ci-plan/recipe-report.py @@ -12,7 +12,7 @@ Subcommands (the /recipe-report agent runs them around its own review/classifica publish OUT.html DATE [KIND] copy to cc-ci:/var/lib/cc-ci-reports/-DATE.html and regen the archive index. KIND is `week` (default, the weekly /recipe-report) or `cve` (a /cve-check advisory sweep). BOTH kinds appear in the SAME archive index, - newest first, each row suffixed "full report" or "CVE check"; the distinct + newest first, each row suffixed "full" or "CVE check"; the distinct filename prefix just stops a sweep overwriting a weekly edition. Page order: short lead → the full wire table (priority-sorted, CVEs column) → Addendum → Security @@ -50,7 +50,7 @@ INFRA = {"cc-ci", "cc-ci-orchestrator", "cc-ci-secrets"} HOST_REPORTS = "/var/lib/cc-ci-reports" # Both kinds live in ONE archive, distinguished by a suffix on a common title. # prefix -> (page title, index label) -KINDS = {"week": ("The Recipe Report", "Week of {d} — full report"), +KINDS = {"week": ("The Recipe Report", "Week of {d} — full"), "cve": ("The Recipe Report — CVE check", "{d} — CVE check")} diff --git a/cc-ci-plan/resolve-images.py b/cc-ci-plan/resolve-images.py new file mode 100644 index 0000000..8721763 --- /dev/null +++ b/cc-ci-plan/resolve-images.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +"""resolve-images — what version is each of a recipe's images on, and what is newest? + +An abra-independent version resolver. `abra recipe upgrade` is the normal path, but it has a hard +failure mode: an image pinned with BOTH a tag and a digest makes it FATA and abandon the WHOLE +recipe — even images it already parsed. immich pins two that way: + + ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0@sha256:bcf6… + docker.io/valkey/valkey:9@sha256:3acc… + +so immich contributes NO version data at all and silently drops out of every survey. That is +indistinguishable from "up to date" unless a human notices the missing row — which is exactly how it +kept getting skipped, and why a CVE sweep reported it as unknown. + +This reads the compose files directly and queries the registries itself, so a digest pin is just a +digest pin. Output is JSON (default) or a table. + + resolve-images.py [--ssh HOST] [--recipe-dir DIR] [--table] [--only IMAGE] + +The cc-ci host has no python3, so `--ssh cc-ci` reads the compose files from that host's checkout +over ssh and does the resolving locally. That keeps the source of truth the SAME tree abra and CI +use, rather than a second copy that can drift. + +TAG SHAPES. Registries mix wildly different tag conventions in one repo, so "newest" is meaningless +without a shape. Each tag is reduced to a signature by replacing digit runs with '#': + + v3.1.0 -> v#.#.# + 1.27.1-rootless -> #.#.#-rootless + 8.10-alpine -> #.#-alpine + 14-vectorchord0.4.3-pgvectors0.2.0 -> #-vectorchord#.#.#-pgvectors#.#.# + +Only tags sharing the CURRENT pin's shape are candidates. That keeps `-alpine` on `-alpine`, and +stops a `latest`/`release`/`sha-…` tag from ever being proposed as an upgrade. + +TWO ANSWERS, NOT ONE. It reports `newest_same_shape` AND `newest_within_major` (same leading number). +For a plain app image they usually agree. For a compatibility-pinned sidecar they do not, and taking +the max would be wrong: immich's postgres tag encodes the pg major plus the vectorchord/pgvectors +versions that immich-server is built against, so jumping pg major because a newer tag exists breaks +the deployment. The caller picks; this tool refuses to guess and shows both. +""" + +from __future__ import annotations + +import argparse +import glob +import shlex +import subprocess +import gzip +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +UA = "cc-ci-resolve-images (+https://git.autonomic.zone/recipe-maintainers/cc-ci)" +TIMEOUT = int(os.environ.get("RESOLVE_IMAGES_TIMEOUT", "45")) +RECIPE_DIR = os.environ.get("ABRA_RECIPE_DIR", os.path.expanduser("~/.abra/recipes")) +MAX_TAG_PAGES = int(os.environ.get("RESOLVE_IMAGES_MAX_PAGES", "40")) + +IMAGE_RE = re.compile(r"""^\s*image:\s*["']?([^"'\s]+)["']?\s*$""", re.M) + + +RETRIES = int(os.environ.get("RESOLVE_IMAGES_RETRIES", "4")) + + +def _fetch(url: str, headers: dict | None = None) -> bytes: + """GET with backoff on rate limits. + + Docker Hub throttles anonymous clients hard, and a sweep re-reads the same popular repos + (nginx, redis, postgres) for recipe after recipe. A 429 mid-sweep used to surface as + 'unresolved', which is indistinguishable from a real lookup failure — so retry, and let the + per-repo cache below remove most of the requests entirely.""" + h = {"User-Agent": UA, "Accept-Encoding": "gzip"} + h.update(headers or {}) + delay = 2.0 + for attempt in range(RETRIES): + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=h), timeout=TIMEOUT) as r: + raw = r.read() + if r.headers.get("Content-Encoding") == "gzip": + raw = gzip.decompress(raw) + return raw + except urllib.error.HTTPError as e: + if e.code in (429, 503) and attempt < RETRIES - 1: + time.sleep(delay) + delay *= 2 + continue + raise + raise RuntimeError("unreachable") + + +_HUB_JWT: list = [] + + +def _hub_auth() -> dict: + """Authenticated Docker Hub calls get a far higher rate limit than anonymous ones. + + Credentials come from /srv/cc-ci/.testenv (DOCKERHUB_USERNAME / DOCKERHUB_TOKEN), the same pair + the CI host already uses. Absent creds are fine — the sweep just runs anonymous and slower.""" + if _HUB_JWT: + return _HUB_JWT[0] + env = {} + try: + for ln in open(os.environ.get("CCCI_TESTENV", "/srv/cc-ci/.testenv")): + if "=" in ln and not ln.strip().startswith("#"): + k, v = ln.strip().split("=", 1) + env[k] = v.strip().strip("\"'") + except OSError: + pass + u = os.environ.get("DOCKERHUB_USERNAME") or env.get("DOCKERHUB_USERNAME") + t = os.environ.get("DOCKERHUB_TOKEN") or env.get("DOCKERHUB_TOKEN") + hdrs = {} + if u and t: + try: + body = json.dumps({"username": u, "password": t}).encode() + req = urllib.request.Request("https://hub.docker.com/v2/users/login", + data=body, method="POST", + headers={"Content-Type": "application/json", "User-Agent": UA}) + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + tokj = json.load(r).get("token") + if tokj: + hdrs = {"Authorization": f"JWT {tokj}"} + except Exception: # noqa: BLE001 — anonymous is a valid fallback + hdrs = {} + _HUB_JWT.append(hdrs) + return hdrs + + +def _json(url: str, headers: dict | None = None): + return json.loads(_fetch(url, headers)) + + +def shape(tag: str) -> str: + """Signature of a tag with every digit run replaced by '#'. See module docstring.""" + return re.sub(r"\d+", "#", tag) + + +def vkey(tag: str) -> tuple: + """Ordering key: every number in the tag, in order. '1.27.10' > '1.27.9'; text ignored.""" + return tuple(int(x) for x in re.findall(r"\d+", tag)) + + +def parse_ref(ref: str) -> dict: + """Split an image reference into registry / repo / tag / digest.""" + digest = None + if "@" in ref: + ref, _, digest = ref.partition("@") + host, repo, tag = "docker.io", ref, "latest" + # A leading component is a REGISTRY only when there is a path after it. Without the slash test, + # a bare `postgres:15.18` looks like host "postgres:15.18" because of the tag's colon — which + # silently sent every library image to a nonexistent registry. + if "/" in ref: + first = ref.split("/")[0] + if "." in first or ":" in first or first == "localhost": + host, _, repo = ref.partition("/") + if ":" in repo.split("/")[-1]: + repo, _, tag = repo.rpartition(":") + if host == "docker.io" and "/" not in repo: + repo = f"library/{repo}" # bare `redis` is really `library/redis` + return {"registry": host, "repo": repo, "tag": tag, "digest": digest} + + +HUB_RECENT_PAGES = int(os.environ.get("RESOLVE_IMAGES_HUB_PAGES", "10")) + + +def _hub_tag_exists(repo: str, tag: str) -> bool: + try: + _json(f"https://hub.docker.com/v2/repositories/{repo}/tags/{tag}", _hub_auth()) + return True + except Exception: # noqa: BLE001 + return False + + +def _hub_tags(repo: str) -> list[str]: + """Recently-pushed tags, newest first. + + Popular Docker Hub repos carry many thousands of tags, so a full enumeration is impractical — + but it is also unnecessary: a tag NEWER than the one we run must have been pushed AFTER it, so + ordering by last_updated and reading a bounded recent window is sufficient to find any upgrade. + (ghcr offers no ordering, which is why that path needs a different strategy.) + """ + tags, url = [], (f"https://hub.docker.com/v2/repositories/{repo}/tags" + f"?page_size=100&ordering=last_updated") + auth = _hub_auth() + for _ in range(HUB_RECENT_PAGES): + d = _json(url, auth) + tags += [r["name"] for r in d.get("results", [])] + url = d.get("next") + if not url: + break + return tags + + +def _oci_bearer(host: str, repo: str) -> dict: + """Token for an OCI registry, discovered from its own auth challenge. + + Registries do NOT share a token endpoint. ghcr answers at /token?scope=…&service=ghcr.io, but + lscr.io and dock.mau.dev advertise different realms, and assuming ghcr's shape made both 401 — + which then read as "could not resolve" rather than "asked the wrong URL". The registry tells us + where to go in its WWW-Authenticate header; use that.""" + try: + urllib.request.urlopen( + urllib.request.Request(f"https://{host}/v2/{repo}/tags/list?n=1", + headers={"User-Agent": UA}), timeout=TIMEOUT) + return {} # no auth needed + except urllib.error.HTTPError as e: + if e.code != 401: + return {} + chal = e.headers.get("WWW-Authenticate", "") or "" + except Exception: # noqa: BLE001 + return {} + if not chal.lower().startswith("bearer"): + return {} + parts = dict(re.findall(r'(\w+)="([^"]*)"', chal)) + realm = parts.get("realm") + if not realm: + return {} + q = {"service": parts.get("service", host), "scope": parts.get("scope", f"repository:{repo}:pull")} + url = realm + ("&" if "?" in realm else "?") + urllib.parse.urlencode(q) + try: + tok = (_json(url) or {}).get("token") or (_json(url) or {}).get("access_token") + return {"Authorization": f"Bearer {tok}"} if tok else {} + except Exception: # noqa: BLE001 + return {} + + +def _oci_tags(host: str, repo: str) -> list[str]: + """Tags from any OCI/v2 registry, with challenge-derived auth and Link pagination. + + ghcr paginates hard — immich-server has >40,000 tags — and a truncated listing silently hides + the newest release line, so follow the cursor and let the caller's integrity check catch a read + that never reached the current pin.""" + hdrs = _oci_bearer(host, repo) + tags, url = [], f"https://{host}/v2/{repo}/tags/list?n=1000" + for _ in range(MAX_TAG_PAGES): + req = urllib.request.Request(url, headers={"User-Agent": UA, **hdrs}) + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + tags += (json.load(r) or {}).get("tags") or [] + link = r.headers.get("Link", "") or "" + m = re.search(r'<([^>]+)>;\s*rel="next"', link) + if not m: + break + nxt = m.group(1) + url = f"https://{host}{nxt}" if nxt.startswith("/") else nxt + return tags + + +def _gh_token() -> str | None: + tok = os.environ.get("GITHUB_TOKEN") + if tok: + return tok.strip() + try: + return open(os.environ.get("GITHUB_TOKEN_FILE", "/srv/cc-ci/.github-token")).read().strip() or None + except OSError: + return None + + +def github_release_tags(owner: str, repo: str, max_pages: int = 4) -> list[str]: + """Release tag names for a GitHub repo, newest first. + + FALLBACK for registries whose tag listing cannot be enumerated. ghcr has no ordering and no + server-side filter, and immich-machine-learning carries >40,000 tags — a full read is impractical + and a partial read silently hides the newest release line. The project's RELEASES are ordered, + small, and authoritative: container tags track them. (The GitHub Packages API would answer this + directly but needs a scoped token; this scan's token deliberately has none.) + """ + hdrs = {"Accept": "application/vnd.github+json"} + tok = _gh_token() + if tok: + hdrs["Authorization"] = f"Bearer {tok}" + out = [] + for page in range(1, max_pages + 1): + try: + rows = _json(f"https://api.github.com/repos/{owner}/{repo}/releases" + f"?per_page=100&page={page}", hdrs) + except Exception: # noqa: BLE001 + break + if not rows: + break + out += [r.get("tag_name") or "" for r in rows] + return [t for t in out if t] + + +def _release_fallback_repos(registry: str, repo: str) -> list[tuple[str, str]]: + """Candidate GitHub repos whose releases track this image's tags.""" + if "ghcr.io" not in registry: + return [] + parts = repo.split("/") + if len(parts) < 2: + return [] + owner, name = parts[0], parts[-1] + cands = [(owner, name)] + # ghcr.io/immich-app/immich-machine-learning is built from immich-app/immich. + if name.startswith(owner.split("-")[0]): + cands.append((owner, owner.split("-")[0])) + return cands + + +_TAG_CACHE: dict[tuple[str, str], tuple[list[str], str | None]] = {} + + +def list_tags(registry: str, repo: str) -> tuple[list[str], str | None]: + if (registry, repo) in _TAG_CACHE: + return _TAG_CACHE[(registry, repo)] + res = _list_tags_uncached(registry, repo) + _TAG_CACHE[(registry, repo)] = res + return res + + +def _list_tags_uncached(registry: str, repo: str) -> tuple[list[str], str | None]: + try: + return (_hub_tags(repo) if registry in ("docker.io", "registry-1.docker.io") + else _oci_tags(registry, repo)), None + except urllib.error.HTTPError as e: + return [], f"HTTP {e.code}" + except Exception as e: # noqa: BLE001 + return [], f"{type(e).__name__}: {e}" + + +def resolve(ref: str) -> dict: + """Current pin -> newest same-shape tag, and newest within the current major.""" + if "${" in ref or "$(" in ref: + # The tag is a compose variable (ghost pins `ghost:${IMAGE_VERSION}-alpine`). Its real value + # lives in .env, not here. Report it as skipped, never as a failed lookup. + return {**parse_ref(ref), "ref": ref, "shape": None, "candidates": 0, + "newest_same_shape": None, "newest_within_major": None, + "upgrade_available": False, + "status": "skipped: templated ref (tag comes from a compose variable)"} + info = parse_ref(ref) + out = {**info, "ref": ref, "shape": shape(info["tag"]), "status": "ok", + "newest_same_shape": None, "newest_within_major": None, "candidates": 0, + "upgrade_available": False} + tags, err = list_tags(info["registry"], info["repo"]) + if err: + out["status"] = f"error: {err}" + return out + out["tags_seen"] = len(set(tags)) + # INTEGRITY CHECK: the tag we are currently running MUST appear in the listing. If it does not, + # the listing is incomplete and any "newest" derived from it is a guess — ghcr paginates to tens + # of thousands of tags and a truncated read silently hides whole release lines. immich's + # machine-learning image is pinned v3.1.0, which EXISTS, yet a short read reported v1.134.0 as + # newest; without this check that becomes a confident, wrong answer. + if info["tag"] not in set(tags): + # Docker Hub: the window is recency-ordered, so the pin being outside it just means the pin + # is old — which is fine, because anything NEWER is necessarily inside the window. Confirm + # the pin genuinely exists (so a typo is still caught) and carry on. + if info["registry"] in ("docker.io", "registry-1.docker.io") and _hub_tag_exists(info["repo"], info["tag"]): + out["source"] = f"docker-hub:recent-{HUB_RECENT_PAGES * 100}" + tags = list(tags) + [info["tag"]] + else: + for owner, name in _release_fallback_repos(info["registry"], info["repo"]): + rel = github_release_tags(owner, name) + if info["tag"] in rel: + tags = rel + out["source"] = f"github-releases:{owner}/{name}" + out["tags_seen"] = len(set(rel)) + break + else: + out["status"] = ("error: tag listing incomplete — the current pin " + f"{info['tag']!r} is absent from {len(set(tags))} registry tags " + f"and from the project's GitHub releases") + return out + want, cur = out["shape"], vkey(info["tag"]) + same = [t for t in set(tags) if shape(t) == want and vkey(t)] + out["candidates"] = len(same) + if not same: + # Not a failure: digest-only pins and `latest`/`stable` have no comparable siblings. + out["status"] = "no comparable tags (shape has no numeric siblings)" + return out + newest = max(same, key=vkey) + out["newest_same_shape"] = newest + if cur: + within = [t for t in same if vkey(t)[:1] == cur[:1]] + if within: + out["newest_within_major"] = max(within, key=vkey) + out["upgrade_available"] = bool(cur and vkey(newest) > cur) + return out + + +def compose_images_ssh(recipe: str, host: str, recipe_dir: str) -> list[str]: + """Same as compose_images, but the recipe tree lives on another host (cc-ci has no python3).""" + # NB: no shell-quoting of the directory — it may legitimately start with ~ or $HOME, and + # quoting it stops the remote shell expanding it, which yields an empty (and silent) result. + d = f"{recipe_dir}/{shlex.quote(recipe)}".replace("~", "$HOME") + cmd = (f'for f in {d}/compose*.yml; do case "$f" in *compose.ccci.yml) continue;; esac; ' + f'[ -f "$f" ] && {{ cat "$f"; echo; }}; done; exit 0') + out = subprocess.run(["ssh", host, cmd], capture_output=True, text=True, timeout=120) + if out.returncode != 0: + raise RuntimeError(f"ssh {host}: {(out.stderr.strip() or 'no output')[:200]}") + if not out.stdout.strip(): + raise RuntimeError(f"ssh {host}: no compose files found under {d}") + refs = [] + for m in IMAGE_RE.finditer(out.stdout): + if m.group(1) not in refs: + refs.append(m.group(1)) + return refs + + +def compose_images(recipe: str, recipe_dir: str) -> list[str]: + """Every `image:` ref in the recipe's own compose files (the cc-ci overlay is NOT the recipe).""" + refs, base = [], os.path.join(recipe_dir, recipe) + for path in sorted(glob.glob(os.path.join(base, "compose*.yml"))): + if os.path.basename(path) == "compose.ccci.yml": + continue + try: + for m in IMAGE_RE.finditer(open(path).read()): + if m.group(1) not in refs: + refs.append(m.group(1)) + except OSError: + continue + return refs + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("recipe") + ap.add_argument("--ssh", default=None, metavar="HOST", + help="read the recipe's compose files from HOST over ssh (e.g. --ssh cc-ci); " + "resolving still happens locally") + ap.add_argument("--recipe-dir", default=RECIPE_DIR) + ap.add_argument("--table", action="store_true", help="human-readable table instead of JSON") + ap.add_argument("--only", default=None, help="resolve just the images whose ref contains this") + a = ap.parse_args() + + rdir = a.recipe_dir if a.recipe_dir != RECIPE_DIR or not a.ssh else "~/.abra/recipes" + refs = (compose_images_ssh(a.recipe, a.ssh, rdir) if a.ssh + else compose_images(a.recipe, a.recipe_dir)) + if a.only: + refs = [r for r in refs if a.only in r] + results = [resolve(r) for r in refs] + report = { + "recipe": a.recipe, + "images": results, + "upgrades_available": [r["ref"] for r in results if r["upgrade_available"]], + "unresolved": [r["ref"] for r in results if r["status"].startswith("error")], + # The whole point: distinguish "checked, current" from "could not check". + "all_resolved": not any(r["status"].startswith("error") for r in results), + } + if not a.table: + print(json.dumps(report, indent=2)) + return 0 + print(f"{a.recipe} — {len(results)} images") + for r in results: + flag = "UPGRADE" if r["upgrade_available"] else ("ERROR" if r["status"].startswith("error") else "current") + print(f" [{flag:7}] {r['repo']}:{r['tag']}" + (" (digest-pinned)" if r["digest"] else "")) + print(f" shape={r['shape']} candidates={r['candidates']}" + f" newest_same_shape={r['newest_same_shape']}" + f" newest_within_major={r['newest_within_major']}") + if r["status"] != "ok": + print(f" status: {r['status']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/cc-ci-plan/test-advisory-scan.py b/cc-ci-plan/test-advisory-scan.py index 69e46c3..80be3ea 100755 --- a/cc-ci-plan/test-advisory-scan.py +++ b/cc-ci-plan/test-advisory-scan.py @@ -503,6 +503,40 @@ class TestReleaseNoteResolution(unittest.TestCase): self.assertNotIn("CVE-OK", rep.get("resolved_by_release_notes") or {}) +class TestReleaseLineSemantics(unittest.TestCase): + """A fix inside the numeric window is not a fix on the branch you actually land on.""" + + def test_fix_later_on_the_targets_own_line_is_not_counted(self): + # ClickHouse fixed CVE-2023-48704 in 23.9.6.20 AND 23.10.5.20. Landing on 23.10.4.25 crosses + # the 23.9 fix numerically but is BELOW its own line's fix, so it does not have it. + rep = run_scan([gh("ClickHouse/ClickHouse", + [adv("CVE-2023-48704", patched="v23.10.5.20; v23.9.6.20; v23.8.8.20")])], + v_from="23.4.2.11", v_to="23.10.4.25", + urls=["https://github.com/ClickHouse/ClickHouse"]) + self.assertEqual(rep["fixed_by_this_upgrade"], []) + + def test_fix_earlier_on_the_targets_own_line_is_counted(self): + rep = run_scan([gh("ClickHouse/ClickHouse", + [adv("CVE-2023-47118", patched="v23.10.2.13; v23.8.6.16")])], + v_from="23.4.2.11", v_to="23.10.4.25", + urls=["https://github.com/ClickHouse/ClickHouse"]) + self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2023-47118"]) + + def test_fix_exactly_at_the_target_is_counted(self): + rep = run_scan([gh("ClickHouse/ClickHouse", + [adv("CVE-2023-48298", patched="v23.10.4.25; v23.9.5.29")])], + v_from="23.4.2.11", v_to="23.10.4.25", + urls=["https://github.com/ClickHouse/ClickHouse"]) + self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2023-48298"]) + + def test_no_fix_on_the_target_line_falls_back_to_the_window(self): + # redis fixes 7.4.6/8.0.4/8.2.2 with no 8.10.x entry; landing on 8.10 still has them, + # because nothing on the 8.10 line is named as a LATER fix. + rep = run_scan([gh("redis/redis", [adv("CVE-2025-49844", patched="7.4.6; 8.0.4; 8.2.2")])], + v_from="7.4", v_to="8.10", urls=["https://github.com/redis/redis"]) + self.assertEqual(rep["fixed_by_this_upgrade"], ["CVE-2025-49844"]) + + class TestAdjudicationEvidenceAssembly(unittest.TestCase): """Pass 2's JUDGEMENT is a model's and not testable; what IS testable is what it gets shown.""" @@ -624,16 +658,22 @@ class TestHistoricReportNumbers(unittest.TestCase): for cve in delta: self.assertIn("redis", both["cves"][cve]["sources"][0]) - def test_mailu_scan_finds_zero_and_says_so_knowingly(self): - # Published as 2 via the UNION with release-note reading; the scan's own contribution is 0, - # and 0 here must mean "checked, none", not "could not check". + def test_mailu_finds_the_roundcube_pair_without_an_agent(self): + # Published as 2 on 2026-08-07, but only because an agent read the release notes; the scan + # itself contributed 0. It now reaches 2 deterministically: the CVEs appear only on + # github.com/Mailu/Mailu/releases, and release 2024.06.56 (inside the window) names them. rep = self._count("mailu", "2024.06.55", "2024.06.57", [("redis", "8.8.0", "8.10.0")]) - self.assertEqual(rep["cve_count_fixed"], 0) - self.assertTrue(rep["count_known"]) + self.assertEqual(rep["cve_count_fixed"], 2) + self.assertEqual(set(rep["fixed_by_this_upgrade"]), {"CVE-2026-54432", "CVE-2026-54433"}) + # The redis bump fixes nothing new — every advisory it crosses was fixed at or before 8.6.3. + self.assertEqual(rep["cve_count_indeterminate"], 0) - def test_keycloak_26_7_0_to_26_7_1_is_7(self): + def test_keycloak_26_7_0_to_26_7_1_is_12(self): + # Was 7 while only the GHSA feed was consulted. keycloak lists five more CVEs in the 26.7.1 + # release notes' fixed-issues section that it never filed as advisories — the gitea pattern. rep = self._count("keycloak", "26.7.0", "26.7.1") - self.assertEqual(rep["cve_count_fixed"], 7) + self.assertEqual(rep["cve_count_fixed"], 12) + self.assertEqual(len(rep.get("resolved_by_release_notes") or {}), 5) def _main():