feat(settings): server settings.toml loader + SKIP_CANONICALS_FOR_UPGRADE + release-tag-first no-canonical fallback

- harness/settings.py: stdlib tomllib loader, [upgrade].skip_canonicals_for_upgrade
  (bool, default false), _SCHEMA single-source defaults+validation; graceful on
  absent/malformed (WARN+defaults), warn-and-ignore unknown keys/tables, TypeError on
  wrong type. Path $CCCI_SETTINGS / /etc/cc-ci/settings.toml. + tracked settings.toml.example.
- resolve_upgrade_base: flag true bypasses the canonical lookup -> no-canonical fallback;
  canonical-present path (incl. samever step-back) unchanged when false.
- _no_canonical_base (always-on, §2.C): newest release tag < head (reuse
  warm_reconcile.newest_older_version) -> main-tip -> skip; replaces jump-to-main-tip.
- unit: full resolution matrix + loader tests; 315 unit pass, ruff clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

runner/harness/settings.py

@@ -0,0 +1,118 @@
+"""Server-level CI settings (phase settings). A minimal, extensible TOML config layer for the cc-ci
+server, read once per harness process. Stdlib-only (`tomllib`). Defaults are baked into this loader,
+so an ABSENT file — or an absent key within a present file — behaves EXACTLY as before (this server
+needs no file to behave as today). The live file is an operator-managed HOST OVERRIDE; it must carry
+NO secrets (config only — secrets stay in sops). See `settings.toml.example` for the documented keys.
+
+Layout (minimal + extensible — one table, one key now; shaped so future CI-server configs slot in
+without a redesign):
+
+    [upgrade]
+    skip_canonicals_for_upgrade = false   # when true, resolve the upgrade BASE without canonicals
+
+Path resolution: `$CCCI_SETTINGS` if set, else `/etc/cc-ci/settings.toml` (co-located with the
+deployed checkout `$CCCI_REPO=/etc/cc-ci`, alongside the tracked `settings.toml.example`; both the
+Drone recipe-CI runner and the nightly sweep read this same absolute host path).
+
+Validation:
+  - absent / unreadable / malformed-TOML file  → WARN + all-defaults (a bad file can NEVER crash the
+    harness — a red config should degrade to today's behavior, not take down CI).
+  - unknown table / unknown key                → WARN-and-ignore (forward/typo tolerant).
+  - present known key of the WRONG TYPE        → raise TypeError (a typo'd value is loud, not silently
+    mis-parsed) — this is the one "errors clearly" case, distinct from a malformed file.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import tomllib
+from dataclasses import dataclass
+
+DEFAULT_PATH = "/etc/cc-ci/settings.toml"
+
+# The full schema: table -> {key: (type, default)}. SINGLE SOURCE OF TRUTH for defaults + validation.
+# Add future settings here (a new key, or a new table); `Settings` field names mirror the keys.
+_SCHEMA: dict[str, dict[str, tuple[type, object]]] = {
+    "upgrade": {
+        "skip_canonicals_for_upgrade": (bool, False),
+    },
+}
+
+
+@dataclass(frozen=True)
+class Settings:
+    # [upgrade].skip_canonicals_for_upgrade — when True, resolve_upgrade_base skips the canonical
+    # lookup entirely and takes the no-canonical release-tag-first fallback (phase settings §2.B/§2.C).
+    skip_canonicals_for_upgrade: bool = False
+
+
+def _warn(msg: str) -> None:
+    print(f"[settings] WARNING: {msg}", file=sys.stderr, flush=True)
+
+
+def _resolve_path() -> str:
+    return os.environ.get("CCCI_SETTINGS") or DEFAULT_PATH
+
+
+def _check_type(table: str, key: str, value: object, typ: type) -> object:
+    # bool is a subclass of int — reject an int given for a bool key (and vice-versa) so a stray
+    # `1`/`0` for a flag is a loud error, not a silent truthy coercion.
+    ok = isinstance(value, typ) and (typ is bool or not isinstance(value, bool))
+    if not ok:
+        raise TypeError(
+            f"settings: [{table}].{key} must be {typ.__name__}, "
+            f"got {type(value).__name__} ({value!r})"
+        )
+    return value
+
+
+def load(path: str | None = None) -> Settings:
+    """Load settings from `path` (default: `$CCCI_SETTINGS` / DEFAULT_PATH). Returns all-defaults on an
+    absent / unreadable / malformed file (WARN, never raises). Raises TypeError only on a present
+    known key whose value is the wrong type."""
+    p = path or _resolve_path()
+    raw: dict = {}
+    if os.path.exists(p):
+        try:
+            with open(p, "rb") as f:
+                raw = tomllib.load(f)
+        except (OSError, tomllib.TOMLDecodeError) as e:
+            _warn(f"could not read {p} ({e}); using defaults")
+            raw = {}
+
+    values: dict[str, object] = {}
+    for table, keys in _SCHEMA.items():
+        section = raw.get(table, {})
+        if not isinstance(section, dict):
+            _warn(f"[{table}] is not a table ({type(section).__name__}); ignoring it")
+            section = {}
+        for key, (typ, default) in keys.items():
+            if key in section:
+                values[key] = _check_type(table, key, section[key], typ)
+            else:
+                values[key] = default
+
+    # Forward/typo tolerant: warn on anything not in the schema, but never fail on it.
+    for table, section in raw.items():
+        if table not in _SCHEMA:
+            _warn(f"unknown table [{table}] ignored")
+            continue
+        if isinstance(section, dict):
+            for key in section:
+                if key not in _SCHEMA[table]:
+                    _warn(f"unknown key [{table}].{key} ignored")
+
+    return Settings(**values)  # type: ignore[arg-type]
+
+
+_CACHE: Settings | None = None
+
+
+def get() -> Settings:
+    """Process-wide cached settings (read the file once). The harness calls this; tests call `load()`
+    with an explicit path (or monkeypatch this) to bypass the cache."""
+    global _CACHE
+    if _CACHE is None:
+        _CACHE = load()
+    return _CACHE

runner/run_recipe_ci.py

@@ -75,6 +75,9 @@ from harness import (  # noqa: E402
 from harness import (  # noqa: E402
     screenshot as screenshot_mod,
 )
+from harness import (  # noqa: E402
+    settings as settings_mod,
+)
 
 ALL_STAGES = ("install", "upgrade", "backup", "restore", "custom")
 
@@ -113,8 +116,14 @@ def resolve_upgrade_base(
     stages, meta, recipe: str, head_ref: str | None = None, head_version: str | None = None
 ) -> BasePlan:
     """Dynamic upgrade-base resolution (phase prevb, replaces the static `recipe_versions[-2]`
-    default). Order: last-green (warm canonical, with same-version step-back) → target-branch (main)
-    tip → skip. EXPECTED_NA[upgrade] / upgrade∉stages short-circuit to a declared skip first.
+    default). Chain: last-green (warm canonical, with same-version step-back) → newest release tag
+    older than head → main-tip → skip. EXPECTED_NA[upgrade] / upgrade∉stages short-circuit to a
+    declared skip first.
+
+    SKIP_CANONICALS_FOR_UPGRADE (phase settings, server settings.toml, default false): when true, the
+    canonical lookup is bypassed entirely — the resolver behaves as if no canonical exists and takes
+    the no-canonical release-tag-first fallback (`_no_canonical_base`). Scope is the upgrade BASE only;
+    canonical promotion and the `--quick` warm-reattach are unaffected (see DECISIONS, phase settings).
 
     `head_version` is the head checkout's published version (the `coop-cloud.<stack>.version` label;
     see abra.head_compose_version). When the last-green warm-canonical version EQUALS it, deploying
@@ -142,8 +151,9 @@ def resolve_upgrade_base(
             flush=True,
         )
         return BasePlan("skip", None, None, f"declared EXPECTED_NA[upgrade]: {declared}")
+    skip_canonicals = settings_mod.get().skip_canonicals_for_upgrade
     rec = canonical.read_registry(recipe)
-    if rec and rec.get("version"):
+    if rec and rec.get("version") and not skip_canonicals:
         canon = rec["version"]
         same = head_version is not None and warm_reconcile.version_key(
             canon
@@ -176,13 +186,46 @@ def resolve_upgrade_base(
             None,
             f"base == head ({head_version}) and no older published predecessor",
         )
+    # No canonical in play — none recorded, OR SKIP_CANONICALS_FOR_UPGRADE=true (canonical lookup
+    # bypassed entirely, behaving as if none exists). Improved fallback (phase settings §2.C): prefer
+    # a REAL published predecessor (newest release tag < head) over the raw main-tip.
+    return _no_canonical_base(recipe, head_ref, head_version)
+
+
+def _no_canonical_base(recipe: str, head_ref: str | None, head_version: str | None) -> BasePlan:
+    """Upgrade base when no canonical is used (none recorded, its promote failed, or
+    SKIP_CANONICALS_FOR_UPGRADE is true). Release-tag-first fallback (phase settings §2.C):
+      1. most recent release TAG with version strictly older than the PR head — a clean published
+         predecessor (reuses samever's `newest_older_version` helper, the single source of version
+         ordering, so this and the step-back never diverge);
+      2. raw `main`-tip (target-branch tip) — only if the recipe has NO prior release tag at all;
+      3. skip — no predecessor (no older tag and head == main-tip, or no main at all).
+    This replaces the old jump-straight-to-main-tip path, so an un-promoted recipe upgrades from a real
+    release base instead of a possibly-untagged WIP commit."""
+    older = (
+        warm_reconcile.newest_older_version(warm_reconcile.recipe_tags(recipe), head_version)
+        if head_version
+        else None
+    )
+    if older:
+        return BasePlan(
+            "version",
+            older,
+            None,
+            f"no-canonical fallback: newest release tag older than head {head_version}",
+        )
     main_tip = lifecycle.recipe_branch_commit(recipe, "main")
     if main_tip and main_tip != head_ref:
-        return BasePlan("ref", None, main_tip, "target-branch (main) tip")
+        return BasePlan(
+            "ref",
+            None,
+            main_tip,
+            "no-canonical fallback: target-branch (main) tip (no prior release tag)",
+        )
     if main_tip and main_tip == head_ref:
         return BasePlan("skip", None, None, "head == main tip (no predecessor delta)")
     return BasePlan(
-        "skip", None, None, "no last-green and no main tip (new recipe / no predecessor)"
+        "skip", None, None, "no release tag and no main tip (new recipe / no predecessor)"
     )