feat(prevb): dynamic upgrade base (last-green→main→skip) + per-recipe previous/ overlay; migrate discourse off static base + leaky overlay

- resolve_upgrade_base: BasePlan(kind=version|ref|skip); last-green (warm canonical) primary,
  main-tip fallback, declared skip else. UPGRADE_BASE_VERSION retained as optional override.
- deploy_app: base_ref path (chaos-deploy a main-tip/last-green commit) + apply_previous wiring.
- lifecycle: previous/ surface (has_previous, previous_target_version, previous_status decision,
  provide/remove overlay, compose_file add/remove, recipe_branch_commit, stack_service_names).
- generic.perform_upgrade: strip previous/ overlay + COMPOSE_FILE entry before head redeploy.
- discourse: compose.ccci.yml now environmental-only (order: stop-first); removed bitnamilegacy
  pins + sidekiq + UPGRADE_BASE_VERSION; test_upgrade.py asserts head image == official 3.5.3 + no sidekiq.
- unit tests: resolve_upgrade_base matrix + previous/ apply/skip/stale + COMPOSE_FILE layering.

runner/harness/lifecycle.py

@@ -154,6 +154,145 @@ def provide_ccci_overlay(recipe: str) -> None:
     )
 
 
+# ---------------------------------------------------------------------------------------------
+# Phase prevb: dynamic upgrade base + per-recipe `previous/` overlay.
+#
+# `previous/` holds the MINIMAL config needed to deploy the *previous (last-green) version* when it
+# can't deploy as-published (e.g. an image relocation). It is applied ONLY to the base deploy and
+# ONLY when the resolved base is that exact published version; NEVER to the PR head; on a main-tip
+# base or version mismatch it is skipped + flagged stale. Mechanism mirrors the environmental
+# compose.ccci.yml overlay (copied untracked into the checkout, referenced via COMPOSE_FILE), but is
+# stripped before the head redeploy so the head runs unmodified. (plan-phase-prevb §2.)
+# ---------------------------------------------------------------------------------------------
+
+PREVIOUS_COMPOSE = "compose.previous.yml"
+
+
+def previous_dir(recipe: str) -> str:
+    return os.path.join(meta_mod.TESTS_DIR, recipe, "previous")
+
+
+def has_previous(recipe: str) -> bool:
+    """True iff the recipe ships a `tests/<recipe>/previous/` folder with a compose.previous.yml."""
+    return os.path.isfile(os.path.join(previous_dir(recipe), PREVIOUS_COMPOSE))
+
+
+def previous_target_version(recipe: str) -> str | None:
+    """The published version the recipe's `previous/` folder targets — declared in a one-line
+    `previous/VERSION` marker (first non-blank, non-`#` line). None if no marker. The harness applies
+    `previous/` ONLY when the resolved base equals this; otherwise the folder is stale and skipped."""
+    marker = os.path.join(previous_dir(recipe), "VERSION")
+    try:
+        with open(marker) as f:
+            for line in f:
+                s = line.strip()
+                if s and not s.startswith("#"):
+                    return s
+    except OSError:
+        return None
+    return None
+
+
+def previous_status(recipe: str, base_kind: str, base_version: str | None) -> dict:
+    """Decide whether `previous/` applies to this base, as a pure decision (unit-tested).
+
+    Returns {apply, stale, reason}:
+      - no `previous/` folder           → apply=False, stale=False (nothing to do).
+      - base is not a pinned published  → apply=False, stale=True  (main-tip / no base): previous/ can
+        version (kind != "version")       only repair a published base; flag for review.
+      - no `previous/VERSION` marker     → apply=False, stale=True  (undeclared: cannot version-guard).
+      - marker != resolved base version  → apply=False, stale=True  (stale: targets X, base is Y).
+      - marker == resolved base version  → apply=True,  stale=False.
+    """
+    if not has_previous(recipe):
+        return {"apply": False, "stale": False, "reason": ""}
+    if base_kind != "version" or not base_version:
+        return {
+            "apply": False,
+            "stale": True,
+            "reason": (
+                f"previous/ present but the resolved base is not a pinned published version "
+                f"(base_kind={base_kind}) — previous/ only repairs a published base; not applied"
+            ),
+        }
+    target = previous_target_version(recipe)
+    if not target:
+        return {
+            "apply": False,
+            "stale": True,
+            "reason": "previous/ has no VERSION marker — cannot version-guard; not applied",
+        }
+    if target != base_version:
+        return {
+            "apply": False,
+            "stale": True,
+            "reason": f"previous/ targets {target}, base is {base_version} — stale, remove it",
+        }
+    return {"apply": True, "stale": False, "reason": ""}
+
+
+def provide_previous_overlay(recipe: str) -> None:
+    """Copy `tests/<recipe>/previous/compose.previous.yml` into THIS run's recipe checkout so a
+    COMPOSE_FILE reference to it resolves (base deploy only). No-op if absent."""
+    src = os.path.join(previous_dir(recipe), PREVIOUS_COMPOSE)
+    if not os.path.isfile(src):
+        return
+    dest_dir = abra.recipe_dir(recipe)
+    if not os.path.isdir(dest_dir):
+        raise RuntimeError(f"recipe checkout missing for {recipe}: {dest_dir}")
+    shutil.copy(src, os.path.join(dest_dir, PREVIOUS_COMPOSE))
+    print(
+        f"  previous-overlay: provided {PREVIOUS_COMPOSE} to the {recipe} base checkout "
+        "(base-only; stripped before the head redeploy)",
+        flush=True,
+    )
+
+
+def remove_previous_overlay(recipe: str) -> None:
+    """Delete compose.previous.yml from this run's recipe checkout (called before the head redeploy so
+    the PR head NEVER sees the previous-version repair). No-op if absent."""
+    p = os.path.join(abra.recipe_dir(recipe), PREVIOUS_COMPOSE)
+    with contextlib.suppress(OSError):
+        if os.path.isfile(p):
+            os.remove(p)
+            print(
+                f"  previous-overlay: removed {PREVIOUS_COMPOSE} from the head checkout", flush=True
+            )
+
+
+def compose_file_add(compose_file: str, overlay: str) -> str:
+    """Append `overlay` to a ':'-separated COMPOSE_FILE value if absent (pure). Defaults a missing
+    value to `compose.yml` first, matching abra."""
+    parts = [p for p in (compose_file or "").split(":") if p] or ["compose.yml"]
+    if overlay not in parts:
+        parts.append(overlay)
+    return ":".join(parts)
+
+
+def compose_file_remove(compose_file: str, overlay: str) -> str:
+    """Remove `overlay` from a ':'-separated COMPOSE_FILE value (pure). Keeps order; defaults a now-
+    empty value to `compose.yml`."""
+    parts = [p for p in (compose_file or "").split(":") if p and p != overlay]
+    return ":".join(parts or ["compose.yml"])
+
+
+def recipe_branch_commit(recipe: str, branch: str = "main") -> str | None:
+    """Resolve the recipe repo's target-branch tip (the predecessor the PR merges onto) to a commit
+    SHA, for the dynamic upgrade-base main-tip fallback. The per-run tree is a full clone of the
+    mirror, so `origin/<branch>` is present. Tries origin/<branch>, then origin/master. None if
+    neither resolves (new recipe / detached state)."""
+    path = abra.recipe_dir(recipe)
+    for ref in (f"origin/{branch}", "origin/master"):
+        proc = subprocess.run(
+            ["git", "-C", path, "rev-parse", "--verify", "--quiet", ref],
+            capture_output=True,
+            text=True,
+        )
+        if proc.returncode == 0 and proc.stdout.strip():
+            return proc.stdout.strip()
+    return None
+
+
 def _run_install_steps(hook: tuple[str, str], recipe: str, domain: str) -> None:
     """Run a recipe's custom install-steps hook (install_steps.sh) during the install tier — after
     `abra app new` + env defaults + secret generate, before deploy (Phase 1d DG5). The hook gets the
@@ -234,6 +373,8 @@ def deploy_app(
     recipe: str,
     domain: str,
     version: str | None = None,
+    base_ref: str | None = None,
+    apply_previous: bool = False,
     secrets: bool = True,
     install_steps_hook: tuple[str, str] | None = None,
     deploy_timeout: int = 900,
@@ -273,7 +414,18 @@ def deploy_app(
     # Adversary F1d-2). Chaos is correct ONLY for the version=None case (deploy the current PR-head
     # checkout). Order matters: checkout before secret_generate (-C) so secrets match the pinned tree.
     chaos = version is None
-    if version:
+    if base_ref:
+        # Dynamic upgrade base = target-branch (main) tip, or a last-green commit (phase prevb): an
+        # arbitrary git ref, not a published tag. Check it out and deploy via chaos — same mechanism
+        # as the head deploy (a non-tag ref would FATA abra's pinned-deploy lint/clean-tree gate).
+        recipe_checkout_ref(recipe, base_ref)
+        chaos = True
+        print(
+            f"  deploy_app({recipe}): base = main-tip/ref {base_ref[:12]} → chaos deploy of the "
+            "checked-out ref (the PR's true predecessor; not a published pin)",
+            flush=True,
+        )
+    elif version:
         abra.recipe_checkout(recipe, version)
         # A pinned (non-chaos) deploy runs `abra recipe lint`, which FATAs R014 ('only annotated
         # tags') if the upstream recipe ships a stray lightweight version tag (e.g. lasuite-meet's
@@ -309,8 +461,19 @@ def deploy_app(
     # it ourselves is recipe-agnostic and canonical (the run domain IS the app's domain).
     abra.env_set(domain, "DOMAIN", domain)
     abra.env_set(domain, "LETS_ENCRYPT_ENV", "")
-    for k, v in meta_mod.extra_env(meta, meta_mod.hook_ctx(domain, meta)).items():
+    extra = meta_mod.extra_env(meta, meta_mod.hook_ctx(domain, meta))
+    for k, v in extra.items():
         abra.env_set(domain, k, v)
+    # Per-recipe `previous/` overlay (phase prevb): version-specific repair to deploy the *previous
+    # (last-green) version*, applied to the BASE deploy ONLY (the caller resolved apply=True only when
+    # the resolved base equals previous/VERSION). Appended on top of the recipe's COMPOSE_FILE (which
+    # may already include the environmental compose.ccci.yml). Stripped before the head redeploy
+    # (generic.perform_upgrade) so the PR head runs unmodified.
+    if apply_previous:
+        cf = compose_file_add(extra.get("COMPOSE_FILE", "compose.yml"), PREVIOUS_COMPOSE)
+        abra.env_set(domain, "COMPOSE_FILE", cf)
+        provide_previous_overlay(recipe)
+        print(f"  previous-overlay: COMPOSE_FILE for base deploy = {cf}", flush=True)
     if secrets:
         abra.secret_generate(domain)
     if install_steps_hook:
@@ -332,6 +495,24 @@ def _stack_name(domain: str) -> str:
     return domain.replace(".", "_")
 
 
+def stack_service_names(domain: str) -> list[str]:
+    """Short service names in this app's swarm stack (stack prefix stripped). Used by recipe overlay
+    assertions to prove a service was added/removed by an upgrade (e.g. discourse drops `sidekiq`)."""
+    stack = _stack_name(domain)
+    proc = subprocess.run(
+        ["docker", "stack", "services", stack, "--format", "{{.Name}}"],
+        capture_output=True,
+        text=True,
+    )
+    names = []
+    for ln in proc.stdout.split("\n"):
+        n = ln.strip()
+        if not n:
+            continue
+        names.append(n[len(stack) + 1 :] if n.startswith(stack + "_") else n)
+    return names
+
+
 def services_converged(domain: str) -> bool:
     """True when every service in the stack reports replicas N/N (N>0) AND no service is
     mid-rolling-update (swarm UpdateStatus settled)."""