M5: upgrade + backup/restore stages green (custom-html); backup-bot-two oneshot

3-stage run green (install/upgrade/backup), clean teardown. backupbot deployed
via reconcile oneshot; PTY (script) for abra backup/restore; -m for secret generate
(no value leak). M5 CLAIMED.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

runner/harness/abra.py

@@ -19,6 +19,19 @@ class AbraError(RuntimeError):
     pass
 
 
+def _run_pty(args: list[str], timeout: int = 900, check: bool = True) -> subprocess.CompletedProcess:
+    """Run abra under a pseudo-TTY (via util-linux `script`). Needed for commands that exec into
+    a container interactively (backup create / restore: 'the input device is not a TTY')."""
+    cmd = "abra " + " ".join(args)
+    proc = subprocess.run(
+        ["script", "-qec", cmd, "/dev/null"],
+        capture_output=True, text=True, timeout=timeout,
+    )
+    if check and proc.returncode != 0:
+        raise AbraError(f"[pty] {cmd} failed ({proc.returncode}):\n{proc.stdout}\n{proc.stderr}")
+    return proc
+
+
 def _run(args: list[str], timeout: int = 300, check: bool = True) -> subprocess.CompletedProcess:
     proc = subprocess.run(
         [ABRA, *args],
@@ -64,7 +77,9 @@ def env_set(domain: str, key: str, value: str) -> None:
 
 
 def secret_generate(domain: str, timeout: int = 300) -> None:
-    _run(["app", "secret", "generate", domain, "--all", "-n"], timeout=timeout, check=False)
+    # -m avoids the TTY/table (ioctl) path; output (which contains the generated values) is
+    # captured by _run and never logged. check=False: recipes with no secrets are a no-op.
+    _run(["app", "secret", "generate", domain, "--all", "-m", "-n"], timeout=timeout, check=False)
 
 
 def deploy(domain: str, chaos: bool = True, timeout: int = 900) -> None:
@@ -74,6 +89,34 @@ def deploy(domain: str, chaos: bool = True, timeout: int = 900) -> None:
     _run(args, timeout=timeout)
 
 
+def upgrade(domain: str, version: Optional[str] = None, timeout: int = 900) -> None:
+    args = ["app", "upgrade", domain]
+    if version:
+        args.append(version)
+    # -f no prompt, -D skip public-DNS checks (our per-run domains route via the gateway).
+    # (upgrade has no --chaos flag.)
+    args += ["-f", "-D", "-n"]
+    _run(args, timeout=timeout)
+
+
+def backup_create(domain: str, timeout: int = 900) -> None:
+    _run_pty(["app", "backup", "create", domain, "-n"], timeout=timeout)
+
+
+def restore(domain: str, timeout: int = 900) -> None:
+    _run_pty(["app", "restore", domain, "-n"], timeout=timeout)
+
+
+def recipe_versions(recipe: str) -> list[str]:
+    """Published versions of a recipe, oldest→newest (from the recipe git tags)."""
+    import os
+    import subprocess
+    path = os.path.expanduser(f"~/.abra/recipes/{recipe}")
+    proc = subprocess.run(["git", "-C", path, "tag", "--sort=creatordate"],
+                          capture_output=True, text=True)
+    return [t for t in proc.stdout.split("\n") if t.strip()]
+
+
 def undeploy(domain: str, timeout: int = 600) -> None:
     # NB: no --chaos here (unsupported).
     _run(["app", "undeploy", domain, "-n"], timeout=timeout, check=False)

runner/harness/lifecycle.py

@@ -87,6 +87,52 @@ def wait_healthy(domain: str, ok_codes=(200, 301, 302), deploy_timeout: int = 60
     raise TimeoutError(f"{domain}: not healthy over HTTPS (last status {last})")
 
 
+def upgrade_app(domain: str, version: str | None = None) -> None:
+    abra.upgrade(domain, version=version)
+
+
+def backup_app(domain: str) -> None:
+    abra.backup_create(domain)
+
+
+def restore_app(domain: str) -> None:
+    abra.restore(domain)
+
+
+def previous_version(recipe: str) -> str | None:
+    """The second-newest published version (to deploy before upgrading to latest)."""
+    vers = abra.recipe_versions(recipe)
+    return vers[-2] if len(vers) >= 2 else None
+
+
+def _app_container(domain: str, service: str = "app") -> str:
+    """The running container id for <stack>_<service>."""
+    name = f"{_stack_name(domain)}_{service}"
+    proc = subprocess.run(
+        ["docker", "ps", "--filter", f"name={name}", "--format", "{{.ID}}"],
+        capture_output=True, text=True,
+    )
+    cid = proc.stdout.strip().split("\n")[0]
+    if not cid:
+        raise RuntimeError(f"no running container for {name}")
+    return cid
+
+
+def exec_in_app(domain: str, cmd: list[str], service: str = "app") -> str:
+    cid = _app_container(domain, service)
+    proc = subprocess.run(["docker", "exec", cid, *cmd], capture_output=True, text=True)
+    return proc.stdout
+
+
+def http_body(domain: str, path: str = "/", timeout: int = 15) -> str:
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    req = urllib.request.Request(f"https://{domain}{path}", method="GET")
+    with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
+        return resp.read().decode(errors="replace")
+
+
 def teardown_app(domain: str) -> None:
     """Idempotent, best-effort full teardown. Never raises (finalizer-safe)."""
     abra.undeploy(domain)

runner/run_recipe_ci.py

@@ -56,24 +56,27 @@ def main() -> int:
     fetch_recipe(recipe, ref, src)
 
     test_dir = os.path.join(ROOT, "tests", recipe)
-    targets = []
+    overall = 0
+    ran = 0
     for stage in stages:
         fname = STAGE_FILES.get(stage)
         if not fname:
             print(f"unknown stage {stage}", file=sys.stderr)
             return 2
         path = os.path.join(test_dir, fname)
-        if os.path.exists(path):
-            targets.append(path)
-        else:
+        if not os.path.exists(path):
             print(f"  (skip {stage}: {path} not present)")
-    # also discover recipe-local tests later (D4); install stage first (M4)
-    if not targets:
+            continue
+        print(f"\n===== STAGE: {stage} =====", flush=True)
+        # each stage is its own pytest invocation => its own reported result (D2 separate stages)
+        rc = subprocess.call([sys.executable, "-m", "pytest", "-v", "-rA", path], cwd=ROOT)
+        ran += 1
+        if rc != 0:
+            overall = rc
+    if ran == 0:
         print("no stage test files found", file=sys.stderr)
         return 1
-
-    rc = subprocess.call([sys.executable, "-m", "pytest", "-v", "-rA", *targets], cwd=ROOT)
-    return rc
+    return overall
 
 
 if __name__ == "__main__":