cc-ci/runner/harness/lifecycle.py

"""App lifecycle for the CI harness: deploy, wait-healthy, teardown, janitor (plan §4.3).

The teardown guarantee is sacred: a failed test must never leak an app/volume/secret into the
next run. Callers wrap deploy()/teardown() in try/finally (or a pytest finalizer).
"""

from __future__ import annotations

import contextlib
import datetime
import json
import os
import re
import ssl
import subprocess
import time
import urllib.request

from . import abra

GATEWAY_IP = "143.244.213.108"  # *.ci.commoninternet.net -> gateway (TLS passthrough to cc-ci)
# A run app domain is "<recipe[:4]>-<6hex>.ci.commoninternet.net" (see DECISIONS.md). Used by the
# janitor to recognise orphaned run apps (infra apps like traefik/drone/backups don't match).
RUN_APP_RE = re.compile(r"^[a-z0-9]{1,4}-[0-9a-f]{6}\.ci\.commoninternet\.net$")


class TeardownError(RuntimeError):
    pass


def _docker_names(kind: str, stack: str) -> list[str]:
    """docker <kind> ls names filtered to a stack (kind: service|volume|secret)."""
    proc = subprocess.run(
        ["docker", kind, "ls", "--filter", f"name={stack}", "--format", "{{.Name}}"],
        capture_output=True,
        text=True,
    )
    return [n for n in proc.stdout.split("\n") if n.strip()]


def _residual(domain: str) -> dict:
    stack = _stack_name(domain)
    return {
        "services": _docker_names("service", stack),
        "volumes": _docker_names("volume", stack),
        "secrets": _docker_names("secret", stack),
    }


def _stack_age_seconds(stack: str) -> float | None:
    """Age of the stack's oldest service, or None if not present."""
    svcs = _docker_names("service", stack)
    if not svcs:
        return None
    oldest = None
    for s in svcs:
        p = subprocess.run(
            ["docker", "service", "inspect", s, "--format", "{{.CreatedAt}}"],
            capture_output=True,
            text=True,
        )
        ts = p.stdout.strip()
        try:
            # docker emits e.g. 2026-05-27 00:12:33.123 +0000 UTC -> take the leading 19 chars
            dt = datetime.datetime.strptime(ts[:19], "%Y-%m-%d %H:%M:%S").replace(
                tzinfo=datetime.UTC
            )
        except ValueError:
            continue
        age = (datetime.datetime.now(datetime.UTC) - dt).total_seconds()
        oldest = age if oldest is None else max(oldest, age)
    return oldest


def _recipe_extra_env(recipe: str, domain: str) -> dict[str, str]:
    """Per-recipe extra .env keys, applied at every deploy (install + upgrade's old_app) so a recipe
    with multi-domain / config needs is enrolled with NO shared-harness change (D5/M6.5). A recipe
    declares `EXTRA_ENV` in tests/<recipe>/recipe_meta.py as either a dict or a callable
    `EXTRA_ENV(domain) -> dict` (callable form lets it derive values from the per-run domain, e.g.
    cryptpad's SANDBOX_DOMAIN). Returns {} if none."""
    path = os.path.join(os.path.dirname(__file__), "..", "..", "tests", recipe, "recipe_meta.py")
    if not os.path.exists(path):
        return {}
    ns: dict = {}
    with open(path) as fh:
        exec(compile(fh.read(), path, "exec"), ns)  # noqa: S102 (trusted, in-repo)
    ee = ns.get("EXTRA_ENV")
    if callable(ee):
        ee = ee(domain)
    return {str(k): str(v) for k, v in (ee or {}).items()}


def _record_deploy() -> None:
    """Increment the per-run deploy counter (DG4.1: one deploy per run). No-op unless the
    orchestrator set CCCI_DEPLOY_COUNT_FILE — so it never affects standalone/manual use."""
    path = os.environ.get("CCCI_DEPLOY_COUNT_FILE")
    if not path:
        return
    n = 0
    with contextlib.suppress(OSError, ValueError), open(path) as f:
        n = int(f.read().strip() or "0")
    with contextlib.suppress(OSError), open(path, "w") as f:
        f.write(str(n + 1))


def _run_install_steps(hook: tuple[str, str], recipe: str, domain: str) -> None:
    """Run a recipe's custom install-steps hook (install_steps.sh) during the install tier — after
    `abra app new` + env defaults + secret generate, before deploy (Phase 1d DG5). The hook gets the
    app .env path + domain so it can insert secrets / set env / seed before the app comes up."""
    source, path = hook
    env_path = os.path.expanduser(f"~/.abra/servers/default/{domain}.env")
    print(f"  install-steps hook ({source}): {path}", flush=True)
    subprocess.run(
        ["bash", path],
        check=True,
        env=dict(
            os.environ,
            CCCI_APP_DOMAIN=domain,
            CCCI_RECIPE=recipe,
            CCCI_APP_ENV=env_path,
        ),
    )


def deploy_app(
    recipe: str,
    domain: str,
    version: str | None = None,
    secrets: bool = True,
    install_steps_hook: tuple[str, str] | None = None,
    deploy_timeout: int = 900,
) -> None:
    """Create + configure + deploy an app. Forces LETS_ENCRYPT_ENV='' so traefik serves the
    wildcard cert via the file provider and NEVER attempts ACME (adversary finding A1). Applies any
    per-recipe EXTRA_ENV (recipe_meta.py) and the custom install-steps hook (Phase 1d) before deploy.

    `deploy_timeout` is the subprocess timeout for `abra app deploy`. Caller (orchestrator) passes
    `recipe_meta.DEPLOY_TIMEOUT` so heavy recipes (ghost, matrix-synapse, lasuite-meet) can extend
    past the 900s default. abra's INTERNAL TIMEOUT (recipe's TIMEOUT env, default 300s) is set via
    EXTRA_ENV; this is the Python subprocess wrapper's timeout so abra doesn't get SIGKILLed mid-deploy."""
    _record_deploy()
    abra.app_config_remove(domain)  # clear any stale .env from a prior crashed run
    abra.app_new(recipe, domain, version=version, secrets=secrets)
    # A pinned version must actually deploy that version: check the recipe out to the tag so the
    # on-disk compose/.env match, and deploy NON-chaos below (chaos ignores the pin → deployed LATEST,
    # Adversary F1d-2). Chaos is correct ONLY for the version=None case (deploy the current PR-head
    # checkout). Order matters: checkout before secret_generate (-C) so secrets match the pinned tree.
    if version:
        abra.recipe_checkout(recipe, version)
    # Pin DOMAIN to the run domain explicitly. `abra app new -D` fills it for recipes whose
    # .env.sample uses a literal placeholder, but NOT for ones using a `{{ .Domain }}` Go-template
    # (this abra version leaves it unexpanded → deploy fails "can't evaluate field Domain"). Setting
    # it ourselves is recipe-agnostic and canonical (the run domain IS the app's domain).
    abra.env_set(domain, "DOMAIN", domain)
    abra.env_set(domain, "LETS_ENCRYPT_ENV", "")
    for k, v in _recipe_extra_env(recipe, domain).items():
        abra.env_set(domain, k, v)
    if secrets:
        abra.secret_generate(domain)
    if install_steps_hook:
        _run_install_steps(install_steps_hook, recipe, domain)
    abra.deploy(domain, chaos=(version is None), timeout=deploy_timeout)


def _stack_name(domain: str) -> str:
    # abra derives the swarm stack name from the domain by replacing dots with underscores
    # and KEEPING hyphens (e.g. custom-html-x.ci.commoninternet.net -> custom-html-x_ci_...).
    return domain.replace(".", "_")


def services_converged(domain: str) -> bool:
    """True when every service in the stack reports replicas N/N (N>0)."""
    stack = _stack_name(domain)
    proc = subprocess.run(
        ["docker", "stack", "services", stack, "--format", "{{.Replicas}}"],
        capture_output=True,
        text=True,
    )
    rows = [r for r in proc.stdout.split("\n") if r.strip()]
    if not rows:
        return False
    for r in rows:
        cur, _, want = r.partition("/")
        # A service at its DESIRED replica count is converged — including a `replicas: 0`
        # on-demand one-shot (e.g. lasuite-drive's `minio-createbuckets`, which is scaled up
        # manually only when buckets need (re)creating), which reports "0/0". The earlier
        # `want == "0"` rejection wrongly treated those as never-converged, hanging the deploy
        # forever. `cur == want` (with `want` present) is the correct convergence test; a service
        # still spinning up shows e.g. "0/1" (cur != want) and is correctly not-yet-converged.
        if not want or cur != want:
            return False
    return True


def http_get(domain: str, path: str = "/", timeout: int = 15) -> int:
    """HTTPS GET the app by its real hostname. On cc-ci the *.ci.commoninternet.net wildcard
    resolves (public DNS) to the gateway, which SNI-passthroughs to cc-ci's traefik — so using
    the real URL keeps SNI correct (connecting to the bare IP would drop SNI and fail to route)."""
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    req = urllib.request.Request(f"https://{domain}{path}", method="GET")
    try:
        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
            return resp.status
    except urllib.error.HTTPError as e:
        return e.code
    except Exception:
        return 0


def http_fetch(domain: str, path: str = "/", timeout: int = 15) -> tuple[int, str]:
    """One HTTPS GET → (status, body) in a SINGLE request, never raising. Lets a caller check the
    status and body together with no race between two requests (assert_serving) — and captures the
    error body on a 4xx/5xx instead of throwing."""
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    req = urllib.request.Request(f"https://{domain}{path}", method="GET")
    try:
        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
            return resp.status, resp.read().decode(errors="replace")
    except urllib.error.HTTPError as e:
        try:
            body = e.read().decode(errors="replace")
        except Exception:  # noqa: BLE001
            body = ""
        return e.code, body
    except Exception:  # noqa: BLE001
        return 0, ""


def wait_healthy(
    domain: str,
    ok_codes=(200, 301, 302),
    path: str = "/",
    deploy_timeout: int = 600,
    http_timeout: int = 300,
) -> None:
    """Wait for stack services converged, then for the app to answer ok over HTTPS at `path`.
    `path` is per-recipe (recipe_meta.HEALTH_PATH), e.g. keycloak uses /realms/master."""
    deadline = time.time() + deploy_timeout
    while time.time() < deadline:
        if services_converged(domain):
            break
        time.sleep(5)
    else:
        raise TimeoutError(f"{domain}: services did not converge in {deploy_timeout}s")

    deadline = time.time() + http_timeout
    last = 0
    while time.time() < deadline:
        last = http_get(domain, path)
        if last in ok_codes:
            return
        time.sleep(5)
    raise TimeoutError(f"{domain}: not healthy over HTTPS {path} (last status {last})")


def deployed_identity(domain: str, service: str = "app") -> dict[str, str | None]:
    """Identity of the running app service: {"version", "image", "chaos"}. Used to prove an upgrade
    actually MOVED the deployment (not a vacuous no-op — Adversary F1d-2), AND (Phase 1e HC1) that an
    `abra app deploy --chaos` upgrade actually deployed the PR-head code under test.

    - `version` = the `coop-cloud.<stack>.version` label (bumped per published recipe version).
    - `image`   = the running container image (usually bumps with a published version).
    - `chaos`   = the chaos deploy's recipe git commit. abra stamps `coop-cloud.<stack>.chaos-version`
      = the deployed recipe commit (e.g. "91b27ceb") + `coop-cloud.<stack>.chaos`="true" on a
      `--chaos` deploy; both are absent on a clean pinned-tag deploy. We prefer the `.chaos-version`
      commit — for prev→PR-head it IS the proof the PR-head code under test was deployed even when the
      version label is unbumped (HC1); fall back to the `.chaos` flag if no commit is present."""
    name = f"{_stack_name(domain)}_{service}"
    proc = subprocess.run(
        [
            "docker",
            "service",
            "inspect",
            name,
            "--format",
            "{{json .Spec.Labels}}|{{.Spec.TaskTemplate.ContainerSpec.Image}}",
        ],
        capture_output=True,
        text=True,
    )
    out = proc.stdout.strip()
    if "|" not in out:
        return {"version": None, "image": None, "chaos": None}
    labels_json, _, image = out.partition("|")
    ver = chaos = chaos_flag = None
    with contextlib.suppress(ValueError, json.JSONDecodeError):
        for k, v in json.loads(labels_json).items():
            if not k.startswith("coop-cloud."):
                continue
            if k.endswith(".version"):
                ver = v
            elif k.endswith(".chaos-version"):
                chaos = v  # the deployed recipe commit — the strongest signal
            elif k.endswith(".chaos"):
                chaos_flag = v
    return {"version": ver, "image": image.strip() or None, "chaos": chaos or chaos_flag}


def upgrade_app(domain: str, version: str | None = None) -> None:
    abra.upgrade(domain, version=version)


def recipe_head_commit(recipe: str) -> str | None:
    """The recipe checkout's current HEAD commit (captured right after fetch, before any version-tag
    checkout) so the upgrade tier can re-checkout the PR head for the chaos redeploy (HC1)."""
    return abra.recipe_head_commit(recipe)


def recipe_checkout_ref(recipe: str, ref: str) -> None:
    """git-checkout the recipe to an arbitrary ref/commit (HC1: restore the PR-head checkout before
    the chaos upgrade — the prev-tag base deploy reset it to the published tag)."""
    abra.recipe_checkout(recipe, ref)


def chaos_redeploy(domain: str) -> None:
    """In-place `abra app deploy --chaos`: redeploy the running app at the CURRENT recipe checkout
    (HC1: the PR-head code under test). This is the upgrade op, not a fresh install — it does NOT go
    through deploy_app, so the deploy-count guard (DG4.1) is not incremented."""
    abra.deploy(domain, chaos=True)


def backup_app(domain: str) -> str:
    """Create a backup; return the abra/restic output (carries the produced snapshot_id)."""
    return abra.backup_create(domain)


def restore_app(domain: str) -> None:
    abra.restore(domain)


def previous_version(recipe: str) -> str | None:
    """The second-newest published version (to deploy before upgrading to latest)."""
    vers = abra.recipe_versions(recipe)
    return vers[-2] if len(vers) >= 2 else None


def _app_container(domain: str, service: str = "app", timeout: int = 60) -> str:
    """The running container id for <stack>_<service>, with a BOUNDED POLL for it to (re)appear.
    A lifecycle op can briefly leave no running task — notably `abra app backup create`, where
    backup-bot-two stops/cycles the app container, so a mutate exec right after backup hit an empty
    `docker ps` and raised. Poll (no bare sleep) until the container is back or timeout."""
    name = f"{_stack_name(domain)}_{service}"
    deadline = time.time() + timeout
    while True:
        proc = subprocess.run(
            ["docker", "ps", "--filter", f"name={name}", "--format", "{{.ID}}"],
            capture_output=True,
            text=True,
        )
        cid = proc.stdout.strip().split("\n")[0]
        if cid:
            return cid
        if time.time() >= deadline:
            raise RuntimeError(f"no running container for {name} after {timeout}s")
        time.sleep(3)


def exec_in_app(domain: str, cmd: list[str], service: str = "app", timeout: int = 90) -> str:
    """Run `docker exec` in the app's container and return stdout. Hardened (Adversary F1e-1): a
    lifecycle op (backup/restore) cycles the container, so a freshly-resolved container can be
    mid-transition and `docker exec` FAILS — poll (re-resolving the container each try) until the exec
    succeeds (returncode 0) or timeout, then RAISE. Never silently return '' on a failed exec: that
    masked a container-cycle race as empty data, flipping a healthy recipe RED under opt-out (no
    accidental generic-pytest timing buffer) — and could mask a real failure as a pass elsewhere."""
    deadline = time.time() + timeout
    last = ""
    while True:
        cid = _app_container(domain, service)
        proc = subprocess.run(["docker", "exec", cid, *cmd], capture_output=True, text=True)
        if proc.returncode == 0:
            return proc.stdout
        last = (proc.stderr or proc.stdout).strip()
        if time.time() >= deadline:
            raise RuntimeError(
                f"docker exec in {domain}/{service} failed (rc={proc.returncode}) after {timeout}s: {last}"
            )
        time.sleep(3)


def http_body(domain: str, path: str = "/", timeout: int = 15) -> str:
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    req = urllib.request.Request(f"https://{domain}{path}", method="GET")
    with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
        return resp.read().decode(errors="replace")


def _force_stack_rm(stack: str, timeout: int = 120) -> None:
    """Remove a stack's services directly (no .env needed) and wait for them to disappear."""
    subprocess.run(["docker", "stack", "rm", stack], capture_output=True, text=True)
    deadline = time.time() + timeout
    while time.time() < deadline and _docker_names("service", stack):
        time.sleep(2)


def teardown_app(domain: str, verify: bool = True) -> None:
    """Full teardown with a docker fallback, then VERIFY nothing is left (raise otherwise).

    Order matters (A3): undeploy, then remove volumes/secrets *while the .env still exists* (abra
    needs it), then drop the .env LAST — and only after the stack is confirmed gone. If abra
    undeploy fails, fall back to `docker stack rm` (which needs no .env)."""
    stack = _stack_name(domain)
    abra.undeploy(domain)
    if _docker_names("service", stack):
        _force_stack_rm(stack)  # fallback: abra undeploy didn't clear it
    abra.volume_remove(domain)  # needs the .env -> before removing it
    abra.secret_remove_all(domain)
    # belt-and-suspenders: drop any volumes/secrets abra missed, by stack name. A volume can be
    # briefly held by a just-stopped task after `stack rm`, so retry the volume removal.
    deadline = time.time() + 60
    while time.time() < deadline:
        vols = _docker_names("volume", stack)
        if not vols:
            break
        for v in vols:
            subprocess.run(["docker", "volume", "rm", v], capture_output=True, text=True)
        if not _docker_names("volume", stack):
            break
        time.sleep(3)
    for s in _docker_names("secret", stack):
        subprocess.run(["docker", "secret", "rm", s], capture_output=True, text=True)
    abra.app_config_remove(domain)  # only now (stack gone) drop the .env

    if verify:
        residual = _residual(domain)
        if any(residual.values()):
            raise TeardownError(f"teardown left residual for {domain}: {residual}")


def janitor(max_age_seconds: int | None = None) -> None:
    """Reap orphaned run apps from crashed/rebooted runs. Matches the real naming scheme and only
    reaps apps older than max_age_seconds (so concurrent in-flight runs are never killed). Reaps via
    docker primitives so it works even when the .env is gone (A2/A3). Default 2h, env-overridable
    via CCCI_JANITOR_MAX_AGE (e.g. 0 to reap all matching orphans immediately)."""
    import os

    if max_age_seconds is None:
        max_age_seconds = int(os.environ.get("CCCI_JANITOR_MAX_AGE", "7200"))
    seen = set()
    for app in abra.app_ls():
        name = app.get("appName") or app.get("domain") or ""
        if RUN_APP_RE.match(name):
            seen.add(name)
    # also catch stacks whose .env was already deleted (abra ls won't list them)
    for svc in _docker_names("service", ""):
        # svc like cust-c95a69_ci_commoninternet_net_app -> reconstruct domain
        m = re.match(r"^([a-z0-9]{1,4}-[0-9a-f]{6})_ci_commoninternet_net_", svc)
        if m:
            seen.add(f"{m.group(1)}.ci.commoninternet.net")

    for name in seen:
        stack = _stack_name(name)
        age = _stack_age_seconds(stack)
        if age is not None and age < max_age_seconds:
            continue  # likely a concurrent in-flight run; leave it
        with contextlib.suppress(Exception):
            teardown_app(name, verify=False)