"""Known-good snapshot/restore of an app's data volumes (Phase 2w / WC3). A snapshot is a **raw copy of every docker volume belonging to an app's stack, taken while the app is UNDEPLOYED** (nothing is writing → consistent). Stored under `/var/lib/ci-warm//` as one last-known-good per slot, replaced atomically. Restore clears each volume and untars it back. Used by: - WC1.1 — snapshot keycloak's data volume BEFORE an auto-upgrade; restore on health-gate rollback (a forward DB migration makes a version-only rollback unsafe). - WC5 — promote-on-green-cold re-snapshots a canonical at teardown. Warm snapshots are **cache, excluded from the D8 reproducibility closure** (WC8) — re-seeded by cold runs, not restored on a VM rebuild. SLOT, not recipe (F-redfix-4). Warm state belongs to a **deployed stack**, not to a recipe name. A recipe can own TWO warm stacks at once — keycloak is both the live-warm shared OIDC provider (`warm-keycloak…`) and a data-warm canonical (`warm-canon-keycloak…`) — and they must never share a snapshot slot: `snapshot()` atomically REPLACES the slot, so a shared slot means each deployment destroys the other's known-good, and `restore()` (which requires every recorded volume to exist in the target stack) then raises. Callers therefore pass a **slot key**, not a bare recipe: - live-warm reconciler → `live_slot(recipe)` (`/`) - data-warm canonical → `canonical.canonical_slot(r)` (`/`, or `canon-/` when the recipe is also a live-warm provider) Every slot key `` maps 1:1 to the stack at `warm.stable_domain()`, so distinct slot ⇔ distinct stack. `canonical.py` derives its domain and its slot from ONE namespace so the two cannot drift apart. Layout (atomic dir swap of the `snapshot/` subdir; one last-good per slot). Sibling per-slot state (e.g. the reconciler's `last_good`) lives in `/` and is NOT clobbered by the swap: $CCCI_WARM_ROOT// last_good # (reconciler) the version known healthy — survives snapshot swaps snapshot/ meta.json # {slot, domain, commit, version, ts, volumes:[...]} volumes/.tar # raw tar of the volume root, one per stack volume Implementation note: volumes are tarred from their host mountpoint (`docker volume inspect -f '{{.Mountpoint}}'`), so no sidecar image pull is needed. The caller runs as root (the reconciler / runner on cc-ci) — direct mountpoint access is available there. """ from __future__ import annotations import json import os import shutil import subprocess from . import lifecycle DEFAULT_WARM_ROOT = "/var/lib/ci-warm" class SnapshotError(RuntimeError): pass def warm_root() -> str: """Root for warm snapshots; overridable via $CCCI_WARM_ROOT (tests).""" return os.environ.get("CCCI_WARM_ROOT", DEFAULT_WARM_ROOT) def live_slot(recipe: str) -> str: """Warm-state slot of the LIVE-warm reconciler's deployment (the stack at `warm-`). The canonical's slot is `canonical.canonical_slot()`, which diverges from this one exactly when the recipe is also a live-warm provider (F-redfix-4).""" return recipe def app_dir(slot: str) -> str: return os.path.join(warm_root(), slot) def snap_dir(slot: str) -> str: """The snapshot subdir — atomically swapped on update. Kept SEPARATE from app_dir so sibling per-slot state (the reconciler's last_good) survives a snapshot swap.""" return os.path.join(app_dir(slot), "snapshot") def meta_path(slot: str) -> str: return os.path.join(snap_dir(slot), "meta.json") def volumes_dir(slot: str) -> str: return os.path.join(snap_dir(slot), "volumes") def has_snapshot(slot: str) -> bool: """True iff a complete last-good snapshot (meta + at least its declared volume tars) exists.""" meta = read_meta(slot) if not meta: return False for v in meta.get("volumes", []): if not os.path.isfile(os.path.join(volumes_dir(slot), f"{v}.tar")): return False return True def read_meta(slot: str) -> dict | None: try: with open(meta_path(slot)) as f: return json.load(f) except (OSError, ValueError): return None def _run(cmd: list[str], timeout: int = 600) -> subprocess.CompletedProcess: return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) def _volume_mountpoint(volume: str) -> str: r = _run(["docker", "volume", "inspect", "-f", "{{.Mountpoint}}", volume], timeout=30) mp = r.stdout.strip() if r.returncode != 0 or not mp: raise SnapshotError(f"cannot inspect volume {volume}: {r.stderr.strip()}") return mp def stack_volumes(domain: str) -> list[str]: """Names of the docker volumes belonging to the app's stack (reuses lifecycle's stack scan).""" stack = lifecycle._stack_name(domain) # noqa: SLF001 — shared internal in our package return sorted(lifecycle._docker_names("volume", stack)) # noqa: SLF001 def _assert_undeployed(domain: str) -> None: """Snapshots/restores must happen while the app is UNDEPLOYED (consistency + safe volume writes). Raise if any service of the stack is still running.""" stack = lifecycle._stack_name(domain) # noqa: SLF001 svcs = lifecycle._docker_names("service", stack) # noqa: SLF001 if svcs: raise SnapshotError( f"refusing to snapshot/restore {domain} while deployed (services: {svcs}); " "undeploy first (WC3: snapshot while undeployed)" ) def _assert_slot_not_foreign(slot: str, domain: str) -> None: """Refuse to touch a slot that already holds another DOMAIN's known-good (F-redfix-4). Defence in depth behind the slot separation itself: naming-scheme-independent, it catches any future caller that pairs a slot with the wrong stack. Fails BEFORE the destructive swap rather than at the next restore. A slot with no snapshot yet is free to claim.""" meta = read_meta(slot) if meta and meta.get("domain") and meta["domain"] != domain: raise SnapshotError( f"warm slot {slot!r} holds the known-good of {meta['domain']!r}, not {domain!r}: " "refusing to clobber another deployment's snapshot (F-redfix-4). " "Live-warm and data-warm deployments of one recipe need distinct slots." ) def snapshot(slot: str, domain: str, commit: str | None = None, version: str | None = None) -> dict: """Take a last-known-good snapshot of every data volume of 's stack into . The app MUST be undeployed. Atomically replaces the prior last-good in that slot. Returns the meta dict. `slot` must be the warm-state slot OWNING `domain` (see module docstring) — passing another deployment's slot would destroy its known-good, so the pairing is asserted below.""" _assert_slot_not_foreign(slot, domain) _assert_undeployed(domain) volumes = stack_volumes(domain) if not volumes: raise SnapshotError(f"no volumes found for {domain} — nothing to snapshot") os.makedirs(app_dir(slot), exist_ok=True) staging = os.path.join(app_dir(slot), ".snapshot.staging") shutil.rmtree(staging, ignore_errors=True) os.makedirs(os.path.join(staging, "volumes"), exist_ok=True) for vol in volumes: mp = _volume_mountpoint(vol) tar_path = os.path.join(staging, "volumes", f"{vol}.tar") # Tar the volume contents (relative to the mountpoint) so restore can untar back in place. r = _run(["tar", "-C", mp, "-cf", tar_path, "."]) if r.returncode != 0: shutil.rmtree(staging, ignore_errors=True) raise SnapshotError(f"tar of volume {vol} failed: {r.stderr.strip()}") meta = { "slot": slot, "domain": domain, "commit": commit, "version": version, "volumes": volumes, "ts": _now(), } with open(os.path.join(staging, "meta.json"), "w") as f: json.dump(meta, f) # Atomic-ish swap of the snapshot subdir only (sibling state like last_good is untouched). target = snap_dir(slot) old = os.path.join(app_dir(slot), ".snapshot.old") shutil.rmtree(old, ignore_errors=True) if os.path.exists(target): os.rename(target, old) os.rename(staging, target) shutil.rmtree(old, ignore_errors=True) return meta def restore(slot: str, domain: str) -> dict: """Restore 's last-known-good snapshot into 's stack volumes. The app MUST be undeployed. Clears each volume then untars the snapshot back. Returns the snapshot meta. Raises if no snapshot exists, the slot belongs to another domain, or a snapshot volume is missing from the current stack.""" _assert_undeployed(domain) meta = read_meta(slot) if not meta or not has_snapshot(slot): raise SnapshotError(f"no complete snapshot in slot {slot!r} to restore") _assert_slot_not_foreign(slot, domain) current = set(stack_volumes(domain)) for vol in meta.get("volumes", []): tar_path = os.path.join(volumes_dir(slot), f"{vol}.tar") if vol not in current: raise SnapshotError( f"snapshot volume {vol} absent from current stack {sorted(current)}" ) mp = _volume_mountpoint(vol) # Clear the volume contents (incl. dotfiles) without removing the mountpoint itself. r = _run(["sh", "-c", f'rm -rf -- "{mp}"/* "{mp}"/.[!.]* "{mp}"/..?* 2>/dev/null; true']) r = _run(["tar", "-C", mp, "-xf", tar_path]) if r.returncode != 0: raise SnapshotError(f"untar of volume {vol} failed: {r.stderr.strip()}") return meta def _now() -> str: # Import here (not module top) — keeps the pure path/meta helpers importable in restricted envs. import datetime return datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"