recipe-maintainer: public snapshot (secrets + deployment plans removed, single commit)

Sanitized single-commit public mirror of recipe-maintainer.
- Removed test-ssh/.testenv (live creds); added test-ssh/.testenv.example placeholders.
- Removed plans/ and planned-updates/ (deployment-planning docs) so no client/
  deployment domains appear in the public repo.
- All other secret stores were already gitignored.
- docs.coopcloud.tech retained as a submodule (public upstream).

lib/__init__.py

@@ -0,0 +1,4 @@
+"""recipe-maintainer-2 core library.
+
+Stdlib-only Python modules for Co-op Cloud recipe maintenance.
+"""

lib/abra.py

@@ -0,0 +1,291 @@
+"""Wrapper for the abra CLI.
+
+Encapsulates TTY wrapping, --chaos flag, --no-input, timeout handling,
+and the Linux `timeout` command guard.
+
+All abra commands go through this module so that learnings (TTY requirements,
+--chaos caveats, timeout behaviors) are encoded in one place.
+"""
+
+import json
+import subprocess
+from dataclasses import dataclass
+
+
+# ---------------------------------------------------------------------------
+# Result type
+# ---------------------------------------------------------------------------
+
+@dataclass
+class AbraResult:
+    """Result of a shell/abra command."""
+    returncode: int
+    stdout: str
+    stderr: str
+    timed_out: bool
+    command: str
+
+    @property
+    def ok(self) -> bool:
+        return self.returncode == 0 and not self.timed_out
+
+    def json(self) -> list | dict | None:
+        """Try to parse stdout as JSON."""
+        try:
+            return json.loads(self.stdout)
+        except (json.JSONDecodeError, ValueError):
+            return None
+
+    def jsonl(self) -> list[list | dict]:
+        """Parse stdout as JSON Lines (one JSON document per line)."""
+        results = []
+        for line in self.stdout.strip().split("\n"):
+            if line.strip():
+                try:
+                    results.append(json.loads(line))
+                except (json.JSONDecodeError, ValueError):
+                    pass
+        return results
+
+
+# ---------------------------------------------------------------------------
+# TTY requirements from learnings.md
+# ---------------------------------------------------------------------------
+
+# These abra subcommands require a TTY wrapper (script -qefc).
+# Without it they fail with "not a TTY" or hang indefinitely.
+_TTY_COMMANDS = {
+    "secret insert", "secret remove", "secret generate",
+    "volume remove",
+    "cmd",
+    "backup create", "backup snapshots", "backup list",
+    "restore",
+    "recipe lint",
+}
+
+
+def _needs_tty(args: str) -> bool:
+    """Check if an abra command needs the TTY wrapper."""
+    # Normalize: "app secret insert ..." → check "secret insert"
+    parts = args.strip()
+    # Strip leading "app " if present
+    if parts.startswith("app "):
+        parts = parts[4:]
+    for cmd in _TTY_COMMANDS:
+        if parts.startswith(cmd):
+            return True
+    return False
+
+
+# ---------------------------------------------------------------------------
+# Core execution
+# ---------------------------------------------------------------------------
+
+def run(cmd: str, *, check: bool = True, timeout: int = 120) -> AbraResult:
+    """Run a shell command with a hard timeout.
+
+    Uses the Linux `timeout` command to guarantee the process tree is
+    killed after `timeout` seconds, even if the process ignores signals.
+    subprocess.run gets a slightly longer timeout as a fallback.
+    """
+    wrapped = f"timeout --kill-after=5 {timeout} {cmd}"
+    print(f"  $ {cmd}", flush=True)
+    try:
+        result = subprocess.run(
+            wrapped, shell=True, capture_output=True, text=True,
+            timeout=timeout + 15,
+        )
+    except subprocess.TimeoutExpired:
+        print(f"    TIMEOUT after {timeout}s (subprocess fallback)", flush=True)
+        if check:
+            raise RuntimeError(f"Command timed out after {timeout}s: {cmd}")
+        return AbraResult(
+            returncode=124, stdout="", stderr="",
+            timed_out=True, command=cmd,
+        )
+
+    timed_out = result.returncode == 124
+    if result.stdout.strip():
+        for line in result.stdout.strip().split("\n"):
+            print(f"    {line}", flush=True)
+    if result.returncode != 0:
+        if result.stderr.strip():
+            for line in result.stderr.strip().split("\n"):
+                print(f"    stderr: {line}", flush=True)
+        if timed_out:
+            print(f"    TIMEOUT after {timeout}s", flush=True)
+            if check:
+                raise RuntimeError(f"Command timed out after {timeout}s: {cmd}")
+        elif check:
+            raise RuntimeError(
+                f"Command failed (exit {result.returncode}): {cmd}"
+            )
+
+    return AbraResult(
+        returncode=result.returncode,
+        stdout=result.stdout,
+        stderr=result.stderr,
+        timed_out=timed_out,
+        command=cmd,
+    )
+
+
+def abra(args: str, *, tty_wrap: bool | None = None,
+         check: bool = True, timeout: int = 120,
+         chaos: bool = True) -> AbraResult:
+    """Run an abra command.
+
+    - tty_wrap: wrap with script -qefc for commands that need TTY.
+                If None, auto-detects from the command.
+    - chaos: append --chaos (default True for local dev)
+    - Always appends --no-input
+    """
+    flags = "--no-input"
+    if chaos and "--chaos" not in args:
+        flags += " --chaos"
+
+    cmd = f"abra {args} {flags}".strip()
+
+    if tty_wrap is None:
+        tty_wrap = _needs_tty(args)
+
+    if tty_wrap:
+        cmd = f'script -qefc "{cmd}" /dev/null 2>&1'
+
+    return run(cmd, check=check, timeout=timeout)
+
+
+# ---------------------------------------------------------------------------
+# High-level abra operations
+# ---------------------------------------------------------------------------
+
+def app_deploy(domain: str, *, force: bool = True,
+               chaos: bool = True, timeout: int = 60) -> AbraResult:
+    """Deploy an app."""
+    flags = f"app deploy {domain}"
+    if force:
+        flags += " --force"
+    return abra(flags, chaos=chaos, check=False, timeout=timeout)
+
+
+def app_undeploy(domain: str, *, timeout: int = 60) -> AbraResult:
+    """Undeploy an app."""
+    return abra(f"app undeploy {domain}", chaos=False, check=False, timeout=timeout)
+
+
+def app_new(recipe: str, server: str, domain: str, *,
+            chaos: bool = True, timeout: int = 60) -> AbraResult:
+    """Create a new app instance."""
+    return abra(
+        f"app new {recipe} --server {server} --domain {domain}",
+        chaos=chaos, timeout=timeout,
+    )
+
+
+def app_ps(domain: str, *, chaos: bool = True) -> AbraResult:
+    """Get app process status (machine-readable)."""
+    return abra(f"app ps {domain} -m", chaos=chaos, check=False, timeout=30)
+
+
+def app_secret_generate(domain: str, *,
+                        chaos: bool = True, timeout: int = 60) -> AbraResult:
+    """Generate all secrets for an app."""
+    return abra(
+        f"app secret generate {domain} --all",
+        chaos=chaos, check=False, timeout=timeout,
+    )
+
+
+def app_secret_insert(domain: str, name: str, version: str,
+                      value: str, *, chaos: bool = True,
+                      timeout: int = 60) -> AbraResult:
+    """Insert a specific secret."""
+    return abra(
+        f"app secret insert {domain} {name} {version} {value}",
+        chaos=chaos, check=False, timeout=timeout,
+    )
+
+
+def app_secret_remove_all(domain: str, *,
+                          chaos: bool = True,
+                          timeout: int = 60) -> AbraResult:
+    """Remove all secrets for an app."""
+    return abra(
+        f"app secret remove {domain} --all",
+        chaos=chaos, check=False, timeout=timeout,
+    )
+
+
+def app_volume_remove(domain: str, *, timeout: int = 60) -> AbraResult:
+    """Remove all volumes for an app."""
+    return abra(
+        f"app volume remove {domain} --force",
+        chaos=False, check=False, timeout=timeout,
+    )
+
+
+def app_cmd(domain: str, service: str, cmd_name: str, *,
+            chaos: bool = True, timeout: int = 120) -> AbraResult:
+    """Run an abra app cmd."""
+    return abra(
+        f"app cmd {domain} {service} {cmd_name}",
+        chaos=chaos, check=False, timeout=timeout,
+    )
+
+
+def app_ls(server: str, *, timeout: int = 30) -> AbraResult:
+    """List all apps on a server (machine-readable)."""
+    return abra(f"app ls -s {server} -S -m", chaos=False, check=False,
+                timeout=timeout)
+
+
+def app_backup_create(domain: str, *,
+                      chaos: bool = True, timeout: int = 120) -> AbraResult:
+    """Create a backup of an app."""
+    return abra(
+        f"app backup create {domain}",
+        chaos=chaos, check=False, timeout=timeout,
+    )
+
+
+def app_backup_restore(domain: str, *,
+                       chaos: bool = True, timeout: int = 120) -> AbraResult:
+    """Restore an app from backup."""
+    return abra(
+        f"app restore {domain}",
+        chaos=chaos, check=False, timeout=timeout,
+    )
+
+
+def recipe_fetch(recipe: str, *, force: bool = True,
+                 timeout: int = 60) -> AbraResult:
+    """Fetch a recipe from upstream."""
+    flags = f"recipe fetch {recipe}"
+    if force:
+        flags += " --force"
+    return abra(flags, chaos=False, timeout=timeout)
+
+
+def recipe_versions(recipe: str) -> AbraResult:
+    """List recipe versions (machine-readable)."""
+    return abra(f"recipe versions {recipe} -m", chaos=False, check=False,
+                timeout=30)
+
+
+def recipe_upgrade(recipe: str, *, dry_run: bool = True) -> AbraResult:
+    """Check for recipe upgrades (machine-readable)."""
+    flags = f"recipe upgrade {recipe} -m"
+    if dry_run:
+        flags += " -n"
+    return abra(flags, chaos=False, check=False, timeout=30)
+
+
+def recipe_lint(recipe: str) -> AbraResult:
+    """Lint a recipe."""
+    return abra(f"recipe lint {recipe} -C", chaos=False, check=False,
+                timeout=30)
+
+
+def recipe_diff(recipe: str) -> AbraResult:
+    """Show local changes in a recipe checkout."""
+    return abra(f"recipe diff {recipe}", chaos=False, check=False, timeout=15)

lib/authentik.py

@@ -0,0 +1,199 @@
+"""Authentik admin API client.
+
+Provides an AuthentikAdmin class for managing OAuth2 providers, applications,
+and users via the Authentik REST API. Used by setup_authentik_integration.py
+and setup_docs_integration.py scripts.
+"""
+
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+class AuthentikAdmin:
+    """Client for the Authentik Admin REST API."""
+
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self.api_url = f"{self.base_url}/api/v3"
+        self.token = token
+
+    def _request(self, method: str, endpoint: str, *,
+                 data: dict | None = None,
+                 timeout: int = 30) -> tuple[int, dict | list | None]:
+        """Make an authenticated API request."""
+        url = f"{self.api_url}{endpoint}"
+        body = json.dumps(data).encode() if data is not None else None
+        req = urllib.request.Request(url, data=body, method=method)
+        req.add_header("Authorization", f"Bearer {self.token}")
+        req.add_header("Content-Type", "application/json")
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                raw = resp.read()
+                if not raw:
+                    return resp.getcode(), None
+                return resp.getcode(), json.loads(raw)
+        except urllib.error.HTTPError as e:
+            try:
+                raw = e.read().decode(errors="replace")
+                return e.code, json.loads(raw) if raw.strip() else None
+            except Exception:
+                return e.code, None
+
+    def _get_first(self, endpoint: str, match_field: str,
+                   match_value: str) -> dict | None:
+        """GET a list endpoint and return the first result matching a field."""
+        _, data = self._request("GET", endpoint)
+        if not data or "results" not in data:
+            return None
+        for r in data["results"]:
+            if r.get(match_field) == match_value:
+                return r
+        return None
+
+    def resolve_uuids(self) -> dict:
+        """Look up Authentik flow/mapping/certificate UUIDs dynamically."""
+        print("=== Resolving Authentik UUIDs ===", flush=True)
+        uuids = {}
+
+        lookups = [
+            ("authorization_flow",
+             "/flows/instances/?slug=default-provider-authorization-implicit-consent"),
+            ("invalidation_flow",
+             "/flows/instances/?slug=default-provider-invalidation-flow"),
+            ("authentication_flow",
+             "/flows/instances/?slug=default-authentication-flow"),
+            ("signing_key",
+             "/crypto/certificatekeypairs/?name=authentik+Self-signed+Certificate"),
+            ("scope_openid",
+             "/propertymappings/all/?managed=goauthentik.io/providers/oauth2/scope-openid"),
+            ("scope_email",
+             "/propertymappings/all/?managed=goauthentik.io/providers/oauth2/scope-email"),
+            ("scope_profile",
+             "/propertymappings/all/?managed=goauthentik.io/providers/oauth2/scope-profile"),
+        ]
+
+        for name, endpoint in lookups:
+            _, data = self._request("GET", endpoint)
+            pk = data["results"][0]["pk"]
+            uuids[name] = pk
+            print(f"  {name}: {pk}", flush=True)
+
+        return uuids
+
+    def ensure_provider(self, name: str, client_id: str,
+                        redirect_uris: list[dict],
+                        uuids: dict) -> tuple[int, str]:
+        """Create an OAuth2 provider if needed. Returns (provider_pk, client_secret)."""
+        print(f"=== Ensure OAuth2 provider '{name}' ===", flush=True)
+
+        existing = self._get_first(
+            f"/providers/oauth2/?search={name}", "name", name)
+
+        if existing:
+            pk = existing["pk"]
+            print(f"  Provider '{name}' already exists (pk: {pk}), fetching secret",
+                  flush=True)
+            _, provider_data = self._request("GET", f"/providers/oauth2/{pk}/")
+            client_secret = provider_data["client_secret"]
+        else:
+            _, resp = self._request("POST", "/providers/oauth2/", data={
+                "name": name,
+                "client_type": "confidential",
+                "client_id": client_id,
+                "authorization_flow": uuids["authorization_flow"],
+                "authentication_flow": uuids["authentication_flow"],
+                "invalidation_flow": uuids["invalidation_flow"],
+                "redirect_uris": redirect_uris,
+                "property_mappings": [
+                    uuids["scope_openid"],
+                    uuids["scope_email"],
+                    uuids["scope_profile"],
+                ],
+                "signing_key": uuids["signing_key"],
+                "include_claims_in_id_token": True,
+            })
+            pk = resp["pk"]
+            client_secret = resp["client_secret"]
+            print(f"  Created provider (pk: {pk})", flush=True)
+
+        print(f"  Client ID: {client_id}", flush=True)
+        print(f"  Client secret: {client_secret}", flush=True)
+        return pk, client_secret
+
+    def ensure_application(self, name: str, slug: str,
+                           provider_pk: int, launch_url: str) -> None:
+        """Create an application if it doesn't exist."""
+        print(f"=== Ensure application '{slug}' ===", flush=True)
+
+        existing = self._get_first(
+            f"/core/applications/?slug={slug}", "slug", slug)
+
+        if existing:
+            print(f"  Application '{slug}' already exists, skipping", flush=True)
+        else:
+            self._request("POST", "/core/applications/", data={
+                "name": name,
+                "slug": slug,
+                "provider": provider_pk,
+                "meta_launch_url": launch_url,
+            })
+            print(f"  Created application '{slug}'", flush=True)
+
+    def ensure_user(self, username: str, email: str,
+                    password: str) -> int:
+        """Create a user if needed, set password. Returns user pk."""
+        print(f"=== Ensure user '{username}' ===", flush=True)
+
+        existing = self._get_first(
+            f"/core/users/?search={username}", "username", username)
+
+        if existing:
+            user_pk = existing["pk"]
+            print(f"  User '{username}' already exists (pk: {user_pk})",
+                  flush=True)
+        else:
+            _, resp = self._request("POST", "/core/users/", data={
+                "username": username,
+                "name": "Test User",
+                "email": email,
+                "is_active": True,
+            })
+            user_pk = resp["pk"]
+            print(f"  Created user '{username}' (pk: {user_pk})", flush=True)
+
+        # Set password
+        print(f"  Setting password for '{username}' ...", flush=True)
+        self._request(
+            "POST", f"/core/users/{user_pk}/set_password/",
+            data={"password": password},
+        )
+        print("  Password set", flush=True)
+        return user_pk
+
+    def ensure_app_password(self, user_pk: int,
+                            identifier: str = "testuser-app-password") -> str:
+        """Create an APP_PASSWORD token for password grant. Returns the key."""
+        print("  Creating APP_PASSWORD token for password grant ...", flush=True)
+
+        # Delete existing token if present
+        existing = self._get_first(
+            f"/core/tokens/?identifier={identifier}", "identifier", identifier)
+        if existing:
+            self._request("DELETE", f"/core/tokens/{identifier}/")
+            print("  Deleted existing APP_PASSWORD token", flush=True)
+
+        self._request("POST", "/core/tokens/", data={
+            "identifier": identifier,
+            "intent": "app_password",
+            "user": user_pk,
+            "description": "Test user app password for OIDC password grant",
+            "expiring": False,
+        })
+
+        _, key_data = self._request(
+            "GET", f"/core/tokens/{identifier}/view_key/")
+        app_password = key_data["key"]
+        print(f"  APP_PASSWORD created: {app_password[:10]}...", flush=True)
+        return app_password

lib/config.py

@@ -0,0 +1,175 @@
+"""Central configuration for the recipe-maintainer-3 project.
+
+Resolves the workspace root, provides paths to key directories,
+and loads instance/deployment configuration from TOML files.
+
+All paths are resolved lazily on first access and cached.
+"""
+
+import os
+import tomllib
+from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# Workspace root detection
+# ---------------------------------------------------------------------------
+
+_workspace: Path | None = None
+
+
+def get_workspace() -> Path:
+    """Auto-detect the workspace root by walking up to find AGENTS.md."""
+    global _workspace
+    if _workspace is not None:
+        return _workspace
+
+    # Start from this file's location (lib/config.py → lib/ → workspace root)
+    candidate = Path(__file__).resolve().parent.parent
+    if (candidate / "AGENTS.md").exists():
+        _workspace = candidate
+        return _workspace
+
+    # Fallback: walk up from cwd
+    candidate = Path.cwd()
+    for _ in range(10):
+        if (candidate / "AGENTS.md").exists():
+            _workspace = candidate
+            return _workspace
+        parent = candidate.parent
+        if parent == candidate:
+            break
+        candidate = parent
+
+    raise RuntimeError(
+        "Could not find workspace root (no AGENTS.md found). "
+        "Are you running from within the recipe-maintainer-3 directory?"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Path constants (lazy)
+# ---------------------------------------------------------------------------
+
+def _ws() -> Path:
+    return get_workspace()
+
+
+class _Paths:
+    """Lazy path accessors for key project directories."""
+
+    @property
+    def WORKSPACE(self) -> Path:
+        return _ws()
+
+    @property
+    def LIB_DIR(self) -> Path:
+        return _ws() / "lib"
+
+    @property
+    def SCRIPTS_DIR(self) -> Path:
+        return _ws() / "scripts"
+
+    @property
+    def RECIPE_INFO_DIR(self) -> Path:
+        return _ws() / "recipe-info"
+
+    @property
+    def TESTSECRETS_DIR(self) -> Path:
+        return _ws() / "recipe-info" / "testsecrets"
+
+    @property
+    def LOGS_DIR(self) -> Path:
+        return _ws() / "logs"
+
+    @property
+    def PLANS_DIR(self) -> Path:
+        return _ws() / "plans"
+
+    @property
+    def PLANNED_UPDATES_DIR(self) -> Path:
+        return _ws() / "planned-updates"
+
+    @property
+    def TERRAFORM_DIR(self) -> Path:
+        return _ws() / "terraform"
+
+    @property
+    def ABRA_DIR(self) -> Path:
+        env = os.environ.get("ABRA_DIR")
+        if env:
+            return Path(env)
+        return Path.home() / ".abra"
+
+    @property
+    def ABRA_RECIPES_DIR(self) -> Path:
+        return self.ABRA_DIR / "recipes"
+
+    @property
+    def ABRA_SERVERS_DIR(self) -> Path:
+        return self.ABRA_DIR / "servers"
+
+
+paths = _Paths()
+
+
+# ---------------------------------------------------------------------------
+# Settings
+# ---------------------------------------------------------------------------
+
+def load_settings() -> dict:
+    """Load settings.toml from the workspace root."""
+    p = paths.WORKSPACE / "settings.toml"
+    if not p.exists():
+        return {}
+    with open(p, "rb") as f:
+        return tomllib.load(f)
+
+
+# ---------------------------------------------------------------------------
+# Instance loading (from settings.toml [instances.*])
+# ---------------------------------------------------------------------------
+
+def _load_instances_from_settings() -> dict:
+    """Load the [instances] section from settings.toml."""
+    settings = load_settings()
+    return settings.get("instances", {})
+
+
+def get_instance_names() -> list[str]:
+    """Return all instance names from settings.toml [instances.*]."""
+    return list(_load_instances_from_settings().keys())
+
+
+def get_default_instance_name() -> str:
+    """Return the default instance from settings.toml."""
+    settings = load_settings()
+    default = settings.get("default_instance")
+    if not default:
+        raise RuntimeError("No default_instance set in settings.toml")
+    if default not in get_instance_names():
+        raise RuntimeError(
+            f"default_instance '{default}' from settings.toml "
+            f"not found in settings.toml [instances]"
+        )
+    return default
+
+
+def load_instance_toml(name: str) -> dict:
+    """Load raw TOML data for a specific instance from settings.toml."""
+    instances = _load_instances_from_settings()
+    if name not in instances:
+        raise ValueError(f"Instance '{name}' not found in settings.toml [instances]")
+    data = dict(instances[name])
+    data["name"] = name
+    return data
+
+
+def load_recipe_toml(recipe_name: str) -> dict:
+    """Load recipe-info/<recipe>/recipe.toml."""
+    p = paths.RECIPE_INFO_DIR / recipe_name / "recipe.toml"
+    if not p.exists():
+        return {"name": recipe_name}
+    with open(p, "rb") as f:
+        data = tomllib.load(f)
+    data.setdefault("name", recipe_name)
+    return data

lib/env.py

@@ -0,0 +1,83 @@
+"""Read and write abra .env files and recipe-info config files.
+
+Handles the specific format of abra .env files: KEY=VALUE lines,
+with support for comments (#) and uncommenting keys.
+"""
+
+from pathlib import Path
+
+
+def read_env_file(path: str | Path) -> dict[str, str]:
+    """Read a KEY=VALUE env file into a dict.
+
+    Ignores empty lines and comments (#).
+    """
+    result = {}
+    p = Path(path)
+    if not p.exists():
+        return result
+    with open(p) as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            if "=" in line:
+                key, value = line.split("=", 1)
+                result[key.strip()] = value.strip()
+    return result
+
+
+def write_env_file(path: str | Path, data: dict[str, str]) -> None:
+    """Write a dict as a KEY=VALUE env file."""
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    with open(p, "w") as f:
+        for key, value in data.items():
+            f.write(f"{key}={value}\n")
+
+
+def apply_env_overrides(env_path: str | Path,
+                        overrides: dict[str, str]) -> None:
+    """Set or uncomment values in an abra .env file.
+
+    For each key in overrides:
+    - If the key exists (even commented out), replace the line
+    - If the key doesn't exist, append it
+    """
+    p = Path(env_path)
+    if not p.exists():
+        write_env_file(p, overrides)
+        return
+
+    with open(p) as f:
+        lines = f.readlines()
+
+    remaining = dict(overrides)
+    new_lines = []
+    for line in lines:
+        stripped = line.strip()
+        matched = False
+        for key, value in list(remaining.items()):
+            # Match "KEY=...", "#KEY=...", or "# KEY=..."
+            if stripped.lstrip("#").strip().startswith(f"{key}="):
+                new_lines.append(f"{key}={value}\n")
+                remaining.pop(key)
+                matched = True
+                print(f"  env: {key}={value}", flush=True)
+                break
+        if not matched:
+            new_lines.append(line)
+
+    # Append any keys that weren't found in the file
+    for key, value in remaining.items():
+        new_lines.append(f"{key}={value}\n")
+        print(f"  env: {key}={value} (appended)", flush=True)
+
+    with open(p, "w") as f:
+        f.writelines(new_lines)
+
+
+def get_abra_env_path(server: str, domain: str) -> Path:
+    """Return the path to an abra app's .env file."""
+    from lib.config import paths
+    return paths.ABRA_SERVERS_DIR / server / f"{domain}.env"

lib/keycloak.py

@@ -0,0 +1,195 @@
+"""Keycloak admin API client.
+
+Provides a KeycloakAdmin class for managing realms, OIDC clients, and users
+via the Keycloak REST API. Used by setup_keycloak_integration.py scripts.
+"""
+
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+class KeycloakAdmin:
+    """Client for the Keycloak Admin REST API."""
+
+    def __init__(self, base_url: str, admin_user: str, admin_password: str):
+        self.base_url = base_url.rstrip("/")
+        self.admin_user = admin_user
+        self.admin_password = admin_password
+
+    def _request(self, method: str, path: str, *,
+                 data: dict | None = None,
+                 headers: dict | None = None,
+                 timeout: int = 30) -> tuple[int, dict | list | None]:
+        """Make an HTTP request, return (status_code, parsed_json)."""
+        url = f"{self.base_url}{path}"
+        body = json.dumps(data).encode() if data is not None else None
+        req = urllib.request.Request(url, data=body, method=method)
+        req.add_header("Content-Type", "application/json")
+        for k, v in (headers or {}).items():
+            req.add_header(k, v)
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                raw = resp.read()
+                if not raw:
+                    return resp.getcode(), None
+                return resp.getcode(), json.loads(raw)
+        except urllib.error.HTTPError as e:
+            try:
+                raw = e.read().decode(errors="replace")
+                return e.code, json.loads(raw) if raw.strip() else None
+            except Exception:
+                return e.code, None
+
+    def get_admin_token(self) -> str:
+        """Get an admin access token from the master realm."""
+        url = f"{self.base_url}/realms/master/protocol/openid-connect/token"
+        body = urllib.parse.urlencode({
+            "username": self.admin_user,
+            "password": self.admin_password,
+            "grant_type": "password",
+            "client_id": "admin-cli",
+        }).encode()
+        req = urllib.request.Request(url, data=body, method="POST")
+        req.add_header("Content-Type", "application/x-www-form-urlencoded")
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            data = json.loads(resp.read())
+        return data["access_token"]
+
+    def _auth_headers(self) -> dict:
+        token = self.get_admin_token()
+        return {"Authorization": f"Bearer {token}"}
+
+    def ensure_realm(self, realm: str) -> None:
+        """Create a realm if it doesn't exist."""
+        print(f"=== Ensure Keycloak realm '{realm}' ===", flush=True)
+        headers = self._auth_headers()
+        status, _ = self._request(
+            "GET", f"/admin/realms/{realm}", headers=headers)
+        if status == 404:
+            status, _ = self._request("POST", "/admin/realms", data={
+                "realm": realm,
+                "enabled": True,
+                "registrationAllowed": False,
+            }, headers=headers)
+            print(f"  Created realm '{realm}'", flush=True)
+        else:
+            print(f"  Realm '{realm}' already exists, skipping", flush=True)
+
+    def ensure_client(self, realm: str, client_id: str,
+                      redirect_uris: list[str],
+                      web_origins: list[str]) -> tuple[str, str]:
+        """Create an OIDC client if needed, return (client_uuid, client_secret)."""
+        print(f"=== Ensure OIDC client '{client_id}' ===", flush=True)
+        headers = self._auth_headers()
+
+        # Check if client exists
+        status, data = self._request(
+            "GET",
+            f"/admin/realms/{realm}/clients?clientId={client_id}",
+            headers=headers,
+        )
+        if data and len(data) > 0:
+            client_uuid = data[0]["id"]
+            print(f"  Client '{client_id}' already exists, skipping", flush=True)
+        else:
+            status, _ = self._request(
+                "POST", f"/admin/realms/{realm}/clients",
+                data={
+                    "clientId": client_id,
+                    "enabled": True,
+                    "protocol": "openid-connect",
+                    "publicClient": False,
+                    "standardFlowEnabled": True,
+                    "directAccessGrantsEnabled": True,
+                    "serviceAccountsEnabled": True,
+                    "authorizationServicesEnabled": True,
+                    "redirectUris": redirect_uris,
+                    "webOrigins": web_origins,
+                    "attributes": {"pkce.code.challenge.method": ""},
+                },
+                headers=headers,
+            )
+            print(f"  Created client '{client_id}'", flush=True)
+
+            # Re-fetch to get UUID
+            headers = self._auth_headers()
+            _, data = self._request(
+                "GET",
+                f"/admin/realms/{realm}/clients?clientId={client_id}",
+                headers=headers,
+            )
+            client_uuid = data[0]["id"]
+
+        # Get client secret
+        headers = self._auth_headers()
+        _, secret_data = self._request(
+            "GET",
+            f"/admin/realms/{realm}/clients/{client_uuid}/client-secret",
+            headers=headers,
+        )
+        client_secret = secret_data["value"]
+        print(f"  Client UUID: {client_uuid}", flush=True)
+        print(f"  Client secret: {client_secret}", flush=True)
+        return client_uuid, client_secret
+
+    def ensure_user(self, realm: str, username: str, email: str,
+                    password: str, *,
+                    first_name: str = "Test",
+                    last_name: str = "User") -> str:
+        """Create a user if needed, set password. Returns user ID."""
+        print(f"=== Ensure user '{username}' ===", flush=True)
+        headers = self._auth_headers()
+
+        _, data = self._request(
+            "GET",
+            f"/admin/realms/{realm}/users?username={username}",
+            headers=headers,
+        )
+        if data and len(data) > 0:
+            user_id = data[0]["id"]
+            print(f"  User '{username}' exists ({user_id}), resetting password",
+                  flush=True)
+            headers = self._auth_headers()
+            self._request(
+                "PUT",
+                f"/admin/realms/{realm}/users/{user_id}/reset-password",
+                data={
+                    "type": "password",
+                    "value": password,
+                    "temporary": False,
+                },
+                headers=headers,
+            )
+        else:
+            headers = self._auth_headers()
+            self._request(
+                "POST", f"/admin/realms/{realm}/users",
+                data={
+                    "username": username,
+                    "email": email,
+                    "firstName": first_name,
+                    "lastName": last_name,
+                    "emailVerified": True,
+                    "enabled": True,
+                    "credentials": [{
+                        "type": "password",
+                        "value": password,
+                        "temporary": False,
+                    }],
+                },
+                headers=headers,
+            )
+            print(f"  Created user '{username}'", flush=True)
+
+            # Re-fetch to get ID
+            headers = self._auth_headers()
+            _, data = self._request(
+                "GET",
+                f"/admin/realms/{realm}/users?username={username}",
+                headers=headers,
+            )
+            user_id = data[0]["id"]
+
+        return user_id

lib/log.py

@@ -0,0 +1,106 @@
+"""Structured logging for skill operations.
+
+Each skill invocation creates a SkillLogger that captures commands,
+their output, and contextual messages. On save(), the log is written
+to logs/<skill>-<recipe>-<date>.md in markdown format.
+"""
+
+from datetime import datetime
+from pathlib import Path
+
+from lib.config import paths
+
+
+class SkillLogger:
+    """Captures structured output from a skill invocation."""
+
+    def __init__(self, skill_name: str, recipe_name: str | None = None):
+        self.skill_name = skill_name
+        self.recipe_name = recipe_name
+        self.started = datetime.now()
+        self._entries: list[dict] = []
+
+    def step(self, description: str) -> None:
+        """Log a major step in the skill execution."""
+        self._entries.append({"type": "step", "text": description})
+        print(f"\n=== {description} ===", flush=True)
+
+    def command(self, cmd: str, output: str, returncode: int) -> None:
+        """Log a command and its result."""
+        self._entries.append({
+            "type": "command",
+            "cmd": cmd,
+            "output": output,
+            "returncode": returncode,
+        })
+
+    def info(self, message: str) -> None:
+        """Log an informational message."""
+        self._entries.append({"type": "info", "text": message})
+        print(f"  {message}", flush=True)
+
+    def warn(self, message: str) -> None:
+        """Log a warning."""
+        self._entries.append({"type": "warn", "text": message})
+        print(f"  WARNING: {message}", flush=True)
+
+    def error(self, message: str) -> None:
+        """Log an error."""
+        self._entries.append({"type": "error", "text": message})
+        print(f"  ERROR: {message}", flush=True)
+
+    def save(self) -> Path:
+        """Write the log to logs/<skill>-<recipe>-<date>.md."""
+        date_str = self.started.strftime("%Y-%m-%d")
+        if self.recipe_name:
+            filename = f"{self.skill_name}-{self.recipe_name}-{date_str}.md"
+        else:
+            filename = f"{self.skill_name}-{date_str}.md"
+
+        log_dir = paths.LOGS_DIR
+        log_dir.mkdir(parents=True, exist_ok=True)
+        log_path = log_dir / filename
+
+        lines = [
+            f"# {self.skill_name}",
+            "",
+        ]
+        if self.recipe_name:
+            lines.append(f"Recipe: {self.recipe_name}")
+            lines.append("")
+        lines.append(
+            f"Started: {self.started.strftime('%Y-%m-%d %H:%M:%S')}"
+        )
+        lines.append(
+            f"Finished: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+        )
+        lines.append("")
+
+        for entry in self._entries:
+            if entry["type"] == "step":
+                lines.append(f"## {entry['text']}")
+                lines.append("")
+            elif entry["type"] == "command":
+                lines.append(f"```")
+                lines.append(f"$ {entry['cmd']}")
+                if entry["output"]:
+                    lines.append(entry["output"])
+                lines.append(f"```")
+                if entry["returncode"] != 0:
+                    lines.append(f"Exit code: {entry['returncode']}")
+                lines.append("")
+            elif entry["type"] == "info":
+                lines.append(f"{entry['text']}")
+                lines.append("")
+            elif entry["type"] == "warn":
+                lines.append(f"**WARNING:** {entry['text']}")
+                lines.append("")
+            elif entry["type"] == "error":
+                lines.append(f"**ERROR:** {entry['text']}")
+                lines.append("")
+
+        with open(log_path, "w") as f:
+            f.write("\n".join(lines))
+
+        print(f"\n  Log saved: {log_path}", flush=True)
+        return log_path

lib/models.py

@@ -0,0 +1,144 @@
+"""Data model for instances, deployments, and recipes.
+
+Provides dataclasses and loader functions that build on lib/config.py
+to give structured access to the project's configuration.
+"""
+
+from dataclasses import dataclass, field
+from lib import config
+
+
+# ---------------------------------------------------------------------------
+# Core dataclasses
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Instance:
+    """A test server where apps can be deployed."""
+    name: str                        # "b1cc"
+    server: str                      # "b1cc.commoninternet.net"
+    domain_suffix: str               # "b1cc.commoninternet.net"
+    protected_recipes: list[str]     # ["traefik", "backup-bot-two"]
+
+    def default_domain(self, recipe: str) -> str:
+        """Compute the default domain for a recipe on this instance."""
+        return f"{recipe}.{self.domain_suffix}"
+
+
+@dataclass
+class Deployment:
+    """A specific app deployment on an instance."""
+    recipe: str                      # "hedgedoc"
+    domain: str                      # "hedgedoc.b1cc.commoninternet.net"
+    instance: Instance
+    name: str = "default"            # "default" or custom name
+    env_overrides: dict = field(default_factory=dict)
+
+    @property
+    def stack_prefix(self) -> str:
+        """Docker stack name: domain with dots replaced by underscores."""
+        return self.domain.replace(".", "_")
+
+    @property
+    def server(self) -> str:
+        return self.instance.server
+
+
+@dataclass
+class Recipe:
+    """Metadata about a recipe we maintain."""
+    name: str
+    dependencies: list[str] = field(default_factory=list)
+    test_requires: list[str] = field(default_factory=list)
+    sso_provider: str | None = None  # "keycloak" | "authentik" | None
+    setup_script: str | None = None
+    post_deploy_script: str | None = None
+
+
+# ---------------------------------------------------------------------------
+# Loader functions
+# ---------------------------------------------------------------------------
+
+def load_instance(name: str) -> Instance:
+    """Load an Instance from settings.toml."""
+    data = config.load_instance_toml(name)
+    return Instance(
+        name=data["name"],
+        server=data.get("server", f"{name}.commoninternet.net"),
+        domain_suffix=data.get("domain_suffix", f"{name}.commoninternet.net"),
+        protected_recipes=data.get("protected_recipes", ["traefik", "backup-bot-two"]),
+    )
+
+
+def load_default_instance() -> Instance:
+    """Load the default instance from settings.toml."""
+    name = config.get_default_instance_name()
+    return load_instance(name)
+
+
+def list_instances() -> list[Instance]:
+    """Load all configured instances."""
+    return [load_instance(name) for name in config.get_instance_names()]
+
+
+def load_recipe(name: str) -> Recipe:
+    """Load a Recipe from recipe-info/<name>/recipe.toml."""
+    data = config.load_recipe_toml(name)
+    deps_section = data.get("dependencies", {})
+    sso_section = data.get("sso", {})
+    post_deploy_section = data.get("post_deploy", {})
+    return Recipe(
+        name=data.get("name", name),
+        dependencies=deps_section.get("requires", []),
+        test_requires=deps_section.get("test_requires", []),
+        sso_provider=sso_section.get("provider"),
+        setup_script=sso_section.get("setup_script"),
+        post_deploy_script=post_deploy_section.get("script"),
+    )
+
+
+def load_deployment(recipe: str, *,
+                    instance: str | None = None,
+                    name: str = "default") -> Deployment:
+    """Load a Deployment for a recipe on an instance.
+
+    If instance is None, uses the default instance.
+    Domain is computed as {recipe}.{instance.domain_suffix}.
+    An optional 'domain' key in recipe.toml overrides this.
+    """
+    if instance is None:
+        inst = load_default_instance()
+    else:
+        inst = load_instance(instance)
+
+    # Check recipe.toml for optional domain override
+    recipe_data = config.load_recipe_toml(recipe)
+    domain = recipe_data.get("domain", inst.default_domain(recipe))
+
+    return Deployment(
+        recipe=recipe,
+        domain=domain,
+        instance=inst,
+        name=name,
+    )
+
+
+def list_deployments(instance_name: str | None = None) -> list[Deployment]:
+    """List all deployments by iterating recipe-info/*/recipe.toml files."""
+    if instance_name is None:
+        inst = load_default_instance()
+        instance_name = inst.name
+
+    deployments = []
+    recipe_info_dir = config.paths.RECIPE_INFO_DIR
+    if not recipe_info_dir.exists():
+        return []
+
+    for d in sorted(recipe_info_dir.iterdir()):
+        recipe_toml = d / "recipe.toml"
+        if d.is_dir() and recipe_toml.exists():
+            recipe_name = d.name
+            dep = load_deployment(recipe_name, instance=instance_name)
+            deployments.append(dep)
+
+    return deployments

lib/recipe.py

@@ -0,0 +1,186 @@
+"""Recipe version parsing, compose.yml reading, and metadata.
+
+Handles the Co-op Cloud version format: <recipe-semver>+v<upstream-version>
+Example: 0.2.6+v4.5.0
+"""
+
+import os
+import re
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from lib.config import paths
+
+
+# ---------------------------------------------------------------------------
+# Version parsing
+# ---------------------------------------------------------------------------
+
+@dataclass
+class RecipeVersion:
+    """Parses and manipulates Co-op Cloud version strings.
+
+    Format: <major>.<minor>.<patch>+v<upstream_version>
+    Example: 0.2.6+v4.5.0
+    """
+    recipe_major: int
+    recipe_minor: int
+    recipe_patch: int
+    upstream_version: str
+
+    @classmethod
+    def parse(cls, version_string: str) -> "RecipeVersion":
+        """Parse a version string like '0.2.6+v4.5.0'."""
+        # Split on '+v' to separate recipe semver from upstream
+        if "+v" in version_string:
+            recipe_part, upstream = version_string.split("+v", 1)
+        elif "+" in version_string:
+            recipe_part, upstream = version_string.split("+", 1)
+            if upstream.startswith("v"):
+                upstream = upstream[1:]
+        else:
+            recipe_part = version_string
+            upstream = ""
+
+        parts = recipe_part.split(".")
+        if len(parts) != 3:
+            raise ValueError(
+                f"Invalid recipe version format: {version_string}"
+            )
+        return cls(
+            recipe_major=int(parts[0]),
+            recipe_minor=int(parts[1]),
+            recipe_patch=int(parts[2]),
+            upstream_version=upstream,
+        )
+
+    def bump(self, level: str) -> "RecipeVersion":
+        """Return a new version with the specified level bumped.
+
+        level: "patch", "minor", or "major"
+        """
+        if level == "patch":
+            return RecipeVersion(
+                self.recipe_major, self.recipe_minor,
+                self.recipe_patch + 1, self.upstream_version,
+            )
+        elif level == "minor":
+            return RecipeVersion(
+                self.recipe_major, self.recipe_minor + 1,
+                0, self.upstream_version,
+            )
+        elif level == "major":
+            return RecipeVersion(
+                self.recipe_major + 1, 0, 0,
+                self.upstream_version,
+            )
+        else:
+            raise ValueError(f"Invalid bump level: {level}")
+
+    def with_upstream(self, upstream: str) -> "RecipeVersion":
+        """Return a new version with a different upstream version."""
+        return RecipeVersion(
+            self.recipe_major, self.recipe_minor,
+            self.recipe_patch, upstream,
+        )
+
+    def __str__(self) -> str:
+        base = f"{self.recipe_major}.{self.recipe_minor}.{self.recipe_patch}"
+        if self.upstream_version:
+            return f"{base}+v{self.upstream_version}"
+        return base
+
+
+# ---------------------------------------------------------------------------
+# Recipe directory operations
+# ---------------------------------------------------------------------------
+
+def recipe_dir(recipe: str) -> Path:
+    """Return the abra recipe checkout directory."""
+    return paths.ABRA_RECIPES_DIR / recipe
+
+
+def compose_path(recipe: str) -> Path:
+    """Return the path to a recipe's compose.yml."""
+    return recipe_dir(recipe) / "compose.yml"
+
+
+def has_local_changes(recipe: str) -> bool:
+    """Check if a recipe checkout has uncommitted changes."""
+    d = recipe_dir(recipe)
+    if not d.exists():
+        return False
+    result = subprocess.run(
+        ["git", "-C", str(d), "status", "--short"],
+        capture_output=True, text=True, timeout=10,
+    )
+    return bool(result.stdout.strip())
+
+
+def read_compose_version(recipe: str) -> RecipeVersion | None:
+    """Read the version label from a recipe's compose.yml.
+
+    Looks for: coop-cloud.${STACK_NAME}.version=X.Y.Z+vU.P.S
+    """
+    p = compose_path(recipe)
+    if not p.exists():
+        return None
+    with open(p) as f:
+        content = f.read()
+    # Match the version label pattern
+    m = re.search(r'coop-cloud\.\$\{STACK_NAME\}\.version=([^\s"]+)', content)
+    if m:
+        return RecipeVersion.parse(m.group(1))
+    return None
+
+
+def write_compose_version(recipe: str, version: RecipeVersion) -> None:
+    """Update the version label in a recipe's compose.yml."""
+    p = compose_path(recipe)
+    with open(p) as f:
+        content = f.read()
+    content = re.sub(
+        r'(coop-cloud\.\$\{STACK_NAME\}\.version=)[^\s"]+',
+        rf'\g<1>{version}',
+        content,
+    )
+    with open(p, "w") as f:
+        f.write(content)
+
+
+def read_compose_images(recipe: str) -> dict[str, str]:
+    """Read service→image:tag mapping from compose.yml.
+
+    Returns a dict like {"app": "cryptpad/cryptpad:version-2025.9.0"}.
+    Simple regex-based parsing (not full YAML).
+    """
+    p = compose_path(recipe)
+    if not p.exists():
+        return {}
+
+    images = {}
+    current_service = None
+    with open(p) as f:
+        in_services = False
+        indent_level = 0
+        for line in f:
+            stripped = line.strip()
+            # Detect services: block
+            if stripped == "services:":
+                in_services = True
+                continue
+            if in_services:
+                # Service name: indented once, ends with ":"
+                if line.startswith("  ") and not line.startswith("    ") and stripped.endswith(":"):
+                    current_service = stripped.rstrip(":")
+                # Image line: indented under a service
+                elif current_service and "image:" in stripped:
+                    image = stripped.split("image:", 1)[1].strip()
+                    images[current_service] = image
+                # New top-level block ends services
+                elif not line.startswith(" ") and stripped and stripped != "---":
+                    in_services = False
+                    current_service = None
+
+    return images

lib/secrets.py

@@ -0,0 +1,82 @@
+"""Secret resolution, syncing, and reading from containers.
+
+Replaces the bash scripts resolve-secrets.sh and sync-secrets.sh
+with Python equivalents that work with the new deployment model.
+"""
+
+from pathlib import Path
+
+from lib.config import paths, load_recipe_toml
+from lib.env import read_env_file
+from lib.ssh import read_container_secrets
+
+
+def secrets_dir() -> Path:
+    """Return the secrets directory: recipe-info/testsecrets/."""
+    return paths.TESTSECRETS_DIR
+
+
+def load_secrets(domain: str) -> dict[str, str]:
+    """Load locally-stored secrets for a deployment.
+
+    Reads from recipe-info/testsecrets/<domain>.
+    Returns empty dict if no secrets file exists.
+    """
+    p = secrets_dir() / domain
+    if not p.exists():
+        return {}
+    return read_env_file(p)
+
+
+def save_secrets(domain: str, secrets: dict[str, str]) -> Path:
+    """Save secrets to recipe-info/testsecrets/<domain>."""
+    d = secrets_dir()
+    d.mkdir(parents=True, exist_ok=True)
+    p = d / domain
+    with open(p, "w") as f:
+        for name in sorted(secrets.keys()):
+            f.write(f"{name}={secrets[name]}\n")
+    return p
+
+
+def sync_secrets_for_deployment(server: str, domain: str) -> dict[str, str]:
+    """Sync secrets from running containers to local file.
+
+    SSHes into the server, reads /run/secrets/ from all containers
+    of the stack, and saves to recipe-info/testsecrets/<domain>.
+
+    Returns the secrets dict.
+    """
+    stack_prefix = domain.replace(".", "_")
+    secrets = read_container_secrets(server, stack_prefix)
+    if secrets:
+        save_secrets(domain, secrets)
+        print(f"  Wrote {len(secrets)} secrets for {domain}", flush=True)
+    else:
+        print(f"  No secrets found for {domain}", flush=True)
+    return secrets
+
+
+def sync_all_secrets(server: str) -> dict[str, dict]:
+    """Sync secrets for all recipes that have a recipe.toml.
+
+    Returns {domain: {secret_name: value}}.
+    """
+    from lib.models import load_default_instance
+
+    inst = load_default_instance()
+    recipe_info_dir = paths.RECIPE_INFO_DIR
+    if not recipe_info_dir.exists():
+        return {}
+
+    results = {}
+    for d in sorted(recipe_info_dir.iterdir()):
+        recipe_toml = d / "recipe.toml"
+        if d.is_dir() and recipe_toml.exists():
+            recipe_name = d.name
+            domain = f"{recipe_name}.{inst.domain_suffix}"
+            print(f"=== {recipe_name} ({domain}) ===", flush=True)
+            results[domain] = sync_secrets_for_deployment(
+                server, domain
+            )
+    return results

lib/ssh.py

@@ -0,0 +1,166 @@
+"""SSH command runner for direct server operations.
+
+Used for docker service ls, reading secrets from containers,
+checking service logs, and other operations that require
+server-side access bypassing abra.
+"""
+
+from lib.abra import run, AbraResult
+
+
+def ssh_run(server: str, command: str, *,
+            timeout: int = 30) -> AbraResult:
+    """Run a command on a remote server via SSH."""
+    return run(f'ssh {server} "{command}"', check=False, timeout=timeout)
+
+
+def docker_service_ls(server: str, stack_prefix: str) -> list[dict]:
+    """List Docker services for a stack, parsed into dicts.
+
+    Returns list of {"name": str, "replicas": str, "image": str}.
+    """
+    result = ssh_run(
+        server,
+        f"docker service ls --filter 'name={stack_prefix}_'"
+        f" --format '{{{{.Name}}}}|{{{{.Replicas}}}}|{{{{.Image}}}}'",
+        timeout=30,
+    )
+    if not result.ok or not result.stdout.strip():
+        return []
+
+    services = []
+    for line in result.stdout.strip().split("\n"):
+        line = line.strip()
+        if not line:
+            continue
+        parts = line.split("|")
+        if len(parts) >= 3:
+            services.append({
+                "name": parts[0],
+                "replicas": parts[1],
+                "image": parts[2],
+            })
+    return services
+
+
+def all_replicas_ready(server: str, stack_prefix: str) -> bool:
+    """Check if all Docker services have their desired replica count.
+
+    Returns True if every service shows N/N replicas (e.g. "1/1").
+    Returns False if any service is not ready or no services found.
+    """
+    result = ssh_run(
+        server,
+        f"docker service ls --filter 'name={stack_prefix}_'"
+        f" --format '{{{{.Replicas}}}}'",
+        timeout=30,
+    )
+    if not result.ok:
+        return False
+
+    lines = [l.strip() for l in result.stdout.strip().split("\n") if l.strip()]
+    if not lines:
+        return False
+
+    for replicas in lines:
+        parts = replicas.split("/")
+        if len(parts) != 2:
+            return False
+        if parts[0] != parts[1]:
+            return False
+    return True
+
+
+def read_container_secrets(server: str, stack_prefix: str) -> dict[str, str]:
+    """Read all Docker secrets from running containers of a stack.
+
+    SSHes into the server, finds all containers for the stack,
+    reads /run/secrets/ from each container, returns {name: value}.
+    """
+    cmd = (
+        f"containers=\\$(docker ps -q --filter 'name={stack_prefix}_' 2>/dev/null); "
+        f"if [ -z \\\"\\$containers\\\" ]; then exit 0; fi; "
+        f"for cid in \\$containers; do "
+        f"  files=\\$(docker exec \\$cid ls /run/secrets/ 2>/dev/null) || continue; "
+        f"  for f in \\$files; do "
+        f"    val=\\$(docker exec \\$cid cat /run/secrets/\\$f 2>/dev/null) || continue; "
+        f"    echo \\\"\\$f=\\$val\\\"; "
+        f"  done; "
+        f"done"
+    )
+    result = ssh_run(server, cmd, timeout=60)
+    if not result.ok or not result.stdout.strip():
+        return {}
+
+    secrets = {}
+    for line in result.stdout.strip().split("\n"):
+        line = line.strip()
+        if "=" in line:
+            name, value = line.split("=", 1)
+            if name not in secrets:  # first occurrence wins
+                secrets[name] = value
+    return secrets
+
+
+def docker_service_logs(server: str, stack_name: str, service: str, *,
+                        since: str = "5m", tail: int = 100) -> str:
+    """Get Docker service logs via SSH (avoids abra app logs which hangs)."""
+    result = ssh_run(
+        server,
+        f"docker service logs {stack_name}_{service} --since {since} --tail {tail} 2>&1",
+        timeout=30,
+    )
+    return result.stdout if result.ok else ""
+
+
+def docker_secret_ls(server: str, stack_prefix: str) -> list[str]:
+    """List Docker secret names matching a stack prefix."""
+    result = ssh_run(
+        server,
+        f"docker secret ls --filter 'name={stack_prefix}_' --format '{{{{.Name}}}}'",
+        timeout=15,
+    )
+    if not result.ok or not result.stdout.strip():
+        return []
+    return [l.strip() for l in result.stdout.strip().split("\n") if l.strip()]
+
+
+def docker_volume_ls(server: str, stack_prefix: str) -> list[str]:
+    """List Docker volume names for a stack.
+
+    Returns a list of volume names. Raises RuntimeError if the SSH
+    command fails (a failed command is a real error, not "no volumes").
+    """
+    result = ssh_run(
+        server,
+        f"docker volume ls --filter 'name={stack_prefix}_' --format '{{{{.Name}}}}'",
+        timeout=15,
+    )
+    if not result.ok:
+        raise RuntimeError(
+            f"Failed to list volumes on {server}: {result.stderr.strip()}"
+        )
+    if not result.stdout.strip():
+        return []
+    return [line.strip() for line in result.stdout.strip().split("\n") if line.strip()]
+
+
+def docker_ps(server: str, *, filter_name: str | None = None) -> list[dict]:
+    """List Docker containers on the server."""
+    cmd = "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'"
+    if filter_name:
+        cmd = f"docker ps --filter 'name={filter_name}' --format '{{{{.Names}}}}|{{{{.Status}}}}|{{{{.Image}}}}'"
+    result = ssh_run(server, cmd, timeout=30)
+    if not result.ok or not result.stdout.strip():
+        return []
+
+    containers = []
+    for line in result.stdout.strip().split("\n"):
+        parts = line.strip().split("|")
+        if len(parts) >= 3:
+            containers.append({
+                "name": parts[0],
+                "status": parts[1],
+                "image": parts[2],
+            })
+    return containers