From cb20bea7cdc50ba18de8f8f2bd31d1fcf4638f09 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Tue, 4 Aug 2026 01:57:34 +0000 Subject: [PATCH] recovery: give the incident tooling a permanent home (scripts/recovery/) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-08-03 cc-ci outage was recovered with ad-hoc tooling living in /tmp (leftover from a PREVIOUS incident, half-evaporated). Promoted to the repo: - scripts/recovery/hetzner.py — Hetzner API helper (status/actions/reboot/reset/ power/rescue-on|off/console), knows cc-ci=134485294 + orchestrator=134487234 by name; token from HCLOUD_TOKEN or /srv/cc-ci/.hcloud-token (0600, never in git). - scripts/recovery/hetzner-console.sh — shell-only VGA console: fresh console session -> websocat bridge -> vncdotool (venv auto-bootstrapped in ~/.cache). screenshot / key / type subcommands; encodes the reset-invalidates-session and single-connection-bridge gotchas. - scripts/recovery/README.md — the condensed 10-minute unreachable-server drill, incl. the GRUB submenu 1>N ids + clear-grubenv-after-switch rule. - hetzner-server-recovery skill: console/API sections now point at the repo tools instead of describing /tmp rebuilds. Smoke-tested: hetzner.py cc-ci status OK. --- .../skills/hetzner-server-recovery/SKILL.md | 69 ++++-------- scripts/recovery/README.md | 37 ++++++ scripts/recovery/hetzner-console.sh | 76 +++++++++++++ scripts/recovery/hetzner.py | 105 ++++++++++++++++++ 4 files changed, 239 insertions(+), 48 deletions(-) create mode 100644 scripts/recovery/README.md create mode 100755 scripts/recovery/hetzner-console.sh create mode 100755 scripts/recovery/hetzner.py diff --git a/.claude/skills/hetzner-server-recovery/SKILL.md b/.claude/skills/hetzner-server-recovery/SKILL.md index 526cd6c..bc1d6aa 100644 --- a/.claude/skills/hetzner-server-recovery/SKILL.md +++ b/.claude/skills/hetzner-server-recovery/SKILL.md @@ -57,60 +57,33 @@ tailscale ping -c 3 If the host still does not come back, continue. -## 3. Request the Hetzner console +## 3. Use the repo recovery tools (permanent home — do NOT rebuild these in /tmp) -Request a remote console session: +The API and console tooling live in **`scripts/recovery/`** (see its README for the condensed +10-minute drill, proven 2026-08-03): ```bash -curl -s -X POST \ - -H "Authorization: Bearer ${HCLOUD_TOKEN}" \ - -H "Content-Type: application/json" \ - "https://api.hetzner.cloud/v1/servers//actions/request_console" +# API: status / actions / reboot / reset / poweroff / poweron / rescue-on / rescue-off / console +python3 /srv/cc-ci-orch/scripts/recovery/hetzner.py cc-ci status +python3 /srv/cc-ci-orch/scripts/recovery/hetzner.py cc-ci actions 10 + +# Shell-only console access (fresh console session + websocat bridge + vncdotool, +# venv auto-bootstrapped at ~/.cache/hetzner-console-venv): +bash /srv/cc-ci-orch/scripts/recovery/hetzner-console.sh cc-ci screenshot /tmp/console.png +bash /srv/cc-ci-orch/scripts/recovery/hetzner-console.sh cc-ci key Down Down Return ``` -The API returns: +Known server names: `cc-ci` (134485294), `orchestrator` (134487234). Token: `HCLOUD_TOKEN` +env or `/srv/cc-ci/.hcloud-token` (0600, not in git; prefer per-incident revocable tokens, +and never paste tokens into a chat transcript). -- `wss_url` -- `password` - -If you have a browser, use the Hetzner console directly. - -If you only have shell access, you can still drive it locally because the console is **raw VNC over -websocket**. - -## 4. Shell-only console access (websocket VNC bridge) - -Install temporary tools: - -```bash -nix shell nixpkgs#websocat -c websocat --version -python3 -m venv /tmp/opencode/hetzner-console-venv -/tmp/opencode/hetzner-console-venv/bin/pip install --disable-pip-version-check pillow websocket-client vncdotool -``` - -Bridge the websocket console to a local VNC TCP port: - -```bash -nohup nix shell nixpkgs#websocat -c \ - websocat -b -E tcp-l:127.0.0.1:5905 '' \ - >/tmp/opencode/hetzner-websockify.log 2>&1 & -``` - -Validate the RFB banner: - -```bash -python3 - <<'PY' -import socket -s=socket.socket(); s.settimeout(5); s.connect(('127.0.0.1',5905)) -print(repr(s.recv(32))) -PY -``` - -Expected: - -```text -b'RFB 003.008\n' -``` +Notes that used to cost time: +- Each console command requests a **fresh** console session — old sessions die on hard reset, + and the websocat bridge is single-connection anyway. +- A GRUB one-shot/default for a NixOS generation needs the **submenu id `1>N`** (top level: + 0 = default entry, 1 = the "All configurations" submenu). A bare index silently falls back + to the default entry. Clear any grubenv override after the next `switch` regenerates + grub.cfg — indices shift. Capture a screenshot from the console: diff --git a/scripts/recovery/README.md b/scripts/recovery/README.md new file mode 100644 index 0000000..1e75b47 --- /dev/null +++ b/scripts/recovery/README.md @@ -0,0 +1,37 @@ +# Recovery tooling — permanent home + +Promoted from `/tmp` ad-hocery after the 2026-08-03 cc-ci 26.05 outage (see +`.cc-ci-logs/server-update-2026-08-03.md` and the `hetzner-server-recovery` skill, which is +the *procedure*; these are the *tools* it uses). + +| Tool | What it does | +|---|---| +| `hetzner.py ` | Hetzner Cloud API: status, actions, reboot/reset/poweroff/poweron, rescue-on/off, console credentials. Knows `cc-ci` (134485294) and `orchestrator` (134487234) by name. | +| `hetzner-console.sh screenshot\|key\|type` | Shell-only access to the VGA console: fresh console session → websocat bridge → vncdotool (venv auto-bootstrapped at `~/.cache/hetzner-console-venv`). | + +**Token:** `HCLOUD_TOKEN` env, or `/srv/cc-ci/.hcloud-token` (chmod 600). Not in git, not in +`.testenv`. Prefer per-incident tokens and revoke after — and never paste tokens into chat +transcripts (the 2026-08-03 incident token was pasted mid-incident and had to be flagged for +rotation). + +## The 10-minute unreachable-server drill (condensed from 2026-08-03) + +1. `hetzner.py cc-ci status` — "running" + no SSH/ping means booted-but-broken, not crashed. +2. `hetzner-console.sh cc-ci screenshot /tmp/console.png` — look at the actual screen: which + generation booted? login prompt or emergency shell? +3. Plain `reboot` first. If the default boot is the broken generation, DON'T fight GRUB + timing over VNC — go to rescue: + `rescue-on` → `poweroff` → wait `status=off` → `poweron` → `ssh root@` (key + 113082420 = `~/.ssh/cc-ci-root-ed25519`; fresh `UserKnownHostsFile`). +4. In rescue: `mount /dev/sda1 /mnt` and fix the boot default: + - Generations live in a GRUB **submenu**: entry ids are `1>N` (top level: 0 = default + entry, 1 = the submenu). A bare index like `2` silently falls back to the broken default. + - Persistent: `grub-editenv /mnt/boot/grub/grubenv set 'default=1>N'` (survives reboots). + - **Clear it after the next successful `nixos-rebuild switch`** — the regenerated grub.cfg + shifts indices and a stale override points at the wrong generation. + - Journal of the failed boot: `journalctl -D /mnt/var/log/journal --list-boots` / `-b `. +5. `rescue-off` → `poweroff` → `poweron` → verify → write the incident up in + `cc-ci-plan/JOURNAL.md` and (if server) `.cc-ci-logs/`. + +**Prevention:** `nixos-rebuild test` before `switch`, always (see AGENTS.md / the update +skills) — `test` leaves the bootloader alone, so a power-cycle recovers by itself. diff --git a/scripts/recovery/hetzner-console.sh b/scripts/recovery/hetzner-console.sh new file mode 100755 index 0000000..e045826 --- /dev/null +++ b/scripts/recovery/hetzner-console.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# hetzner-console.sh — drive a Hetzner web-console (raw VNC over websocket) from a shell. +# The screenshot/keyboard half of hetzner-server-recovery; proven in the 2026-08-03 +# cc-ci 26.05 outage (console screenshots identified the wrong-generation boot). +# +# Usage: +# hetzner-console.sh screenshot +# hetzner-console.sh key [key ...] # e.g. key Down Down Return +# hetzner-console.sh type "" +# +# = name/id understood by hetzner.py. Each invocation requests a FRESH console +# session (they are cheap, and hard resets invalidate old ones), bridges it to a local +# TCP port with websocat, and runs vncdo against it. The bridge is single-connection — +# that is why every command re-requests + re-bridges. +# +# Gotchas encoded here so nobody rediscovers them at 2am: +# - The wss_url from the API contains literal '&' — when it arrives via JSON it may be +# &-escaped; parse the JSON properly (as below), never paste from raw output. +# - A hard `reset` drops the console websocket mid-session: re-request and reconnect. +# - GRUB menus: generations live in a SUBMENU — one-shot boot ids are "1>N", and any +# persistent grubenv `default` must be cleared after the next switch regenerates +# grub.cfg (indices shift). See the hetzner-server-recovery skill. +set -o errexit -o nounset -o pipefail + +HERE="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +SERVER="${1:?usage: hetzner-console.sh screenshot|key|type ...}" +CMD="${2:?need a command: screenshot|key|type}" +shift 2 + +PORT="${CONSOLE_PORT:-5905}" +VENV="${HOME}/.cache/hetzner-console-venv" + +# 1. vncdotool venv (bootstrap once; durable across incidents, unlike /tmp) +if [ ! -x "${VENV}/bin/vncdo" ]; then + echo "bootstrapping vncdotool venv at ${VENV}..." >&2 + python3 -m venv "${VENV}" + "${VENV}/bin/pip" -q install vncdotool +fi + +# 2. fresh console session +CREDS="$(python3 "${HERE}/hetzner.py" "${SERVER}" console)" +WSS="$(printf '%s' "${CREDS}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["wss_url"])')" +PW="$(printf '%s' "${CREDS}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["password"])')" + +# 3. bridge (single-connection; killed on exit) +pkill -f "websocat.*${PORT}" 2>/dev/null || true +sleep 0.5 +nix shell nixpkgs#websocat -c websocat --binary "tcp-listen:127.0.0.1:${PORT}" "${WSS}" \ + > /tmp/hetzner-console-websocat.log 2>&1 & +BRIDGE=$! +trap 'kill ${BRIDGE} 2>/dev/null || true; pkill -f "websocat.*${PORT}" 2>/dev/null || true' EXIT +sleep 2 + +# 4. run the vncdo command +case "${CMD}" in + screenshot) + OUT="${1:?screenshot needs an output path}" + timeout 40 "${VENV}/bin/vncdo" -s "127.0.0.1::${PORT}" -p "${PW}" capture "${OUT}" + echo "captured ${OUT}" + ;; + key) + [ $# -ge 1 ] || { echo "key needs at least one key name" >&2; exit 1; } + ARGS=() + for k in "$@"; do ARGS+=(key "$k" pause 0.3); done + timeout 60 "${VENV}/bin/vncdo" -s "127.0.0.1::${PORT}" -p "${PW}" "${ARGS[@]}" + echo "sent: $*" + ;; + type) + TEXT="${1:?type needs text}" + timeout 60 "${VENV}/bin/vncdo" -s "127.0.0.1::${PORT}" -p "${PW}" type "${TEXT}" + echo "typed" + ;; + *) + echo "unknown command ${CMD} (screenshot|key|type)" >&2; exit 1 + ;; +esac diff --git a/scripts/recovery/hetzner.py b/scripts/recovery/hetzner.py new file mode 100755 index 0000000..3fb7b78 --- /dev/null +++ b/scripts/recovery/hetzner.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Hetzner Cloud recovery helper — the API half of hetzner-server-recovery. + +Promoted to the repo 2026-08-04 after the cc-ci 26.05 outage recovery was done with +ad-hoc tooling living in /tmp (which had evaporated from the previous incident). + +Token: $HCLOUD_TOKEN, else the file $HCLOUD_TOKEN_FILE, else /srv/cc-ci/.hcloud-token +(chmod 600; ask the operator for a token if absent — and prefer a per-incident token +that gets revoked afterwards). + +Usage: + hetzner.py status + hetzner.py actions [n] # recent actions, newest first + hetzner.py reboot|reset|poweroff|poweron + hetzner.py rescue-on [ssh_key_id ...] # then poweroff+poweron to enter it + hetzner.py rescue-off + hetzner.py console # prints wss_url + password (JSON) + + is a name from SERVERS below or a numeric Hetzner server id. +""" + +from __future__ import annotations + +import json +import os +import sys +import urllib.error +import urllib.request + +SERVERS = { + "cc-ci": 134485294, # the CI server (nixos, `ssh cc-ci`, tailnet 100.95.31.88) + "orchestrator": 134487234, # this host (cc-ci-orchestrator-1, tailnet 100.84.190.30) +} +# SSH keys registered in the Hetzner project (for rescue-mode injection): +# 113082219 cc-ci-deploy · 113082420 cc-ci-orchestrator-deploy (= ~/.ssh/cc-ci-root-ed25519) +DEFAULT_RESCUE_KEYS = [113082219, 113082420] + + +def token() -> str: + tok = os.environ.get("HCLOUD_TOKEN") + if not tok: + path = os.environ.get("HCLOUD_TOKEN_FILE", "/srv/cc-ci/.hcloud-token") + try: + tok = open(path).read().strip() + except OSError: + sys.exit( + "ERROR: no Hetzner token. Set HCLOUD_TOKEN, or put one in " + f"{path} (chmod 600). Ask the operator; prefer a revocable per-incident token." + ) + return tok + + +def api(path: str, method: str = "GET", body: dict | None = None) -> dict: + req = urllib.request.Request( + "https://api.hetzner.cloud/v1" + path, + method=method, + headers={"Authorization": "Bearer " + token(), "Content-Type": "application/json"}, + data=json.dumps(body).encode() if body is not None else None, + ) + try: + with urllib.request.urlopen(req, timeout=30) as r: + raw = r.read() + return json.loads(raw) if raw.strip() else {} + except urllib.error.HTTPError as e: + sys.exit(f"ERROR: HTTP {e.code} on {method} {path}: {e.read()[:300]!r}") + + +def main() -> None: + if len(sys.argv) < 3: + sys.exit(__doc__) + server, cmd, args = sys.argv[1], sys.argv[2], sys.argv[3:] + sid = SERVERS.get(server) or (int(server) if server.isdigit() else None) + if sid is None: + sys.exit(f"ERROR: unknown server {server!r} (known: {', '.join(SERVERS)} or numeric id)") + + if cmd == "status": + d = api(f"/servers/{sid}")["server"] + print( + f"status={d['status']} rescue_enabled={d.get('rescue_enabled')} " + f"locked={d['locked']} public_ip={d['public_net']['ipv4']['ip']}" + ) + elif cmd == "actions": + n = int(args[0]) if args else 10 + for a in api(f"/servers/{sid}/actions?sort=started:desc&per_page={n}")["actions"]: + print(a["started"], a["command"], a["status"], a["progress"]) + elif cmd in ("reboot", "reset", "poweroff", "poweron"): + r = api(f"/servers/{sid}/actions/{cmd}", "POST") + print(cmd, r["action"]["status"], r["action"]["started"]) + elif cmd == "rescue-on": + keys = [int(k) for k in args] or DEFAULT_RESCUE_KEYS + r = api(f"/servers/{sid}/actions/enable_rescue", "POST", {"type": "linux64", "ssh_keys": keys}) + print("enable_rescue", r["action"]["status"], "| root password:", r.get("root_password")) + print("NOTE: rescue boots on the next power cycle — run poweroff, wait for status=off, poweron.") + elif cmd == "rescue-off": + r = api(f"/servers/{sid}/actions/disable_rescue", "POST") + print("disable_rescue", r["action"]["status"]) + elif cmd == "console": + r = api(f"/servers/{sid}/actions/request_console", "POST") + print(json.dumps({"wss_url": r["wss_url"], "password": r["password"]})) + else: + sys.exit(f"ERROR: unknown command {cmd!r}\n{__doc__}") + + +if __name__ == "__main__": + main()