autonomic-bot
cb20bea7cd
The 2026-08-03 cc-ci outage was recovered with ad-hoc tooling living in /tmp (leftover from a PREVIOUS incident, half-evaporated). Promoted to the repo: - scripts/recovery/hetzner.py — Hetzner API helper (status/actions/reboot/reset/ power/rescue-on|off/console), knows cc-ci=134485294 + orchestrator=134487234 by name; token from HCLOUD_TOKEN or /srv/cc-ci/.hcloud-token (0600, never in git). - scripts/recovery/hetzner-console.sh — shell-only VGA console: fresh console session -> websocat bridge -> vncdotool (venv auto-bootstrapped in ~/.cache). screenshot / key / type subcommands; encodes the reset-invalidates-session and single-connection-bridge gotchas. - scripts/recovery/README.md — the condensed 10-minute unreachable-server drill, incl. the GRUB submenu 1>N ids + clear-grubenv-after-switch rule. - hetzner-server-recovery skill: console/API sections now point at the repo tools instead of describing /tmp rebuilds. Smoke-tested: hetzner.py cc-ci status OK.
106 lines
4.2 KiB
Python
Executable File
106 lines
4.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Hetzner Cloud recovery helper — the API half of hetzner-server-recovery.
|
|
|
|
Promoted to the repo 2026-08-04 after the cc-ci 26.05 outage recovery was done with
|
|
ad-hoc tooling living in /tmp (which had evaporated from the previous incident).
|
|
|
|
Token: $HCLOUD_TOKEN, else the file $HCLOUD_TOKEN_FILE, else /srv/cc-ci/.hcloud-token
|
|
(chmod 600; ask the operator for a token if absent — and prefer a per-incident token
|
|
that gets revoked afterwards).
|
|
|
|
Usage:
|
|
hetzner.py <server> status
|
|
hetzner.py <server> actions [n] # recent actions, newest first
|
|
hetzner.py <server> reboot|reset|poweroff|poweron
|
|
hetzner.py <server> rescue-on [ssh_key_id ...] # then poweroff+poweron to enter it
|
|
hetzner.py <server> rescue-off
|
|
hetzner.py <server> console # prints wss_url + password (JSON)
|
|
|
|
<server> is a name from SERVERS below or a numeric Hetzner server id.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
SERVERS = {
|
|
"cc-ci": 134485294, # the CI server (nixos, `ssh cc-ci`, tailnet 100.95.31.88)
|
|
"orchestrator": 134487234, # this host (cc-ci-orchestrator-1, tailnet 100.84.190.30)
|
|
}
|
|
# SSH keys registered in the Hetzner project (for rescue-mode injection):
|
|
# 113082219 cc-ci-deploy · 113082420 cc-ci-orchestrator-deploy (= ~/.ssh/cc-ci-root-ed25519)
|
|
DEFAULT_RESCUE_KEYS = [113082219, 113082420]
|
|
|
|
|
|
def token() -> str:
|
|
tok = os.environ.get("HCLOUD_TOKEN")
|
|
if not tok:
|
|
path = os.environ.get("HCLOUD_TOKEN_FILE", "/srv/cc-ci/.hcloud-token")
|
|
try:
|
|
tok = open(path).read().strip()
|
|
except OSError:
|
|
sys.exit(
|
|
"ERROR: no Hetzner token. Set HCLOUD_TOKEN, or put one in "
|
|
f"{path} (chmod 600). Ask the operator; prefer a revocable per-incident token."
|
|
)
|
|
return tok
|
|
|
|
|
|
def api(path: str, method: str = "GET", body: dict | None = None) -> dict:
|
|
req = urllib.request.Request(
|
|
"https://api.hetzner.cloud/v1" + path,
|
|
method=method,
|
|
headers={"Authorization": "Bearer " + token(), "Content-Type": "application/json"},
|
|
data=json.dumps(body).encode() if body is not None else None,
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
raw = r.read()
|
|
return json.loads(raw) if raw.strip() else {}
|
|
except urllib.error.HTTPError as e:
|
|
sys.exit(f"ERROR: HTTP {e.code} on {method} {path}: {e.read()[:300]!r}")
|
|
|
|
|
|
def main() -> None:
|
|
if len(sys.argv) < 3:
|
|
sys.exit(__doc__)
|
|
server, cmd, args = sys.argv[1], sys.argv[2], sys.argv[3:]
|
|
sid = SERVERS.get(server) or (int(server) if server.isdigit() else None)
|
|
if sid is None:
|
|
sys.exit(f"ERROR: unknown server {server!r} (known: {', '.join(SERVERS)} or numeric id)")
|
|
|
|
if cmd == "status":
|
|
d = api(f"/servers/{sid}")["server"]
|
|
print(
|
|
f"status={d['status']} rescue_enabled={d.get('rescue_enabled')} "
|
|
f"locked={d['locked']} public_ip={d['public_net']['ipv4']['ip']}"
|
|
)
|
|
elif cmd == "actions":
|
|
n = int(args[0]) if args else 10
|
|
for a in api(f"/servers/{sid}/actions?sort=started:desc&per_page={n}")["actions"]:
|
|
print(a["started"], a["command"], a["status"], a["progress"])
|
|
elif cmd in ("reboot", "reset", "poweroff", "poweron"):
|
|
r = api(f"/servers/{sid}/actions/{cmd}", "POST")
|
|
print(cmd, r["action"]["status"], r["action"]["started"])
|
|
elif cmd == "rescue-on":
|
|
keys = [int(k) for k in args] or DEFAULT_RESCUE_KEYS
|
|
r = api(f"/servers/{sid}/actions/enable_rescue", "POST", {"type": "linux64", "ssh_keys": keys})
|
|
print("enable_rescue", r["action"]["status"], "| root password:", r.get("root_password"))
|
|
print("NOTE: rescue boots on the next power cycle — run poweroff, wait for status=off, poweron.")
|
|
elif cmd == "rescue-off":
|
|
r = api(f"/servers/{sid}/actions/disable_rescue", "POST")
|
|
print("disable_rescue", r["action"]["status"])
|
|
elif cmd == "console":
|
|
r = api(f"/servers/{sid}/actions/request_console", "POST")
|
|
print(json.dumps({"wss_url": r["wss_url"], "password": r["password"]}))
|
|
else:
|
|
sys.exit(f"ERROR: unknown command {cmd!r}\n{__doc__}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|