#!/usr/bin/env python3 """Hetzner Cloud recovery helper โ€” the API half of hetzner-server-recovery. Promoted to the repo 2026-08-04 after the cc-ci 26.05 outage recovery was done with ad-hoc tooling living in /tmp (which had evaporated from the previous incident). Token: $HCLOUD_TOKEN, else the file $HCLOUD_TOKEN_FILE, else /srv/cc-ci/.hcloud-token (chmod 600; ask the operator for a token if absent โ€” and prefer a per-incident token that gets revoked afterwards). Usage: hetzner.py status hetzner.py actions [n] # recent actions, newest first hetzner.py reboot|reset|poweroff|poweron hetzner.py rescue-on [ssh_key_id ...] # then poweroff+poweron to enter it hetzner.py rescue-off hetzner.py console # prints wss_url + password (JSON) is a name from SERVERS below or a numeric Hetzner server id. """ from __future__ import annotations import json import os import sys import urllib.error import urllib.request SERVERS = { "cc-ci": 134485294, # the CI server (nixos, `ssh cc-ci`, tailnet 100.95.31.88) "orchestrator": 134487234, # this host (cc-ci-orchestrator-1, tailnet 100.84.190.30) } # SSH keys registered in the Hetzner project (for rescue-mode injection): # 113082219 cc-ci-deploy ยท 113082420 cc-ci-orchestrator-deploy (= ~/.ssh/cc-ci-root-ed25519) DEFAULT_RESCUE_KEYS = [113082219, 113082420] def token() -> str: tok = os.environ.get("HCLOUD_TOKEN") if not tok: path = os.environ.get("HCLOUD_TOKEN_FILE", "/srv/cc-ci/.hcloud-token") try: tok = open(path).read().strip() except OSError: sys.exit( "ERROR: no Hetzner token. Set HCLOUD_TOKEN, or put one in " f"{path} (chmod 600). Ask the operator; prefer a revocable per-incident token." ) return tok def api(path: str, method: str = "GET", body: dict | None = None) -> dict: req = urllib.request.Request( "https://api.hetzner.cloud/v1" + path, method=method, headers={"Authorization": "Bearer " + token(), "Content-Type": "application/json"}, data=json.dumps(body).encode() if body is not None else None, ) try: with urllib.request.urlopen(req, timeout=30) as r: raw = r.read() return json.loads(raw) if raw.strip() else {} except urllib.error.HTTPError as e: sys.exit(f"ERROR: HTTP {e.code} on {method} {path}: {e.read()[:300]!r}") def main() -> None: if len(sys.argv) < 3: sys.exit(__doc__) server, cmd, args = sys.argv[1], sys.argv[2], sys.argv[3:] sid = SERVERS.get(server) or (int(server) if server.isdigit() else None) if sid is None: sys.exit(f"ERROR: unknown server {server!r} (known: {', '.join(SERVERS)} or numeric id)") if cmd == "status": d = api(f"/servers/{sid}")["server"] print( f"status={d['status']} rescue_enabled={d.get('rescue_enabled')} " f"locked={d['locked']} public_ip={d['public_net']['ipv4']['ip']}" ) elif cmd == "actions": n = int(args[0]) if args else 10 for a in api(f"/servers/{sid}/actions?sort=started:desc&per_page={n}")["actions"]: print(a["started"], a["command"], a["status"], a["progress"]) elif cmd in ("reboot", "reset", "poweroff", "poweron"): r = api(f"/servers/{sid}/actions/{cmd}", "POST") print(cmd, r["action"]["status"], r["action"]["started"]) elif cmd == "rescue-on": keys = [int(k) for k in args] or DEFAULT_RESCUE_KEYS r = api(f"/servers/{sid}/actions/enable_rescue", "POST", {"type": "linux64", "ssh_keys": keys}) print("enable_rescue", r["action"]["status"], "| root password:", r.get("root_password")) print("NOTE: rescue boots on the next power cycle โ€” run poweroff, wait for status=off, poweron.") elif cmd == "rescue-off": r = api(f"/servers/{sid}/actions/disable_rescue", "POST") print("disable_rescue", r["action"]["status"]) elif cmd == "console": r = api(f"/servers/{sid}/actions/request_console", "POST") print(json.dumps({"wss_url": r["wss_url"], "password": r["password"]})) else: sys.exit(f"ERROR: unknown command {cmd!r}\n{__doc__}") if __name__ == "__main__": main()