# cc-ci-orchestrator-hetzner — NixOS config for the Hetzner loops runtime host.
#
# Purpose: run the cc-ci Builder/Adversary/Watchdog loops + orchestrator/assistant sessions
# on a Hetzner cpx11 (2 vCPU / 2 GB dedicated AMD / 40 GB NVMe), replacing the slow b1 Incus VM.
#
# Provision with terraform/ then converge with: nixos-rebuild switch --flake .#cc-ci-orchestrator-hetzner
# See terraform/README.md for the full Stage 2 procedure.
{ config, pkgs, lib, ... }:
{
  # hardware.nix is the nixos-infect generated hardware-configuration.nix (see README Stage 2a).

  # atproto-likes — the notplants-atproto "most-liked accounts" web UI, served at
  # atproto.commoninternet.net. Brings in Docker + a compose stack + an nginx vhost
  # with ACME, and opens 80/443 (previously only 22 was public). Canonical source of
  # the module is the project repo; ../atproto-likes.nix is a copy kept in this tree
  # because pure evaluation cannot import a path outside the flake.
  # cc-ci.nix holds everything that exists FOR cc-ci (the loops, the cc-ci orchestrator session,
  # the weekly recipe-upgrade run and its hourly supervisor). Split out 2026-08-20 so this host —
  # a general agent box that also serves unrelated projects — can be reasoned about separately
  # from cc-ci. Also exported as `nixosModules.cc-ci` from this repo's flake for other hosts.
  imports = [ ../../atproto-likes.nix ../../modules/cc-ci.nix ];

  services.openssh = {
    enable = true;
    settings.PermitRootLogin = "yes";
  };

  # Root SSH access — all keys from the current orchestrator VM's /root/.ssh/authorized_keys.
  users.users.root.openssh.authorizedKeys.keys = [
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOk8NaeBdPbS2gfUvbny8h0AkZlVjGYHzx4QPXSJ38gd claude@claude-vm"
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJVlfoLBPseQ9fA9534KmRg2KWcksKZGzAJIpHJ2JpsI mfowler.email@protonmail.com"
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAcyTGb/wVgdhg5oBCZZvBaR1RuUQRY/3WHnOQpNDCsp claude-cc-ci-sandbox@20260526"
  ];
  networking.useDHCP = true;
  networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
  networking.firewall = {
    enable = true;
    trustedInterfaces = [ "tailscale0" ];
    # Port 80 open only on the tailscale interface (trusted) — nginx binds there for oc.commoninternet.net.
    allowedTCPPorts = [ 22 ];
  };
  nix.settings.experimental-features = [ "nix-command" "flakes" ];
  system.stateVersion = "24.11";

  # Tailscale — auth key at /etc/ts-auth-key (placed manually in Stage 2, not in git).
  services.tailscale = {
    enable = true;
    authKeyFile = "/etc/ts-auth-key";
    extraUpFlags = [ "--hostname=cc-ci-orchestrator" ];
  };

  # 4 GB disk swap — claude session memory safety net (2 GB RAM is tight for 3+ sessions).
  swapDevices = [ { device = "/swapfile"; size = 4096; } ];

  # nix-ld — lets the standalone Claude Code CLI (foreign dynamic ELF / Bun) run on NixOS.
  programs.nix-ld.enable = true;
  programs.nix-ld.libraries = with pkgs; [
    stdenv.cc.cc.lib
    zlib
    openssl
    curl
    glibc
  ];

  environment.systemPackages = with pkgs; [
    git tmux python3 jq curl cacert
    gnused gawk coreutils gnugrep findutils util-linux
    nettools openssh
    age sops                # key management (same toolchain as cc-ci server)
  ];

  # loops user — claude sessions run as non-root (--dangerously-skip-permissions blocked for root).
  users.users.loops = {
    isNormalUser = true;
    home = "/home/loops";
    shell = pkgs.bash;
    extraGroups = [ "wheel" ];
  };
  security.sudo.wheelNeedsPassword = false;
  security.sudo.extraRules = [{
    users = [ "loops" ];
    commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
  }];

  # Ensure /home/loops/.local/bin (claude + opencode) is on the loops user PATH.
  # opencode binary is installed there manually (not yet in nixpkgs); re-install if missing:
  #   curl -sL https://github.com/anomalyco/opencode/releases/download/v1.15.13/opencode-linux-x64.tar.gz \
  #     | tar -xz -C /home/loops/.local/bin opencode && chmod +x /home/loops/.local/bin/opencode
  environment.variables.PATH = lib.mkForce
    "/home/loops/.local/bin:/run/current-system/sw/bin:/run/wrappers/bin:/usr/bin:/bin";

  # SSH config for the loops user — points to the cc-ci Hetzner server via tailnet.
  # HostName is the Hetzner cc-ci server's tailnet IP (cutover settled 2026-05-31).
  system.activationScripts.loopsSshConfig = ''
    mkdir -p /home/loops/.ssh && chown loops:users /home/loops/.ssh && chmod 700 /home/loops/.ssh
    # Only write if not already present (preserves manual customisation).
    if [ ! -f /home/loops/.ssh/config ]; then
      cat > /home/loops/.ssh/config <<'SSHCFG'
Host cc-ci
    HostName 100.95.31.88
    User root
    IdentityFile /home/loops/.ssh/cc-ci-root-ed25519
    IdentitiesOnly yes
    StrictHostKeyChecking accept-new
    ServerAliveInterval 30
SSHCFG
      chmod 600 /home/loops/.ssh/config
      chown loops:users /home/loops/.ssh/config
    fi
  '';

  # claude-install — fetch the standalone Claude Code CLI for the loops user if missing.
  systemd.services.claude-install = {
    description = "Install Claude Code CLI for loops user (idempotent)";
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      Type = "oneshot"; RemainAfterExit = true;
      User = "loops"; Group = "users";
    };
    environment = { HOME = "/home/loops"; };
    path = [ pkgs.curl pkgs.bash pkgs.coreutils pkgs.gnutar pkgs.gzip ];
    script = ''
      if [ ! -x "$HOME/.local/bin/claude" ]; then
        echo "installing Claude Code CLI for loops user..."
        curl -fsSL https://claude.ai/install.sh | bash || echo "install failed — retry on next activation"
      fi
    '';
  };

  # opencode web server — one shared instance; agent sessions attach to it for web visibility.
  # Serves the web UI at http://oc.commoninternet.net (via nginx below, tailscale-only).
  # Provider creds are read from /srv/cc-ci/.testenv at startup.
  systemd.services.opencode-web = {
    description = "opencode web server for cc-ci agents";
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" "tailscaled.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      Type = "simple";
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/cc-ci-orch/cc-ci";
      EnvironmentFile = "/srv/cc-ci/.testenv";
      ExecStartPre = "${pkgs.coreutils}/bin/rm -rf /tmp/opencode";
      ExecStart = "/home/loops/.local/bin/opencode serve --hostname 127.0.0.1 --port 4096";
      Restart = "on-failure";
      RestartSec = "5s";
    };
    environment = {
      HOME = "/home/loops";
      PATH = lib.mkForce "/run/wrappers/bin:/home/loops/.local/bin:/run/current-system/sw/bin:/usr/bin:/bin:/home/loops/.nix-profile/bin:/nix/profile/bin:/home/loops/.local/state/nix/profile/bin:/etc/profiles/per-user/loops/bin:/nix/var/nix/profiles/default/bin";
    };
    path = [ pkgs.bash pkgs.coreutils pkgs.git pkgs.python3 pkgs.openssh pkgs.tmux pkgs.nettools ];
  };

  # nginx — reverse-proxy oc.commoninternet.net → opencode web server.
  # Bound to the tailscale IP so it is only reachable on the tailnet.
  # DNS: add A record  oc.commoninternet.net → 100.84.190.30  (operator step if hostname access is wanted).
  services.nginx = {
    enable = true;
    recommendedProxySettings = true;
    virtualHosts."oc.commoninternet.net" = {
      # Listen on the tailscale interface only — not the public IP. Both 80 and 443.
      # 443 uses a SELF-SIGNED cert (below): this name resolves to a CGNAT tailscale
      # IP, so Let's Encrypt HTTP-01 can't validate it and there is no DNS-01 provider
      # configured on this host. The trusted *.ci.commoninternet.net wildcard lives on
      # the coop-cloud swarm (91.98.47.73), not here, and doesn't cover this label.
      # Self-signed is fine because the vhost is tailnet-only (trusted network); the
      # browser shows a one-time trust prompt. (Chosen by operator, 2026-08-03.)
      #
      # The cert/key are created out of band — same convention as oc-htpasswd, NOT in
      # git and NOT in the nix store (a store path would be world-readable):
      #
      #   /etc/nginx/oc-selfsigned.crt    root:nginx 0644
      #   /etc/nginx/oc-selfsigned.key    root:nginx 0640
      #
      # Regenerate (10y, SANs = hostname + tailscale IP + ts.net name) with:
      #   openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \
      #     -keyout /etc/nginx/oc-selfsigned.key -out /etc/nginx/oc-selfsigned.crt \
      #     -subj /CN=oc.commoninternet.net \
      #     -addext "subjectAltName=DNS:oc.commoninternet.net,DNS:cc-ci-orchestrator-1.taila4a0bf.ts.net,IP:100.84.190.30"
      #   sudo chown root:nginx /etc/nginx/oc-selfsigned.{crt,key}
      #   sudo chmod 644 /etc/nginx/oc-selfsigned.crt && sudo chmod 640 /etc/nginx/oc-selfsigned.key
      #
      # NOTE: like oc-htpasswd, if these files go missing nginx fails to START — which
      # would take the atproto vhost down with it. Recreate them before rebuilding on
      # a fresh host.
      listen = [
        { addr = "100.84.190.30"; port = 80;  ssl = false; }
        { addr = "100.84.190.30"; port = 443; ssl = true;  }
      ];
      # addSSL (serve BOTH http+https) is required, not cosmetic: the NixOS nginx
      # module only renders the `ssl_certificate` directives when a vhost is flagged
      # as an SSL vhost. An explicit `listen ... ssl` + sslCertificate WITHOUT this
      # flag produces an SSL listener with no cert → `nginx -t` fails and the whole
      # service (atproto included) won't start.
      addSSL = true;
      sslCertificate    = "/etc/nginx/oc-selfsigned.crt";
      sslCertificateKey = "/etc/nginx/oc-selfsigned.key";

      # HTTP basic auth in front of opencode. The opencode web UI has no
      # authentication of its own and can drive agent sessions, so since this host
      # started serving 80/443 publicly (atproto-likes, 2026-08-01) it is worth a
      # second layer even though this vhost is tailnet-only.
      #
      # basicAuthFile, NOT basicAuth: the `basicAuth` attrset writes the password
      # into the nix store, which is world-readable. This file is created out of
      # band — same convention as /etc/ts-auth-key — and is not in git:
      #
      #   /etc/nginx/oc-htpasswd          root:nginx 0640, `oc:<bcrypt hash>`
      #   /secrets/files/oc-basic-auth.txt the plaintext, loops-only 0600
      #
      # Rotate with:
      #   P=$(python3 -c "import secrets,string;a=string.ascii_letters+string.digits;print(''.join(secrets.choice(a) for _ in range(32)))")
      #   printf 'oc:%s\n' "$(mkpasswd -m bcrypt "$P")" | sudo tee /etc/nginx/oc-htpasswd
      #   sudo chown root:nginx /etc/nginx/oc-htpasswd && sudo chmod 640 /etc/nginx/oc-htpasswd
      #   sudo systemctl reload nginx
      #
      # NOTE: if this file goes missing, nginx fails to START — which would take
      # the atproto vhost down with it. Recreate it before rebuilding on a fresh
      # host.
      basicAuthFile = "/etc/nginx/oc-htpasswd";

      locations."/" = {
        proxyPass = "http://127.0.0.1:4096";
        proxyWebsockets = true;
      };
    };
  };



  # p-lichen-orchestrator supervisor — the SEPARATE lichen.page testing/hardening orchestrator
  # (distinct from cc-ci-loops above). Reboot-resilience: on boot, resume the orchestrator's Remote
  # Control session + watchdog + pipeline via `engine/agents.py up`. Added 2026-07-08 after a reboot
  # (Hetzner rollback) left this orchestrator down while cc-ci-loops auto-recovered.
  # NOTE: still points at the /home path — will be re-pointed to /srv when that migration happens.
  systemd.services.lichen-orchestrator = {
    description = "p-lichen-orchestrator (lichen.page testing) — orchestrator + watchdog + pipeline";
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" "tailscaled.service" "claude-install.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      # KillMode=process: this unit only LAUNCHES the tmux server, it does not own it. With the
      # default (control-group) systemd kills every leftover process in the cgroup when the unit
      # stops — and since one tmux server hosts every agent session on this host, a rebuild that
      # merely touched this unit wiped all of them (operator 2026-08-01). Only the (already
      # exited) main process is killed now; `systemctl stop` therefore does NOT tear down agents.
      KillMode = "process";
      Type = "oneshot"; RemainAfterExit = true;
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/lichen-orchestrator";
    };
    environment = { HOME = "/home/loops"; CLAUDE_BIN = "/home/loops/.local/bin/claude"; };
    path = [ pkgs.bash pkgs.tmux pkgs.git pkgs.python3 pkgs.openssh pkgs.nettools ];
    script = ''
      export PATH="/home/loops/.local/bin:$PATH"
      proj="/srv/lichen-orchestrator"
      # boot marker (best-effort; boot_id-gated logging can be added later)
      echo "$(cat /proc/sys/kernel/random/boot_id) boot $(date -u +%FT%TZ) — lichen-orchestrator up" \
        >> "$proj/.ao-state/boot.log" 2>/dev/null || true
      cd "$proj" && python3 engine/agents.py up || echo "p-lichen agents.py up failed"
    '';
  };

  # project-orchestrator (fleet manager) — always-on so the operator can reach it over Remote
  # Control at any time (operator 2026-08-01). Same shape as lichen-orchestrator above; the PO's
  # own agents.toml declares NO `wake`, so the watchdog only heals a dead session — it never sends
  # periodic prompts. Starting it is `agents.py up`; that also starts its watchdog.
  systemd.services.project-orchestrator = {
    description = "project-orchestrator (fleet manager) — PO agent + watchdog, remote-control always up";
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" "tailscaled.service" "claude-install.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      # KillMode=process: this unit only LAUNCHES the tmux server, it does not own it. With the
      # default (control-group) systemd kills every leftover process in the cgroup when the unit
      # stops — and since one tmux server hosts every agent session on this host, a rebuild that
      # merely touched this unit wiped all of them (operator 2026-08-01). Only the (already
      # exited) main process is killed now; `systemctl stop` therefore does NOT tear down agents.
      KillMode = "process";
      Type = "oneshot"; RemainAfterExit = true;
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/project-orchestrator";
    };
    environment = { HOME = "/home/loops"; CLAUDE_BIN = "/home/loops/.local/bin/claude"; };
    path = [ pkgs.bash pkgs.tmux pkgs.git pkgs.python3 pkgs.openssh pkgs.nettools ];
    script = ''
      export PATH="/home/loops/.local/bin:$PATH"
      proj="/srv/project-orchestrator"
      echo "$(cat /proc/sys/kernel/random/boot_id) boot $(date -u +%FT%TZ) — project-orchestrator up" \
        >> "$proj/.ao-state/boot.log" 2>/dev/null || true
      cd "$proj" && python3 engine/agents.py up || echo "PO agents.py up failed"
    '';
  };




}