# orchestrator-host.nix — the host contract that nix/modules/cc-ci.nix (the orchestrator's
# loops/timers) silently assumes, made explicit and reusable: the `loops` user the agents run as,
# the standalone opencode CLI, the shared opencode web server and its basic-auth web UI,
# nix-ld so foreign binaries run on NixOS, and the tool set agents reach for.
#
# Exported from flake.nix as `nixosModules.orchestrator-host`. A host imports this together with
# `nixosModules.cc-ci-orchestrator`; the combined CI-server + orchestrator host (`#cc-ci`) also
# imports recipe-maintainers/cc-ci's `nixosModules.cc-ci-server`.
#
# History: until 2026-09 this lived (twice, drifting) in nix/hosts/cc-ci-orchestrator-hetzner/
# configuration.nix here and in notplants-nix's hosts/notplants-orchestrator/configuration.nix,
# the shared agent box that also ran lichen + project-orchestrator. The cc-ci half moved to its
# own host; this file is that half.
{ config, lib, pkgs, ... }:
let
  cfg = config.cc-ci-orchestrator;
in
{
  options.cc-ci-orchestrator = {
    ciSshHost = lib.mkOption {
      type = lib.types.str;
      default = "127.0.0.1";
      example = "100.95.31.88";
      description = ''
        Where `ssh cc-ci` (used by every skill and script that drives the CI server) connects to,
        as root with ~loops/.ssh/cc-ci-local-ed25519 (a key generated ON the host — nothing
        copied from another machine). On the combined host the CI server IS this
        machine, so the default is loopback; a standalone orchestrator points it at the CI
        server's tailnet address.
      '';
    };

    opencodeUiBackendPort = lib.mkOption {
      type = lib.types.port;
      default = 8090;   # not 8080: acme-dns's local API has it on the combined host
      description = ''
        Plain-HTTP port nginx listens on for the opencode UI, reachable ONLY from the docker
        bridge (firewall rule on docker_gwbridge). Traefik — which owns the public 443 on the
        combined host — terminates TLS for opencodeUiHost and forwards here; nginx adds the basic
        auth and logs failures for fail2ban with the real client IP.
      '';
    };

    opencodeUiTraefikNetwork = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      example = "proxy";
      description = ''
        Name of the swarm overlay network the cc-ci traefik watches (cc-ci's swarm.nix creates
        `proxy`). When set, a one-container swarm stack `ccci-opencode-ui` (a socat TCP relay to
        nginx on the docker bridge) is deployed with traefik labels routing opencodeUiHost on
        the `web-secure` entrypoint — the same label mechanism every cc-ci service and recipe
        uses, so it coexists with the traefik recipe's own file provider (the wildcard cert and
        the `security` middleware live there; switching traefik to a file *directory* replaces
        that file and takes every front door down — learned 2026-09-07). null = no route.
      '';
    };

    opencodeUiHost = lib.mkOption {
      type = lib.types.str;
      default = "oc.commoninternet.net";
      description = "nginx server_name for the opencode web UI (TLS + basic auth).";
    };

    opencodeUiExtraHosts = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [ ];
      description = ''
        Extra hostnames routed to the opencode UI alongside opencodeUiHost, rendered as
        `||`-joined Host conditions on the same router. Domain cutover 2026-09: the
        legacy oc.ci.commoninternet.net keeps answering here during the bake window.
      '';
    };

    opencodeUiHtpasswdFile = lib.mkOption {
      type = lib.types.str;
      default = "/secrets/nginx/oc-htpasswd";
      description = ''
        htpasswd file for the opencode UI (`oc:<bcrypt>`), created out of band — a store path
        would be world-readable. Default is under /secrets, the authoritative location for this
        host's secrets; it must be readable by the `nginx` user (root:nginx 0640 in a directory
        nginx can traverse). **nginx refuses to start if it is missing**, and its config check
        runs as the nginx user, so a root-only file fails the check even though the path exists.
      '';
    };

  };

  config = {
    # ---- the loops user -------------------------------------------------------------------
    # claude sessions run as non-root (--dangerously-skip-permissions is refused for root).
    users.users.loops = {
      isNormalUser = true;
      uid = 1000;                 # fixed: workspace files are rsynced between hosts by uid
      home = "/home/loops";
      shell = pkgs.bash;
      extraGroups = [ "wheel" "docker" ];
    };
    security.sudo.wheelNeedsPassword = false;
    security.sudo.extraRules = [{
      users = [ "loops" ];
      commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
    }];

    # /home/loops/.local/bin holds the standalone opencode binary; it must be first on every PATH
    # (interactive shells, tmux, the systemd units in cc-ci.nix prepend it too).
    environment.variables.PATH = lib.mkForce
      "/home/loops/.local/bin:/run/current-system/sw/bin:/run/wrappers/bin:/usr/bin:/bin";

    # ---- nix-ld: the standalone opencode CLI is a foreign dynamic ELF binary --------------------
    programs.nix-ld.enable = true;
    programs.nix-ld.libraries = with pkgs; [ stdenv.cc.cc.lib zlib openssl curl glibc ];

    # ---- the toolbox every agent (and every operator ssh session) on this box gets ----------
    # Bar for adding something: an agent or an operator doing ordinary work would otherwise waste
    # a turn discovering it is absent. Installed system-wide, so it is on PATH for BOTH root and
    # loops via /run/current-system/sw/bin (which the forced PATH below keeps for every user).
    environment.systemPackages = with pkgs; [
      git tmux python3 jq curl cacert
      gnused gawk coreutils gnugrep findutils util-linux nettools openssh
      age sops ssh-to-age
      wget gnutar gzip unzip zip xz
      ripgrep fd tree file less which
      procps psmisc htop lsof strace ncdu
      dnsutils socat netcat-gnu iproute2 iputils
      openssl gnumake gcc pkg-config
      yq-go diffutils patch rsync bubblewrap
      # Editors: `vim` ships `vi` too, so both names resolve for anyone who types either.
      # `nano` comes from the base system. EDITOR is set below so git/systemctl/visudo agree.
      vim nano
      # `sqlite` is not a nicety here: Drone's build/step logs live in its sqlite volume and the
      # runbook (and /cc-ci-status) tell you to read them there; without it every such check
      # needed an ad-hoc `nix-shell -p sqlite`.
      sqlite
      bat bc moreutils pv man-pages
    ];

    # So `git commit`, `systemctl edit`, `visudo` etc. open something that exists on this host.
    environment.variables.EDITOR = "vim";

    # ---- ssh config for the loops user: `ssh cc-ci` = the CI server (root) -----------------
    # Written only if absent so a manual customisation survives rebuilds.
    system.activationScripts.loopsSshConfig = ''
      mkdir -p /home/loops/.ssh && chown loops:users /home/loops/.ssh && chmod 700 /home/loops/.ssh
      if [ ! -f /home/loops/.ssh/config ]; then
        cat > /home/loops/.ssh/config <<'SSHCFG'
Host cc-ci
    HostName ${cfg.ciSshHost}
    User root
    IdentityFile /home/loops/.ssh/cc-ci-local-ed25519
    IdentitiesOnly yes
    StrictHostKeyChecking accept-new
    ServerAliveInterval 30

Host git.autonomic.zone
    HostName git.autonomic.zone
    Port 2222
    User git
    IdentityFile /home/loops/.ssh/autonomic-bot-cc-ci-ed25519
    IdentitiesOnly yes
SSHCFG
        chmod 600 /home/loops/.ssh/config
        chown loops:users /home/loops/.ssh/config
      fi
    '';

    # ---- standalone CLIs (idempotent installers; re-run on every activation, no-op if present) --
    # No Claude Code on this host (operator 2026-09-07): the orchestrator and the weekly upgrader
    # are opencode agents; Claude sessions run on the notplants-orchestrator box and reach this
    # host over ssh.
    systemd.services.opencode-install = {
      description = "Install opencode CLI for loops user (idempotent)";
      wantedBy = [ "multi-user.target" ];
      after = [ "network-online.target" ];
      wants = [ "network-online.target" ];
      serviceConfig = { Type = "oneshot"; RemainAfterExit = true; User = "loops"; Group = "users"; };
      environment = { HOME = "/home/loops"; };
      path = [ pkgs.curl pkgs.bash pkgs.coreutils pkgs.gnutar pkgs.gzip pkgs.unzip ];
      script = ''
        if [ ! -x "$HOME/.local/bin/opencode" ]; then
          echo "installing opencode CLI for loops user..."
          curl -fsSL https://opencode.ai/install | bash || echo "install failed — retry on next activation"
          # The installer puts the binary in ~/.opencode/bin; every unit here expects ~/.local/bin.
          if [ -x "$HOME/.opencode/bin/opencode" ]; then
            mkdir -p "$HOME/.local/bin" && ln -sfn "$HOME/.opencode/bin/opencode" "$HOME/.local/bin/opencode"
          fi
        fi
      '';
    };

    # ---- weekly opencode CLI auto-upgrade ----------------------------------------------------
    # opencode is not in nixpkgs: opencode-install (above) only installs the standalone CLI when
    # the binary is MISSING, so the installed version just ages in place (it sat on 1.18.29 for
    # weeks while 1.18.33 was out). opencode's built-in autoupdate does not cover this host: it
    # auto-applies patch releases only and only fires on a fresh interactive TUI start, and every
    # agent here lives inside the long-lived `opencode serve` (opencode-web). This unit is the
    # mechanism instead. It runs weekly in the Tuesday maintenance window (an hour BEFORE the
    # host auto-update at Tue 03:00 UTC, and clear of the Thursday recipe-upgrade run and the
    # Sunday canonical sweep), and:
    #   1. compares the installed version with the latest GitHub release (no-op when equal);
    #   2. reinstalls via the official installer (same code path as opencode-install) when they
    #      differ, and re-links ~/.local/bin/opencode;
    #   3. restarts opencode-web so the new binary actually takes effect — sessions attached to
    #      it (orchestrator, upgrader, report) drop and their supervisors re-attach/resume, which
    #      is why the busy gate below must hold: a mid-flight CI/upgrade run is never cut, and a
    #      skipped run simply retries next week.
    # Deploying this module never itself bumps opencode: boot/activation installs stay
    # install-if-missing in opencode-install, and this unit is timer-driven only.
    # The outcome of each run lands in .cc-ci-logs/opencode-update-state (read by /cc-ci-status).
    systemd.services.opencode-upgrade = {
      description = "Weekly opencode CLI auto-upgrade (latest release → reinstall → restart opencode-web)";
      after = [ "network-online.target" "opencode-install.service" "opencode-web.service" ];
      wants = [ "network-online.target" ];
      serviceConfig = { Type = "oneshot"; TimeoutStartSec = "30min"; };
      path = with pkgs; [ curl bash coreutils gnugrep gnutar gzip unzip systemd util-linux procps ];
      environment = { HOME = "/home/loops"; OPENCODE_UI_HOST = cfg.opencodeUiHost; };
      script = ''
        set -u
        BIN=/home/loops/.local/bin/opencode
        STATE=/srv/cc-ci-orch/.cc-ci-logs/opencode-update-state
        ocver() { "$BIN" --version 2>/dev/null | tail -1 || true; }
        state() { printf '%s result=%s installed=%s note=%s\n' "$(date -u +%FT%TZ)" "$1" "$2" "$3" > "$STATE"; chown loops:users "$STATE" 2>/dev/null || true; }
        busy() { echo "BUSY: $1 — skipping this week's opencode upgrade (retries next week)"; state skipped "$(ocver)" "$1"; exit 0; }

        # Same busy gate as auto-update.nix: never cut a CI run, the weekly upgrade or the sweep.
        pgrep -f run_recipe_ci >/dev/null && busy "a CI run is in flight"
        systemctl is-active --quiet nightly-sweep.service && busy "the canonical sweep is running"
        runuser -u loops -- tmux has-session -t cc-ci-upgrader 2>/dev/null && busy "the weekly recipe-upgrade run is in flight (tmux cc-ci-upgrader)"
        runuser -u loops -- tmux has-session -t cc-ci-report 2>/dev/null && busy "the weekly report is being written (tmux cc-ci-report)"
        if [ -r /run/secrets/bridge_drone_token ]; then
          running=$(curl -s -m 20 -H "Authorization: Bearer $(cat /run/secrets/bridge_drone_token)" \
            "https://drone.ci.commoninternet.net/api/repos/recipe-maintainers/cc-ci/builds?per_page=10" \
            | grep -o '"status":"running"' | wc -l)
          [ "''${running:-0}" -eq 0 ] || busy "$running Drone build(s) running"
        fi

        INSTALLED=$(ocver)
        LATEST=$(curl -fsSL -m 30 https://api.github.com/repos/anomalyco/opencode/releases/latest \
          | grep -Po '"tag_name":\s*"v?\K[0-9][0-9.]*' || true)
        [ -n "$LATEST" ] || { echo "could not determine the latest opencode release"; state failed "''${INSTALLED:-none}" "latest-unresolved"; exit 1; }
        echo "installed: ''${INSTALLED:-none}   latest: $LATEST"
        if [ "$INSTALLED" = "$LATEST" ]; then
          echo "opencode is up to date"
          state ok "$INSTALLED" "up-to-date"
          exit 0
        fi

        echo "upgrading opencode: ''${INSTALLED:-none} -> $LATEST"
        runuser -u loops -- env HOME=/home/loops bash -c 'curl -fsSL https://opencode.ai/install | bash' \
          || { echo "install failed — the previously installed version is untouched"; state failed "''${INSTALLED:-none}" "install-failed"; exit 1; }
        mkdir -p /home/loops/.local/bin
        ln -sfn /home/loops/.opencode/bin/opencode /home/loops/.local/bin/opencode
        NEWVER=$(ocver)
        [ "$NEWVER" = "$LATEST" ] || { echo "install ran but opencode reports $NEWVER (wanted $LATEST) — not restarting"; state failed "$NEWVER" "install-mismatch"; exit 1; }

        echo "restarting opencode-web so the new binary takes effect (attached sessions drop; their supervisors resume)"
        systemctl restart opencode-web.service
        sleep 10
        systemctl is-active --quiet opencode-web.service || { echo "opencode-web did not come back after the upgrade"; state failed "$NEWVER" "web-restart-failed"; exit 1; }
        code=$(curl -s -m 20 -o /dev/null -w '%{http_code}' --resolve "$OPENCODE_UI_HOST:443:127.0.0.1" "https://$OPENCODE_UI_HOST/")
        [ "$code" = "401" ] || { echo "opencode UI answered $code, not the 401 auth challenge"; state failed "$NEWVER" "ui-check-$code"; exit 1; }
        echo "opencode upgraded to $NEWVER; opencode-web restarted and healthy"
        state ok "$NEWVER" "upgraded; web restarted"
      '';
    };

    systemd.timers.opencode-upgrade = {
      wantedBy = [ "timers.target" ];
      timerConfig = {
        # Weekly slot in the Tuesday maintenance window, an hour BEFORE the host auto-update
        # (cc-ci-auto-update.timer: Tue 03:00 UTC) so a restarted opencode-web has settled before
        # that run's health check; Persistent=false, like the auto-update — no catch-up at boot.
        OnCalendar = "Tue *-*-* 02:00:00 UTC";
        Persistent = false;
        RandomizedDelaySec = "10min";
      };
    };

    # ---- opencode web server: one shared instance the opencode-backed agents attach to -------
    # Provider creds come from /srv/cc-ci/.testenv (out of band, see README).
    systemd.services.opencode-web = {
      description = "opencode web server for cc-ci agents";
      wantedBy = [ "multi-user.target" ];
      after = [ "network-online.target" "opencode-install.service" ];
      wants = [ "network-online.target" ];
      serviceConfig = {
        Type = "simple";
        User = "loops"; Group = "users";
        WorkingDirectory = "/srv/cc-ci-orch/cc-ci";
        EnvironmentFile = [ "-/srv/cc-ci/cc-ci/.env.public" "/srv/cc-ci/.testenv" ];
        ExecStartPre = "${pkgs.coreutils}/bin/rm -rf /tmp/opencode";
        ExecStart = "/home/loops/.local/bin/opencode serve --hostname 127.0.0.1 --port 4096";
        Restart = "on-failure";
        RestartSec = "5s";
      };
      environment = {
        HOME = "/home/loops";
        PATH = lib.mkForce "/run/wrappers/bin:/home/loops/.local/bin:/run/current-system/sw/bin:/usr/bin:/bin:/etc/profiles/per-user/loops/bin:/nix/var/nix/profiles/default/bin";
      };
      path = [ pkgs.bash pkgs.coreutils pkgs.git pkgs.python3 pkgs.openssh pkgs.tmux pkgs.nettools ];
    };

    # ---- nginx: basic auth for the opencode UI, behind traefik -----------------------------
    # Traefik (public 443, the CI wildcard cert) routes opencodeUiHost to this plain-HTTP vhost
    # on the docker bridge address. nginx enforces HTTP basic auth (the opencode web UI has no
    # auth of its own and can drive agent sessions), and — via real_ip from traefik's
    # X-Forwarded-For — logs the CLIENT address on a 401, which is what the fail2ban jail bans.
    # The htpasswd is created out of band (a store path would be world readable); nginx FAILS TO
    # START without it, and its config check runs as the nginx user:
    #   /etc/nginx/oc-htpasswd   root:nginx 0640   (`oc:<bcrypt>`; plaintext kept in /secrets)
    # Rotate with: printf 'oc:%s\n' "$(mkpasswd -m bcrypt "$P")" > /etc/nginx/oc-htpasswd && systemctl reload nginx
    services.nginx = {
      enable = true;
      recommendedProxySettings = true;
      virtualHosts.${cfg.opencodeUiHost} = {
        listen = [ { addr = "0.0.0.0"; port = cfg.opencodeUiBackendPort; } ];
        serverAliases = cfg.opencodeUiExtraHosts;
        basicAuthFile = cfg.opencodeUiHtpasswdFile;
        extraConfig = ''
          # traefik sits on the docker networks (ingress 10.0.0.0/24, gwbridge 172.18.0.0/16)
          set_real_ip_from 172.16.0.0/12;
          set_real_ip_from 10.0.0.0/8;
          real_ip_header X-Forwarded-For;
        '';
        locations."/" = {
          proxyPass = "http://127.0.0.1:4096";
          proxyWebsockets = true;
        };
      };
    };
    # Only docker's bridge may reach the plain-HTTP backend; the public interface stays closed.
    networking.firewall.interfaces.docker_gwbridge.allowedTCPPorts = [ cfg.opencodeUiBackendPort ];

    # The traefik side of the route: a swarm service carrying the router labels. Named ccci-* so
    # the weekly run's orphan sweep (skills/upgrade-all/sweep-orphans.sh keep-list) leaves it be. A plain
    # TCP relay (socat) from the overlay network to nginx on the docker bridge; traefik's
    # X-Forwarded-For passes through untouched, which is what nginx's real_ip reads.
    systemd.services.opencode-ui-route = lib.mkIf (cfg.opencodeUiTraefikNetwork != null) {
      description = "swarm stack ccci-opencode-ui: traefik labels ${cfg.opencodeUiHost} -> nginx basic auth";
      wantedBy = [ "multi-user.target" ];
      after = [ "deploy-proxy.service" "docker.service" "nginx.service" ];
      wants = [ "deploy-proxy.service" ];
      path = [ pkgs.docker ];
      serviceConfig = { Type = "oneshot"; RemainAfterExit = true; };
      script = ''
        docker stack deploy --detach=true -c ${pkgs.writeText "opencode-ui-stack.yml" ''
          # Deployed by opencode-ui-route.service (nix/modules/orchestrator-host.nix). Do not edit.
          version: "3.8"
          services:
            relay:
              image: alpine/socat:1.8.0.3
              command: ["TCP-LISTEN:${toString cfg.opencodeUiBackendPort},fork,reuseaddr", "TCP:172.18.0.1:${toString cfg.opencodeUiBackendPort}"]
              networks: [ ${cfg.opencodeUiTraefikNetwork} ]
              deploy:
                replicas: 1
                labels:
                  - "traefik.enable=true"
                  - "traefik.http.routers.opencode-ui.rule=${lib.concatStringsSep " || " (map (h: "Host(`${h}`)") ([ cfg.opencodeUiHost ] ++ cfg.opencodeUiExtraHosts))}"
                  - "traefik.http.routers.opencode-ui.entrypoints=web-secure"
                  - "traefik.http.routers.opencode-ui.tls=true"
                  - "traefik.http.services.opencode-ui.loadbalancer.server.port=${toString cfg.opencodeUiBackendPort}"
          networks:
            ${cfg.opencodeUiTraefikNetwork}:
              external: true
        ''} ccci-opencode-ui
      '';
    };
  };
}