cc-ci-orchestrator/nix/hosts/cc-ci/configuration.nix

# cc-ci — ONE Hetzner Cloud host running both the cc-ci CI server and the cc-ci orchestrator.
#
# This file is only what is physical or identity about the machine: hardware, networking,
# root SSH keys, firewall + fail2ban, swap, stateVersion. Everything functional comes from modules:
#   cc-ci.nixosModules.cc-ci-server        recipe-maintainers/cc-ci — swarm, traefik, drone,
#                                          runner, bridge, dashboard, reports, acme-dns, harness
#   self.nixosModules.cc-ci-orchestrator   nix/modules/cc-ci.nix — loops, orchestrator, timers
#   self.nixosModules.orchestrator-host    nix/modules/orchestrator-host.nix — loops user, CLIs
# See README.md for provisioning (Hetzner Debian → nixos-infect → this flake) and staging.
{ lib, pkgs, ... }:
{
  imports = [
    ./hardware.nix
    ./networking.nix
  ];

  networking.hostName = "cc-ci";

  # ---- cc-ci server identity --------------------------------------------------------------
  # Public address: acme-dns binds to it and publishes it as the `ns-acme` glue record; the
  # Gandi A records for ci / *.ci / ns-acme .commoninternet.net point here.
  cc-ci.publicIPv4 = "195.201.88.249";
  # cc-ci is a plain flake input here (no private submodule), so the sops file is the one in
  # the deployed --recursive checkout the weekly sweep runs from (README "Stage the workspace").
  cc-ci.sopsFile = "/etc/cc-ci/secrets/secrets.yaml";

  # ---- orchestrator identity --------------------------------------------------------------
  # The CI server is this very host, so `ssh cc-ci` goes to loopback (the module default).
  cc-ci-orchestrator.ciSshHost = "127.0.0.1";
  # Weekly self-update (Tue 03:00 UTC; skips itself while CI is busy; see nix/modules/auto-update.nix).
  cc-ci-orchestrator.autoUpdate.enable = true;

  # The opencode UI: traefik (public 443, the *.ci.commoninternet.net cert) → nginx basic auth.
  cc-ci-orchestrator.opencodeUiHost = "oc.ci.commoninternet.net";
  cc-ci-orchestrator.opencodeUiTraefikNetwork = "proxy";

  # ---- no tailscale on this host (operator 2026-09-07) --------------------------------------
  # Everything is reached over the public IP: ssh (keys only), the CI front doors via traefik,
  # and the opencode UI on 443 (traefik → nginx basic auth). fail2ban below guards the two logins.

  # ---- /secrets is THE authoritative location for this host's secret material ---------------
  # Operator rule (2026-09-08): every secret lives under /secrets; anything that needs one either
  # reads it from there directly (where we own the path) or reaches it by a symlink (where the
  # consuming software fixes the path). One directory to audit, back up, and reason about.
  #
  #   /secrets/files   loops:users 0700  the agent's secrets (testenv, opencode auth, its ssh keys)
  #   /secrets/host    root:root   0700  host identity: ssh host keys + the sops age identity
  #   /secrets/nginx   root:nginx  0750  the opencode UI htpasswd (nginx must read it)
  #
  # /secrets itself is 0711: traversable so nginx can reach its own subdirectory, not listable.
  systemd.tmpfiles.rules = [
    "d /secrets       0711 root root  -"
    "d /secrets/host  0700 root root  -"
    "d /secrets/nginx 0750 root nginx -"
    # Convenience symlinks at the conventional paths, so an operator (or a tool that assumes the
    # usual location) still finds the host keys. NOT load-bearing: sshd and sops-nix below are
    # pointed at /secrets directly, precisely so nothing depends on symlink/activation ordering.
    "L+ /etc/ssh/ssh_host_ed25519_key     - - - - /secrets/host/ssh_host_ed25519_key"
    "L+ /etc/ssh/ssh_host_ed25519_key.pub - - - - /secrets/host/ssh_host_ed25519_key.pub"
    "L+ /etc/ssh/ssh_host_rsa_key         - - - - /secrets/host/ssh_host_rsa_key"
    "L+ /etc/ssh/ssh_host_rsa_key.pub     - - - - /secrets/host/ssh_host_rsa_key.pub"
  ];

  # sops-nix: the cc-ci server module hard-codes /etc/ssh/... and /var/lib/sops-nix/key.txt.
  # Override both to the authoritative copies. THE ED25519 HOST KEY IS LOAD-BEARING BEYOND SSH:
  # its age identity (age1tmvg…) is a recipient of cc-ci-secrets, so replacing or regenerating it
  # makes every cc-ci secret undecryptable. Move it, never re-create it.
  sops.age.sshKeyPaths = lib.mkForce [ "/secrets/host/ssh_host_ed25519_key" ];
  sops.age.keyFile = lib.mkForce "/secrets/host/sops-age-key.txt";

  # ---- ssh ----------------------------------------------------------------------------------
  services.openssh = {
    enable = true;
    settings.PermitRootLogin = "yes";
    # Host keys live in /secrets (above). sshd is pointed here directly rather than through the
    # /etc/ssh symlinks, so it can never write a NEW key through a dangling link — that would
    # silently rotate the age identity that decrypts cc-ci-secrets.
    hostKeys = [
      { path = "/secrets/host/ssh_host_ed25519_key"; type = "ed25519"; }
      { path = "/secrets/host/ssh_host_rsa_key"; type = "rsa"; bits = 4096; }
    ];
  };
  # Root keys: PUBLIC keys, tracked deliberately in ./ssh-keys (one per line, blank lines ok).
  users.users.root.openssh.authorizedKeys.keys =
    builtins.filter (s: s != "") (lib.splitString "\n" (builtins.readFile ./ssh-keys));
  # The loops user can also be reached directly (same keys) — handy for rsync of its workspace.
  users.users.loops.openssh.authorizedKeys.keys =
    builtins.filter (s: s != "") (lib.splitString "\n" (builtins.readFile ./ssh-keys));
  services.openssh.settings.PasswordAuthentication = false;
  services.openssh.settings.KbdInteractiveAuthentication = false;

  # ---- firewall -------------------------------------------------------------------------------
  # 80/443 (traefik) and 53 (acme-dns) are opened by the cc-ci-server module; the opencode UI
  # rides 443 through traefik (orchestrator-host.nix), so only ssh is opened here.
  networking.firewall = {
    enable = true;
    allowedTCPPorts = [ 22 ];
  };

  # ---- fail2ban: sshd (password auth is off, this stops the log noise and slow brute force) and
  # the opencode UI's basic auth (nginx logs 401s with the real client IP to the journal; the
  # built-in nginx-http-auth filter matches them). Those clients arrive through traefik's
  # docker-published 443, which iptables FORWARDs rather than INPUTs, so the ban for that jail
  # goes into the DOCKER-USER chain — an INPUT rule would never see the traffic.
  services.fail2ban = {
    enable = true;
    maxretry = 5;
    bantime = "1h";
    bantime-increment = { enable = true; maxtime = "48h"; factor = "4"; };
    ignoreIP = [ "127.0.0.0/8" "::1" ];
    jails.nginx-http-auth.settings = {
      enabled = true;
      filter = "nginx-http-auth";
      # NixOS nginx logs errors to stderr → the journal, not /var/log/nginx/error.log (which
      # exists but stays empty). Read the unit's journal instead.
      backend = "systemd";
      journalmatch = "_SYSTEMD_UNIT=nginx.service";
      banaction = "iptables-allports";
      chain = "DOCKER-USER";
    };
  };

  networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
  # This host's own public names resolve to itself regardless of external DNS state (host
  # processes: the drone runner, the harness, the orchestrator; containers use the resolvers
  # above). Per-run recipe domains are random and cannot be pinned — those follow public DNS.
  networking.hosts."195.201.88.249" = [
    "ci.commoninternet.net"
    "drone.ci.commoninternet.net"
    "report.ci.commoninternet.net"
    "traefik.ci.commoninternet.net"
    "warm-keycloak.ci.commoninternet.net"
    "oc.ci.commoninternet.net"
  ];

  # ---- memory: 8 GB RAM shared by the swarm (recipe deploys) and 3–6 agent sessions ---------
  swapDevices = [ { device = "/swapfile"; size = 8192; } ];

  # ssh client for root (the orchestrator's `ssh cc-ci` goes through the loops user's own config).
  environment.systemPackages = [ pkgs.openssh ];

  # Fresh NixOS 26.05 install (nixos-infect, 2026-09-07). Never change this on an existing host.
  system.stateVersion = "26.05";
}