nix: one Hetzner host for the CI server + the orchestrator (#cc-ci)
The orchestrator's flake now builds the machine it shares with the cc-ci CI
server: `nixosConfigurations.cc-ci` composes cc-ci's nixosModules.cc-ci-server
(new flake input, nixpkgs + sops-nix follow ours), this repo's orchestrator
module (nix/modules/cc-ci.nix, exported as cc-ci-orchestrator, `cc-ci` kept
as an alias for notplants-nix) and the new nix/modules/orchestrator-host.nix
— the host contract those units always assumed (loops user, claude/opencode
CLIs, opencode web server + tailnet-only UI on 8443 since traefik owns
80/443, nix-ld, tool set, `ssh cc-ci` → loopback).
nix/hosts/cc-ci/{hardware,networking}.nix are PROVISIONAL copies of the old
server's layout so the flake evaluates; they get replaced by the
nixos-infect output of 195.201.88.249.
README.md is the deploy guide (Hetzner Debian → nixos-infect → this flake →
staging → data restore → cutover). archive/ holds the retired Incus/Hetzner
orchestrator host configs, the old terraform and the migration plans;
references updated. cc-ci-plan/plan-cc-ci-combined-host.md is the working
plan for the move.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FqkQq3CDmFWcQ7u1LzoyRz
This commit is contained in:
Claude Fable 5.1
+16
-16
@@ -1,15 +1,16 @@
|
||||
# cc-ci.nix — everything on this host that exists FOR cc-ci, and nothing else.
|
||||
# cc-ci.nix — the cc-ci ORCHESTRATOR: the Builder/Adversary loops supervisor, the operator's
|
||||
# steering session, and the weekly-upgrade + hourly-supervisor timers. Nothing else.
|
||||
#
|
||||
# Split out of the orchestrator host config on 2026-08-20. The host it runs on is a general
|
||||
# agent/orchestration box that also serves several unrelated projects; this module is the cc-ci
|
||||
# part of it, so that the two can evolve (and be reviewed) independently. It is exported from this
|
||||
# repo's flake as `nixosModules.cc-ci` and imported by whichever host runs cc-ci.
|
||||
# Exported from this repo's flake as `nixosModules.cc-ci-orchestrator` (and, for the host that
|
||||
# used to import it under the old name, `nixosModules.cc-ci`). Split out of the shared agent
|
||||
# host config on 2026-08-20; since 2026-09 it runs on the same Hetzner host as the CI server
|
||||
# itself (`#cc-ci` in flake.nix), next to recipe-maintainers/cc-ci's `nixosModules.cc-ci-server`.
|
||||
#
|
||||
# All of it assumes the cc-ci workspaces exist on the host:
|
||||
# /srv/cc-ci the loops workspace (+ .cc-ci-logs, upgrader.env)
|
||||
# /srv/cc-ci-orch this repo (the orchestrator's own working dir)
|
||||
# /srv/cc-ci the loops workspace (+ .cc-ci-logs, upgrader.env) — a symlink to
|
||||
# /srv/cc-ci-orch this repo (the orchestrator's own working dir), with cc-ci/ checked out
|
||||
# and that a `loops` user, tmux, python3 and the standalone claude/opencode CLIs are present —
|
||||
# those are host concerns, provided by the host config, not by this module.
|
||||
# those are host concerns, provided by nix/modules/orchestrator-host.nix, not by this module.
|
||||
{ config, pkgs, lib, ... }:
|
||||
{
|
||||
# cc-ci-loops supervisor — workspace staged 2026-05-31, so ENABLED for reboot-resilience.
|
||||
@@ -48,15 +49,14 @@
|
||||
# cc-ci-orchestrator supervisor — the operator's steering session. Same shape as
|
||||
# lichen-orchestrator / project-orchestrator above: this unit only LAUNCHES the orchestrator's
|
||||
# tmux session via the agent-orchestrator harness (cc-ci-plan/agents.py); it does not own the
|
||||
# session or the tmux server. The orchestrator agent is declared in cc-ci-plan/agents.toml on
|
||||
# the OPencode backend (backend = "opencode", model = "opencode/glm-5.2"), so on boot it
|
||||
# attaches to the shared opencode web server (opencode-web.service below) and is reachable for
|
||||
# Remote Control at https://oc.commoninternet.net under the /srv/cc-ci-orch project. The harness
|
||||
# watchdog (started by `agents.py up`) keeps it alive: heal-only (no stall reboots — a persistent
|
||||
# supervisor must not be killed just for idling). Added 2026-08-03 to give the cc-ci orchestrator
|
||||
# the same reboot-resilience the other two orchestrators already have.
|
||||
# session or the tmux server. The orchestrator agent is declared in cc-ci-plan/agents.toml
|
||||
# (backend/model chosen there — Claude Code under Remote Control since 2026-09-07; before that
|
||||
# opencode/glm-5.2 attached to the shared opencode web server, opencode-web.service in
|
||||
# orchestrator-host.nix, which the upgrader still uses). The harness watchdog (started by
|
||||
# `agents.py up`) keeps it alive: heal-only (no stall reboots — a persistent supervisor must not
|
||||
# be killed just for idling). Added 2026-08-03 for reboot-resilience.
|
||||
systemd.services.cc-ci-orchestrator = {
|
||||
description = "cc-ci orchestrator (operator steering session) — agents.py up orchestrator, opencode backend";
|
||||
description = "cc-ci orchestrator (operator steering session) — agents.py up orchestrator";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" "tailscaled.service" "opencode-web.service" ];
|
||||
wants = [ "network-online.target" ];
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
# orchestrator-host.nix — the host contract that nix/modules/cc-ci.nix (the orchestrator's
|
||||
# loops/timers) silently assumes, made explicit and reusable: the `loops` user the agents run as,
|
||||
# the standalone claude/opencode CLIs, the shared opencode web server and its tailnet-only UI,
|
||||
# nix-ld so foreign binaries run on NixOS, and the tool set agents reach for.
|
||||
#
|
||||
# Exported from flake.nix as `nixosModules.orchestrator-host`. A host imports this together with
|
||||
# `nixosModules.cc-ci-orchestrator`; the combined CI-server + orchestrator host (`#cc-ci`) also
|
||||
# imports recipe-maintainers/cc-ci's `nixosModules.cc-ci-server`.
|
||||
#
|
||||
# History: until 2026-09 this lived (twice, drifting) in nix/hosts/cc-ci-orchestrator-hetzner/
|
||||
# configuration.nix here and in notplants-nix's hosts/notplants-orchestrator/configuration.nix,
|
||||
# the shared agent box that also ran lichen + project-orchestrator. The cc-ci half moved to its
|
||||
# own host; this file is that half.
|
||||
{ config, lib, pkgs, ... }:
|
||||
let
|
||||
cfg = config.cc-ci-orchestrator;
|
||||
in
|
||||
{
|
||||
options.cc-ci-orchestrator = {
|
||||
ciSshHost = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "127.0.0.1";
|
||||
example = "100.95.31.88";
|
||||
description = ''
|
||||
Where `ssh cc-ci` (used by every skill and script that drives the CI server) connects to,
|
||||
as root with ~loops/.ssh/cc-ci-root-ed25519. On the combined host the CI server IS this
|
||||
machine, so the default is loopback; a standalone orchestrator points it at the CI
|
||||
server's tailnet address.
|
||||
'';
|
||||
};
|
||||
|
||||
opencodeUiPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8443;
|
||||
description = ''
|
||||
TLS port of the nginx front door for the opencode web UI. Not 443: on the combined host
|
||||
Traefik (docker swarm) owns 80/443. The port is not opened in the firewall, so it is
|
||||
reachable only over the trusted tailscale interface.
|
||||
'';
|
||||
};
|
||||
|
||||
opencodeUiHost = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "oc.commoninternet.net";
|
||||
description = "nginx server_name for the opencode web UI (self-signed, basic auth).";
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
# ---- the loops user -------------------------------------------------------------------
|
||||
# claude sessions run as non-root (--dangerously-skip-permissions is refused for root).
|
||||
users.users.loops = {
|
||||
isNormalUser = true;
|
||||
home = "/home/loops";
|
||||
shell = pkgs.bash;
|
||||
extraGroups = [ "wheel" "docker" ];
|
||||
};
|
||||
security.sudo.wheelNeedsPassword = false;
|
||||
security.sudo.extraRules = [{
|
||||
users = [ "loops" ];
|
||||
commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
|
||||
}];
|
||||
|
||||
# /home/loops/.local/bin holds the standalone claude + opencode binaries; it must be first on
|
||||
# every PATH (interactive shells, tmux, the systemd units in cc-ci.nix prepend it too).
|
||||
environment.variables.PATH = lib.mkForce
|
||||
"/home/loops/.local/bin:/run/current-system/sw/bin:/run/wrappers/bin:/usr/bin:/bin";
|
||||
|
||||
# ---- nix-ld: the standalone Claude Code / opencode CLIs are foreign dynamic ELF binaries ---
|
||||
programs.nix-ld.enable = true;
|
||||
programs.nix-ld.libraries = with pkgs; [ stdenv.cc.cc.lib zlib openssl curl glibc ];
|
||||
|
||||
# ---- the toolbox every agent on this box gets ----------------------------------------
|
||||
# Bar for adding something: an agent doing ordinary work would otherwise waste a turn
|
||||
# discovering it is absent.
|
||||
environment.systemPackages = with pkgs; [
|
||||
git tmux python3 jq curl cacert
|
||||
gnused gawk coreutils gnugrep findutils util-linux nettools openssh
|
||||
age sops ssh-to-age
|
||||
wget gnutar gzip unzip zip xz
|
||||
ripgrep fd tree file less which
|
||||
procps psmisc htop lsof strace ncdu
|
||||
dnsutils socat netcat-gnu iproute2 iputils
|
||||
openssl gnumake gcc pkg-config
|
||||
yq-go diffutils patch rsync bubblewrap
|
||||
];
|
||||
|
||||
# ---- ssh config for the loops user: `ssh cc-ci` = the CI server (root) -----------------
|
||||
# Written only if absent so a manual customisation survives rebuilds.
|
||||
system.activationScripts.loopsSshConfig = ''
|
||||
mkdir -p /home/loops/.ssh && chown loops:users /home/loops/.ssh && chmod 700 /home/loops/.ssh
|
||||
if [ ! -f /home/loops/.ssh/config ]; then
|
||||
cat > /home/loops/.ssh/config <<'SSHCFG'
|
||||
Host cc-ci
|
||||
HostName ${cfg.ciSshHost}
|
||||
User root
|
||||
IdentityFile /home/loops/.ssh/cc-ci-root-ed25519
|
||||
IdentitiesOnly yes
|
||||
StrictHostKeyChecking accept-new
|
||||
ServerAliveInterval 30
|
||||
|
||||
Host git.autonomic.zone
|
||||
HostName git.autonomic.zone
|
||||
Port 2222
|
||||
User git
|
||||
IdentityFile /home/loops/.ssh/autonomic-bot-gitea-ed25519
|
||||
IdentitiesOnly yes
|
||||
|
||||
Host tangled.org
|
||||
IdentityFile /home/loops/.ssh/tangled-ed25519
|
||||
IdentitiesOnly yes
|
||||
SSHCFG
|
||||
chmod 600 /home/loops/.ssh/config
|
||||
chown loops:users /home/loops/.ssh/config
|
||||
fi
|
||||
'';
|
||||
|
||||
# ---- standalone CLIs (idempotent installers; re-run on every activation, no-op if present) --
|
||||
systemd.services.claude-install = {
|
||||
description = "Install Claude Code CLI for loops user (idempotent)";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
serviceConfig = { Type = "oneshot"; RemainAfterExit = true; User = "loops"; Group = "users"; };
|
||||
environment = { HOME = "/home/loops"; };
|
||||
path = [ pkgs.curl pkgs.bash pkgs.coreutils pkgs.gnutar pkgs.gzip ];
|
||||
script = ''
|
||||
if [ ! -x "$HOME/.local/bin/claude" ]; then
|
||||
echo "installing Claude Code CLI for loops user..."
|
||||
curl -fsSL https://claude.ai/install.sh | bash || echo "install failed — retry on next activation"
|
||||
fi
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.services.opencode-install = {
|
||||
description = "Install opencode CLI for loops user (idempotent)";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
serviceConfig = { Type = "oneshot"; RemainAfterExit = true; User = "loops"; Group = "users"; };
|
||||
environment = { HOME = "/home/loops"; };
|
||||
path = [ pkgs.curl pkgs.bash pkgs.coreutils pkgs.gnutar pkgs.gzip pkgs.unzip ];
|
||||
script = ''
|
||||
if [ ! -x "$HOME/.local/bin/opencode" ]; then
|
||||
echo "installing opencode CLI for loops user..."
|
||||
curl -fsSL https://opencode.ai/install | bash || echo "install failed — retry on next activation"
|
||||
fi
|
||||
'';
|
||||
};
|
||||
|
||||
# ---- opencode web server: one shared instance the opencode-backed agents attach to -------
|
||||
# Provider creds come from /srv/cc-ci/.testenv (out of band, see README).
|
||||
systemd.services.opencode-web = {
|
||||
description = "opencode web server for cc-ci agents";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" "tailscaled.service" "opencode-install.service" ];
|
||||
wants = [ "network-online.target" ];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
User = "loops"; Group = "users";
|
||||
WorkingDirectory = "/srv/cc-ci-orch/cc-ci";
|
||||
EnvironmentFile = [ "-/srv/cc-ci/cc-ci/.env.public" "/srv/cc-ci/.testenv" ];
|
||||
ExecStartPre = "${pkgs.coreutils}/bin/rm -rf /tmp/opencode";
|
||||
ExecStart = "/home/loops/.local/bin/opencode serve --hostname 127.0.0.1 --port 4096";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "5s";
|
||||
};
|
||||
environment = {
|
||||
HOME = "/home/loops";
|
||||
PATH = lib.mkForce "/run/wrappers/bin:/home/loops/.local/bin:/run/current-system/sw/bin:/usr/bin:/bin:/etc/profiles/per-user/loops/bin:/nix/var/nix/profiles/default/bin";
|
||||
};
|
||||
path = [ pkgs.bash pkgs.coreutils pkgs.git pkgs.python3 pkgs.openssh pkgs.tmux pkgs.nettools ];
|
||||
};
|
||||
|
||||
# ---- tailnet-only nginx front door for the opencode UI -------------------------------
|
||||
# Self-signed cert + basic auth, both created out of band (a store path would be world
|
||||
# readable) — see README "Secrets to stage". nginx FAILS TO START if they are missing.
|
||||
# /etc/nginx/oc-selfsigned.crt root:nginx 0644
|
||||
# /etc/nginx/oc-selfsigned.key root:nginx 0640
|
||||
# /etc/nginx/oc-htpasswd root:nginx 0640 (`oc:<bcrypt>`; plaintext in /secrets)
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
recommendedProxySettings = true;
|
||||
virtualHosts.${cfg.opencodeUiHost} = {
|
||||
listen = [ { addr = "0.0.0.0"; port = cfg.opencodeUiPort; ssl = true; } ];
|
||||
# onlySSL flags the vhost as SSL so the module renders ssl_certificate for the listener.
|
||||
onlySSL = true;
|
||||
sslCertificate = "/etc/nginx/oc-selfsigned.crt";
|
||||
sslCertificateKey = "/etc/nginx/oc-selfsigned.key";
|
||||
basicAuthFile = "/etc/nginx/oc-htpasswd";
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:4096";
|
||||
proxyWebsockets = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user