diff --git a/nix/hosts/cc-ci-orchestrator-hetzner/configuration.nix b/nix/hosts/cc-ci-orchestrator-hetzner/configuration.nix index 276e061..a39a24e 100644 --- a/nix/hosts/cc-ci-orchestrator-hetzner/configuration.nix +++ b/nix/hosts/cc-ci-orchestrator-hetzner/configuration.nix @@ -213,6 +213,31 @@ SSHCFG ''; }; + # project-orchestrator (fleet manager) — always-on so the operator can reach it over Remote + # Control at any time (operator 2026-08-01). Same shape as lichen-orchestrator above; the PO's + # own agents.toml declares NO `wake`, so the watchdog only heals a dead session — it never sends + # periodic prompts. Starting it is `agents.py up`; that also starts its watchdog. + systemd.services.project-orchestrator = { + description = "project-orchestrator (fleet manager) — PO agent + watchdog, remote-control always up"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" "tailscaled.service" "claude-install.service" ]; + wants = [ "network-online.target" ]; + serviceConfig = { + Type = "oneshot"; RemainAfterExit = true; + User = "loops"; Group = "users"; + WorkingDirectory = "/srv/project-orchestrator"; + }; + environment = { HOME = "/home/loops"; CLAUDE_BIN = "/home/loops/.local/bin/claude"; }; + path = [ pkgs.bash pkgs.tmux pkgs.git pkgs.python3 pkgs.openssh pkgs.nettools ]; + script = '' + export PATH="/home/loops/.local/bin:$PATH" + proj="/srv/project-orchestrator" + echo "$(cat /proc/sys/kernel/random/boot_id) boot $(date -u +%FT%TZ) — project-orchestrator up" \ + >> "$proj/.ao-state/boot.log" 2>/dev/null || true + cd "$proj" && python3 engine/agents.py up || echo "PO agents.py up failed" + ''; + }; + # Weekly recipe upgrade — runs /upgrade-all over every enrolled recipe (opens recipe PRs # verified by !testme, never merges). Replaces the boot-fragile busybox-crond-in-tmux from # phase 5 §4 with a reboot-safe systemd timer. The service is timer-triggered only (NOT diff --git a/nix/hosts/cc-ci-orchestrator-hetzner/hardware.nix b/nix/hosts/cc-ci-orchestrator-hetzner/hardware.nix index dfea2be..842ef56 100644 --- a/nix/hosts/cc-ci-orchestrator-hetzner/hardware.nix +++ b/nix/hosts/cc-ci-orchestrator-hetzner/hardware.nix @@ -13,4 +13,21 @@ boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" "vmw_pvscsi" ]; boot.initrd.kernelModules = [ "nvme" ]; fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; }; + + # 150G Hetzner volume (scsi-0HC_Volume_106342723) — holds the Rust build trees, which repeatedly + # filled the 75G root. A full disk here does NOT error: it silently truncates whatever is being + # written (it destroyed a plan file mid-write before anyone noticed). + # + # by-uuid, not /dev/sdb: device names are not stable across reboots and attaching another volume + # would silently mount the wrong disk here. + # + # nofail is REQUIRED, not decoration: without it, a detached/failed volume makes the mount unit a + # boot dependency, systemd drops to emergency mode, and the box comes up WITHOUT SSH. That failure + # has already happened once on this host from a bad nixos config, and it needed a recovery to the + # previous generation. The build cache is not worth risking access to the machine. + fileSystems."/mnt/data" = { + device = "/dev/disk/by-uuid/16cd6650-1399-4cb9-a696-54b70d83203a"; + fsType = "ext4"; + options = [ "defaults" "nofail" "x-systemd.device-timeout=10s" ]; + }; }