nix: one Hetzner host for the CI server + the orchestrator (#cc-ci)
The orchestrator's flake now builds the machine it shares with the cc-ci CI
server: `nixosConfigurations.cc-ci` composes cc-ci's nixosModules.cc-ci-server
(new flake input, nixpkgs + sops-nix follow ours), this repo's orchestrator
module (nix/modules/cc-ci.nix, exported as cc-ci-orchestrator, `cc-ci` kept
as an alias for notplants-nix) and the new nix/modules/orchestrator-host.nix
— the host contract those units always assumed (loops user, claude/opencode
CLIs, opencode web server + tailnet-only UI on 8443 since traefik owns
80/443, nix-ld, tool set, `ssh cc-ci` → loopback).
nix/hosts/cc-ci/{hardware,networking}.nix are PROVISIONAL copies of the old
server's layout so the flake evaluates; they get replaced by the
nixos-infect output of 195.201.88.249.
README.md is the deploy guide (Hetzner Debian → nixos-infect → this flake →
staging → data restore → cutover). archive/ holds the retired Incus/Hetzner
orchestrator host configs, the old terraform and the migration plans;
references updated. cc-ci-plan/plan-cc-ci-combined-host.md is the working
plan for the move.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FqkQq3CDmFWcQ7u1LzoyRz
This commit is contained in:
Claude Fable 5.1
Generated
+23
@@ -0,0 +1,23 @@
|
||||
# This file is maintained automatically by "tofu init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.opentofu.org/hetznercloud/hcloud" {
|
||||
version = "1.64.0"
|
||||
constraints = "1.64.0"
|
||||
hashes = [
|
||||
"h1:FUkTfFrWlmv0JhsbjQvTk3zY7A2Q0LuoSs0PKEzaLpk=",
|
||||
"zh:5bf7f8f429b1a8f485988d199f46295676a6cdf7d84ad11f1f4613faecfa89d5",
|
||||
"zh:63b3d182474dd5afd0d5ab3f5f66228b752504436bcb2f4721bd6f1233d0f2ae",
|
||||
"zh:6867da2d89d297b6760d80dde373e74df511bea72f7daccf6a944a9de4b4d4ed",
|
||||
"zh:766fdcea1b03038a92414eafaa430b9ac0c57b36ce4c1573e6e291431659d528",
|
||||
"zh:7f3186dfcae4028eac4f2c9c2c382b49c1fad0b63d0471b50748ee6817fbd8d2",
|
||||
"zh:bb8a33b6ff9a4d3bce87628c49b08a4780e2c034762f40112058d96f5a4e52bd",
|
||||
"zh:cc93751c7c90a37f180cf3e5439ed34f3154e60de5920a13d153d93954938239",
|
||||
"zh:d6e2abf05a0eb8fe0544eb099960a4962db61532e7757016ccacbf0b83bcd1ae",
|
||||
"zh:da9e3adedd8d33623aac4929fa8b1210f98d2931d5737c201da0dda992dd25ab",
|
||||
"zh:dffc931aec4d7b0733690e115b1aabdf5c157b7d347a09a9d149ee6b7e9d8ce3",
|
||||
"zh:e565dea4f28182099a271f794e3b781f069ea54976f5f05dbb79a1c2b6627459",
|
||||
"zh:e79411287af28ccf6187bd418b7ea2ee217e642026392ddc8027bf3e3287fb80",
|
||||
"zh:f5102d7141a04c193dffbb5cbc3f7e3588c41b87e11877d2e20d57ea5ef64123",
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
# terraform — Hetzner cc-ci-orchestrator server
|
||||
|
||||
Provisions a Hetzner **cpx11** (2 vCPU / 2 GB dedicated AMD / 40 GB NVMe) for the cc-ci loops
|
||||
runtime (Builder + Adversary + Watchdog + Orchestrator sessions), replacing the slow b1 Incus VM.
|
||||
Uses nixos-infect to convert Debian → NixOS, then converges via the cc-ci-orchestrator flake.
|
||||
|
||||
---
|
||||
|
||||
## Stage 1 — provision the server
|
||||
|
||||
```bash
|
||||
# from /srv/cc-ci/terraform/
|
||||
source /srv/cc-ci/.testenv # loads HCLOUD_TOKEN
|
||||
export TF_VAR_ssh_public_key="$(cat /home/loops/.ssh/cc-ci-root-ed25519.pub)"
|
||||
|
||||
tofu init
|
||||
tofu plan
|
||||
tofu apply
|
||||
```
|
||||
|
||||
Note the `server_ipv4` output. nixos-infect runs on first boot — wait ~5 min, then:
|
||||
|
||||
```bash
|
||||
# confirm NixOS is up (may need to retry while infect reboots)
|
||||
ssh root@<server_ipv4> 'nixos-version'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Stage 2 — converge to cc-ci-orchestrator-hetzner
|
||||
|
||||
### 2a. Capture hardware config
|
||||
|
||||
```bash
|
||||
ssh root@<server_ipv4> 'cat /etc/nixos/hardware-configuration.nix'
|
||||
```
|
||||
|
||||
Copy the output to `nix/hosts/cc-ci-orchestrator-hetzner/hardware.nix` in this repo, commit, push.
|
||||
|
||||
### 2b. Stage workspace on the new server
|
||||
|
||||
```bash
|
||||
ssh root@<server_ipv4>
|
||||
|
||||
# Install Tailscale auth key (from .testenv TS_AUTH_KEY)
|
||||
echo "<TS_AUTH_KEY>" > /etc/ts-auth-key && chmod 600 /etc/ts-auth-key
|
||||
|
||||
# Clone this repo as the loops user workspace
|
||||
git clone --recursive \
|
||||
https://autonomic-bot:<token>@git.autonomic.zone/recipe-maintainers/cc-ci-orchestrator.git \
|
||||
/srv/cc-ci-orch
|
||||
ln -sfn /srv/cc-ci-orch /srv/cc-ci # loops expect /srv/cc-ci
|
||||
|
||||
# Place master age key (copied from current VM .sops/master-age.txt)
|
||||
mkdir -p /srv/cc-ci/.sops
|
||||
scp loops@<old-vm>:/srv/cc-ci/.sops/master-age.txt /srv/cc-ci/.sops/master-age.txt
|
||||
chmod 600 /srv/cc-ci/.sops/master-age.txt
|
||||
```
|
||||
|
||||
### 2c. Run nixos-rebuild
|
||||
|
||||
```bash
|
||||
# on the new server
|
||||
cd /srv/cc-ci
|
||||
nixos-rebuild switch --flake .#cc-ci-orchestrator-hetzner
|
||||
```
|
||||
|
||||
### 2d. Stage credentials (not in git — placed once)
|
||||
|
||||
```bash
|
||||
# SSH key for reaching cc-ci
|
||||
mkdir -p /home/loops/.ssh && chmod 700 /home/loops/.ssh
|
||||
# scp cc-ci-root-ed25519 from current VM or copy content
|
||||
chmod 600 /home/loops/.ssh/cc-ci-root-ed25519
|
||||
|
||||
# .testenv (GITEA creds, etc.)
|
||||
cp /path/to/.testenv /srv/cc-ci/.testenv && chmod 600 /srv/cc-ci/.testenv
|
||||
```
|
||||
|
||||
### 2e. Auth claude and start loops
|
||||
|
||||
```bash
|
||||
# as loops user on new server
|
||||
sudo -u loops /home/loops/.local/bin/claude auth login # device code — operator step
|
||||
|
||||
# start the loops
|
||||
cd /srv/cc-ci && sudo -u loops ./cc-ci-plan/launch.sh start
|
||||
```
|
||||
|
||||
### 2f. Verify
|
||||
|
||||
```bash
|
||||
tmux ls # should show cc-ci-builder, cc-ci-adv, cc-ci-watchdog
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Cutover
|
||||
|
||||
Once the new server is running and the loops are verified:
|
||||
|
||||
1. Update the `Host cc-ci` entry in the current VM's `/home/loops/.ssh/config` if needed
|
||||
2. Stop the old Incus VM (or just leave it idle — it costs nothing in disk)
|
||||
|
||||
---
|
||||
|
||||
## Variables
|
||||
|
||||
| Variable | Default | Notes |
|
||||
|---|---|---|
|
||||
| `location` | `nbg1` | Nuremberg |
|
||||
| `server_type` | `cpx11` | 2 vCPU / 2 GB dedicated AMD. Upgrade to `cpx21` (4 GB) if OOM. |
|
||||
| `image` | `debian-12` | nixos-infect base |
|
||||
| `server_name` | `cc-ci-orchestrator` | |
|
||||
| `ssh_public_key` | required | Pass via `TF_VAR_ssh_public_key` |
|
||||
|
||||
---
|
||||
|
||||
## State
|
||||
|
||||
`terraform.tfstate` and `terraform.tfstate.backup` are gitignored. Keep the state file locally or
|
||||
in a remote backend — losing it means `tofu destroy` can't find the server (use `tofu import` to
|
||||
recover, or delete directly via the Hetzner console).
|
||||
@@ -0,0 +1,32 @@
|
||||
resource "hcloud_ssh_key" "cc_ci_orch" {
|
||||
name = "cc-ci-orchestrator-deploy"
|
||||
public_key = var.ssh_public_key
|
||||
|
||||
labels = {
|
||||
project = "cc-ci-orchestrator"
|
||||
managed = "terraform"
|
||||
}
|
||||
}
|
||||
|
||||
resource "hcloud_server" "cc_ci_orch" {
|
||||
name = var.server_name
|
||||
server_type = var.server_type
|
||||
image = var.image
|
||||
location = var.location
|
||||
ssh_keys = [hcloud_ssh_key.cc_ci_orch.id]
|
||||
|
||||
# Stage 1: cloud-init runs nixos-infect on first boot, converting Debian to NixOS, then reboots.
|
||||
# Wait ~5 min after apply, then SSH in and run Stage 2 per README.md.
|
||||
user_data = file("${path.module}/user-data.sh")
|
||||
|
||||
public_net {
|
||||
ipv4_enabled = true
|
||||
ipv6_enabled = false
|
||||
}
|
||||
|
||||
labels = {
|
||||
project = "cc-ci-orchestrator"
|
||||
managed = "terraform"
|
||||
stage = "infect"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
output "server_ipv4" {
|
||||
description = "Public IPv4 address of the cc-ci-orchestrator Hetzner server"
|
||||
value = hcloud_server.cc_ci_orch.ipv4_address
|
||||
}
|
||||
|
||||
output "server_id" {
|
||||
description = "Hetzner internal server ID"
|
||||
value = hcloud_server.cc_ci_orch.id
|
||||
}
|
||||
|
||||
output "ssh_connect" {
|
||||
description = "SSH command to connect as root (after nixos-infect)"
|
||||
value = "ssh root@${hcloud_server.cc_ci_orch.ipv4_address}"
|
||||
}
|
||||
|
||||
output "nixos_infect_log" {
|
||||
description = "Check infect progress"
|
||||
value = "ssh root@${hcloud_server.cc_ci_orch.ipv4_address} 'cat /var/log/nixos-infect.log'"
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# Stage 1 — convert Debian 12 → NixOS via nixos-infect (pinned revision).
|
||||
#
|
||||
# nixos-infect generates /etc/nixos/{configuration.nix,hardware-configuration.nix,networking.nix}
|
||||
# with Hetzner-correct bootloader (GRUB) and networking, then reboots into NixOS.
|
||||
#
|
||||
# After the reboot SSH as root is available. Run Stage 2 per terraform/README.md.
|
||||
# Logs: /var/log/nixos-infect.log
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Same pinned revision as the cc-ci server terraform (2026-03-22).
|
||||
INFECT_SHA="40f62a680bb0e8f2f607d79abfaaecd99d59401c"
|
||||
|
||||
export NIX_CHANNEL="nixos-24.11"
|
||||
export PROVIDER="hetzner"
|
||||
export NIXOS_IMPORT=""
|
||||
|
||||
curl -fsSL "https://raw.githubusercontent.com/elitak/nixos-infect/${INFECT_SHA}/nixos-infect" \
|
||||
| bash -x 2>&1 | tee /var/log/nixos-infect.log
|
||||
@@ -0,0 +1,39 @@
|
||||
variable "location" {
|
||||
description = "Hetzner datacenter (nbg1=Nuremberg, fsn1=Falkenstein, hel1=Helsinki)"
|
||||
type = string
|
||||
default = "nbg1"
|
||||
}
|
||||
|
||||
variable "server_type" {
|
||||
description = <<-EOT
|
||||
Hetzner server type. Must be x86 — the flake is x86_64-linux; NEVER use cax* (ARM).
|
||||
cpx22 = AMD 2 vCPU / 4 GB (default; dedicated vCPU, NVMe — the orchestrator loops runtime).
|
||||
cpx32 = AMD 4 vCPU / 8 GB (upgrade if more CPU headroom needed).
|
||||
cx23 = Intel 2 vCPU / 4 GB (shared vCPU, cheaper alternative).
|
||||
Note: cpx11/cpx21 are retired in nbg1.
|
||||
EOT
|
||||
type = string
|
||||
default = "cpx22"
|
||||
|
||||
validation {
|
||||
condition = !startswith(var.server_type, "cax")
|
||||
error_message = "ARM server types (cax*) are not supported — the flake is x86_64-linux only."
|
||||
}
|
||||
}
|
||||
|
||||
variable "image" {
|
||||
description = "Base OS image. nixos-infect supports debian-12 and ubuntu-24.04. debian-12 preferred."
|
||||
type = string
|
||||
default = "debian-12"
|
||||
}
|
||||
|
||||
variable "ssh_public_key" {
|
||||
description = "SSH public key content (the full line). Registered with Hetzner for root access post-infect. Pass via TF_VAR_ssh_public_key."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "server_name" {
|
||||
description = "Hetzner server name and initial NixOS hostname"
|
||||
type = string
|
||||
default = "cc-ci-orchestrator"
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
terraform {
|
||||
required_version = ">= 1.0"
|
||||
required_providers {
|
||||
hcloud = {
|
||||
source = "hetznercloud/hcloud"
|
||||
version = "1.64.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# The hcloud provider reads HCLOUD_TOKEN from the environment automatically.
|
||||
# Never put the token value in any .tf file or .tfvars — keep it in the shell
|
||||
# environment (export HCLOUD_TOKEN=...) or pass via TF_VAR_hcloud_token.
|
||||
provider "hcloud" {}
|
||||
Reference in New Issue
Block a user