From f105808bed03da2a2a0a09c1ecef69ca6156d505 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Mon, 7 Sep 2026 20:15:16 +0000 Subject: [PATCH] nix/hosts/cc-ci: real hardware + networking from the box; document the infect fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hardware.nix / networking.nix are now the nixos-infect output of 195.201.88.249 (ESP E079-7D41, eth0 MAC-pinned), with the one edit that matters: defaultGateway as an attrset with interface = "eth0". README §2a records what it actually took to get NixOS running on the Debian 13 image: NO_SWAP=true (tmpfs /tmp), a manual lustration from rescue mode because 26.05's systemd initrd did not honour NIXOS_LUSTRATE (Debian's unit files shadowed every NixOS service), and the gateway fix applied via a chroot rebuild with the nix sandbox off. flake: cc-ci input back on main (PR #32 merged as f6dbfa3); loops uid pinned to 1000 so workspace rsyncs by uid line up. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FqkQq3CDmFWcQ7u1LzoyRz --- README.md | 52 +++++++++++++++++++++++++---- flake.lock | 9 +++--- flake.nix | 2 +- nix/hosts/cc-ci/hardware.nix | 46 ++++++++------------------ nix/hosts/cc-ci/networking.nix | 54 +++++++++++++------------------ nix/modules/orchestrator-host.nix | 1 + 6 files changed, 86 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 0cffe1e..23d5fe1 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,48 @@ It downloads Nix, builds a NixOS system (5–10 min; follow with `ssh root@ tail -f /var/log/nixos-infect.log`), then reboots. The SSH host key changes: `ssh-keygen -R ` and confirm `ssh root@ nixos-version` prints a 26.05 version. -> If the box does not come back within ~5 minutes, open the Hetzner console (or rescue mode) and -> look at the boot messages — the two known failure modes are a broken bootloader install and a -> wrong `networking.nix` (see §3, the `defaultGateway.interface` note). +### 2a. What went wrong on 2026-09-07, and the fixes (Debian 13 image, NixOS 26.05) + +All three bit on the first attempt; the script above and §3 already include the fixes, this is +so you recognise them if they come back in another form. + +1. **`swapon: /tmp/nixos-infect.XXXX.swp: Invalid argument`** right at the start, script exits. + The Debian 13 cloud image mounts `/tmp` as tmpfs and a swapfile cannot live there. + Fix: `NO_SWAP=true` (in the script above). An 8 GB box does not need the temporary swap. +2. **The box never comes back after the reboot: it boots NixOS, but nearly every unit fails** + (`dbus`, `systemd-logind`, `sshd`, networking …) with + `Could not start dynamically linked executable: /usr/bin/dbus-daemon` in the journal. + nixos-infect leaves the old Debian root in place and relies on NixOS's first boot to move it + to `/old-root` (`/etc/NIXOS_LUSTRATE`). With NixOS 26.05's systemd-based initrd that + lustration did not happen, so Debian's `/etc/systemd/system/*.service` files shadowed the + NixOS units and started Debian binaries. Fix, from Hetzner **rescue mode** + (`enable_rescue` + `reset` in the API/console, ssh in, `mount /dev/sda1 /mnt/root`): + move everything except `nix`, `boot`, `swapfile`, `lost+found`, `var/log`, `var/empty`, + `etc/nixos`, `etc/resolv.conf`, `etc/NIXOS`, `etc/machine-id`, `etc/ssh/ssh_host_*`, + `root/.nix-*`, `root/.ssh` into `/mnt/root/old-root`, delete `etc/NIXOS_LUSTRATE`, unmount, + `disable_rescue`, `reset`. (`/old-root`, ~1 GB, can be deleted once the host is in service.) +3. **Boots, units fine, but no network.** The generated `networking.nix` has + `defaultGateway = "172.31.1.1";` — a bare string. Since NixOS 25.05 that yields no default + route. Fix: `defaultGateway = { address = "172.31.1.1"; interface = "eth0"; };` (this is what + `nix/hosts/cc-ci/networking.nix` carries). To apply it from rescue mode, chroot into the + mounted root and rebuild the boot entry — the nix sandbox cannot `pivot_root` inside a chroot, + so turn it off for that one build: + ```bash + for d in proc sys dev dev/pts; do mount --bind /$d /mnt/root/$d; done + mount -t tmpfs tmpfs /mnt/root/run; cp -L /etc/resolv.conf /mnt/root/etc/resolv.conf + chroot /mnt/root /nix/var/nix/profiles/system/sw/bin/bash -c ' + export PATH=/nix/var/nix/profiles/system/sw/bin NIX_REMOTE= HOME=/root + export NIX_PATH=nixos-config=/etc/nixos/configuration.nix:nixpkgs=/root/.nix-defexpr/channels/nixos + ln -sfn /nix/var/nix/profiles/system /run/current-system + nixos-rebuild boot --option sandbox false' + ``` + The `journalctl -D /mnt/root/var/log/journal -b 0` trick (reading the dead system's journal + from rescue mode) is what told these apart. + +> Rescue mode without a console: `POST /servers//actions/enable_rescue` with your ssh key +> id, then `…/actions/reset`; afterwards `disable_rescue` **and check `rescue_enabled` is false +> before** the next `reset`, or it boots the rescue image again. `scripts/recovery/hetzner.py` +> wraps these (token in `/srv/cc-ci/.hcloud-token`). ## 3. Capture the machine-specific config into this repo @@ -101,10 +140,9 @@ Then in `nix/hosts/cc-ci/`: - `hardware.nix`: keep as generated (GRUB EFI with `efiInstallAsRemovable`, `/boot/efi` by UUID, `/dev/sda1` root). Do not copy another host's file — the UUIDs are per machine. - `networking.nix`: keep the static IPv4 + Hetzner gateway `172.31.1.1`. Make sure - `networking.defaultGateway` has **both** `address` and `interface = "eth0"` — since NixOS 25.05 - omitting the interface leaves the host without a default route after a rebuild (this took the - old CI server offline once; rescue mode fixed it). If the generated IPv6 block has an empty - address, delete the IPv6 parts. + `networking.defaultGateway` has **both** `address` and `interface = "eth0"` (§2a item 3). If + the generated IPv6 block has an empty address, delete the IPv6 parts; a real global address + (as on the 2026-09 box) can stay. - `configuration.nix`: set `cc-ci.publicIPv4` to the server's IPv4 and check `system.stateVersion` is the release you installed (never change it later). - `ssh-keys`: the root keys. diff --git a/flake.lock b/flake.lock index 2544a65..d3827ac 100644 --- a/flake.lock +++ b/flake.lock @@ -10,16 +10,15 @@ ] }, "locked": { - "lastModified": 1788811018, + "lastModified": 1788812004, "narHash": "sha256-Vc7RSeqFHCwlVRhnEEjavuJoyIID+RQdJSygHNA8s8Y=", - "ref": "feat/nixos-module-export", - "rev": "9b99f81f5f20bde865549912a03ff59ac89b2d7c", - "revCount": 1532, + "ref": "refs/heads/main", + "rev": "f6dbfa368995f4d45de09f4052631fd433c87d5b", + "revCount": 1533, "type": "git", "url": "https://git.autonomic.zone/recipe-maintainers/cc-ci.git" }, "original": { - "ref": "feat/nixos-module-export", "type": "git", "url": "https://git.autonomic.zone/recipe-maintainers/cc-ci.git" } diff --git a/flake.nix b/flake.nix index 9564b25..517e491 100644 --- a/flake.nix +++ b/flake.nix @@ -15,7 +15,7 @@ # the host reads the deployed --recursive checkout's secrets.yaml at activation instead # (`cc-ci.sopsFile`). Both `follows` are REQUIRED: without them cc-ci's own nixpkgs/sops-nix # pins would produce a second sops-nix module tree and a second nixpkgs in one system. - cc-ci.url = "git+https://git.autonomic.zone/recipe-maintainers/cc-ci.git?ref=feat/nixos-module-export"; + cc-ci.url = "git+https://git.autonomic.zone/recipe-maintainers/cc-ci.git"; cc-ci.inputs.nixpkgs.follows = "nixpkgs"; cc-ci.inputs.sops-nix.follows = "sops-nix"; }; diff --git a/nix/hosts/cc-ci/hardware.nix b/nix/hosts/cc-ci/hardware.nix index f3f97cf..9f6235a 100644 --- a/nix/hosts/cc-ci/hardware.nix +++ b/nix/hosts/cc-ci/hardware.nix @@ -1,39 +1,19 @@ -# PROVISIONAL — copied from the old cc-ci server (same Hetzner cloud layout) so the flake can be -# evaluated before the new box yielded its own file. MUST be replaced by /etc/nixos/hardware-configuration.nix -# from the new host (README §3): the ESP UUID below belongs to the OLD machine. -# Hardware configuration for cc-ci on Hetzner Cloud (cpx32: AMD 4 vCPU / 8 GB / x86_64). -# Generated by nixos-infect from a Debian 12 base image, then committed here. -# -# nixos-infect uses GRUB + EFI on Hetzner (not systemd-boot), with a qemu-guest profile -# because Hetzner Cloud uses KVM virtualisation. -# -# IMPORTANT: networking.nix (below) contains the server's static public IP. -# When provisioning a new server via `terraform apply`, copy the fresh networking.nix -# from /etc/nixos/networking.nix on the new host and commit it here before rebuilding. +# Generated by nixos-infect on this machine (2026-09-07), captured verbatim per README §3. +# The ESP UUID is specific to THIS server; a new server gets a new file. { modulesPath, ... }: { imports = [ (modulesPath + "/profiles/qemu-guest.nix") ]; - - boot = { - loader = { - efi.efiSysMountPoint = "/boot/efi"; - grub = { - efiSupport = true; - efiInstallAsRemovable = true; - device = "nodev"; - }; + boot.loader = { + efi.efiSysMountPoint = "/boot/efi"; + grub = { + efiSupport = true; + efiInstallAsRemovable = true; + device = "nodev"; }; - initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" "vmw_pvscsi" ]; - initrd.kernelModules = [ "nvme" ]; - }; - - fileSystems."/boot/efi" = { - device = "/dev/disk/by-uuid/D978-69EE"; - fsType = "vfat"; - }; - - fileSystems."/" = { - device = "/dev/sda1"; - fsType = "ext4"; }; + fileSystems."/boot/efi" = { device = "/dev/disk/by-uuid/E079-7D41"; fsType = "vfat"; }; + boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" "vmw_pvscsi" ]; + boot.initrd.kernelModules = [ "nvme" ]; + fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; }; + } diff --git a/nix/hosts/cc-ci/networking.nix b/nix/hosts/cc-ci/networking.nix index 8b1a211..d3aa712 100644 --- a/nix/hosts/cc-ci/networking.nix +++ b/nix/hosts/cc-ci/networking.nix @@ -1,48 +1,38 @@ -# PROVISIONAL — derived from the old cc-ci server networking.nix with the new address; replace with the -# nixos-infect output of the new host (README §3), keeping defaultGateway.interface. -# Hetzner static networking — generated by nixos-infect at provision time. -# -# This file is server-specific: the IP, gateway, and MAC address are tied to a -# particular Hetzner instance. When provisioning a new server: -# 1. After `terraform apply` + nixos-infect completes, run: -# ssh root@ 'cat /etc/nixos/networking.nix' -# 2. Replace this file's contents with the output and commit. -# 3. Then: `nixos-rebuild switch --flake .#cc-ci-hetzner --target-host root@` -# -# Current instance: 195.201.88.249 (fsn1, Hetzner server 134485294, provisioned 2026-05-31). +# Generated by nixos-infect on this machine (2026-09-07), captured per README §3, with ONE edit: +# `defaultGateway` as an attrset WITH `interface = "eth0"`. The generated bare-string form leaves +# NixOS ≥25.05 without a default route (the host boots and is unreachable) — see README §2. { lib, ... }: { + # This file was populated at runtime with the networking + # details gathered from the active system. networking = { - nameservers = [ - "185.12.64.1" - "185.12.64.2" - ]; - # The interface MUST be explicit here. Since NixOS 25.05 the scripted-networking - # module installs the default route from the gateway interface's - # network-addresses-.service, and it finds that interface either by - # `defaultGateway.interface` or by the gateway address being inside one of the - # interface's subnets. With Hetzner's off-subnet point-to-point gateway - # (195.201.88.249/32 on eth0, gateway 172.31.1.1) neither matched when this was a - # bare string, so NO default route was installed and the 26.05 rebuild on - # 2026-08-03 took the host off the network (recovered via rescue mode). - defaultGateway = { - address = "172.31.1.1"; + nameservers = [ "2a01:4ff:ff00::add:2" + "2a01:4ff:ff00::add:1" + "185.12.64.2" + ]; + defaultGateway = { address = "172.31.1.1"; interface = "eth0"; }; + defaultGateway6 = { + address = "fe80::1"; interface = "eth0"; }; - # No IPv6 on this Hetzner instance (link-local only) — nixos-infect emitted an empty - # defaultGateway6/ipv6.route which made network-addresses-eth0.service fail - # ("ip route add /128" with no prefix). v4-only box, so no IPv6 gateway/route declared. dhcpcd.enable = false; usePredictableInterfaceNames = lib.mkForce false; interfaces = { eth0 = { ipv4.addresses = [ - { address = "195.201.88.249"; prefixLength = 32; } + { address="195.201.88.249"; prefixLength=32; } ]; - ipv4.routes = [{ address = "172.31.1.1"; prefixLength = 32; }]; + ipv6.addresses = [ + { address="2a01:4f8:1c1c:a9b::1"; prefixLength=64; } +{ address="fe80::2ff8:e3ea:bbb8:aa39"; prefixLength=64; } + ]; + ipv4.routes = [ { address = "172.31.1.1"; prefixLength = 32; } ]; + ipv6.routes = [ { address = "fe80::1"; prefixLength = 128; } ]; }; + }; }; services.udev.extraRules = '' - ATTR{address}=="00:00:00:00:00:00", NAME="eth0" + ATTR{address}=="92:00:09:d5:ec:0d", NAME="eth0" + ''; } diff --git a/nix/modules/orchestrator-host.nix b/nix/modules/orchestrator-host.nix index be160eb..ef7d420 100644 --- a/nix/modules/orchestrator-host.nix +++ b/nix/modules/orchestrator-host.nix @@ -51,6 +51,7 @@ in # claude sessions run as non-root (--dangerously-skip-permissions is refused for root). users.users.loops = { isNormalUser = true; + uid = 1000; # fixed: workspace files are rsynced between hosts by uid home = "/home/loops"; shell = pkgs.bash; extraGroups = [ "wheel" "docker" ];