notplantsandClaude Fable 5.1
a8af8429f2
nix/modules/auto-update.nix (own module, no notplants-nix dependency): Tuesday 03:00 UTC, busy-gated (CI run, weekly upgrader, report, sweep, running Drone builds → skip), `nix flake update` → build → switch-to-configuration test → cc-ci health checks (sshd, 0 failed units, core units, every swarm service at replica count, sops decrypted, dashboard/reports/drone 200, opencode UI 401) → profile + bootloader → flake.lock committed and pushed to main → /etc/cc-ci fast-forwarded; revert + lock restore on failure; one-line state file for /cc-ci-status. Skills (.opencode canonical, .claude pointers' descriptions synced): - cc-ci-orchestrator-update: THE host update — drives the auto-update unit by hand; --cc-ci-only for a cc-ci-main-only move; failure playbook. - cc-ci-server-update: delegates to it and explains why the old procedure (rebuilding the cc-ci repo's standalone #cc-ci) must not be run on this host. - cc-ci-update: chains orchestrator-update then tests-update. - cc-ci-status: §5/§6 for one host — auto-update state, generation vs boot generation, front doors (oc.ci = 401), sops, fail2ban, timers, orchestrator agent session, secrets inventory; verdict updated. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FqkQq3CDmFWcQ7u1LzoyRz
108 lines
5.4 KiB
Nix
108 lines
5.4 KiB
Nix
# cc-ci — ONE Hetzner Cloud host running both the cc-ci CI server and the cc-ci orchestrator.
|
||
#
|
||
# This file is only what is physical or identity about the machine: hardware, networking,
|
||
# root SSH keys, firewall + fail2ban, swap, stateVersion. Everything functional comes from modules:
|
||
# cc-ci.nixosModules.cc-ci-server recipe-maintainers/cc-ci — swarm, traefik, drone,
|
||
# runner, bridge, dashboard, reports, acme-dns, harness
|
||
# self.nixosModules.cc-ci-orchestrator nix/modules/cc-ci.nix — loops, orchestrator, timers
|
||
# self.nixosModules.orchestrator-host nix/modules/orchestrator-host.nix — loops user, CLIs
|
||
# See README.md for provisioning (Hetzner Debian → nixos-infect → this flake) and staging.
|
||
{ lib, pkgs, ... }:
|
||
{
|
||
imports = [
|
||
./hardware.nix
|
||
./networking.nix
|
||
];
|
||
|
||
networking.hostName = "cc-ci";
|
||
|
||
# ---- cc-ci server identity --------------------------------------------------------------
|
||
# Public address: acme-dns binds to it and publishes it as the `ns-acme` glue record; the
|
||
# Gandi A records for ci / *.ci / ns-acme .commoninternet.net point here.
|
||
cc-ci.publicIPv4 = "195.201.88.249";
|
||
# cc-ci is a plain flake input here (no private submodule), so the sops file is the one in
|
||
# the deployed --recursive checkout the weekly sweep runs from (README "Stage the workspace").
|
||
cc-ci.sopsFile = "/etc/cc-ci/secrets/secrets.yaml";
|
||
|
||
# ---- orchestrator identity --------------------------------------------------------------
|
||
# The CI server is this very host, so `ssh cc-ci` goes to loopback (the module default).
|
||
cc-ci-orchestrator.ciSshHost = "127.0.0.1";
|
||
# Weekly self-update (Tue 03:00 UTC; skips itself while CI is busy; see nix/modules/auto-update.nix).
|
||
cc-ci-orchestrator.autoUpdate.enable = true;
|
||
|
||
# The opencode UI: traefik (public 443, the *.ci.commoninternet.net cert) → nginx basic auth.
|
||
cc-ci-orchestrator.opencodeUiHost = "oc.ci.commoninternet.net";
|
||
cc-ci-orchestrator.opencodeUiTraefikNetwork = "proxy";
|
||
|
||
# ---- no tailscale on this host (operator 2026-09-07) --------------------------------------
|
||
# Everything is reached over the public IP: ssh (keys only), the CI front doors via traefik,
|
||
# and the opencode UI on 443 (traefik → nginx basic auth). fail2ban below guards the two logins.
|
||
|
||
# ---- ssh ----------------------------------------------------------------------------------
|
||
services.openssh = {
|
||
enable = true;
|
||
settings.PermitRootLogin = "yes";
|
||
};
|
||
# Root keys: PUBLIC keys, tracked deliberately in ./ssh-keys (one per line, blank lines ok).
|
||
users.users.root.openssh.authorizedKeys.keys =
|
||
builtins.filter (s: s != "") (lib.splitString "\n" (builtins.readFile ./ssh-keys));
|
||
# The loops user can also be reached directly (same keys) — handy for rsync of its workspace.
|
||
users.users.loops.openssh.authorizedKeys.keys =
|
||
builtins.filter (s: s != "") (lib.splitString "\n" (builtins.readFile ./ssh-keys));
|
||
services.openssh.settings.PasswordAuthentication = false;
|
||
services.openssh.settings.KbdInteractiveAuthentication = false;
|
||
|
||
# ---- firewall -------------------------------------------------------------------------------
|
||
# 80/443 (traefik) and 53 (acme-dns) are opened by the cc-ci-server module; the opencode UI
|
||
# rides 443 through traefik (orchestrator-host.nix), so only ssh is opened here.
|
||
networking.firewall = {
|
||
enable = true;
|
||
allowedTCPPorts = [ 22 ];
|
||
};
|
||
|
||
# ---- fail2ban: sshd (password auth is off, this stops the log noise and slow brute force) and
|
||
# the opencode UI's basic auth (nginx logs 401s with the real client IP to the journal; the
|
||
# built-in nginx-http-auth filter matches them). Those clients arrive through traefik's
|
||
# docker-published 443, which iptables FORWARDs rather than INPUTs, so the ban for that jail
|
||
# goes into the DOCKER-USER chain — an INPUT rule would never see the traffic.
|
||
services.fail2ban = {
|
||
enable = true;
|
||
maxretry = 5;
|
||
bantime = "1h";
|
||
bantime-increment = { enable = true; maxtime = "48h"; factor = "4"; };
|
||
ignoreIP = [ "127.0.0.0/8" "::1" ];
|
||
jails.nginx-http-auth.settings = {
|
||
enabled = true;
|
||
filter = "nginx-http-auth";
|
||
# NixOS nginx logs errors to stderr → the journal, not /var/log/nginx/error.log (which
|
||
# exists but stays empty). Read the unit's journal instead.
|
||
backend = "systemd";
|
||
journalmatch = "_SYSTEMD_UNIT=nginx.service";
|
||
banaction = "iptables-allports";
|
||
chain = "DOCKER-USER";
|
||
};
|
||
};
|
||
|
||
networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
|
||
# This host's own public names resolve to itself regardless of external DNS state (host
|
||
# processes: the drone runner, the harness, the orchestrator; containers use the resolvers
|
||
# above). Per-run recipe domains are random and cannot be pinned — those follow public DNS.
|
||
networking.hosts."195.201.88.249" = [
|
||
"ci.commoninternet.net"
|
||
"drone.ci.commoninternet.net"
|
||
"report.ci.commoninternet.net"
|
||
"traefik.ci.commoninternet.net"
|
||
"warm-keycloak.ci.commoninternet.net"
|
||
"oc.ci.commoninternet.net"
|
||
];
|
||
|
||
# ---- memory: 8 GB RAM shared by the swarm (recipe deploys) and 3–6 agent sessions ---------
|
||
swapDevices = [ { device = "/swapfile"; size = 8192; } ];
|
||
|
||
# ssh client for root (the orchestrator's `ssh cc-ci` goes through the loops user's own config).
|
||
environment.systemPackages = [ pkgs.openssh ];
|
||
|
||
# Fresh NixOS 26.05 install (nixos-infect, 2026-09-07). Never change this on an existing host.
|
||
system.stateVersion = "26.05";
|
||
}
|