weekly health-gated auto-update of the cc-ci host; skills rewritten for the combined host
nix/modules/auto-update.nix (own module, no notplants-nix dependency): Tuesday 03:00 UTC, busy-gated (CI run, weekly upgrader, report, sweep, running Drone builds → skip), `nix flake update` → build → switch-to-configuration test → cc-ci health checks (sshd, 0 failed units, core units, every swarm service at replica count, sops decrypted, dashboard/reports/drone 200, opencode UI 401) → profile + bootloader → flake.lock committed and pushed to main → /etc/cc-ci fast-forwarded; revert + lock restore on failure; one-line state file for /cc-ci-status. Skills (.opencode canonical, .claude pointers' descriptions synced): - cc-ci-orchestrator-update: THE host update — drives the auto-update unit by hand; --cc-ci-only for a cc-ci-main-only move; failure playbook. - cc-ci-server-update: delegates to it and explains why the old procedure (rebuilding the cc-ci repo's standalone #cc-ci) must not be run on this host. - cc-ci-update: chains orchestrator-update then tests-update. - cc-ci-status: §5/§6 for one host — auto-update state, generation vs boot generation, front doors (oc.ci = 401), sops, fail2ban, timers, orchestrator agent session, secrets inventory; verdict updated. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FqkQq3CDmFWcQ7u1LzoyRz
This commit is contained in:
Claude Fable 5.1
@@ -27,6 +27,9 @@
|
||||
# ---- orchestrator identity --------------------------------------------------------------
|
||||
# The CI server is this very host, so `ssh cc-ci` goes to loopback (the module default).
|
||||
cc-ci-orchestrator.ciSshHost = "127.0.0.1";
|
||||
# Weekly self-update (Tue 03:00 UTC; skips itself while CI is busy; see nix/modules/auto-update.nix).
|
||||
cc-ci-orchestrator.autoUpdate.enable = true;
|
||||
|
||||
# The opencode UI: traefik (public 443, the *.ci.commoninternet.net cert) → nginx basic auth.
|
||||
cc-ci-orchestrator.opencodeUiHost = "oc.ci.commoninternet.net";
|
||||
cc-ci-orchestrator.opencodeUiTraefikNetwork = "proxy";
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
# auto-update.nix — weekly, health-gated self-update of the combined cc-ci host.
|
||||
#
|
||||
# Modelled on notplants-nix's modules/auto-update.nix (the nightly updater the old orchestrator
|
||||
# box had), with the two things the cc-ci host needs on top:
|
||||
# * a BUSY GATE — a `switch` restarts docker/traefik/drone/the runner, so the update skips
|
||||
# (and simply retries next week) while a CI run, the weekly recipe-upgrade run or the
|
||||
# Sunday canonical sweep is in flight;
|
||||
# * cc-ci HEALTH CHECKS — beyond "sshd + no failed units": every swarm service at its replica
|
||||
# count, the front doors (dashboard, reports, drone, the opencode UI) answering, sops having
|
||||
# decrypted, acme-dns and fail2ban up.
|
||||
#
|
||||
# Sequence (as in the notplants module — read its comments for the whys):
|
||||
# flake update (all inputs: nixpkgs, sops-nix, cc-ci) → nixos-rebuild build (as loops) →
|
||||
# switch-to-configuration test (bootloader untouched) → settle → health check →
|
||||
# on failure: re-activate the previous generation, restore flake.lock, exit 1
|
||||
# on success: set the system profile + bootloader, commit flake.lock as "auto-update", push
|
||||
# main (the repo stays the source of truth), refresh /etc/cc-ci.
|
||||
# Nothing is committed that did not pass the health check. Never fires a missed run at boot.
|
||||
# `/cc-ci-status` reads the outcome from the state file this writes.
|
||||
{ config, lib, pkgs, ... }:
|
||||
let
|
||||
cfg = config.cc-ci-orchestrator.autoUpdate;
|
||||
builderHome = config.users.users.${cfg.buildUser}.home;
|
||||
stateFile = "${cfg.flakePath}/.cc-ci-logs/auto-update-state";
|
||||
curl = "${pkgs.curl}/bin/curl";
|
||||
|
||||
busyGate = pkgs.writeShellScript "cc-ci-auto-update-busy-gate" ''
|
||||
set -u
|
||||
busy() { echo "BUSY: $1 — skipping this week's update"; exit 3; }
|
||||
${pkgs.procps}/bin/pgrep -f run_recipe_ci >/dev/null && busy "a CI run is in flight"
|
||||
${pkgs.systemd}/bin/systemctl is-active --quiet nightly-sweep.service && busy "the canonical sweep is running"
|
||||
${pkgs.util-linux}/bin/runuser -u ${cfg.buildUser} -- ${pkgs.tmux}/bin/tmux has-session -t cc-ci-upgrader 2>/dev/null \
|
||||
&& busy "the weekly recipe-upgrade run is in flight (tmux cc-ci-upgrader)"
|
||||
${pkgs.util-linux}/bin/runuser -u ${cfg.buildUser} -- ${pkgs.tmux}/bin/tmux has-session -t cc-ci-report 2>/dev/null \
|
||||
&& busy "the weekly report is being written (tmux cc-ci-report)"
|
||||
if [ -r /run/secrets/bridge_drone_token ]; then
|
||||
running=$(${curl} -s -m 20 -H "Authorization: Bearer $(cat /run/secrets/bridge_drone_token)" \
|
||||
"https://drone.${cfg.ciDomain}/api/repos/recipe-maintainers/cc-ci/builds?per_page=10" \
|
||||
| ${pkgs.gnugrep}/bin/grep -o '"status":"running"' | ${pkgs.coreutils}/bin/wc -l)
|
||||
[ "''${running:-0}" -eq 0 ] || busy "$running Drone build(s) running"
|
||||
fi
|
||||
echo "not busy"
|
||||
'';
|
||||
|
||||
healthCheck = pkgs.writeShellScript "cc-ci-auto-update-health-check" ''
|
||||
set -u
|
||||
fail() { echo "HEALTH FAIL: $1"; exit 1; }
|
||||
${pkgs.systemd}/bin/systemctl is-active --quiet sshd || fail "sshd not active"
|
||||
${pkgs.iproute2}/bin/ss -tlnH | ${pkgs.gnugrep}/bin/grep -q ':22 ' || fail "nothing listening on :22"
|
||||
n=$(${pkgs.systemd}/bin/systemctl --failed --no-legend | ${pkgs.coreutils}/bin/wc -l)
|
||||
[ "$n" -eq 0 ] || fail "$n failed unit(s): $(${pkgs.systemd}/bin/systemctl --failed --no-legend --plain | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.coreutils}/bin/tr '\n' ' ')"
|
||||
for u in docker acme-dns fail2ban nginx opencode-web drone-runner-exec; do
|
||||
${pkgs.systemd}/bin/systemctl is-active --quiet "$u" || fail "$u not active"
|
||||
done
|
||||
# The reconcile oneshots re-run on activation; give the swarm up to 10 minutes to converge.
|
||||
for i in $(${pkgs.coreutils}/bin/seq 1 40); do
|
||||
short=$(${pkgs.docker}/bin/docker service ls --format '{{.Name}} {{.Replicas}}' \
|
||||
| ${pkgs.gawk}/bin/awk '{ split($2,a,"/"); if (a[1] != a[2]) print $1 }')
|
||||
[ -z "$short" ] && break
|
||||
${pkgs.coreutils}/bin/sleep 15
|
||||
done
|
||||
[ -z "$short" ] || fail "swarm services not at their replica count: $(echo "$short" | ${pkgs.coreutils}/bin/tr '\n' ' ')"
|
||||
[ -s /run/secrets/test_secret ] || fail "sops did not decrypt (/run/secrets/test_secret missing)"
|
||||
code() { ${curl} -s -m 20 -o /dev/null -w '%{http_code}' --resolve "$1:443:127.0.0.1" "https://$1/$2"; }
|
||||
[ "$(code ${cfg.ciDomain} "")" = 200 ] || fail "dashboard not 200"
|
||||
[ "$(code report.${cfg.ciDomain} "")" = 200 ] || fail "reports not 200"
|
||||
[ "$(code drone.${cfg.ciDomain} version)" = 200 ] || fail "drone /version not 200"
|
||||
[ "$(code ${cfg.opencodeUiHost} "")" = 401 ] || fail "opencode UI not answering with its auth challenge"
|
||||
echo "HEALTH OK"
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.cc-ci-orchestrator.autoUpdate = {
|
||||
enable = lib.mkEnableOption "weekly health-gated self-update of the cc-ci host";
|
||||
flakePath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/srv/cc-ci-orch";
|
||||
description = "The cc-ci-orchestrator checkout (owned by buildUser) whose flake.lock is updated, committed and pushed.";
|
||||
};
|
||||
flakeRef = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "${cfg.flakePath}#cc-ci";
|
||||
description = "Flake reference to build.";
|
||||
};
|
||||
buildUser = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "loops";
|
||||
description = "Owner of the checkout; runs the update, build, commit and push (root activates).";
|
||||
};
|
||||
onCalendar = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "Tue *-*-* 03:00:00 UTC";
|
||||
description = ''
|
||||
When to run. Must stay clear of the weekly recipe-upgrade run (Thu 22:00 America/New_York
|
||||
= Fri 02:00/03:00 UTC, several hours) and the Sunday 03:00 UTC canonical sweep (up to 6 h);
|
||||
the busy gate covers the rest.
|
||||
'';
|
||||
};
|
||||
remote = lib.mkOption { type = lib.types.str; default = "origin"; };
|
||||
branch = lib.mkOption { type = lib.types.str; default = "main"; };
|
||||
ciDomain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "ci.commoninternet.net";
|
||||
description = "The CI apex; report./drone. are checked under it.";
|
||||
};
|
||||
opencodeUiHost = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = config.cc-ci-orchestrator.opencodeUiHost;
|
||||
description = "The opencode UI host name (expected to answer 401).";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
systemd.services.cc-ci-auto-update = {
|
||||
description = "Weekly cc-ci host self-update (flake update → build → test → health check → commit)";
|
||||
after = [ "network-online.target" "docker.service" ];
|
||||
wants = [ "network-online.target" ];
|
||||
path = with pkgs; [ nixos-rebuild nix git openssh coreutils gnugrep gawk systemd util-linux bash python3 docker curl ];
|
||||
serviceConfig = { Type = "oneshot"; TimeoutStartSec = "3h"; };
|
||||
# A unit that performs the switch must never be stopped or restarted BY that switch
|
||||
# (notplants-orchestrator, 2026-09-07: it was, and the host ran half-switched for 12 h).
|
||||
stopIfChanged = false;
|
||||
restartIfChanged = false;
|
||||
script = ''
|
||||
set -u
|
||||
cd ${cfg.flakePath}
|
||||
as_builder() { runuser -u ${cfg.buildUser} -- env HOME=${builderHome} "$@"; }
|
||||
state() { printf '%s result=%s generation=%s note=%s\n' "$(date -u +%FT%TZ)" "$1" "$2" "$3" > ${stateFile}; chown ${cfg.buildUser} ${stateFile} || true; }
|
||||
PREV=$(readlink -f /run/current-system)
|
||||
LOCK_BAK=${cfg.flakePath}/.flake.lock.auto-update-prev
|
||||
|
||||
if ! ${busyGate}; then
|
||||
state skipped "$PREV" busy
|
||||
exit 0
|
||||
fi
|
||||
|
||||
revert() {
|
||||
echo "REVERT: $1"
|
||||
as_builder cp "$LOCK_BAK" flake.lock 2>/dev/null || true
|
||||
"$PREV"/bin/switch-to-configuration test || echo "re-activation of $PREV failed"
|
||||
state failed "$PREV" "$1"
|
||||
exit 1
|
||||
}
|
||||
|
||||
as_builder cp flake.lock "$LOCK_BAK"
|
||||
echo "=== checkout at: $(as_builder git rev-parse --short HEAD) on $(as_builder git rev-parse --abbrev-ref HEAD)"
|
||||
echo "=== flake update (all inputs)"
|
||||
as_builder nix flake update || revert "flake update failed"
|
||||
if as_builder git diff --quiet -- flake.lock; then
|
||||
echo "=== nothing moved; done"
|
||||
state unchanged "$PREV" "lock already current"
|
||||
exit 0
|
||||
fi
|
||||
as_builder git --no-pager diff --stat -- flake.lock
|
||||
|
||||
echo "=== build (cannot affect the running system)"
|
||||
as_builder nixos-rebuild build --flake ${cfg.flakeRef} || revert "build failed"
|
||||
NEW=$(readlink -f result) || revert "no build result"
|
||||
as_builder rm -f result
|
||||
echo "=== built: $NEW"
|
||||
|
||||
echo "=== activate WITHOUT touching the bootloader"
|
||||
"$NEW"/bin/switch-to-configuration test || revert "test activation failed"
|
||||
|
||||
echo "=== settle, then health check"
|
||||
sleep 45
|
||||
${healthCheck} || revert "health check failed"
|
||||
|
||||
echo "=== healthy: committing (system profile + bootloader)"
|
||||
nix-env -p /nix/var/nix/profiles/system --set "$NEW" || revert "could not set system profile"
|
||||
"$NEW"/bin/switch-to-configuration boot || revert "bootloader update failed"
|
||||
state ok "$NEW" "committed"
|
||||
|
||||
# Bookkeeping from here on — never revert a healthy, committed generation over it.
|
||||
if as_builder git -c user.name="auto-update" -c user.email="auto-update@cc-ci" \
|
||||
commit -q -m "flake.lock: weekly auto-update, health-checked" -- flake.lock; then
|
||||
echo "=== flake.lock committed: $(as_builder git rev-parse --short HEAD)"
|
||||
as_builder git push -q ${cfg.remote} HEAD:${cfg.branch} \
|
||||
&& echo "=== pushed ${cfg.branch}" \
|
||||
|| echo "WARNING: push failed; the lock is committed locally, push it by hand"
|
||||
else
|
||||
echo "WARNING: could not commit flake.lock; the system IS committed and healthy"
|
||||
fi
|
||||
# Keep the deployed cc-ci checkout (sweep + sops file) on the same cc-ci rev as the flake input.
|
||||
git -C /etc/cc-ci pull -q --ff-only --recurse-submodules || echo "WARNING: /etc/cc-ci pull failed"
|
||||
echo "=== auto-update committed: $NEW"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.cc-ci-auto-update = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.onCalendar;
|
||||
Persistent = false;
|
||||
RandomizedDelaySec = "10min";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user