cc-ci-orchestrator/nix/hosts/cc-ci-orchestrator-hetzner/configuration.nix

# cc-ci-orchestrator-hetzner — NixOS config for the Hetzner loops runtime host.
#
# Purpose: run the cc-ci Builder/Adversary/Watchdog loops + orchestrator/assistant sessions
# on a Hetzner cpx11 (2 vCPU / 2 GB dedicated AMD / 40 GB NVMe), replacing the slow b1 Incus VM.
#
# Provision with terraform/ then converge with: nixos-rebuild switch --flake .#cc-ci-orchestrator-hetzner
# See terraform/README.md for the full Stage 2 procedure.
{ config, pkgs, lib, ... }:
{
  # hardware.nix is the nixos-infect generated hardware-configuration.nix (see README Stage 2a).

  # atproto-likes — the notplants-atproto "most-liked accounts" web UI, served at
  # atproto.commoninternet.net. Brings in Docker + a compose stack + an nginx vhost
  # with ACME, and opens 80/443 (previously only 22 was public). Canonical source of
  # the module is the project repo; ../atproto-likes.nix is a copy kept in this tree
  # because pure evaluation cannot import a path outside the flake.
  imports = [ ../../atproto-likes.nix ];

  services.openssh = {
    enable = true;
    settings.PermitRootLogin = "yes";
  };

  # Root SSH access — all keys from the current orchestrator VM's /root/.ssh/authorized_keys.
  users.users.root.openssh.authorizedKeys.keys = [
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOk8NaeBdPbS2gfUvbny8h0AkZlVjGYHzx4QPXSJ38gd claude@claude-vm"
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJVlfoLBPseQ9fA9534KmRg2KWcksKZGzAJIpHJ2JpsI mfowler.email@protonmail.com"
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAcyTGb/wVgdhg5oBCZZvBaR1RuUQRY/3WHnOQpNDCsp claude-cc-ci-sandbox@20260526"
  ];
  networking.useDHCP = true;
  networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
  networking.firewall = {
    enable = true;
    trustedInterfaces = [ "tailscale0" ];
    # Port 80 open only on the tailscale interface (trusted) — nginx binds there for oc.commoninternet.net.
    allowedTCPPorts = [ 22 ];
  };
  nix.settings.experimental-features = [ "nix-command" "flakes" ];
  system.stateVersion = "24.11";

  # Tailscale — auth key at /etc/ts-auth-key (placed manually in Stage 2, not in git).
  services.tailscale = {
    enable = true;
    authKeyFile = "/etc/ts-auth-key";
    extraUpFlags = [ "--hostname=cc-ci-orchestrator" ];
  };

  # 4 GB disk swap — claude session memory safety net (2 GB RAM is tight for 3+ sessions).
  swapDevices = [ { device = "/swapfile"; size = 4096; } ];

  # nix-ld — lets the standalone Claude Code CLI (foreign dynamic ELF / Bun) run on NixOS.
  programs.nix-ld.enable = true;
  programs.nix-ld.libraries = with pkgs; [
    stdenv.cc.cc.lib
    zlib
    openssl
    curl
    glibc
  ];

  environment.systemPackages = with pkgs; [
    git tmux python3 jq curl cacert
    gnused gawk coreutils gnugrep findutils util-linux
    nettools openssh
    age sops                # key management (same toolchain as cc-ci server)
  ];

  # loops user — claude sessions run as non-root (--dangerously-skip-permissions blocked for root).
  users.users.loops = {
    isNormalUser = true;
    home = "/home/loops";
    shell = pkgs.bash;
    extraGroups = [ "wheel" ];
  };
  security.sudo.wheelNeedsPassword = false;
  security.sudo.extraRules = [{
    users = [ "loops" ];
    commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
  }];

  # Ensure /home/loops/.local/bin (claude + opencode) is on the loops user PATH.
  # opencode binary is installed there manually (not yet in nixpkgs); re-install if missing:
  #   curl -sL https://github.com/anomalyco/opencode/releases/download/v1.15.13/opencode-linux-x64.tar.gz \
  #     | tar -xz -C /home/loops/.local/bin opencode && chmod +x /home/loops/.local/bin/opencode
  environment.variables.PATH = lib.mkForce
    "/home/loops/.local/bin:/run/current-system/sw/bin:/run/wrappers/bin:/usr/bin:/bin";

  # SSH config for the loops user — points to the cc-ci Hetzner server via tailnet.
  # HostName is the Hetzner cc-ci server's tailnet IP (cutover settled 2026-05-31).
  system.activationScripts.loopsSshConfig = ''
    mkdir -p /home/loops/.ssh && chown loops:users /home/loops/.ssh && chmod 700 /home/loops/.ssh
    # Only write if not already present (preserves manual customisation).
    if [ ! -f /home/loops/.ssh/config ]; then
      cat > /home/loops/.ssh/config <<'SSHCFG'
Host cc-ci
    HostName 100.95.31.88
    User root
    IdentityFile /home/loops/.ssh/cc-ci-root-ed25519
    IdentitiesOnly yes
    StrictHostKeyChecking accept-new
    ServerAliveInterval 30
SSHCFG
      chmod 600 /home/loops/.ssh/config
      chown loops:users /home/loops/.ssh/config
    fi
  '';

  # claude-install — fetch the standalone Claude Code CLI for the loops user if missing.
  systemd.services.claude-install = {
    description = "Install Claude Code CLI for loops user (idempotent)";
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      Type = "oneshot"; RemainAfterExit = true;
      User = "loops"; Group = "users";
    };
    environment = { HOME = "/home/loops"; };
    path = [ pkgs.curl pkgs.bash pkgs.coreutils pkgs.gnutar pkgs.gzip ];
    script = ''
      if [ ! -x "$HOME/.local/bin/claude" ]; then
        echo "installing Claude Code CLI for loops user..."
        curl -fsSL https://claude.ai/install.sh | bash || echo "install failed — retry on next activation"
      fi
    '';
  };

  # opencode web server — one shared instance; agent sessions attach to it for web visibility.
  # Serves the web UI at http://oc.commoninternet.net (via nginx below, tailscale-only).
  # Provider creds are read from /srv/cc-ci/.testenv at startup.
  systemd.services.opencode-web = {
    description = "opencode web server for cc-ci agents";
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" "tailscaled.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      Type = "simple";
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/cc-ci-orch/cc-ci";
      EnvironmentFile = "/srv/cc-ci/.testenv";
      ExecStartPre = "${pkgs.coreutils}/bin/rm -rf /tmp/opencode";
      ExecStart = "/home/loops/.local/bin/opencode serve --hostname 127.0.0.1 --port 4096";
      Restart = "on-failure";
      RestartSec = "5s";
    };
    environment = {
      HOME = "/home/loops";
      PATH = lib.mkForce "/run/wrappers/bin:/home/loops/.local/bin:/run/current-system/sw/bin:/usr/bin:/bin:/home/loops/.nix-profile/bin:/nix/profile/bin:/home/loops/.local/state/nix/profile/bin:/etc/profiles/per-user/loops/bin:/nix/var/nix/profiles/default/bin";
    };
    path = [ pkgs.bash pkgs.coreutils pkgs.git pkgs.python3 pkgs.openssh pkgs.tmux pkgs.nettools ];
  };

  # nginx — reverse-proxy oc.commoninternet.net → opencode web server.
  # Bound to the tailscale IP so it is only reachable on the tailnet.
  # DNS: add A record  oc.commoninternet.net → 100.84.190.30  (operator step if hostname access is wanted).
  services.nginx = {
    enable = true;
    recommendedProxySettings = true;
    virtualHosts."oc.commoninternet.net" = {
      # Listen on the tailscale interface only — not the public IP. Both 80 and 443.
      # 443 uses a SELF-SIGNED cert (below): this name resolves to a CGNAT tailscale
      # IP, so Let's Encrypt HTTP-01 can't validate it and there is no DNS-01 provider
      # configured on this host. The trusted *.ci.commoninternet.net wildcard lives on
      # the coop-cloud swarm (91.98.47.73), not here, and doesn't cover this label.
      # Self-signed is fine because the vhost is tailnet-only (trusted network); the
      # browser shows a one-time trust prompt. (Chosen by operator, 2026-08-03.)
      #
      # The cert/key are created out of band — same convention as oc-htpasswd, NOT in
      # git and NOT in the nix store (a store path would be world-readable):
      #
      #   /etc/nginx/oc-selfsigned.crt    root:nginx 0644
      #   /etc/nginx/oc-selfsigned.key    root:nginx 0640
      #
      # Regenerate (10y, SANs = hostname + tailscale IP + ts.net name) with:
      #   openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \
      #     -keyout /etc/nginx/oc-selfsigned.key -out /etc/nginx/oc-selfsigned.crt \
      #     -subj /CN=oc.commoninternet.net \
      #     -addext "subjectAltName=DNS:oc.commoninternet.net,DNS:cc-ci-orchestrator-1.taila4a0bf.ts.net,IP:100.84.190.30"
      #   sudo chown root:nginx /etc/nginx/oc-selfsigned.{crt,key}
      #   sudo chmod 644 /etc/nginx/oc-selfsigned.crt && sudo chmod 640 /etc/nginx/oc-selfsigned.key
      #
      # NOTE: like oc-htpasswd, if these files go missing nginx fails to START — which
      # would take the atproto vhost down with it. Recreate them before rebuilding on
      # a fresh host.
      listen = [
        { addr = "100.84.190.30"; port = 80;  ssl = false; }
        { addr = "100.84.190.30"; port = 443; ssl = true;  }
      ];
      # addSSL (serve BOTH http+https) is required, not cosmetic: the NixOS nginx
      # module only renders the `ssl_certificate` directives when a vhost is flagged
      # as an SSL vhost. An explicit `listen ... ssl` + sslCertificate WITHOUT this
      # flag produces an SSL listener with no cert → `nginx -t` fails and the whole
      # service (atproto included) won't start.
      addSSL = true;
      sslCertificate    = "/etc/nginx/oc-selfsigned.crt";
      sslCertificateKey = "/etc/nginx/oc-selfsigned.key";

      # HTTP basic auth in front of opencode. The opencode web UI has no
      # authentication of its own and can drive agent sessions, so since this host
      # started serving 80/443 publicly (atproto-likes, 2026-08-01) it is worth a
      # second layer even though this vhost is tailnet-only.
      #
      # basicAuthFile, NOT basicAuth: the `basicAuth` attrset writes the password
      # into the nix store, which is world-readable. This file is created out of
      # band — same convention as /etc/ts-auth-key — and is not in git:
      #
      #   /etc/nginx/oc-htpasswd          root:nginx 0640, `oc:<bcrypt hash>`
      #   /secrets/files/oc-basic-auth.txt the plaintext, loops-only 0600
      #
      # Rotate with:
      #   P=$(python3 -c "import secrets,string;a=string.ascii_letters+string.digits;print(''.join(secrets.choice(a) for _ in range(32)))")
      #   printf 'oc:%s\n' "$(mkpasswd -m bcrypt "$P")" | sudo tee /etc/nginx/oc-htpasswd
      #   sudo chown root:nginx /etc/nginx/oc-htpasswd && sudo chmod 640 /etc/nginx/oc-htpasswd
      #   sudo systemctl reload nginx
      #
      # NOTE: if this file goes missing, nginx fails to START — which would take
      # the atproto vhost down with it. Recreate it before rebuilding on a fresh
      # host.
      basicAuthFile = "/etc/nginx/oc-htpasswd";

      locations."/" = {
        proxyPass = "http://127.0.0.1:4096";
        proxyWebsockets = true;
      };
    };
  };

  # cc-ci-loops supervisor — workspace staged 2026-05-31, so ENABLED for reboot-resilience.
  systemd.services.cc-ci-loops = {
    description = "cc-ci Builder/Adversary loops + watchdog (launch.sh start)";
    wantedBy = [ "multi-user.target" ];  # enabled after workspace staged (Hetzner cutover)
    after = [ "network-online.target" "tailscaled.service" "claude-install.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      # KillMode=process: this unit only LAUNCHES the tmux server, it does not own it. With the
      # default (control-group) systemd kills every leftover process in the cgroup when the unit
      # stops — and since one tmux server hosts every agent session on this host, a rebuild that
      # merely touched this unit wiped all of them (operator 2026-08-01). Only the (already
      # exited) main process is killed now; `systemctl stop` therefore does NOT tear down agents.
      KillMode = "process";
      Type = "oneshot"; RemainAfterExit = true;
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/cc-ci/cc-ci";
      # Append one line to REBOOTS.md per genuine reboot (boot_id-gated; not on manual restart).
      ExecStartPre = "${pkgs.bash}/bin/bash /srv/cc-ci/cc-ci-plan/reboot-log.sh";
    };
    # CLAUDE_BIN points at the standalone CLI installed by claude-install.service; the loops
    # backend defaults to claude (persisted in .loop-backend). Without this, launch.py's preflight
    # `which(claude)` fails because the systemd `path` below has no /home/loops/.local/bin.
    environment = { RESUME_PHASE = "1"; HOME = "/home/loops"; CLAUDE_BIN = "/home/loops/.local/bin/claude"; };
    path = [ pkgs.bash pkgs.tmux pkgs.git pkgs.python3 pkgs.openssh pkgs.nettools ];
    script = ''
      # Put the standalone claude/opencode binaries on PATH. On a cold boot this is the env the
      # tmux server (and thus every agent session) inherits, so bare `claude` resolves everywhere.
      export PATH="/home/loops/.local/bin:$PATH"
      [ -x /srv/cc-ci/cc-ci-plan/launch.sh ] && /srv/cc-ci/cc-ci-plan/launch.sh start || \
        echo "workspace not staged yet — skipping loop start"
    '';
  };

  # cc-ci-orchestrator supervisor — the operator's steering session. Same shape as
  # lichen-orchestrator / project-orchestrator above: this unit only LAUNCHES the orchestrator's
  # tmux session via the agent-orchestrator harness (cc-ci-plan/agents.py); it does not own the
  # session or the tmux server. The orchestrator agent is declared in cc-ci-plan/agents.toml on
  # the OPencode backend (backend = "opencode", model = "opencode/glm-5.2"), so on boot it
  # attaches to the shared opencode web server (opencode-web.service below) and is reachable for
  # Remote Control at https://oc.commoninternet.net under the /srv/cc-ci-orch project. The harness
  # watchdog (started by `agents.py up`) keeps it alive: heal-only (no stall reboots — a persistent
  # supervisor must not be killed just for idling). Added 2026-08-03 to give the cc-ci orchestrator
  # the same reboot-resilience the other two orchestrators already have.
  systemd.services.cc-ci-orchestrator = {
    description = "cc-ci orchestrator (operator steering session) — agents.py up orchestrator, opencode backend";
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" "tailscaled.service" "opencode-web.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      # KillMode=process: see the note on cc-ci-loops — a rebuild that merely touches this unit
      # must not tear down the (shared) tmux server and every agent session with it.
      KillMode = "process";
      Type = "oneshot"; RemainAfterExit = true;
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/cc-ci-orch";
    };
    environment = { HOME = "/home/loops"; };
    path = [ pkgs.bash pkgs.tmux pkgs.git pkgs.python3 pkgs.openssh pkgs.nettools ];
    script = ''
      export PATH="/home/loops/.local/bin:$PATH"
      proj="/srv/cc-ci-orch"
      echo "$(cat /proc/sys/kernel/random/boot_id) boot $(date -u +%FT%TZ) — cc-ci-orchestrator up" \
        >> "$proj/cc-ci-plan/.ao-boot.log" 2>/dev/null || true
      cd "$proj" && python3 cc-ci-plan/agents.py up orchestrator || echo "cc-ci orchestrator agents.py up failed"
    '';
  };

  # p-lichen-orchestrator supervisor — the SEPARATE lichen.page testing/hardening orchestrator
  # (distinct from cc-ci-loops above). Reboot-resilience: on boot, resume the orchestrator's Remote
  # Control session + watchdog + pipeline via `engine/agents.py up`. Added 2026-07-08 after a reboot
  # (Hetzner rollback) left this orchestrator down while cc-ci-loops auto-recovered.
  # NOTE: still points at the /home path — will be re-pointed to /srv when that migration happens.
  systemd.services.lichen-orchestrator = {
    description = "p-lichen-orchestrator (lichen.page testing) — orchestrator + watchdog + pipeline";
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" "tailscaled.service" "claude-install.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      # KillMode=process: this unit only LAUNCHES the tmux server, it does not own it. With the
      # default (control-group) systemd kills every leftover process in the cgroup when the unit
      # stops — and since one tmux server hosts every agent session on this host, a rebuild that
      # merely touched this unit wiped all of them (operator 2026-08-01). Only the (already
      # exited) main process is killed now; `systemctl stop` therefore does NOT tear down agents.
      KillMode = "process";
      Type = "oneshot"; RemainAfterExit = true;
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/lichen-orchestrator";
    };
    environment = { HOME = "/home/loops"; CLAUDE_BIN = "/home/loops/.local/bin/claude"; };
    path = [ pkgs.bash pkgs.tmux pkgs.git pkgs.python3 pkgs.openssh pkgs.nettools ];
    script = ''
      export PATH="/home/loops/.local/bin:$PATH"
      proj="/srv/lichen-orchestrator"
      # boot marker (best-effort; boot_id-gated logging can be added later)
      echo "$(cat /proc/sys/kernel/random/boot_id) boot $(date -u +%FT%TZ) — lichen-orchestrator up" \
        >> "$proj/.ao-state/boot.log" 2>/dev/null || true
      cd "$proj" && python3 engine/agents.py up || echo "p-lichen agents.py up failed"
    '';
  };

  # project-orchestrator (fleet manager) — always-on so the operator can reach it over Remote
  # Control at any time (operator 2026-08-01). Same shape as lichen-orchestrator above; the PO's
  # own agents.toml declares NO `wake`, so the watchdog only heals a dead session — it never sends
  # periodic prompts. Starting it is `agents.py up`; that also starts its watchdog.
  systemd.services.project-orchestrator = {
    description = "project-orchestrator (fleet manager) — PO agent + watchdog, remote-control always up";
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" "tailscaled.service" "claude-install.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      # KillMode=process: this unit only LAUNCHES the tmux server, it does not own it. With the
      # default (control-group) systemd kills every leftover process in the cgroup when the unit
      # stops — and since one tmux server hosts every agent session on this host, a rebuild that
      # merely touched this unit wiped all of them (operator 2026-08-01). Only the (already
      # exited) main process is killed now; `systemctl stop` therefore does NOT tear down agents.
      KillMode = "process";
      Type = "oneshot"; RemainAfterExit = true;
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/project-orchestrator";
    };
    environment = { HOME = "/home/loops"; CLAUDE_BIN = "/home/loops/.local/bin/claude"; };
    path = [ pkgs.bash pkgs.tmux pkgs.git pkgs.python3 pkgs.openssh pkgs.nettools ];
    script = ''
      export PATH="/home/loops/.local/bin:$PATH"
      proj="/srv/project-orchestrator"
      echo "$(cat /proc/sys/kernel/random/boot_id) boot $(date -u +%FT%TZ) — project-orchestrator up" \
        >> "$proj/.ao-state/boot.log" 2>/dev/null || true
      cd "$proj" && python3 engine/agents.py up || echo "PO agents.py up failed"
    '';
  };

  # Weekly recipe upgrade — runs /upgrade-all over every enrolled recipe (opens recipe PRs
  # verified by !testme, never merges). Replaces the boot-fragile busybox-crond-in-tmux from
  # phase 5 §4 with a reboot-safe systemd timer. The service is timer-triggered only (NOT
  # wantedBy multi-user.target) so it never runs on boot/activation — only on the schedule.
  systemd.services.cc-ci-upgrade-all = {
    description = "cc-ci weekly /upgrade-all run (recipe upgrade survey + PRs, never merges)";
    after = [ "network-online.target" "tailscaled.service" "claude-install.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      Type = "oneshot";  # launch-upgrader.py spawns the cc-ci-upgrader tmux session and returns
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/cc-ci";
      # Optional per-run overrides for backend/model (LOOP_BACKEND, LOOP_MODEL, OPENCODE_SHARE,
      # UPGRADER_ARGS, …). The leading "-" makes it optional: absent file → claude/sonnet defaults
      # (current behavior). To run the weekly job on e.g. opencode-go/glm-5.2, drop a file with
      #   LOOP_BACKEND=opencode
      #   LOOP_MODEL=opencode-go/glm-5.2
      # No rebuild needed to switch — the env file is read at each timer fire. Holds no secrets
      # (the opencode-go API key lives in ~/.local/share/opencode/auth.json, mode 600).
      EnvironmentFile = "-/srv/cc-ci/upgrader.env";
    };
    environment = { HOME = "/home/loops"; CLAUDE_BIN = "/home/loops/.local/bin/claude"; };
    path = [ pkgs.bash pkgs.tmux pkgs.git pkgs.python3 pkgs.openssh pkgs.nettools ];
    script = ''
      export PATH="/home/loops/.local/bin:$PATH"
      python3 /srv/cc-ci/cc-ci-plan/launch-upgrader.py start >> /srv/cc-ci/.cc-ci-logs/upgrader-cron.log 2>&1
    '';
  };

  systemd.timers.cc-ci-upgrade-all = {
    description = "Weekly trigger for cc-ci-upgrade-all (Thursdays 22:00 America/New_York — Boston 10pm)";
    wantedBy = [ "timers.target" ];
    timerConfig = {
      # 10pm Thursday Boston time — DST-aware (EDT→02:00 UTC, EST→03:00 UTC) via the tz in OnCalendar.
      OnCalendar = "Thu *-*-* 22:00:00 America/New_York";
      Persistent = true;  # if the box was down at the scheduled time, run once on next boot
    };
  };

  # Hourly SUPERVISOR — a glm-5.2 orchestrator wake-up that keeps the weekly run on track. The
  # log-idle/429 watchdog only handles opencode-go usage-limit stalls; it does NOT cover a host
  # disk-full crash (which killed the 2026-07-03 run) or any other environmental wedge. This is a
  # CHEAP deterministic gate: if the weekly run is complete or actively progressing it does NOTHING
  # (zero model tokens). Only when a run has stalled/died before completing does it launch a
  # short-lived glm-5.2 agent that diagnoses the blockage and drives the run to a clean DONE.
  systemd.services.cc-ci-upgrade-supervisor = {
    description = "cc-ci hourly weekly-run supervisor (glm-5.2 — drives a stalled /upgrade-all to completion)";
    after = [ "network-online.target" "tailscaled.service" ];
    wants = [ "network-online.target" ];
    serviceConfig = {
      Type = "oneshot";  # launch-supervisor.py check: gate now, spawn the agent into tmux, return
      User = "loops"; Group = "users";
      WorkingDirectory = "/srv/cc-ci";
      # Shares the weekly run's optional override file (e.g. SUPERVISOR_MODEL=…); "-" = optional.
      EnvironmentFile = "-/srv/cc-ci/upgrader.env";
    };
    environment = { HOME = "/home/loops"; };
    path = [ pkgs.bash pkgs.tmux pkgs.git pkgs.python3 pkgs.openssh pkgs.nettools ];
    script = ''
      export PATH="/home/loops/.local/bin:$PATH"
      python3 /srv/cc-ci/cc-ci-plan/launch-supervisor.py check >> /srv/cc-ci/.cc-ci-logs/supervisor-cron.log 2>&1
    '';
  };

  systemd.timers.cc-ci-upgrade-supervisor = {
    description = "Hourly trigger for cc-ci-upgrade-supervisor (weekly-run health check + drive)";
    wantedBy = [ "timers.target" ];
    timerConfig = {
      OnCalendar = "*-*-* *:07:00";  # every hour at :07 (offset from the weekly :00 fire)
      Persistent = false;            # a missed hourly check is moot — the next hour re-checks
    };
  };
}