# auto-update.nix — weekly, health-gated self-update of the combined cc-ci host.
#
# Modelled on notplants-nix's modules/auto-update.nix (the nightly updater the old orchestrator
# box had), with the two things the cc-ci host needs on top:
#   * a BUSY GATE — a `switch` restarts docker/traefik/drone/the runner, so the update skips
#     (and simply retries next week) while a CI run, the weekly recipe-upgrade run or the
#     Sunday canonical sweep is in flight;
#   * cc-ci HEALTH CHECKS — beyond "sshd + no failed units": every swarm service at its replica
#     count, the front doors (dashboard, reports, drone, the opencode UI) answering, sops having
#     decrypted, acme-dns and fail2ban up.
#
# Sequence (as in the notplants module — read its comments for the whys):
#   flake update (all inputs: nixpkgs, sops-nix, cc-ci) → nixos-rebuild build (as loops) →
#   switch-to-configuration test (bootloader untouched) → settle → health check →
#   on failure: re-activate the previous generation, restore flake.lock, exit 1
#   on success: set the system profile + bootloader, commit flake.lock as "auto-update", push
#   main (the repo stays the source of truth), refresh /etc/cc-ci.
# Nothing is committed that did not pass the health check. Never fires a missed run at boot.
# `/cc-ci-status` reads the outcome from the state file this writes.
{ config, lib, pkgs, ... }:
let
  cfg = config.cc-ci-orchestrator.autoUpdate;
  builderHome = config.users.users.${cfg.buildUser}.home;
  stateFile = "${cfg.flakePath}/.cc-ci-logs/auto-update-state";
  curl = "${pkgs.curl}/bin/curl";

  busyGate = pkgs.writeShellScript "cc-ci-auto-update-busy-gate" ''
    set -u
    busy() { echo "BUSY: $1 — skipping this week's update"; exit 3; }
    ${pkgs.procps}/bin/pgrep -f run_recipe_ci >/dev/null && busy "a CI run is in flight"
    ${pkgs.systemd}/bin/systemctl is-active --quiet nightly-sweep.service && busy "the canonical sweep is running"
    ${pkgs.util-linux}/bin/runuser -u ${cfg.buildUser} -- ${pkgs.tmux}/bin/tmux has-session -t cc-ci-upgrader 2>/dev/null \
      && busy "the weekly recipe-upgrade run is in flight (tmux cc-ci-upgrader)"
    ${pkgs.util-linux}/bin/runuser -u ${cfg.buildUser} -- ${pkgs.tmux}/bin/tmux has-session -t cc-ci-report 2>/dev/null \
      && busy "the weekly report is being written (tmux cc-ci-report)"
    if [ -r /run/secrets/bridge_drone_token ]; then
      running=$(${curl} -s -m 20 -H "Authorization: Bearer $(cat /run/secrets/bridge_drone_token)" \
        "https://drone.${cfg.ciDomain}/api/repos/recipe-maintainers/cc-ci/builds?per_page=10" \
        | ${pkgs.gnugrep}/bin/grep -o '"status":"running"' | ${pkgs.coreutils}/bin/wc -l)
      [ "''${running:-0}" -eq 0 ] || busy "$running Drone build(s) running"
    fi
    echo "not busy"
  '';

  healthCheck = pkgs.writeShellScript "cc-ci-auto-update-health-check" ''
    set -u
    fail() { echo "HEALTH FAIL: $1"; exit 1; }
    ${pkgs.systemd}/bin/systemctl is-active --quiet sshd || fail "sshd not active"
    ${pkgs.iproute2}/bin/ss -tlnH | ${pkgs.gnugrep}/bin/grep -q ':22 ' || fail "nothing listening on :22"
    n=$(${pkgs.systemd}/bin/systemctl --failed --no-legend | ${pkgs.coreutils}/bin/wc -l)
    [ "$n" -eq 0 ] || fail "$n failed unit(s): $(${pkgs.systemd}/bin/systemctl --failed --no-legend --plain | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.coreutils}/bin/tr '\n' ' ')"
    for u in docker acme-dns fail2ban nginx opencode-web drone-runner-exec; do
      ${pkgs.systemd}/bin/systemctl is-active --quiet "$u" || fail "$u not active"
    done
    # The reconcile oneshots re-run on activation; give the swarm up to 10 minutes to converge.
    for i in $(${pkgs.coreutils}/bin/seq 1 40); do
      short=$(${pkgs.docker}/bin/docker service ls --format '{{.Name}} {{.Replicas}}' \
        | ${pkgs.gawk}/bin/awk '{ split($2,a,"/"); if (a[1] != a[2]) print $1 }')
      [ -z "$short" ] && break
      ${pkgs.coreutils}/bin/sleep 15
    done
    [ -z "$short" ] || fail "swarm services not at their replica count: $(echo "$short" | ${pkgs.coreutils}/bin/tr '\n' ' ')"
    [ -s /run/secrets/test_secret ] || fail "sops did not decrypt (/run/secrets/test_secret missing)"
    code() { ${curl} -s -m 20 -o /dev/null -w '%{http_code}' --resolve "$1:443:127.0.0.1" "https://$1/$2"; }
    [ "$(code ${cfg.ciDomain} "")" = 200 ] || fail "dashboard not 200"
    [ "$(code report.${cfg.ciDomain} "")" = 200 ] || fail "reports not 200"
    [ "$(code drone.${cfg.ciDomain} version)" = 200 ] || fail "drone /version not 200"
    [ "$(code ${cfg.opencodeUiHost} "")" = 401 ] || fail "opencode UI not answering with its auth challenge"
    echo "HEALTH OK"
  '';
in
{
  options.cc-ci-orchestrator.autoUpdate = {
    enable = lib.mkEnableOption "weekly health-gated self-update of the cc-ci host";
    flakePath = lib.mkOption {
      type = lib.types.str;
      default = "/srv/cc-ci-orch";
      description = "The cc-ci-orchestrator checkout (owned by buildUser) whose flake.lock is updated, committed and pushed.";
    };
    flakeRef = lib.mkOption {
      type = lib.types.str;
      default = "${cfg.flakePath}#cc-ci";
      description = "Flake reference to build.";
    };
    buildUser = lib.mkOption {
      type = lib.types.str;
      default = "loops";
      description = "Owner of the checkout; runs the update, build, commit and push (root activates).";
    };
    onCalendar = lib.mkOption {
      type = lib.types.str;
      default = "Tue *-*-* 03:00:00 UTC";
      description = ''
        When to run. Must stay clear of the weekly recipe-upgrade run (Thu 22:00 America/New_York
        = Fri 02:00/03:00 UTC, several hours) and the Sunday 03:00 UTC canonical sweep (up to 6 h);
        the busy gate covers the rest.
      '';
    };
    remote = lib.mkOption { type = lib.types.str; default = "origin"; };
    branch = lib.mkOption { type = lib.types.str; default = "main"; };
    ciDomain = lib.mkOption {
      type = lib.types.str;
      default = "ci.commoninternet.net";
      description = "The CI apex; report./drone. are checked under it.";
    };
    opencodeUiHost = lib.mkOption {
      type = lib.types.str;
      default = config.cc-ci-orchestrator.opencodeUiHost;
      description = "The opencode UI host name (expected to answer 401).";
    };
  };

  config = lib.mkIf cfg.enable {
    systemd.services.cc-ci-auto-update = {
      description = "Weekly cc-ci host self-update (flake update → build → test → health check → commit)";
      after = [ "network-online.target" "docker.service" ];
      wants = [ "network-online.target" ];
      path = with pkgs; [ nixos-rebuild nix git openssh coreutils gnugrep gawk systemd util-linux bash python3 docker curl ];
      serviceConfig = { Type = "oneshot"; TimeoutStartSec = "3h"; };
      # A unit that performs the switch must never be stopped or restarted BY that switch
      # (notplants-orchestrator, 2026-09-07: it was, and the host ran half-switched for 12 h).
      stopIfChanged = false;
      restartIfChanged = false;
      script = ''
        set -u
        cd ${cfg.flakePath}
        as_builder() { runuser -u ${cfg.buildUser} -- env HOME=${builderHome} "$@"; }
        state() { printf '%s result=%s generation=%s note=%s\n' "$(date -u +%FT%TZ)" "$1" "$2" "$3" > ${stateFile}; chown ${cfg.buildUser} ${stateFile} || true; }
        PREV=$(readlink -f /run/current-system)
        LOCK_BAK=${cfg.flakePath}/.flake.lock.auto-update-prev

        if ! ${busyGate}; then
          state skipped "$PREV" busy
          exit 0
        fi

        revert() {
          echo "REVERT: $1"
          as_builder cp "$LOCK_BAK" flake.lock 2>/dev/null || true
          "$PREV"/bin/switch-to-configuration test || echo "re-activation of $PREV failed"
          state failed "$PREV" "$1"
          exit 1
        }

        as_builder cp flake.lock "$LOCK_BAK"
        echo "=== checkout at: $(as_builder git rev-parse --short HEAD) on $(as_builder git rev-parse --abbrev-ref HEAD)"
        echo "=== flake update (all inputs)"
        as_builder nix flake update || revert "flake update failed"
        if as_builder git diff --quiet -- flake.lock; then
          echo "=== nothing moved; done"
          state unchanged "$PREV" "lock already current"
          exit 0
        fi
        as_builder git --no-pager diff --stat -- flake.lock

        echo "=== build (cannot affect the running system)"
        as_builder nixos-rebuild build --flake ${cfg.flakeRef} || revert "build failed"
        NEW=$(readlink -f result) || revert "no build result"
        as_builder rm -f result
        echo "=== built: $NEW"

        echo "=== activate WITHOUT touching the bootloader"
        "$NEW"/bin/switch-to-configuration test || revert "test activation failed"

        echo "=== settle, then health check"
        sleep 45
        ${healthCheck} || revert "health check failed"

        echo "=== healthy: committing (system profile + bootloader)"
        nix-env -p /nix/var/nix/profiles/system --set "$NEW" || revert "could not set system profile"
        "$NEW"/bin/switch-to-configuration boot || revert "bootloader update failed"
        state ok "$NEW" "committed"

        # Bookkeeping from here on — never revert a healthy, committed generation over it.
        if as_builder git -c user.name="auto-update" -c user.email="auto-update@cc-ci" \
             commit -q -m "flake.lock: weekly auto-update, health-checked" -- flake.lock; then
          echo "=== flake.lock committed: $(as_builder git rev-parse --short HEAD)"
          as_builder git push -q ${cfg.remote} HEAD:${cfg.branch} \
            && echo "=== pushed ${cfg.branch}" \
            || echo "WARNING: push failed; the lock is committed locally, push it by hand"
        else
          echo "WARNING: could not commit flake.lock; the system IS committed and healthy"
        fi
        # Keep the deployed cc-ci checkout (sweep + sops file) on the same cc-ci rev as the flake input.
        git -C /etc/cc-ci pull -q --ff-only --recurse-submodules || echo "WARNING: /etc/cc-ci pull failed"
        echo "=== auto-update committed: $NEW"
      '';
    };

    systemd.timers.cc-ci-auto-update = {
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnCalendar = cfg.onCalendar;
        Persistent = false;
        RandomizedDelaySec = "10min";
      };
    };
  };
}