feat(cleanup): guarantee step-2b dev deploys get reaped
- /recipe-upgrade step 2b: teardown is now MANDATORY on every exit path (finally), with a verify-no-leak check; tear down even on failure before reporting. - reap-dev-deploys.sh: safe, age-gated backstop that removes only idle dev-* stacks (never CI per-run stacks, warm-*, infra; an active dev loop stays fresh). - orchestrator: hourly cc-ci-reap-dev-deploys systemd timer runs it against cc-ci, bounding any leaked dev deploy from a crashed/abandoned loop. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
77ba7ee075
commit
23bba98be4
@@ -220,4 +220,37 @@ SSHCFG
|
||||
Persistent = true; # if the box was down at the scheduled time, run once on next boot
|
||||
};
|
||||
};
|
||||
|
||||
# Hourly reaper for LEAKED /recipe-upgrade step-2b dev deploys (`dev-*` stacks on the cc-ci server).
|
||||
# The upgrader must tear down its own dev deploy; this is the automated backstop for a missed
|
||||
# teardown (crashed/abandoned loop). reap-dev-deploys.sh is scoped + age-gated so it is safe to run
|
||||
# even mid-CI: it only touches `dev-*`, and only when idle > THRESHOLD (an active dev loop keeps
|
||||
# redeploying and is never reaped). cc-ci-plan/IDEAS.md tracks the eventual separate-infra fix; this
|
||||
# just bounds the leak window in the meantime.
|
||||
systemd.services.cc-ci-reap-dev-deploys = {
|
||||
description = "Reap leaked step-2b dev deploys (dev-* stacks) on the cc-ci server";
|
||||
after = [ "network-online.target" "tailscaled.service" ];
|
||||
wants = [ "network-online.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = "loops"; Group = "users";
|
||||
WorkingDirectory = "/srv/cc-ci";
|
||||
};
|
||||
environment = { HOME = "/home/loops"; };
|
||||
path = [ pkgs.bash pkgs.openssh pkgs.coreutils ];
|
||||
script = ''
|
||||
ssh cc-ci 'THRESHOLD=14400 bash -s' \
|
||||
< /srv/cc-ci/.claude/skills/upgrade-all/reap-dev-deploys.sh \
|
||||
>> /srv/cc-ci/.cc-ci-logs/reap-dev-deploys.log 2>&1
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.cc-ci-reap-dev-deploys = {
|
||||
description = "Hourly reaper for leaked step-2b dev deploys on cc-ci";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = "hourly";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user