feat(cleanup): guarantee step-2b dev deploys get reaped

- /recipe-upgrade step 2b: teardown is now MANDATORY on every exit path (finally),
  with a verify-no-leak check; tear down even on failure before reporting.
- reap-dev-deploys.sh: safe, age-gated backstop that removes only idle dev-* stacks
  (never CI per-run stacks, warm-*, infra; an active dev loop stays fresh).
- orchestrator: hourly cc-ci-reap-dev-deploys systemd timer runs it against cc-ci,
  bounding any leaked dev deploy from a crashed/abandoned loop.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
autonomic-bot
2026-06-09 15:42:23 +00:00
co-authored by Claude Opus 4.8
parent 77ba7ee075
commit 23bba98be4
3 changed files with 96 additions and 3 deletions
@@ -220,4 +220,37 @@ SSHCFG
Persistent = true; # if the box was down at the scheduled time, run once on next boot
};
};
# Hourly reaper for LEAKED /recipe-upgrade step-2b dev deploys (`dev-*` stacks on the cc-ci server).
# The upgrader must tear down its own dev deploy; this is the automated backstop for a missed
# teardown (crashed/abandoned loop). reap-dev-deploys.sh is scoped + age-gated so it is safe to run
# even mid-CI: it only touches `dev-*`, and only when idle > THRESHOLD (an active dev loop keeps
# redeploying and is never reaped). cc-ci-plan/IDEAS.md tracks the eventual separate-infra fix; this
# just bounds the leak window in the meantime.
systemd.services.cc-ci-reap-dev-deploys = {
description = "Reap leaked step-2b dev deploys (dev-* stacks) on the cc-ci server";
after = [ "network-online.target" "tailscaled.service" ];
wants = [ "network-online.target" ];
serviceConfig = {
Type = "oneshot";
User = "loops"; Group = "users";
WorkingDirectory = "/srv/cc-ci";
};
environment = { HOME = "/home/loops"; };
path = [ pkgs.bash pkgs.openssh pkgs.coreutils ];
script = ''
ssh cc-ci 'THRESHOLD=14400 bash -s' \
< /srv/cc-ci/.claude/skills/upgrade-all/reap-dev-deploys.sh \
>> /srv/cc-ci/.cc-ci-logs/reap-dev-deploys.log 2>&1
'';
};
systemd.timers.cc-ci-reap-dev-deploys = {
description = "Hourly reaper for leaked step-2b dev deploys on cc-ci";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "hourly";
Persistent = true;
};
};
}