fix(harness): make concurrent recipe runs safe (per-recipe flock + active-run registry)
continuous-integration/drone/push Build is passing
continuous-integration/drone/push Build is passing
capacity=2 went live with three stale capacity=1-era assumptions that corrupted concurrent runs (immich 229/230 '/pg_backup.sh: No such file'): - ~/.abra/recipes/<recipe> is ONE shared working tree that fetch_recipe rm-rf's/ reclones and the upgrade tier git-checkouts mid-run. Same-recipe runs now serialise on an exclusive flock (/run/lock/cc-ci-recipe-<recipe>.lock), taken in main() BEFORE fetch_recipe and held for the whole run; the kernel releases it on any process death, so there is no stale-lock failure mode. Different recipes still run in parallel. - CCCI_JANITOR_MAX_AGE=0 made a starting build reap ANY in-flight run app. Every run now registers its app domain + pid in /run/cc-ci-active/<domain> before app creation; the janitor checks the owner: alive (pid is a live run_recipe_ci process) -> never reaped; dead -> reaped immediately; unknown (pre-registry or post-reboot) -> age fallback (default 2h). The MAX_AGE=0 env override is gone from .drone.yml. - .drone.yml: concurrency.limit 1 -> 2 to match DRONE_RUNNER_CAPACITY=2; the 'safe because capacity=1' comments now describe the flock+registry model. lint: PASS, unit tests: 138 passed.
This commit is contained in:
+22
-6
@@ -44,17 +44,25 @@ sys.path.insert(0, os.path.join(ROOT, "runner"))
|
||||
from harness import ( # noqa: E402
|
||||
abra,
|
||||
canonical,
|
||||
card as card_mod,
|
||||
deps as deps_mod,
|
||||
discovery,
|
||||
generic,
|
||||
lifecycle,
|
||||
naming,
|
||||
results as results_mod,
|
||||
screenshot as screenshot_mod,
|
||||
warm,
|
||||
warmsnap,
|
||||
)
|
||||
from harness import ( # noqa: E402
|
||||
card as card_mod,
|
||||
)
|
||||
from harness import ( # noqa: E402
|
||||
deps as deps_mod,
|
||||
)
|
||||
from harness import ( # noqa: E402
|
||||
results as results_mod,
|
||||
)
|
||||
from harness import ( # noqa: E402
|
||||
screenshot as screenshot_mod,
|
||||
)
|
||||
|
||||
ALL_STAGES = ("install", "upgrade", "backup", "restore", "custom")
|
||||
|
||||
@@ -827,6 +835,12 @@ def main() -> int:
|
||||
print(
|
||||
f"== cc-ci run: recipe={recipe} ref={ref} pr={os.environ.get('PR', '0')} stages={sorted(stages)}"
|
||||
)
|
||||
# Concurrent-run safety: runs of the SAME recipe serialise on a per-recipe flock — they share
|
||||
# ONE ~/.abra/recipes/<recipe> working tree which fetch_recipe (below) rm-rf's/reclones and the
|
||||
# upgrade tier git-checkouts mid-run. Must be taken BEFORE fetch_recipe. Different recipes run
|
||||
# in parallel (capacity=2). The reference must stay alive for the whole run: the kernel drops
|
||||
# the flock when the fd closes (including on any crash/SIGKILL — no stale-lock failure mode).
|
||||
_recipe_lock = lifecycle.acquire_recipe_lock(recipe) # noqa: F841
|
||||
fetch_recipe(recipe, ref, src)
|
||||
# The PR-head commit the upgrade tier re-checks out for the chaos redeploy to the code under test
|
||||
# (HC1). Prefer the explicit PR head sha ($REF) — robust + exact; fall back to the recipe checkout
|
||||
@@ -1285,8 +1299,10 @@ def main() -> int:
|
||||
capped = data.get("level_cap_rung")
|
||||
sk = data.get("skips", {})
|
||||
cap_skip = (
|
||||
"intentional" if capped in (sk.get("intentional") or {})
|
||||
else "unintentional" if capped in (sk.get("unintentional") or [])
|
||||
"intentional"
|
||||
if capped in (sk.get("intentional") or {})
|
||||
else "unintentional"
|
||||
if capped in (sk.get("unintentional") or [])
|
||||
else ""
|
||||
)
|
||||
with open(os.path.join(run_artifact_dir, "badge.svg"), "w", encoding="utf-8") as f:
|
||||
|
||||
Reference in New Issue
Block a user