From 2a7cdcdee4d02a3660c3df9acc7ea4d7c52dd9b2 Mon Sep 17 00:00:00 2001 From: autonomic-bot <64+autonomic-bot@noreply.git.autonomic.zone> Date: Mon, 5 Oct 2026 16:45:10 +0000 Subject: [PATCH 1/2] gitea: fix LFS round-trip post-restart wait (stale on 28.0.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_lfs_roundtrip's post-restart poll timed out on gitea 28.0.0 (PR #10 run #31): _api() caught only HTTPError, so a single urlopen(timeout=20) socket timeout inside the poll raised and aborted the poll itself instead of being retried; the poll also hard-coded a 120s deadline + a 20s request timeout, contradicting the recipe's declared HTTP_TIMEOUT=600 (tests/STYLE.md §5-sized for the warm custom tier; the 28.0.0 boot runs a longer migration window than 1.27.3). Fix, assertion untouched (§3): - _api() is never-raising like lifecycle.http_fetch: transient URLError/socket timeout/OSError -> (0, {}) so the bounded poll simply retries (each request still bounded at 20s; the poll's deadline is what bounds the wait). - poll deadline derives from recipe_meta.HTTP_TIMEOUT (600) instead of a hard-coded 120, probing the same /api/v1/version path recipe_meta.READY_PROBE declares. Evidence: gitea 28.0.0 dev deploy converged + served /api/healthz and /version (gitea-upgrade-2026-10-02.md 2b); the RED run #31 failed only in this poll while install/upgrade/backup/restore and every other custom test passed. --- tests/gitea/custom/test_lfs_roundtrip.py | 32 +++++++++++++++++------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/tests/gitea/custom/test_lfs_roundtrip.py b/tests/gitea/custom/test_lfs_roundtrip.py index f118066..fb9544d 100644 --- a/tests/gitea/custom/test_lfs_roundtrip.py +++ b/tests/gitea/custom/test_lfs_roundtrip.py @@ -23,6 +23,7 @@ import shutil import subprocess import sys import tempfile +import time import urllib.error import urllib.request @@ -45,7 +46,15 @@ def _lfs_available() -> bool: return os.path.exists(os.path.join(abra_dir, "recipes", "gitea", _LFS_OVERLAY)) -def _api(domain, path, method="GET", body=None, user="", password=""): +API_TIMEOUT = 20 + + +def _api(domain, path, method="GET", body=None, user="", password="", timeout=API_TIMEOUT): + """One API call → (status, JSON). Never raises: transient errors (connection refused / + dropped mid-restart, DNS, socket timeout) return (0, {}) so a caller's bounded poll can + simply retry, mirroring lifecycle.http_fetch. Only HTTPError was caught before; a socket + timeout on a slow post-restart boot (gitea 28.0.0 migration window, PR #10 run #31) escaped + as TimeoutError and aborted the restart poll itself.""" data = json.dumps(body).encode() if body is not None else None auth = base64.b64encode(f"{user}:{password}".encode()).decode() headers = {"Authorization": f"Basic {auth}"} @@ -55,7 +64,7 @@ def _api(domain, path, method="GET", body=None, user="", password=""): f"https://{domain}/api/v1{path}", data=data, headers=headers, method=method ) try: - with urllib.request.urlopen(req, timeout=20, context=_CTX) as r: + with urllib.request.urlopen(req, timeout=timeout, context=_CTX) as r: raw = r.read() return r.status, (json.loads(raw) if raw else {}) except urllib.error.HTTPError as e: @@ -64,6 +73,8 @@ def _api(domain, path, method="GET", body=None, user="", password=""): return e.code, json.loads(raw) except (ValueError, json.JSONDecodeError): return e.code, {} + except (urllib.error.URLError, TimeoutError, OSError): # TimeoutError ⊃ socket.timeout (UP041) + return 0, {} def _run_git(args, cwd, env=None): @@ -164,6 +175,8 @@ def test_lfs_roundtrip(live_app): finally: shutil.rmtree(fresh_dir, ignore_errors=True) + import recipe_meta # noqa: E402 — per-recipe declared timeouts (tests/STYLE.md §5) + # 7. JWT-secret stability: restart gitea + assert LFS_JWT_SECRET unchanged. # Read the current secret from inside the container's rendered app.ini. current_jwt = lifecycle.exec_in_app( @@ -182,14 +195,15 @@ def test_lfs_roundtrip(live_app): capture_output=True, timeout=120, ) - # Wait for gitea to come back up - - # Re-read meta from the live_app fixture (meta is not in scope here — use the stored meta) - import time - - deadline = time.time() + 120 + # Wait for gitea to come back up. Window + probe are DERIVED from the recipe's declared + # readiness (recipe_meta.HTTP_TIMEOUT / READY_PROBE — the /api/v1/version readiness that + # recipe_meta already declares for the harness, not a guess) — tests/STYLE.md §5. + wait_timeout = int(getattr(recipe_meta, "HTTP_TIMEOUT", 300)) + probe_path = "/version" # READY_PROBE's readiness path (rules carry only ok + domain) + deadline = time.time() + wait_timeout + status2 = 0 while time.time() < deadline: - status2, _ = _api(live_app, "/version", user=user, password=password) + status2, _ = _api(live_app, probe_path, user=user, password=password) if status2 == 200: break time.sleep(5) -- 2.54.0 From e83cd46806abd07b7c275a3cdb83fba356c87fb1 Mon Sep 17 00:00:00 2001 From: autonomic-bot <64+autonomic-bot@noreply.git.autonomic.zone> Date: Mon, 5 Oct 2026 16:59:33 +0000 Subject: [PATCH 2/2] lint: fix pre-existing failures so the self-test pipeline passes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Push build #33 (this branch) failed at the lint gate on failures inherited from main, which block this PR from merging: - runner/harness/warm.py: ruff format (long regex line — no code change) - nix/modules/acme-dns.nix: statix W201 'avoid repeated keys' — fold the three service definitions (acme-dns, cc-ci-acme-storage-seed, cc-ci-acme-traefik-handoff) into one services = { ... } attrset. Pure restructure: systemd.services eval of all three units is byte-identical to main (nix eval --json diff, all three IDENTICAL). scripts/lint.sh now: PASS. --- nix/modules/acme-dns.nix | 149 ++++++++++++++++++++------------------- runner/harness/warm.py | 4 +- 2 files changed, 81 insertions(+), 72 deletions(-) diff --git a/nix/modules/acme-dns.nix b/nix/modules/acme-dns.nix index caa8b5e..c4f9d49 100644 --- a/nix/modules/acme-dns.nix +++ b/nix/modules/acme-dns.nix @@ -111,82 +111,89 @@ in "f /var/lib/ci-certs/acme-production-enabled 0600 root root -" ]; - services.acme-dns = { - description = "Restricted authoritative DNS for cc-ci ACME DNS-01"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" ]; - wants = [ "network-online.target" ]; - serviceConfig = { - User = "acme-dns"; - Group = "acme-dns"; - StateDirectory = "acme-dns"; - StateDirectoryMode = "0700"; - WorkingDirectory = "/var/lib/acme-dns"; - ExecStart = "${pkgs.acme-dns}/bin/acme-dns -c ${acmeDnsConfig}"; - Restart = "on-failure"; - RestartSec = "5s"; - AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; - CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; - NoNewPrivileges = true; - PrivateTmp = true; - PrivateDevices = true; - ProtectHome = true; - ProtectSystem = "strict"; - ReadWritePaths = [ "/var/lib/acme-dns" ]; - RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ]; + # Three systemd units (W201 statix): one attrset, distinct unit names — avoids + # `services = { ... }` attribute sets repeating the `services` key. + services = { + acme-dns = { + description = "Restricted authoritative DNS for cc-ci ACME DNS-01"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + serviceConfig = { + User = "acme-dns"; + Group = "acme-dns"; + StateDirectory = "acme-dns"; + StateDirectoryMode = "0700"; + WorkingDirectory = "/var/lib/acme-dns"; + ExecStart = "${pkgs.acme-dns}/bin/acme-dns -c ${acmeDnsConfig}"; + Restart = "on-failure"; + RestartSec = "5s"; + AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; + CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; + NoNewPrivileges = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectHome = true; + ProtectSystem = "strict"; + ReadWritePaths = [ "/var/lib/acme-dns" ]; + RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ]; + }; }; - }; - # Seed the new cert's acmedns storage before the ACME unit first runs (see - # acmeStorageSeed above). Ordering via the generated acme unit name. - services.cc-ci-acme-storage-seed = { - description = "Seed ci.autonomic.zone acme-dns storage from the legacy account"; - after = [ "acme-dns.service" ]; - wantedBy = [ "acme-ci.autonomic.zone.service" ]; - before = [ "acme-ci.autonomic.zone.service" ]; - serviceConfig = { - Type = "oneshot"; - UMask = "0077"; + # Seed the new cert's acmedns storage before the ACME unit first runs (see + # acmeStorageSeed above). Ordering via the generated acme unit name. + cc-ci-acme-storage-seed = { + description = "Seed ci.autonomic.zone acme-dns storage from the legacy account"; + after = [ "acme-dns.service" ]; + wantedBy = [ "acme-ci.autonomic.zone.service" ]; + before = [ "acme-ci.autonomic.zone.service" ]; + serviceConfig = { + Type = "oneshot"; + UMask = "0077"; + }; + script = "${acmeStorageSeed}/bin/cc-ci-acme-storage-seed"; }; - script = "${acmeStorageSeed}/bin/cc-ci-acme-storage-seed"; - }; - # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed - # host certificate must be copied and reconciled rather than merely reloaded. - # This service is started only by the production-mode ACME postRun hook. - services.cc-ci-acme-traefik-handoff = { - description = "Install renewed cc-ci wildcard into Traefik Swarm secrets"; - after = [ "docker.service" "deploy-proxy.service" ]; - requires = [ "docker.service" ]; - path = [ pkgs.coreutils pkgs.docker pkgs.systemd pkgs.gnugrep ]; - serviceConfig = { - Type = "oneshot"; - UMask = "0077"; + # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed + # host certificate must be copied and reconciled rather than merely reloaded. + # This service is started only by the production-mode ACME postRun hook. + # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed + # host certificate must be copied and reconciled rather than merely reloaded. + # This service is started only by the production-mode ACME postRun hook. + cc-ci-acme-traefik-handoff = { + description = "Install renewed cc-ci wildcard into Traefik Swarm secrets"; + after = [ "docker.service" "deploy-proxy.service" ]; + requires = [ "docker.service" ]; + path = [ pkgs.coreutils pkgs.docker pkgs.systemd pkgs.gnugrep ]; + serviceConfig = { + Type = "oneshot"; + UMask = "0077"; + }; + script = '' + src=/var/lib/acme/ci.autonomic.zone + dst=/var/lib/ci-certs/live + test -s "$src/fullchain.pem" + test -s "$src/key.pem" + install -d -m 0700 "$dst" + install -m 0444 "$src/fullchain.pem" "$dst/fullchain.pem.new" + install -m 0400 "$src/key.pem" "$dst/privkey.pem.new" + mv -f "$dst/fullchain.pem.new" "$dst/fullchain.pem" + mv -f "$dst/privkey.pem.new" "$dst/privkey.pem" + + # deploy-proxy performs the health-gated Swarm rollout. Its reconciler + # derives a fresh version from the public certificate chain and inserts + # the matching ssl_cert/ssl_key secrets before deploying Traefik. + systemctl restart deploy-proxy.service + + # A successful rollout no longer references old wildcard versions. Best + # effort removal retains any secret Docker still reports as in use. + keep="v$(sha256sum "$dst/fullchain.pem" | cut -c1-16)" + docker secret ls --format '{{.Name}}' | \ + grep -E '^traefik_ci_commoninternet_net_ssl_(cert|key)_v' | \ + grep -v -E "_(ssl_cert|ssl_key)_$keep\$" | \ + while IFS= read -r stale; do docker secret rm "$stale" || true; done + ''; }; - script = '' - src=/var/lib/acme/ci.autonomic.zone - dst=/var/lib/ci-certs/live - test -s "$src/fullchain.pem" - test -s "$src/key.pem" - install -d -m 0700 "$dst" - install -m 0444 "$src/fullchain.pem" "$dst/fullchain.pem.new" - install -m 0400 "$src/key.pem" "$dst/privkey.pem.new" - mv -f "$dst/fullchain.pem.new" "$dst/fullchain.pem" - mv -f "$dst/privkey.pem.new" "$dst/privkey.pem" - - # deploy-proxy performs the health-gated Swarm rollout. Its reconciler - # derives a fresh version from the public certificate chain and inserts - # the matching ssl_cert/ssl_key secrets before deploying Traefik. - systemctl restart deploy-proxy.service - - # A successful rollout no longer references old wildcard versions. Best - # effort removal retains any secret Docker still reports as in use. - keep="v$(sha256sum "$dst/fullchain.pem" | cut -c1-16)" - docker secret ls --format '{{.Name}}' | \ - grep -E '^traefik_ci_commoninternet_net_ssl_(cert|key)_v' | \ - grep -v -E "_(ssl_cert|ssl_key)_$keep\$" | \ - while IFS= read -r stale; do docker secret rm "$stale" || true; done - ''; }; }; diff --git a/runner/harness/warm.py b/runner/harness/warm.py index 988afcc..f7f7a3f 100644 --- a/runner/harness/warm.py +++ b/runner/harness/warm.py @@ -41,7 +41,9 @@ _CTX.check_hostname = False _CTX.verify_mode = ssl.CERT_NONE # A cold per-run stack name looks like "-<6hex>_ci_commoninternet_net_"; extract the hex. -_STACK_HEX_RE = re.compile(r"^[a-z0-9]{1,4}-([0-9a-f]{6})_ci_(?:autonomic_zone|commoninternet_net)_") +_STACK_HEX_RE = re.compile( + r"^[a-z0-9]{1,4}-([0-9a-f]{6})_ci_(?:autonomic_zone|commoninternet_net)_" +) def stable_domain(recipe: str) -> str: -- 2.54.0