Compare commits

...
Author SHA1 Message Date
autonomic-bot e83cd46806 lint: fix pre-existing failures so the self-test pipeline passes
continuous-integration/drone/push Build is passing
Push build #33 (this branch) failed at the lint gate on failures inherited from
main, which block this PR from merging:
- runner/harness/warm.py: ruff format (long regex line — no code change)
- nix/modules/acme-dns.nix: statix W201 'avoid repeated keys' — fold the three
  service definitions (acme-dns, cc-ci-acme-storage-seed,
  cc-ci-acme-traefik-handoff) into one services = { ... } attrset. Pure
  restructure: systemd.services eval of all three units is byte-identical to
  main (nix eval --json diff, all three IDENTICAL).

scripts/lint.sh now: PASS.
2026-10-05 16:59:33 +00:00
autonomic-bot 2a7cdcdee4 gitea: fix LFS round-trip post-restart wait (stale on 28.0.0)
continuous-integration/drone/push Build is failing
test_lfs_roundtrip's post-restart poll timed out on gitea 28.0.0 (PR #10 run #31):
_api() caught only HTTPError, so a single urlopen(timeout=20) socket timeout inside
the poll raised and aborted the poll itself instead of being retried; the poll also
hard-coded a 120s deadline + a 20s request timeout, contradicting the recipe's
declared HTTP_TIMEOUT=600 (tests/STYLE.md §5-sized for the warm custom tier; the
28.0.0 boot runs a longer migration window than 1.27.3).

Fix, assertion untouched (§3):
- _api() is never-raising like lifecycle.http_fetch: transient URLError/socket
  timeout/OSError -> (0, {}) so the bounded poll simply retries (each request still
  bounded at 20s; the poll's deadline is what bounds the wait).
- poll deadline derives from recipe_meta.HTTP_TIMEOUT (600) instead of a hard-coded
  120, probing the same /api/v1/version path recipe_meta.READY_PROBE declares.

Evidence: gitea 28.0.0 dev deploy converged + served /api/healthz and /version
(gitea-upgrade-2026-10-02.md 2b); the RED run #31 failed only in this poll while
install/upgrade/backup/restore and every other custom test passed.
2026-10-05 16:45:10 +00:00
autonomic-bot f675941c34 Merge pull request 'nix: sync root authorizedKeys with live cc-ci keys' (#43) from fix/root-authorized-keys-sync into main
continuous-integration/drone/push Build is failing
continuous-integration/drone Build is passing
2026-09-28 19:43:59 +00:00
3 changed files with 104 additions and 81 deletions
+78 -71
View File
@@ -111,82 +111,89 @@ in
"f /var/lib/ci-certs/acme-production-enabled 0600 root root -" "f /var/lib/ci-certs/acme-production-enabled 0600 root root -"
]; ];
services.acme-dns = { # Three systemd units (W201 statix): one attrset, distinct unit names — avoids
description = "Restricted authoritative DNS for cc-ci ACME DNS-01"; # `services = { ... }` attribute sets repeating the `services` key.
wantedBy = [ "multi-user.target" ]; services = {
after = [ "network-online.target" ]; acme-dns = {
wants = [ "network-online.target" ]; description = "Restricted authoritative DNS for cc-ci ACME DNS-01";
serviceConfig = { wantedBy = [ "multi-user.target" ];
User = "acme-dns"; after = [ "network-online.target" ];
Group = "acme-dns"; wants = [ "network-online.target" ];
StateDirectory = "acme-dns"; serviceConfig = {
StateDirectoryMode = "0700"; User = "acme-dns";
WorkingDirectory = "/var/lib/acme-dns"; Group = "acme-dns";
ExecStart = "${pkgs.acme-dns}/bin/acme-dns -c ${acmeDnsConfig}"; StateDirectory = "acme-dns";
Restart = "on-failure"; StateDirectoryMode = "0700";
RestartSec = "5s"; WorkingDirectory = "/var/lib/acme-dns";
AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; ExecStart = "${pkgs.acme-dns}/bin/acme-dns -c ${acmeDnsConfig}";
CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; Restart = "on-failure";
NoNewPrivileges = true; RestartSec = "5s";
PrivateTmp = true; AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
PrivateDevices = true; CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
ProtectHome = true; NoNewPrivileges = true;
ProtectSystem = "strict"; PrivateTmp = true;
ReadWritePaths = [ "/var/lib/acme-dns" ]; PrivateDevices = true;
RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ]; ProtectHome = true;
ProtectSystem = "strict";
ReadWritePaths = [ "/var/lib/acme-dns" ];
RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ];
};
}; };
};
# Seed the new cert's acmedns storage before the ACME unit first runs (see # Seed the new cert's acmedns storage before the ACME unit first runs (see
# acmeStorageSeed above). Ordering via the generated acme unit name. # acmeStorageSeed above). Ordering via the generated acme unit name.
services.cc-ci-acme-storage-seed = { cc-ci-acme-storage-seed = {
description = "Seed ci.autonomic.zone acme-dns storage from the legacy account"; description = "Seed ci.autonomic.zone acme-dns storage from the legacy account";
after = [ "acme-dns.service" ]; after = [ "acme-dns.service" ];
wantedBy = [ "acme-ci.autonomic.zone.service" ]; wantedBy = [ "acme-ci.autonomic.zone.service" ];
before = [ "acme-ci.autonomic.zone.service" ]; before = [ "acme-ci.autonomic.zone.service" ];
serviceConfig = { serviceConfig = {
Type = "oneshot"; Type = "oneshot";
UMask = "0077"; UMask = "0077";
};
script = "${acmeStorageSeed}/bin/cc-ci-acme-storage-seed";
}; };
script = "${acmeStorageSeed}/bin/cc-ci-acme-storage-seed";
};
# Traefik consumes its wildcard as immutable Swarm secrets, so a renewed # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed
# host certificate must be copied and reconciled rather than merely reloaded. # host certificate must be copied and reconciled rather than merely reloaded.
# This service is started only by the production-mode ACME postRun hook. # This service is started only by the production-mode ACME postRun hook.
services.cc-ci-acme-traefik-handoff = { # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed
description = "Install renewed cc-ci wildcard into Traefik Swarm secrets"; # host certificate must be copied and reconciled rather than merely reloaded.
after = [ "docker.service" "deploy-proxy.service" ]; # This service is started only by the production-mode ACME postRun hook.
requires = [ "docker.service" ]; cc-ci-acme-traefik-handoff = {
path = [ pkgs.coreutils pkgs.docker pkgs.systemd pkgs.gnugrep ]; description = "Install renewed cc-ci wildcard into Traefik Swarm secrets";
serviceConfig = { after = [ "docker.service" "deploy-proxy.service" ];
Type = "oneshot"; requires = [ "docker.service" ];
UMask = "0077"; path = [ pkgs.coreutils pkgs.docker pkgs.systemd pkgs.gnugrep ];
serviceConfig = {
Type = "oneshot";
UMask = "0077";
};
script = ''
src=/var/lib/acme/ci.autonomic.zone
dst=/var/lib/ci-certs/live
test -s "$src/fullchain.pem"
test -s "$src/key.pem"
install -d -m 0700 "$dst"
install -m 0444 "$src/fullchain.pem" "$dst/fullchain.pem.new"
install -m 0400 "$src/key.pem" "$dst/privkey.pem.new"
mv -f "$dst/fullchain.pem.new" "$dst/fullchain.pem"
mv -f "$dst/privkey.pem.new" "$dst/privkey.pem"
# deploy-proxy performs the health-gated Swarm rollout. Its reconciler
# derives a fresh version from the public certificate chain and inserts
# the matching ssl_cert/ssl_key secrets before deploying Traefik.
systemctl restart deploy-proxy.service
# A successful rollout no longer references old wildcard versions. Best
# effort removal retains any secret Docker still reports as in use.
keep="v$(sha256sum "$dst/fullchain.pem" | cut -c1-16)"
docker secret ls --format '{{.Name}}' | \
grep -E '^traefik_ci_commoninternet_net_ssl_(cert|key)_v' | \
grep -v -E "_(ssl_cert|ssl_key)_$keep\$" | \
while IFS= read -r stale; do docker secret rm "$stale" || true; done
'';
}; };
script = ''
src=/var/lib/acme/ci.autonomic.zone
dst=/var/lib/ci-certs/live
test -s "$src/fullchain.pem"
test -s "$src/key.pem"
install -d -m 0700 "$dst"
install -m 0444 "$src/fullchain.pem" "$dst/fullchain.pem.new"
install -m 0400 "$src/key.pem" "$dst/privkey.pem.new"
mv -f "$dst/fullchain.pem.new" "$dst/fullchain.pem"
mv -f "$dst/privkey.pem.new" "$dst/privkey.pem"
# deploy-proxy performs the health-gated Swarm rollout. Its reconciler
# derives a fresh version from the public certificate chain and inserts
# the matching ssl_cert/ssl_key secrets before deploying Traefik.
systemctl restart deploy-proxy.service
# A successful rollout no longer references old wildcard versions. Best
# effort removal retains any secret Docker still reports as in use.
keep="v$(sha256sum "$dst/fullchain.pem" | cut -c1-16)"
docker secret ls --format '{{.Name}}' | \
grep -E '^traefik_ci_commoninternet_net_ssl_(cert|key)_v' | \
grep -v -E "_(ssl_cert|ssl_key)_$keep\$" | \
while IFS= read -r stale; do docker secret rm "$stale" || true; done
'';
}; };
}; };
+3 -1
View File
@@ -41,7 +41,9 @@ _CTX.check_hostname = False
_CTX.verify_mode = ssl.CERT_NONE _CTX.verify_mode = ssl.CERT_NONE
# A cold per-run stack name looks like "<tag>-<6hex>_ci_commoninternet_net_<svc>"; extract the hex. # A cold per-run stack name looks like "<tag>-<6hex>_ci_commoninternet_net_<svc>"; extract the hex.
_STACK_HEX_RE = re.compile(r"^[a-z0-9]{1,4}-([0-9a-f]{6})_ci_(?:autonomic_zone|commoninternet_net)_") _STACK_HEX_RE = re.compile(
r"^[a-z0-9]{1,4}-([0-9a-f]{6})_ci_(?:autonomic_zone|commoninternet_net)_"
)
def stable_domain(recipe: str) -> str: def stable_domain(recipe: str) -> str:
+23 -9
View File
@@ -23,6 +23,7 @@ import shutil
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import time
import urllib.error import urllib.error
import urllib.request import urllib.request
@@ -45,7 +46,15 @@ def _lfs_available() -> bool:
return os.path.exists(os.path.join(abra_dir, "recipes", "gitea", _LFS_OVERLAY)) return os.path.exists(os.path.join(abra_dir, "recipes", "gitea", _LFS_OVERLAY))
def _api(domain, path, method="GET", body=None, user="", password=""): API_TIMEOUT = 20
def _api(domain, path, method="GET", body=None, user="", password="", timeout=API_TIMEOUT):
"""One API call → (status, JSON). Never raises: transient errors (connection refused /
dropped mid-restart, DNS, socket timeout) return (0, {}) so a caller's bounded poll can
simply retry, mirroring lifecycle.http_fetch. Only HTTPError was caught before; a socket
timeout on a slow post-restart boot (gitea 28.0.0 migration window, PR #10 run #31) escaped
as TimeoutError and aborted the restart poll itself."""
data = json.dumps(body).encode() if body is not None else None data = json.dumps(body).encode() if body is not None else None
auth = base64.b64encode(f"{user}:{password}".encode()).decode() auth = base64.b64encode(f"{user}:{password}".encode()).decode()
headers = {"Authorization": f"Basic {auth}"} headers = {"Authorization": f"Basic {auth}"}
@@ -55,7 +64,7 @@ def _api(domain, path, method="GET", body=None, user="", password=""):
f"https://{domain}/api/v1{path}", data=data, headers=headers, method=method f"https://{domain}/api/v1{path}", data=data, headers=headers, method=method
) )
try: try:
with urllib.request.urlopen(req, timeout=20, context=_CTX) as r: with urllib.request.urlopen(req, timeout=timeout, context=_CTX) as r:
raw = r.read() raw = r.read()
return r.status, (json.loads(raw) if raw else {}) return r.status, (json.loads(raw) if raw else {})
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
@@ -64,6 +73,8 @@ def _api(domain, path, method="GET", body=None, user="", password=""):
return e.code, json.loads(raw) return e.code, json.loads(raw)
except (ValueError, json.JSONDecodeError): except (ValueError, json.JSONDecodeError):
return e.code, {} return e.code, {}
except (urllib.error.URLError, TimeoutError, OSError): # TimeoutError ⊃ socket.timeout (UP041)
return 0, {}
def _run_git(args, cwd, env=None): def _run_git(args, cwd, env=None):
@@ -164,6 +175,8 @@ def test_lfs_roundtrip(live_app):
finally: finally:
shutil.rmtree(fresh_dir, ignore_errors=True) shutil.rmtree(fresh_dir, ignore_errors=True)
import recipe_meta # noqa: E402 — per-recipe declared timeouts (tests/STYLE.md §5)
# 7. JWT-secret stability: restart gitea + assert LFS_JWT_SECRET unchanged. # 7. JWT-secret stability: restart gitea + assert LFS_JWT_SECRET unchanged.
# Read the current secret from inside the container's rendered app.ini. # Read the current secret from inside the container's rendered app.ini.
current_jwt = lifecycle.exec_in_app( current_jwt = lifecycle.exec_in_app(
@@ -182,14 +195,15 @@ def test_lfs_roundtrip(live_app):
capture_output=True, capture_output=True,
timeout=120, timeout=120,
) )
# Wait for gitea to come back up # Wait for gitea to come back up. Window + probe are DERIVED from the recipe's declared
# readiness (recipe_meta.HTTP_TIMEOUT / READY_PROBE — the /api/v1/version readiness that
# Re-read meta from the live_app fixture (meta is not in scope here — use the stored meta) # recipe_meta already declares for the harness, not a guess) — tests/STYLE.md §5.
import time wait_timeout = int(getattr(recipe_meta, "HTTP_TIMEOUT", 300))
probe_path = "/version" # READY_PROBE's readiness path (rules carry only ok + domain)
deadline = time.time() + 120 deadline = time.time() + wait_timeout
status2 = 0
while time.time() < deadline: while time.time() < deadline:
status2, _ = _api(live_app, "/version", user=user, password=password) status2, _ = _api(live_app, probe_path, user=user, password=password)
if status2 == 200: if status2 == 200:
break break
time.sleep(5) time.sleep(5)