recipe-maintainers/cc-ci
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style(1b): auto-format + lint-clean the whole codebase (RL1)

Browse Source 
Mechanical, semantics-preserving cleanup so the codebase passes the new lint stage:
- ruff format: all 32 Python files (wraps long signatures, normalizes quotes/blank lines).
- nixpkgs-fmt: modules/drone-runner.nix.
- shfmt (-i 2 -ci): scripts/*.sh.

Lint fixes (reviewed, behavior-preserving — no test weakened):
- ruff SIM105: try/except-pass -> contextlib.suppress (abra.py app_config rm; lifecycle.py janitor).
- ruff SIM115: open().read() -> with open() (run_recipe_ci.py redaction-values + gitea-token).
- statix: merge repeated sops `secrets.*` keys into one `secrets = { ... }` (comments kept);
  empty fn pattern `{ ... }:` -> `_:` (packages.nix).
- deadnix: drop unused lambda args (flake `self`; configuration.nix `lib`; overlay `final` -> `_`).

Verified on cc-ci: `scripts/lint.sh` -> lint: PASS; nixosConfigurations.cc-ci evaluates;
all Python byte-compiles. The deployed bridge/dashboard/runner source changes hash (reformat),
so cc-ci will be rebuilt to the new closure in W2 before the cold D1-D10 re-verification.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
autonomic-bot
2026-05-27 20:52:05 +01:00
parent a0ea2f0aa9
commit 2cede01ed7
35 changed files with 431 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
runner/harness/abra.py
View File

@ -6,11 +6,11 @@ Bakes in the known abra gotchas (re-verify per installed abra version, currently
- `abra app ls -S -m` returns nested {server: {apps: [...]}} — parse the inner structure.
- run non-interactively with `-n` (`--no-input`) everywhere.
"""
from __future__ import annotations
import json
import subprocess
from typing import Optional
ABRA = "abra"
@ -19,13 +19,17 @@ class AbraError(RuntimeError):
pass
def _run_pty(args: list[str], timeout: int = 900, check: bool = True) -> subprocess.CompletedProcess:
def _run_pty(
args: list[str], timeout: int = 900, check: bool = True
) -> subprocess.CompletedProcess:
"""Run abra under a pseudo-TTY (via util-linux `script`). Needed for commands that exec into
a container interactively (backup create / restore: 'the input device is not a TTY')."""
cmd = "abra " + " ".join(args)
proc = subprocess.run(
["script", "-qec", cmd, "/dev/null"],
capture_output=True, text=True, timeout=timeout,
capture_output=True,
text=True,
timeout=timeout,
)
if check and proc.returncode != 0:
raise AbraError(f"[pty] {cmd} failed ({proc.returncode}):\n{proc.stdout}\n{proc.stderr}")
@ -40,12 +44,19 @@ def _run(args: list[str], timeout: int = 300, check: bool = True) -> subprocess.
timeout=timeout,
)
if check and proc.returncode != 0:
raise AbraError(f"abra {' '.join(args)} failed ({proc.returncode}):\n{proc.stdout}\n{proc.stderr}")
raise AbraError(
f"abra {' '.join(args)} failed ({proc.returncode}):\n{proc.stdout}\n{proc.stderr}"
)
return proc
def app_new(recipe: str, domain: str, server: str = "default", version: Optional[str] = None,
secrets: bool = False) -> None:
def app_new(
recipe: str,
domain: str,
server: str = "default",
version: str | None = None,
secrets: bool = False,
) -> None:
args = ["app", "new", recipe]
args += ["-s", server, "-D", domain, "-o", "-n"]
if version:
@ -64,6 +75,7 @@ def env_set(domain: str, key: str, value: str) -> None:
"""Set a key in the app's .env (abra has no setter; edit the file directly)."""
import os
import re
path = os.path.expanduser(f"~/.abra/servers/default/{domain}.env")
with open(path) as fh:
lines = fh.read().splitlines()
@ -86,8 +98,11 @@ def secret_generate(domain: str, timeout: int = 300) -> None:
# captured by _run and never logged. -C -o keep the recipe at the PR checkout (without -o it
# re-resolves to a version tag, dropping the PR's files incl. tests/). check=False: recipes with
# no secrets are a no-op.
_run(["app", "secret", "generate", domain, "--all", "-m", "-C", "-o", "-n"],
timeout=timeout, check=False)
_run(
["app", "secret", "generate", domain, "--all", "-m", "-C", "-o", "-n"],
timeout=timeout,
check=False,
)
def deploy(domain: str, chaos: bool = True, timeout: int = 900) -> None:
@ -97,7 +112,7 @@ def deploy(domain: str, chaos: bool = True, timeout: int = 900) -> None:
_run(args, timeout=timeout)
def upgrade(domain: str, version: Optional[str] = None, timeout: int = 900) -> None:
def upgrade(domain: str, version: str | None = None, timeout: int = 900) -> None:
args = ["app", "upgrade", domain]
if version:
args.append(version)
@ -127,9 +142,11 @@ def recipe_versions(recipe: str) -> list[str]:
"""Published versions of a recipe, oldest→newest (from the recipe git tags)."""
import os
import subprocess
path = os.path.expanduser(f"~/.abra/recipes/{recipe}")
proc = subprocess.run(["git", "-C", path, "tag", "--sort=creatordate"],
capture_output=True, text=True)
proc = subprocess.run(
["git", "-C", path, "tag", "--sort=creatordate"], capture_output=True, text=True
)
return [t for t in proc.stdout.split("\n") if t.strip()]
@ -149,12 +166,12 @@ def secret_remove_all(domain: str, timeout: int = 300) -> None:
def app_config_remove(domain: str, server: str = "default") -> None:
"""Delete the app's .env config so a re-run can recreate it (teardown completeness)."""
import contextlib
import os
path = os.path.expanduser(f"~/.abra/servers/{server}/{domain}.env")
try:
with contextlib.suppress(FileNotFoundError):
os.remove(path)
except FileNotFoundError:
pass
def app_ls(server: str = "default") -> list[dict]:

43
runner/harness/lifecycle.py
View File

@ -3,8 +3,10 @@
The teardown guarantee is sacred: a failed test must never leak an app/volume/secret into the
next run. Callers wrap deploy()/teardown() in try/finally (or a pytest finalizer).
"""
from __future__ import annotations
import contextlib
import datetime
import os
import re
@ -29,7 +31,8 @@ def _docker_names(kind: str, stack: str) -> list[str]:
"""docker <kind> ls names filtered to a stack (kind: service|volume|secret)."""
proc = subprocess.run(
["docker", kind, "ls", "--filter", f"name={stack}", "--format", "{{.Name}}"],
capture_output=True, text=True,
capture_output=True,
text=True,
)
return [n for n in proc.stdout.split("\n") if n.strip()]
@ -50,16 +53,20 @@ def _stack_age_seconds(stack: str) -> float | None:
return None
oldest = None
for s in svcs:
p = subprocess.run(["docker", "service", "inspect", s, "--format", "{{.CreatedAt}}"],
capture_output=True, text=True)
p = subprocess.run(
["docker", "service", "inspect", s, "--format", "{{.CreatedAt}}"],
capture_output=True,
text=True,
)
ts = p.stdout.strip()
try:
# docker emits e.g. 2026-05-27 00:12:33.123 +0000 UTC -> take the leading 19 chars
dt = datetime.datetime.strptime(ts[:19], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=datetime.timezone.utc)
tzinfo=datetime.UTC
)
except ValueError:
continue
age = (datetime.datetime.now(datetime.timezone.utc) - dt).total_seconds()
age = (datetime.datetime.now(datetime.UTC) - dt).total_seconds()
oldest = age if oldest is None else max(oldest, age)
return oldest
@ -107,7 +114,8 @@ def services_converged(domain: str) -> bool:
stack = _stack_name(domain)
proc = subprocess.run(
["docker", "stack", "services", stack, "--format", "{{.Replicas}}"],
capture_output=True, text=True,
capture_output=True,
text=True,
)
rows = [r for r in proc.stdout.split("\n") if r.strip()]
if not rows:
@ -136,8 +144,13 @@ def http_get(domain: str, path: str = "/", timeout: int = 15) -> int:
return 0
def wait_healthy(domain: str, ok_codes=(200, 301, 302), path: str = "/",
deploy_timeout: int = 600, http_timeout: int = 300) -> None:
def wait_healthy(
domain: str,
ok_codes=(200, 301, 302),
path: str = "/",
deploy_timeout: int = 600,
http_timeout: int = 300,
) -> None:
"""Wait for stack services converged, then for the app to answer ok over HTTPS at `path`.
`path` is per-recipe (recipe_meta.HEALTH_PATH), e.g. keycloak uses /realms/master."""
deadline = time.time() + deploy_timeout
@ -181,7 +194,8 @@ def _app_container(domain: str, service: str = "app") -> str:
name = f"{_stack_name(domain)}_{service}"
proc = subprocess.run(
["docker", "ps", "--filter", f"name={name}", "--format", "{{.ID}}"],
capture_output=True, text=True,
capture_output=True,
text=True,
)
cid = proc.stdout.strip().split("\n")[0]
if not cid:
@ -221,8 +235,8 @@ def teardown_app(domain: str, verify: bool = True) -> None:
stack = _stack_name(domain)
abra.undeploy(domain)
if _docker_names("service", stack):
_force_stack_rm(stack) # fallback: abra undeploy didn't clear it
abra.volume_remove(domain) # needs the .env -> before removing it
_force_stack_rm(stack) # fallback: abra undeploy didn't clear it
abra.volume_remove(domain) # needs the .env -> before removing it
abra.secret_remove_all(domain)
# belt-and-suspenders: drop any volumes/secrets abra missed, by stack name. A volume can be
# briefly held by a just-stopped task after `stack rm`, so retry the volume removal.
@ -238,7 +252,7 @@ def teardown_app(domain: str, verify: bool = True) -> None:
time.sleep(3)
for s in _docker_names("secret", stack):
subprocess.run(["docker", "secret", "rm", s], capture_output=True, text=True)
abra.app_config_remove(domain) # only now (stack gone) drop the .env
abra.app_config_remove(domain) # only now (stack gone) drop the .env
if verify:
residual = _residual(domain)
@ -252,6 +266,7 @@ def janitor(max_age_seconds: int | None = None) -> None:
docker primitives so it works even when the .env is gone (A2/A3). Default 2h, env-overridable
via CCCI_JANITOR_MAX_AGE (e.g. 0 to reap all matching orphans immediately)."""
import os
if max_age_seconds is None:
max_age_seconds = int(os.environ.get("CCCI_JANITOR_MAX_AGE", "7200"))
seen = set()
@ -271,7 +286,5 @@ def janitor(max_age_seconds: int | None = None) -> None:
age = _stack_age_seconds(stack)
if age is not None and age < max_age_seconds:
continue # likely a concurrent in-flight run; leave it
try:
with contextlib.suppress(Exception):
teardown_app(name, verify=False)
except Exception:
pass

1
runner/harness/naming.py
View File

@ -3,6 +3,7 @@
Domain = "<recipe[:4]>-<6hex(recipe|pr|ref)>.ci.commoninternet.net" — short enough for Docker's
64-char swarm config/secret name limit, unique per run, collision-safe across recipes (DECISIONS.md).
"""
from __future__ import annotations
import hashlib

29
runner/run_recipe_ci.py
View File

@ -14,6 +14,7 @@ tests/<recipe>/. Teardown is guaranteed by the conftest fixture finalizer.
Run env (python with pytest+playwright, PLAYWRIGHT_BROWSERS_PATH) is provided by `cc-ci-run`
(modules/harness.nix); invoke as: cc-ci-run runner/run_recipe_ci.py
"""
from __future__ import annotations
import glob
@ -26,6 +27,7 @@ import tempfile
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(ROOT, "runner"))
from harness import lifecycle, naming # noqa: E402
STAGE_FILES = {
"install": "test_install.py",
"upgrade": "test_upgrade.py",
@ -40,7 +42,8 @@ def _redact_values() -> list[str]:
vals = set()
for p in glob.glob("/run/secrets/*"):
try:
v = open(p).read().strip()
with open(p) as f:
v = f.read().strip()
except OSError:
continue
if len(v) >= 8:
@ -55,8 +58,15 @@ def run_stage_redacted(cmd: list[str], env: dict | None = None) -> int:
"""Run a stage subprocess, streaming its output live (so Drone logs stay tail-able) but masking
any known infra-secret value first. Belt-and-suspenders: the harness already never prints
secrets and abra doesn't echo generated ones."""
proc = subprocess.Popen(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, bufsize=1)
proc = subprocess.Popen(
cmd,
cwd=ROOT,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
bufsize=1,
)
assert proc.stdout is not None
for line in proc.stdout:
for v in _REDACT:
@ -70,7 +80,8 @@ def run_stage_redacted(cmd: list[str], env: dict | None = None) -> int:
def _gitea_token() -> str | None:
tok = os.environ.get("GITEA_TOKEN")
if not tok and os.path.exists("/run/secrets/bridge_gitea_token"):
tok = open("/run/secrets/bridge_gitea_token").read().strip()
with open("/run/secrets/bridge_gitea_token") as f:
tok = f.read().strip()
return tok or None
@ -97,8 +108,10 @@ def fetch_recipe(recipe: str, ref: str | None, src: str | None) -> None:
# to a foreign host). Non-fatal: if upstream is unreachable, upgrade degrades to a skip.
upstream = f"https://git.coopcloud.tech/coop-cloud/{recipe}.git"
# Explicit tags refspec — a bare `fetch --tags <url>` errors "couldn't find remote ref HEAD".
subprocess.run(["git", "-C", dest, "fetch", "--quiet", upstream,
"refs/tags/*:refs/tags/*"], check=False)
subprocess.run(
["git", "-C", dest, "fetch", "--quiet", upstream, "refs/tags/*:refs/tags/*"],
check=False,
)
else:
# Clean re-fetch from the catalogue. rm first so a leftover dir from a prior SRC+REF run
# (which points origin at the private mirror and may lack version tags) can't poison the
@ -178,7 +191,9 @@ def run_recipe_local(recipe: str, local_tests: str | None) -> int | None:
lifecycle.deploy_app(recipe, domain, version=os.environ.get("VERSION") or None)
lifecycle.wait_healthy(domain)
env = dict(os.environ, CCCI_APP_DOMAIN=domain, CCCI_BASE_URL=f"https://{domain}")
return run_stage_redacted([sys.executable, "-m", "pytest", "-v", "-rA", local_tests], env=env)
return run_stage_redacted(
[sys.executable, "-m", "pytest", "-v", "-rA", local_tests], env=env
)
finally:
lifecycle.teardown_app(domain, verify=False)