#!/usr/bin/env python3 """Discover and run tests for a recipe. Finds all Python test scripts in recipe-info//tests/, runs each one, and reports pass/fail results. Usage: python3 scripts/test_runner.py --recipe hedgedoc python3 scripts/test_runner.py --recipe hedgedoc --instance t1cc python3 scripts/test_runner.py --recipe hedgedoc --domain custom.example.com """ import argparse import os import subprocess import sys from pathlib import Path sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from lib.config import paths from lib.models import load_deployment from lib.log import SkillLogger def discover_tests(recipe: str) -> list[Path]: """Find all test scripts for a recipe.""" test_dir = paths.RECIPE_INFO_DIR / recipe / "tests" if not test_dir.exists(): return [] tests = sorted(test_dir.glob("*.py")) # Exclude __init__.py and similar return [t for t in tests if not t.name.startswith("_")] def run_test(test_path: Path, domain: str, server: str, timeout: int = 120) -> dict: """Run a single test script. Returns {"name": str, "passed": bool, "output": str, "returncode": int}. """ name = test_path.stem print(f"\n--- Running: {name} ---", flush=True) cmd = [sys.executable, str(test_path), "--domain", domain] try: result = subprocess.run( cmd, capture_output=True, text=True, timeout=timeout, env={**os.environ, "TEST_SERVER": server, "TEST_DOMAIN": domain}, ) output = result.stdout if result.stderr: output += "\n" + result.stderr passed = result.returncode == 0 except subprocess.TimeoutExpired: output = f"TIMEOUT after {timeout}s" passed = False result = None status = "PASS" if passed else "FAIL" print(f" {status}: {name}", flush=True) if output.strip(): for line in output.strip().split("\n"): print(f" {line}", flush=True) return { "name": name, "passed": passed, "output": output.strip(), "returncode": result.returncode if result else -1, } def main(): parser = argparse.ArgumentParser(description="Run tests for a recipe") parser.add_argument("--recipe", required=True, help="Recipe name") parser.add_argument("--instance", default=None, help="Instance name (default: active)") parser.add_argument("--deployment", default="default", help="Deployment name") parser.add_argument("--domain", default=None, help="Override domain") parser.add_argument("--server", default=None, help="Override server (use with --domain)") parser.add_argument("--timeout", type=int, default=120, help="Timeout per test (seconds)") args = parser.parse_args() log = SkillLogger("test", args.recipe) # Resolve deployment if args.domain: domain = args.domain server = args.server or "localhost" else: dep = load_deployment(args.recipe, instance=args.instance, name=args.deployment) domain = dep.domain server = dep.server log.info(f"Testing: {domain} on {server}") # Discover tests log.step("Discover tests") tests = discover_tests(args.recipe) if not tests: log.warn(f"No tests found in recipe-info/{args.recipe}/tests/") log.info("Create test scripts as Python files in that directory.") log.save() print(f"\nNo tests found for {args.recipe}", flush=True) sys.exit(0) log.info(f"Found {len(tests)} test(s): {', '.join(t.stem for t in tests)}") # Run tests log.step("Run tests") results = [] for test_path in tests: result = run_test(test_path, domain, server, timeout=args.timeout) results.append(result) log.command( f"python3 {test_path.name} --domain {domain}", result["output"], result["returncode"], ) # Summary log.step("Summary") passed = [r for r in results if r["passed"]] failed = [r for r in results if not r["passed"]] print(f"\n{'='*50}", flush=True) print(f"Results: {len(passed)}/{len(results)} passed", flush=True) for r in results: status = "PASS" if r["passed"] else "FAIL" print(f" [{status}] {r['name']}", flush=True) if failed: log.error(f"{len(failed)} test(s) failed: " f"{', '.join(r['name'] for r in failed)}") else: log.info("All tests passed") log.save() if failed: sys.exit(1) if __name__ == "__main__": main()