scripts/run_all.py

"""Run every preset (or a chosen subset) through runner.py, one after another.

Presets whose API key is missing are skipped. Extra arguments after ``--`` are
forwarded to runner.py, e.g. ``python scripts/run_all.py kev-4b -- --limit 5``.
"""
import argparse
import os
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

from benchmark.adapters import ADAPTERS  # noqa: E402
from benchmark.models import PRESETS, load_dotenv  # noqa: E402


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("presets", nargs="*", help=f"Default: all ({', '.join(PRESETS)})")
    parser.add_argument("--skip-local", action="store_true", help="Skip laya / julia.")
    argv = sys.argv[1:]
    split = argv.index("--") if "--" in argv else len(argv)
    args, forwarded = parser.parse_args(argv[:split]), argv[split + 1:]

    unknown = set(args.presets) - PRESETS.keys()
    if unknown:
        raise SystemExit(f"Unknown preset(s): {sorted(unknown)}")

    load_dotenv()
    summary = []
    for name in args.presets or PRESETS:
        provider, _model = PRESETS[name]
        key_env = getattr(ADAPTERS[provider], "api_key_env", None)
        if key_env and not os.getenv(key_env):
            summary.append((name, f"skipped ({key_env} missing)"))
            continue
        if not key_env and args.skip_local:
            summary.append((name, "skipped (local)"))
            continue

        print(f"\n######## {name} ########", flush=True)
        cmd = [sys.executable, "runner.py", "--preset", name, *forwarded]
        code = subprocess.call(cmd, cwd=ROOT)
        summary.append((name, "ok" if code == 0 else f"failed (exit {code})"))

    print("\n" + "=" * 60)
    for name, status in summary:
        print(f"{name:22} {status}")


if __name__ == "__main__":
    main()