scripts/check_groq_models.py

"""Check which models in configs/ are still live on the Groq API."""

import json
import os
from pathlib import Path
from urllib.request import Request, urlopen

import yaml
from dotenv import load_dotenv

load_dotenv()

CONFIGS_DIR = Path(__file__).resolve().parents[1] / "configs"


def fetch_live_models(api_key: str) -> set[str]:
    req = Request(
        "https://api.groq.com/openai/v1/models",
        headers={
            "Authorization": f"Bearer {api_key}",
            "Accept": "application/json",
            "User-Agent": "llm-prompt-injection-security-eval-framework/0.1",
        },
    )
    data = json.loads(urlopen(req).read())
    return {m["id"] for m in data["data"]}


def main() -> None:
    api_key = os.getenv("GROQ_APIKEY") or os.getenv("GROQ_API_KEY")
    if not api_key:
        raise SystemExit("No Groq API key found in environment.")

    live_models = fetch_live_models(api_key)

    print(f"Live Groq models ({len(live_models)}):")
    for m in sorted(live_models):
        print(f"  {m}")

    print("\nChecking configs/ for decommissioned models...")
    broken: list[tuple[str, str]] = []
    for config_path in sorted(CONFIGS_DIR.glob("*.yaml")):
        with config_path.open() as f:
            raw = yaml.safe_load(f)
        model_id = raw.get("model", {}).get("name", "")
        provider = raw.get("model", {}).get("provider", "")
        if provider == "groq" and model_id and model_id not in live_models:
            broken.append((config_path.name, model_id))

    if broken:
        print(f"\n  {len(broken)} config(s) reference decommissioned models:")
        for name, model_id in broken:
            print(f"    {name}  →  {model_id}")
    else:
        print("\n  All groq configs reference live models.")


if __name__ == "__main__":
    main()