scripts/evaluate_model_providers.py

"""
Provider evaluation harness (spec #6).

Runs the SAME jobs through each configured provider (Kimi / NVIDIA / ... / Stub)
with the same base resume, JD set, and automation mode, then prints a per-job
table and a per-provider summary so you can decide which model performs best.

Usage:
    PYTHONPATH=. python scripts/evaluate_model_providers.py            # all JD fixtures
    PYTHONPATH=. python scripts/evaluate_model_providers.py --jobs 4   # first 4 JDs

This is the script to run with REAL provider keys (Kimi 2.6 / NVIDIA) — stub-only
runs validate plumbing, NOT production quality.
"""

import os
import sys
import io
import json
import argparse

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
if hasattr(sys.stdout, "buffer"):
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")

from src.provider_eval import (
    load_base_resume, load_jd_fixtures, run_provider_matrix, summarize,
)
from src.providers import build_provider_chain
from src.llm_client import LLMClient


def _fmt_row(r):
    return (f"{r['provider'][:16]:<16} {r['job'][:18]:<18} "
            f"int={r['internal']:>3} ind={r['independent']:>3} "
            f"read={r['readability']:>3} {r['status'][:22]:<22} "
            f"rep={r['repair_attempts']} risk={r['risk_terms']} "
            f"sch_err={r['schema_errors']} dl={'Y' if r['download_allowed'] else 'n'} "
            f"{r['runtime_s']}s")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--jobs", type=int, default=None, help="limit number of JD fixtures")
    args = ap.parse_args()

    base = load_base_resume()
    if base is None:
        print("ERROR: no base resume. Add data/resume/resume.pdf or data/resume/_parsed.json.")
        sys.exit(1)

    jobs = load_jd_fixtures(limit=args.jobs)
    if not jobs:
        print("ERROR: no JD fixtures in tests/fixtures/jds/.")
        sys.exit(1)

    llm = LLMClient.__new__(LLMClient)
    providers = build_provider_chain(llm)
    real = [p for p in providers if getattr(p, "cfg", {}).get("api_key")]
    print(f"Providers under test: {[p.name for p in providers]}")
    if not real:
        print("\n*** NO PROVIDER API KEYS DETECTED — running StubProvider only. ***")
        print("*** This validates plumbing ONLY. Run again with Kimi/NVIDIA keys")
        print("*** in .env for a real production-quality comparison. ***\n")

    out_root = os.path.join("data", "output", "eval")
    rows = run_provider_matrix(providers, base, jobs, out_root, llm)

    print("\n=== PER-JOB RESULTS ===")
    print("Provider         Job                int ind read Status                 rep risk sch dl  time")
    for r in rows:
        print(_fmt_row(r))

    print("\n=== PER-PROVIDER SUMMARY ===")
    summ = summarize(rows)
    print(f"{'Provider':<16} {'Ready%':>6} {'AvgInd':>7} {'AvgInt':>7} "
          f"{'AvgRep':>7} {'Sch%':>5} {'Err%':>5} {'Risk':>5} {'Runtime':>8}  Recommendation")
    for s in summ:
        print(f"{s['provider'][:16]:<16} {s['ready_rate']:>6} {s['avg_independent']:>7} "
              f"{s['avg_internal']:>7} {s['avg_repairs']:>7} {s['schema_error_rate']:>5} "
              f"{s['provider_error_rate']:>5} {s['risk_overuse']:>5} "
              f"{s['avg_runtime_s']:>7}s  {s['recommendation']}")

    os.makedirs(out_root, exist_ok=True)
    out_json = os.path.join(out_root, "provider_eval.json")
    with open(out_json, "w", encoding="utf-8") as f:
        json.dump({"rows": rows, "summary": summ}, f, ensure_ascii=False, indent=2)
    print(f"\nSaved full results -> {out_json}")


if __name__ == "__main__":
    main()