src/provider_eval.py

"""
Provider evaluation harness (spec #6, #7).

Reusable functions to run the EXACT production generation logic
(`ResumeCustomizer._generate_resume_v4`) once per provider on a shared set of
JDs, then collect comparable metrics. The scripts in scripts/ format the tables.

Model-independent: works with any provider (Claude / Kimi / NVIDIA) and with the
deterministic StubProvider for offline smoke runs.
"""

from __future__ import annotations

import os
import re
import json
import time
import statistics
from typing import List, Optional

from .resume_model import Resume


# ── loaders ──────────────────────────────────────────────────────────────────

def load_base_resume() -> Optional[Resume]:
    """Load the candidate base resume from the parsed cache or the real PDF."""
    cache = os.path.join("data", "resume", "_parsed.json")
    if os.path.exists(cache):
        try:
            with open(cache, encoding="utf-8") as f:
                data = json.load(f)
            return Resume.from_dict(data.get("resume", data))
        except Exception:
            pass
    pdf = os.path.join("data", "resume", "resume.pdf")
    if os.path.exists(pdf):
        from .resume_parser_v2 import parse_resume_pdf_cached
        return parse_resume_pdf_cached(pdf)
    return None


def load_jd_fixtures(limit: int = None) -> List[dict]:
    """Load JD fixtures as job dicts from tests/fixtures/jds/*.txt."""
    jd_dir = os.path.join("tests", "fixtures", "jds")
    jobs: List[dict] = []
    if not os.path.isdir(jd_dir):
        return jobs
    for fn in sorted(os.listdir(jd_dir)):
        if not fn.endswith(".txt") or fn.lower() == "readme.txt":
            continue
        path = os.path.join(jd_dir, fn)
        try:
            with open(path, encoding="utf-8") as f:
                text = f.read()
        except Exception:
            continue
        slug = os.path.splitext(fn)[0]
        title, company = _title_company_from_slug(slug, text)
        jobs.append({"title": title, "company": company, "description": text,
                     "ats_keywords": "", "_raw_assessment": {}, "_jd_slug": slug})
    if limit:
        jobs = jobs[:limit]
    return jobs


def _title_company_from_slug(slug: str, text: str) -> tuple:
    first = (text.strip().splitlines() or [""])[0][:60]
    pretty = slug.replace("_", " ").title()
    return (f"Product Manager ({pretty})", pretty)


# ── single evaluation ──────────────────────────────────────────────────────────

def _make_customizer(llm, base_text: str, output_dir: str):
    from .resume_customizer import ResumeCustomizer
    c = ResumeCustomizer.__new__(ResumeCustomizer)
    c.llm = llm
    c.resume_text = base_text
    c.fast_model_cfg = None
    c.output_dir = output_dir
    os.makedirs(output_dir, exist_ok=True)
    return c


def evaluate_provider_on_job(provider, base_resume: Resume, job: dict,
                             output_dir: str, llm=None) -> dict:
    """Run one provider on one job through the production v4 path; return a row."""
    c = _make_customizer(llm, base_resume.to_flat_text(), output_dir)
    safe = re.sub(r'[\\/*?:"<>|]', "", f"{provider.name}_{job.get('_jd_slug', job.get('company',''))}")[:90]
    filepath = os.path.join(output_dir, safe + ".docx")

    row = {"provider": getattr(provider, "name", "?"),
           "job": job.get("_jd_slug", job.get("company", "")),
           "internal": 0, "independent": 0, "readability": 0,
           "status": "", "repair_attempts": 0, "risk_terms": 0,
           "schema_errors": 0, "provider_errors": 0, "download_allowed": False,
           "runtime_s": 0.0}

    t0 = time.time()
    try:
        path = c._generate_resume_v4(job, cfg=None, filepath=filepath,
                                     provider=provider, base_resume_override=base_resume)
    except Exception as e:
        row["provider_errors"] = 1
        row["status"] = f"ERROR:{str(e)[:40]}"
        row["runtime_s"] = round(time.time() - t0, 1)
        return row
    row["runtime_s"] = round(time.time() - t0, 1)

    report = job.get("_v2_report", {}) or {}
    est = report.get("estimated_scores", {}) or {}
    row["internal"] = est.get("jd_match", 0)
    row["independent"] = report.get("independent_jd_match", 0)
    row["readability"] = est.get("ats_readability", 0)
    row["status"] = report.get("status", job.get("_v2_status", ""))
    row["repair_attempts"] = len(report.get("repair_attempts", []) or [])
    row["risk_terms"] = (len(report.get("review_terms_for_user_review", []) or [])
                         + len(report.get("high_risk_terms_for_confirmation", []) or []))
    row["download_allowed"] = bool(report.get("download_allowed"))
    pq = report.get("provider_response_quality", "ok")
    row["schema_errors"] = 0 if pq == "ok" else 1
    return row


def run_provider_matrix(providers: List, base_resume: Resume, jobs: List[dict],
                        output_root: str, llm=None) -> List[dict]:
    rows: List[dict] = []
    for provider in providers:
        out = os.path.join(output_root, getattr(provider, "name", "provider"))
        for job in jobs:
            rows.append(evaluate_provider_on_job(provider, base_resume, job, out, llm))
    return rows


def summarize(rows: List[dict]) -> List[dict]:
    """Per-provider summary table (spec #6)."""
    by_provider: dict = {}
    for r in rows:
        by_provider.setdefault(r["provider"], []).append(r)
    out: List[dict] = []
    for prov, rs in by_provider.items():
        n = len(rs) or 1
        ready = sum(1 for r in rs if r.get("download_allowed"))
        out.append({
            "provider": prov,
            "ready_rate": round(100 * ready / n),
            "avg_independent": round(statistics.mean(r["independent"] for r in rs), 1),
            "avg_internal": round(statistics.mean(r["internal"] for r in rs), 1),
            "avg_repairs": round(statistics.mean(r["repair_attempts"] for r in rs), 1),
            "schema_error_rate": round(100 * sum(r["schema_errors"] for r in rs) / n),
            "provider_error_rate": round(100 * sum(r["provider_errors"] for r in rs) / n),
            "risk_overuse": round(statistics.mean(r["risk_terms"] for r in rs), 1),
            "avg_runtime_s": round(statistics.mean(r["runtime_s"] for r in rs), 1),
            "recommendation": "preferred" if (ready / n) >= 0.8 else (
                "usable" if (ready / n) >= 0.5 else "deprioritize"),
        })
    out.sort(key=lambda x: (x["ready_rate"], x["avg_independent"]), reverse=True)
    return out


# ── consistency (spec #7) ──────────────────────────────────────────────────────

def consistency_runs(provider, base_resume: Resume, job: dict,
                     output_dir: str, n: int = 3, llm=None) -> dict:
    """Run the same provider on the same JD n times; report variance + stability."""
    scores: List[int] = []
    for i in range(n):
        out = os.path.join(output_dir, f"run{i+1}")
        row = evaluate_provider_on_job(provider, base_resume, dict(job), out, llm)
        scores.append(row["independent"])
    variance = round(statistics.pvariance(scores), 1) if len(scores) > 1 else 0.0
    spread = max(scores) - min(scores) if scores else 0
    return {"provider": getattr(provider, "name", "?"),
            "job": job.get("_jd_slug", job.get("company", "")),
            "scores": scores, "variance": variance, "spread": spread,
            "stable": spread <= 5}