src/ats_safe.py

"""Safe, evidence-gated résumé/ATS-alignment orchestrator.

This is the ONE entry point every résumé-generation path must use. It replaces
the old "extract run-grams → inject into every bullet" pipeline that fabricated
experience and leaked scraped page noise.

Guarantees (all enforced by construction, all covered by tests):
  1. Every JD is preprocessed server-side (contamination stripped) before use.
  2. Extraction output is schema-validated and JD-traceable; hallucinated or
     prompt-injected phrases are dropped deterministically.
  3. No keyword is ever inserted without résumé evidence — the résumé is
     PRESERVED verbatim; gaps are disclosed, never filled.
  4. When the JD can't be isolated or extraction is unusable, the pipeline
     returns a typed `manual_review_required` status and preserves the résumé —
     it NEVER falls back to the unvalidated run-gram extractor to modify output.
  5. The generated PDF is re-parsed and validated before success is claimed.

The value delivered is an honest, evidence-backed alignment report — not keyword
stuffing. Any "alignment estimate" is explicitly an internal estimate, never a
Greenhouse score.
"""
from __future__ import annotations

import os
import re
import tempfile
from typing import Dict, List, Optional

from .jd_preprocess import preprocess_jd
from .keyword_schema import validate_and_repair, calibrate
from .evidence_gate import map_evidence
from .resume_rewrite import plan_and_apply_rewrites
from .ats_score import score_alignment, max_supported_score, compute_coverage
from .keyword_schema import _norm as _knorm


STATUS_OK = "ATS_ALIGNMENT_REPORT_READY"
STATUS_MANUAL = "manual_review_required"


def _deterministic_report_only_items(clean_jd: str) -> List[dict]:
    """Fallback extractor for REPORTING ONLY (no LLM available).

    Uses the curated taxonomy extractor on the CLEANED JD to produce structured
    items. These are used solely for evidence mapping / gap reporting — they are
    NEVER injected (the evidence gate + this orchestrator never insert anything).
    This is deliberately NOT the old run-gram extractor.
    """
    try:
        from .external_ats import extract_jd_keywords as _tax_fn
    except Exception:
        return []
    items = []
    for term in (_tax_fn(clean_jd) or []):
        t = (term or "").strip()
        if len(t) < 2:
            continue
        items.append({
            "exact_phrase": t,
            "normalized_concept": t.lower(),
            "category": "hard_skill",
            "requirement_type": "preferred",
            "importance": "medium",
            "source_text": t,
            "semantic_variants": [],
            "confidence": 0.5,
            "requires_resume_evidence": True,
        })
    return items


def _resume_section_order(latex_src: str, resume_text: str) -> List[str]:
    """Actual section order as it appears in the résumé SOURCE (so PDF-order
    validation compares against the résumé's own order, not a fixed assumption)."""
    candidates = ["summary", "experience", "projects", "education", "skills",
                  "certifications"]
    found = []
    for c in candidates:
        m = re.search(r"\\section\*?\{[^}]*" + re.escape(c) + r"[^}]*\}", latex_src, re.I)
        pos = m.start() if m else (latex_src.lower().find(c.upper().lower())
                                   if c.upper() in latex_src else -1)
        if pos >= 0:
            found.append((pos, c))
    ordered = [c for _, c in sorted(found)]
    return ordered or ["experience", "education", "skills"]


def generate_alignment_safe(
    latex_src: str,
    jd_text: str,
    *,
    company: str = "",
    job_title: str = "",
    llm_client=None,
    rewrite_fn=None,
    summary_fn=None,
    criteria: Optional[list] = None,
    selected_model: Optional[str] = None,
    model_health: Optional[list] = None,
    run_audit: bool = False,
    out_dir: Optional[str] = None,
    compile_pdf: bool = True,
    progress_callback=None,
) -> Dict:
    """Evidence-gated alignment + evidence-backed rewriting. Never fabricates.

    `rewrite_fn(original, target_phrase, concept, category)->str` overrides the
    rewriter (tests/demo). In production it defaults to llm_client.rewrite_bullet.

    Returns a report dict with: status, jd_diagnostics, extraction, evidence,
    pdf_validation, internal_alignment_estimate, tex, pdf_path.
    """
    from .latex_resume import latex_to_text, compile_latex_to_pdf, _safe_jobname

    def _prog(stage, pct):
        if progress_callback:
            try:
                progress_callback(stage, pct)
            except Exception:
                pass

    latex_src = latex_src or ""
    resume_text = latex_to_text(latex_src)

    report: Dict = {
        "source": "ats_safe",
        "status": STATUS_MANUAL,
        "resume_preserved": True,
        "tex": latex_src,
        "pdf_path": None,
        "engine": None,
        "compiled": False,
        "jd_diagnostics": {},
        "extraction": {"valid": [], "rejected_count": 0, "used_fallback": False},
        "evidence": {},
        "pdf_validation": {},
        "internal_alignment_estimate": None,
        "live_model": {"selected_model": selected_model, "health": model_health or []},
        "reason": "",
    }
    # All models failed health-check → live optimization unavailable (honest).
    if model_health is not None and selected_model is None and llm_client is None:
        report["live_model"]["status"] = "live_model_unavailable"

    # 1. MANDATORY preprocessing — treat all input as untrusted.
    _prog("Cleaning job description…", 10)
    pre = preprocess_jd(jd_text, company=company)
    report["jd_diagnostics"] = {
        "ok": pre.ok, "confidence": pre.confidence,
        "reason": pre.reason, **pre.diagnostics,
        "dropped_samples": pre.dropped_samples[:8],
    }
    if not pre.ok:
        report["reason"] = f"jd_isolation_failed:{pre.reason}"
        _compile_preserved(report, latex_src, out_dir, job_title, compile_pdf,
                           compile_latex_to_pdf, _safe_jobname, _prog)
        return report

    clean_jd = pre.clean_text

    # 2. DETERMINISTIC source-grounded extraction (NO LLM). This is the primary
    #    path — extraction/ranking/evidence/scoring never depend on an external
    #    model. The LLM (if any) is optional wording polish only, applied later.
    _prog("Extracting hiring criteria…", 30)
    if criteria is not None:                # test/eval injection (bypass extraction)
        raw_items = list(criteria)
        report["extraction"]["mode"] = "injected"
    else:
        from .deterministic_extract import extract_criteria
        raw_items = extract_criteria(clean_jd, pre.sections)
        report["extraction"]["mode"] = "deterministic"
    report["extraction"]["used_fallback"] = False

    # 3. Validate + traceability gate (defense-in-depth; deterministic items are
    #    already traceable, but this normalizes/guards uniformly).
    _prog("Validating extraction…", 40)
    valid, rejected = validate_and_repair(raw_items, clean_jd)
    report["extraction"]["rejected_count"] = len(rejected)
    report["extraction"]["rejected_samples"] = [
        {"exact_phrase": r.get("exact_phrase", ""),
         "reason": r.get("_reject_reason", "")}
        for r in rejected[:10]
    ]
    if not valid:
        report["reason"] = "no_valid_criteria_extracted"
        _compile_preserved(report, latex_src, out_dir, job_title, compile_pdf,
                           compile_latex_to_pdf, _safe_jobname, _prog)
        return report

    # 3.5. Calibrate — pick the 4-6 match-critical criteria (weights total 100).
    valid = calibrate(valid)
    report["extraction"]["valid"] = valid
    report["calibration"] = [
        {"exact_phrase": c["exact_phrase"], "concept": c["normalized_concept"],
         "requirement_type": c["requirement_type"], "importance": c["importance"],
         "calibration_weight": c.get("calibration_weight", 0)}
        for c in valid if (c.get("calibration_weight") or 0) > 0
    ]

    # 4. Evidence gate (BEFORE) — classify covered / partial / gap.
    _prog("Mapping résumé evidence…", 55)
    ev_before = map_evidence(valid, resume_text)
    # Score BEFORE on the ORIGINAL résumé rendered through the SAME PDF pipeline as
    # the final, so before/after is a fair apples-to-apples (PDF-parsed) comparison.
    before_text = resume_text
    try:
        from .latex_resume import render_text_to_pdf
        from .pdf_validate import _extract_pdf_text
        _bpdf = os.path.join(out_dir or tempfile.mkdtemp(prefix="ats_before_"),
                             "_before.pdf")
        if render_text_to_pdf(resume_text, _bpdf) and os.path.exists(_bpdf):
            _bt = _extract_pdf_text(_bpdf)
            if _bt and len(_bt) > 200:
                before_text = _bt
    except Exception:
        pass
    score_before = score_alignment(valid, ev_before.to_dict(), before_text,
                                    stuffing_penalty=_detect_stuffing(before_text, valid))

    # 5. Evidence-backed rewriting (pass 1) — align supported criteria to the JD's
    #    exact wording. DEFAULT is DETERMINISTIC (no LLM); an LLM, if supplied,
    #    only refines wording and its output is re-verified. Every rewrite passes
    #    the deterministic verifier; rejects fall back to the deterministic result.
    from .resume_rewrite import make_deterministic_rewrite_fn, make_deterministic_summary_fn
    candidates = ev_before.rewrite_candidates()
    if rewrite_fn is not None:
        effective_rewrite_fn = rewrite_fn                 # test/demo override
        report["rewrite_mode"] = "override"
    elif llm_client is not None and hasattr(llm_client, "rewrite_bullet"):
        effective_rewrite_fn = llm_client.rewrite_bullet  # optional LLM polish
        report["rewrite_mode"] = "llm_enhanced"
    else:
        effective_rewrite_fn = make_deterministic_rewrite_fn(candidates)
        report["rewrite_mode"] = "deterministic"

    final_latex = latex_src
    rewrite_records = []
    _prog("Optimizing résumé (evidence-backed)…", 66)
    final_latex, recs = plan_and_apply_rewrites(latex_src, candidates, effective_rewrite_fn)
    # If an LLM run produced zero applied rewrites (unreliable model), fall back to
    # the deterministic rewriter so the résumé still improves (deterministic_rewrite_mode).
    if report["rewrite_mode"] == "llm_enhanced" and not any(r.applied for r in recs):
        det = make_deterministic_rewrite_fn(candidates)
        final_latex, recs = plan_and_apply_rewrites(latex_src, candidates, det)
        report["rewrite_mode"] = "deterministic_rewrite_mode"
    rewrite_records = [r.to_dict() for r in recs]

    # 5.5. Headline/summary optimization — DETERMINISTIC by default (swap supported
    #      variants for JD exact phrases within the existing summary; corpus-verified).
    if summary_fn is not None:
        eff_summary_fn = summary_fn
    elif (report["rewrite_mode"] == "llm_enhanced"
          and llm_client is not None and hasattr(llm_client, "rewrite_summary")):
        eff_summary_fn = llm_client.rewrite_summary
    else:
        eff_summary_fn = make_deterministic_summary_fn(candidates)
    report["summary_rewrite"] = None
    if eff_summary_fn is not None:
        from .resume_rewrite import optimize_summary
        top_phrases = [c["exact_phrase"] for c in valid
                       if (c.get("calibration_weight") or 0) > 0
                       and any(cc.keyword == c["normalized_concept"]
                               for cc in ev_before.covered)][:6]
        new_latex, srec = optimize_summary(
            final_latex, job_title or "", top_phrases, resume_text, eff_summary_fn)
        final_latex = new_latex
        report["summary_rewrite"] = srec

    # 6. Compile + PDF-parse (so scoring can use PARSED text, not just LaTeX).
    _compile_preserved(report, final_latex, out_dir, job_title, compile_pdf,
                       compile_latex_to_pdf, _safe_jobname, _prog,
                       resume_text=latex_to_text(final_latex))
    score_text = _scoring_text(report, final_latex, latex_to_text)

    # 6.5. INDEPENDENT evaluation from the parsed text (not the rewrite flags):
    #      which supported-critical criteria are actually missing from the résumé?
    ev_after = map_evidence(valid, latex_to_text(final_latex))
    cov = compute_coverage(valid, ev_after.to_dict(), score_text)
    missing_critical = cov.get("missing_supported_critical", [])

    # 6.6. ONE controlled correction pass for supported-critical terms still absent.
    correction_applied = False
    if missing_critical and effective_rewrite_fn is not None:
        miss = {_knorm(m) for m in missing_critical}
        done = {_knorm(r.get("exact_jd_phrase", "")) for r in rewrite_records
                if r.get("applied")}
        retry = [c for c in ev_before.rewrite_candidates()
                 if _knorm(c.exact_phrase) in miss and _knorm(c.exact_phrase) not in done]
        if retry:
            _prog("Correction pass…", 80)
            corrected, recs2 = plan_and_apply_rewrites(final_latex, retry,
                                                       effective_rewrite_fn)
            if corrected != final_latex:
                final_latex = corrected
                rewrite_records += [r.to_dict() for r in recs2]
                _compile_preserved(report, final_latex, out_dir, job_title,
                                   compile_pdf, compile_latex_to_pdf, _safe_jobname,
                                   _prog, resume_text=latex_to_text(final_latex))
                score_text = _scoring_text(report, final_latex, latex_to_text)
                ev_after = map_evidence(valid, latex_to_text(final_latex))
            correction_applied = True

    report["rewrites"] = rewrite_records
    applied = [r for r in rewrite_records if r.get("applied")]
    report["tex"] = final_latex
    report["resume_preserved"] = (final_latex == latex_src)
    report["correction_pass_applied"] = correction_applied
    report["evidence"] = ev_after.to_dict()

    # 7. FINAL score — computed from the parsed résumé text, with stuffing penalty
    #    and the 90% gate.
    stuffing = _detect_stuffing(score_text, valid)
    final_score = score_alignment(valid, ev_after.to_dict(), score_text,
                                  pdf_validation=report.get("pdf_validation") or None,
                                  applied_rewrites=len(applied),
                                  stuffing_penalty=stuffing)
    ceiling = max_supported_score(valid, ev_after.to_dict(), score_text)
    report["status"] = STATUS_OK
    report["reason"] = "ok"
    report["internal_alignment_estimate"] = {
        "label": final_score["label"],
        "before": score_before["score"],
        "after": final_score["score"],
        "max_evidence_supported": ceiling,
        "gate_90_passed": final_score["gate_90_passed"],
        "components": final_score["components"],
        "coverage": final_score["coverage"],
        "penalties": final_score["penalties"],
        "supported_integrations": len(applied),
        "unsupported_insertions": 0,
        "coverage_rate": ev_after.metrics().get("coverage_rate"),
        "mandatory_recall": ev_after.metrics().get("mandatory_recall"),
        "scored_from": "parsed_pdf" if report.get("_pdf_text_used") else "latex_text",
    }

    # 8. INDEPENDENT audit + two-parser PDF verification (acceptance gate). The
    #    audit re-extracts from the JD independently and scores from the PARSED
    #    PDF; the reported acceptance score cannot exceed what the audit confirms.
    if run_audit and llm_client is not None and report.get("_pdf_text_used"):
        try:
            from .ats_evaluate import independent_audit, acceptance_verdict
            from .pdf_validate import _extract_pdf_text, verify_keywords_two_parsers
            pdf_text = _extract_pdf_text(report["pdf_path"]) or score_text
            audit = independent_audit(clean_jd, resume_text, pdf_text, llm_client)
            accepted_kw = [r.get("exact_jd_phrase") for r in rewrite_records
                           if r.get("applied")]
            two_parser = verify_keywords_two_parsers(report["pdf_path"], accepted_kw) \
                if accepted_kw else {}
            verdict = acceptance_verdict(audit, 0, stuffing, two_parser)
            report["independent_audit"] = audit
            report["pdf_two_parser"] = two_parser
            report["acceptance"] = verdict
        except Exception as e:
            report["audit_error"] = str(e)[:160]
    return report


def _scoring_text(report: Dict, final_latex: str, latex_to_text) -> str:
    """Prefer PDF-extracted text for scoring (that is what an ATS reads); fall back
    to LaTeX-derived text when no engine compiled a PDF."""
    report["_pdf_text_used"] = False
    path = report.get("pdf_path")
    if path:
        try:
            from .pdf_validate import _extract_pdf_text
            txt = _extract_pdf_text(path)
            if txt and len(txt) > 200:
                report["_pdf_text_used"] = True
                return txt
        except Exception:
            pass
    return latex_to_text(final_latex)


def _detect_stuffing(text: str, criteria: List[dict]) -> float:
    """Deterministic keyword-stuffing penalty: (a) a criterion phrase repeated
    unnaturally often (>3x) or (b) a dense comma-dump line of bare keywords. Note:
    this penalty is applied to BOTH the before and after résumé, so a résumé's own
    Skills section never creates an unfair before/after delta — only a résumé that
    is MORE stuffed than another scores lower (see test_keyword_stuffed…)."""
    low = (text or "").lower()
    penalty = 0.0
    for c in criteria:
        p = (c.get("exact_phrase") or "").lower().strip()
        if len(p) >= 5 and low.count(p) > 3:
            penalty += 5
    for line in low.splitlines():
        if line.count(",") >= 6 and len(line.split()) < line.count(",") * 4:
            penalty += 5
    return min(penalty, 25.0)


def to_legacy_report(safe: Dict) -> Dict:
    """Map a safe-orchestrator report onto the legacy payload keys the API/endpoints
    and the extension popup already consume.

    `injected` now lists the truthful, evidence-backed integrations that were
    applied (exact phrases aligned into existing bullets). `unsupported_insertions`
    stays 0 — gaps are disclosed, never inserted.
    """
    ev = safe.get("evidence") or {}
    covered = ev.get("covered", [])
    gaps = ev.get("gaps", [])
    partial = ev.get("partial", [])
    metrics = ev.get("metrics", {}) or {}
    total = (metrics.get("total_criteria")
             or (len(covered) + len(partial) + len(gaps)) or 0)
    found = metrics.get("covered", len(covered))
    est = safe.get("internal_alignment_estimate") or {}
    # Prefer the AFTER alignment score for the headline pct; fall back to coverage.
    pct = int(round(est.get("after")
                    if est.get("after") is not None
                    else (metrics.get("coverage_rate") or 0) * 100))

    applied = [r for r in (safe.get("rewrites") or []) if r.get("applied")]
    injected = [r.get("exact_jd_phrase") or r.get("normalized_concept")
                for r in applied]

    keywords = []
    for c in covered:
        applied_here = any(
            (r.get("normalized_concept") == c.get("keyword")) for r in applied)
        keywords.append({
            "keyword": c.get("exact_phrase") or c.get("keyword"),
            "found_in_export": True,
            "section": ("Integrated (evidence-backed rewrite)" if applied_here
                        else "Résumé (evidence-backed)"),
            "reason": "",
            "evidence": c.get("resume_evidence", ""),
            "status": c.get("status", ""),
            "requirement_type": c.get("requirement_type", ""),
        })
    for p in partial:
        keywords.append({
            "keyword": p.get("exact_phrase") or p.get("keyword"),
            "found_in_export": False,
            "section": "(partially supported — not inserted)",
            "reason": "concept present but not as a clear capability; not inserted",
            "requirement_type": p.get("requirement_type", ""),
        })
    for g in gaps:
        keywords.append({
            "keyword": g.get("exact_phrase") or g.get("keyword"),
            "found_in_export": False,
            "section": "(gap — not in résumé)",
            "reason": "no résumé evidence — not inserted (honest gap, no fabrication)",
            "requirement_type": g.get("requirement_type", ""),
        })

    manual = safe.get("status") == STATUS_MANUAL
    return {
        "source": "latex",
        "status": safe.get("status"),
        "manual_review_required": manual,
        "pct": pct,
        "expected": total,
        "found": found,
        "missing": [g.get("exact_phrase") or g.get("keyword") for g in gaps],
        "keywords": keywords,
        "coverage_count": f"{found}/{total}",
        "injected": injected,           # truthful evidence-backed integrations
        "rewrites": safe.get("rewrites", []),
        "calibration": safe.get("calibration", []),
        "gated": {},
        "tex": safe.get("tex"),
        "engine": safe.get("engine"),
        "compiled": safe.get("compiled"),
        "pdf_path": safe.get("pdf_path"),
        "compile_log": safe.get("compile_log", ""),
        # New, honest fields (extension may ignore or surface these):
        "evidence": ev,
        "jd_diagnostics": safe.get("jd_diagnostics", {}),
        "extraction_diagnostics": {
            "valid_count": len(safe.get("extraction", {}).get("valid", [])),
            "rejected_count": safe.get("extraction", {}).get("rejected_count", 0),
            "used_fallback": safe.get("extraction", {}).get("used_fallback", False),
            "rejected_samples": safe.get("extraction", {}).get("rejected_samples", []),
        },
        "pdf_validation": safe.get("pdf_validation", {}),
        "internal_alignment_estimate": safe.get("internal_alignment_estimate"),
        "reason": safe.get("reason", ""),
    }


def _compile_preserved(report, latex_src, out_dir, job_title, compile_pdf,
                       compile_latex_to_pdf, _safe_jobname, _prog,
                       resume_text: str = "") -> None:
    """Compile the unmodified résumé and validate the resulting PDF."""
    if not compile_pdf:
        return
    _prog("Compiling PDF…", 75)
    out_dir = out_dir or tempfile.mkdtemp(prefix="ats_safe_")
    slug = _safe_jobname(job_title)
    jobname = f"Saiteja_Tirunagari_{slug}_Resume" if slug else "Saiteja_Tirunagari_Resume"
    try:
        comp = compile_latex_to_pdf(latex_src, out_dir, jobname=jobname, timeout=420)
        report["engine"] = comp.get("engine")
        report["compiled"] = comp.get("compiled")
        report["pdf_path"] = comp.get("pdf_path")
        report["compile_log"] = comp.get("log", "")
    except Exception as e:
        report["compile_log"] = f"compile_error: {e}"

    # ATS-safe fallback: when no LaTeX engine compiled a PDF, render the résumé as
    # a single-column plain-text PDF (reportlab). This is what an ATS reads anyway,
    # and it guarantees a real, parseable PDF on any host (incl. no-tectonic).
    if not report.get("pdf_path"):
        try:
            from .latex_resume import latex_to_text, render_text_to_pdf
            fb = os.path.join(out_dir, f"{jobname}.pdf")
            if render_text_to_pdf(latex_to_text(latex_src), fb) and os.path.exists(fb):
                report["pdf_path"] = fb
                report["engine"] = report.get("engine") or "reportlab-atsafe"
                report["compiled"] = True
                report["pdf_fallback"] = True
        except Exception as e:
            report["compile_log"] = (report.get("compile_log", "") + f" | fallback: {e}")

    if report.get("pdf_path"):
        from .pdf_validate import validate_pdf
        _prog("Validating PDF…", 90)
        try:
            from .ats_safe import _resume_section_order
            sections = (_resume_section_order(latex_src, resume_text)
                        if resume_text else None)
            report["pdf_validation"] = validate_pdf(
                report["pdf_path"], expected_sections=sections)
        except Exception as e:
            report["pdf_validation"] = {"ok": False, "warnings": [f"validate_error:{e}"]}