scripts/verify_anticheat.py

"""
ANTI-CHEAT regression — prove the system is not gaming its own scorer.

Runs the REAL uploaded resume through the full pipeline (stub LLM = honest floor)
and checks the INDEPENDENT validator (not candidate_fit) agrees:

  1. A PM JD reaches READY_90_PLUS for the PM resume (both scores >= 90).
  2. A cybersecurity PM JD is REVIEW_REQUIRED with security tools flagged.
  3. A backend-engineer JD does NOT become CLEAN_90_PLUS for a PM (engineering
     hard skills not auto-claimed; download blocked or quality != CLEAN).
  4. A JD needing a specific cert does not get that cert invented.
  5. A 12-year/Director JD is NOT matched as READY for a mid-level candidate
     (independent seniority check fails).
  6. Final score is from parsed EXPORTED text (independent re-parses the file).
  7. If the renderer drops a keyword, the independent score reflects it.
"""
import os, sys, io, shutil
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
sys.path.insert(0, os.path.abspath("."))

from src.resume_customizer import ResumeCustomizer, _read_docx_text
from src.llm_client import LLMClient
from src.ats_validator import validate_resume
from src.resume_parser_v2 import parse_resume_pdf
# Isolate the vault so prior user confirmations don't change anti-cheat results.
import src.candidate_vault as _cv
_cv._VAULT_PATH = "data/_test_vault_anticheat.json"
if os.path.exists(_cv._VAULT_PATH):
    os.remove(_cv._VAULT_PATH)

src_pdf = r"C:\Users\Nxtwave\Desktop\resume\Saiteja_Tirunagari_Resume A 26 - Copy.pdf"
dst = "data/resume/resume.pdf"
os.makedirs("data/resume", exist_ok=True)
if not os.path.exists(dst) and os.path.exists(src_pdf):
    shutil.copyfile(src_pdf, dst)
if os.path.exists("data/resume/_parsed.json"):
    os.remove("data/resume/_parsed.json")


def stub(self, cfg, resume_dict, jd_text, job_title, company, assessment, **kwargs):
    o = dict(resume_dict)
    o["summary"] = (f"Strong-fit candidate for {job_title} at {company}: "
                    + (resume_dict.get("summary") or "PM 5+ years."))
    return o


LLMClient.tailor_resume_v4 = stub
rc = ResumeCustomizer.__new__(ResumeCustomizer)
rc.llm = LLMClient.__new__(LLMClient); rc.resume_text = ""
rc.output_dir = "data/output/resumes/_anticheat"; os.makedirs(rc.output_dir, exist_ok=True)
rc.fast_model_cfg = {"model": "fake", "api_key": "fake", "base_url": "https://fake"}
rc._pending_summary_inject = []

ok = True
def check(name, cond, detail=""):
    global ok; ok = ok and cond
    print(f"  [{'PASS' if cond else 'FAIL'}] {name}  {detail}")


def gen(jf, co):
    jd = open(f"tests/fixtures/jds/{jf}.txt", encoding="utf-8").read()
    job = {"title": "Product Manager", "company": co, "description": jd, "_raw_assessment": {}}
    rc._generate_resume_v4(job, cfg=rc.fast_model_cfg,
                           filepath=os.path.join(rc.output_dir, f"{co}.docx"))
    return jd, job, job.get("_v2_report", {})


print("1. PM JD -> READY_90_PLUS, both scores >= 90:")
jd, job, r = gen("generic_pm_3_7yrs", "PM")
internal = r.get("estimated_scores", {}).get("jd_match", 0)
indep = r.get("independent_jd_match", 0)
check("internal >= 90", internal >= 90, f"internal={internal}")
check("independent >= 90", indep >= 90, f"independent={indep}")
check("download allowed", job.get("download_allowed") is True)

print("2. Cybersecurity PM -> security tooling is HIGH-risk (confirm), not auto-faked:")
jd, job, r = gen("sumo_logic_pm", "Sec")
_sec = ("siem", "soar", "xdr", "secops", "threat intelligence",
        "threat detection", "security operations")
_high = [t.lower() for t in r.get("high_risk_terms_for_confirmation", [])]
check("security terms classified HIGH-risk (need confirmation)",
      any(t in _high for t in _sec), str(_high)[:90])
parsed_sec = _read_docx_text(os.path.join(rc.output_dir, "Sec.docx")).lower()
auto_claimed = [t for t in _sec if t in parsed_sec]
check("security tooling NOT auto-claimed in resume (HIGH not auto-included)",
      not auto_claimed, f"auto-claimed={auto_claimed}")

print("3. Backend-engineer JD must NOT be CLEAN_90_PLUS for a PM:")
jd, job, r = gen("backend_engineer", "Backend")
q = r.get("quality_flag", "")
indep = r.get("independent_jd_match", 0)
check("quality is not CLEAN_90_PLUS", q != "CLEAN_90_PLUS", f"quality={q} independent={indep}")
parsed = _read_docx_text(os.path.join(rc.output_dir, "Backend.docx")).lower()
check("did not auto-claim 'java'/'spring boot' as evidenced",
      "spring boot" not in parsed or "java" not in parsed,
      "engineering hard skills not stuffed")

print("4. Specific cert not invented:")
# Backend JD has no cert; use a cert check on parsed text — no fake 'AWS Certified'
check("no invented 'aws certified'/'cissp'",
      "aws certified" not in parsed and "cissp" not in parsed)

print("5. 12-year/Director JD not READY for mid-level candidate:")
jd, job, r = gen("senior_pm_10yrs", "SrPM")
indep = r.get("independent_jd_match", 0)
_base = parse_resume_pdf(dst).to_flat_text()
val = validate_resume(jd, _read_docx_text(os.path.join(rc.output_dir, "SrPM.docx")),
                      base_resume_text=_base)
check("independent seniority check fails (12y vs candidate)", not val.seniority_ok,
      f"seniority_ok={val.seniority_ok} notes={val.notes}")
check("not downloadable as READY", job.get("download_allowed") is not True,
      f"download={job.get('download_allowed')} status={job.get('_v2_status')}")

print("6. Score is from parsed EXPORTED text:")
# validate_resume re-parses the file independently; if it scores, it parsed.
check("independent validator scored parsed export", isinstance(indep, int))

print("7. Dropping a keyword lowers the independent score:")
full = _read_docx_text(os.path.join(rc.output_dir, "PM.docx"))
jd_pm = open("tests/fixtures/jds/generic_pm_3_7yrs.txt", encoding="utf-8").read()
base_score = validate_resume(jd_pm, full).independent_jd_match
# Simulate a renderer that dropped the Skills + most of Experience (keep only
# the header + summary). The independent score MUST fall.
lines = [l for l in full.splitlines() if l.strip()]
truncated = "\n".join(lines[: max(4, int(len(lines) * 0.25))])
drop_score = validate_resume(jd_pm, truncated).independent_jd_match
check("truncated resume scores lower", drop_score < base_score,
      f"{base_score} -> {drop_score}")

print("\n" + ("✓ ALL ANTI-CHEAT CHECKS PASS" if ok else "✗ SOME ANTI-CHEAT CHECKS FAILED"))
sys.exit(0 if ok else 1)