JAA-ATS-Tool / src /ats_validator.py
saitejatirunagari's picture
feat(ats): independent validator + anti-cheat — break circular self-scoring
6d3a6a8
Raw
History Blame
5.54 kB
"""
INDEPENDENT validation scorer (anti-circular).
The generation pipeline uses candidate_fit ("plausible" expansion) to decide what
to include, and ats_scoring_v2 to score. If the same "plausible" logic both
inserts a term AND credits it, the 90%+ is self-confirming. This module scores
the FINAL EXPORTED, RE-PARSED resume with a deliberately DIFFERENT, stricter
ruleset that knows nothing about candidate_fit:
- exact / alias presence ONLY in the parsed text — never "plausible" credit
- EVIDENCE-WEIGHTED: a required skill that appears only in the Skills list but
is NOT evidenced in an Experience bullet earns partial credit (0.5)
- NO seniority fuzzing: if the JD asks for N years, the resume must actually
show >= N years, else the seniority bucket fails
- NO education fuzzing beyond a real degree token being present
- title/domain require the actual term in the parsed text
It answers one question: "If a stranger's ATS read ONLY this exported file, would
it still score 90%+?" Used to label WEAK_90_INTERNAL_ONLY and to gate downloads.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from typing import List, Dict
from .ats_scorer import _kw_in_text, _detect_sections
from .jd_analyzer import analyze_jd, JDRequirements, Requirement
from .ats_scoring_v2 import score_ats_readability, _WEIGHTS
@dataclass
class ValidationResult:
independent_jd_match: int
ats_readability: int
breakdown: Dict[str, dict] = field(default_factory=dict)
evidenced_terms: List[str] = field(default_factory=list)
skills_only_terms: List[str] = field(default_factory=list)
missing_terms: List[str] = field(default_factory=list)
seniority_ok: bool = True
notes: List[str] = field(default_factory=list)
def to_dict(self):
return self.__dict__
def _max_years(text: str) -> int:
yrs = [int(m) for m in re.findall(r"(\d{1,2})\+?\s*years?", text.lower())]
return max(yrs) if yrs else 0
def _coverage(reqs: List[Requirement], full_low: str, exp_low: str) -> tuple:
"""Evidence-weighted coverage. Returns (credit, total, evidenced, skills_only,
missing). A term in experience = 1.0; only elsewhere (skills) = 0.5; absent = 0."""
if not reqs:
return (0.0, 0, [], [], [])
credit = 0.0
evidenced, skills_only, missing = [], [], []
for r in reqs:
terms = [r.term] + list(r.aliases or [])
in_exp = any(_kw_in_text(t, exp_low) for t in terms)
in_full = any(_kw_in_text(t, full_low) for t in terms)
if in_exp:
credit += 1.0
evidenced.append(r.term)
elif in_full:
credit += 0.5
skills_only.append(r.term)
else:
missing.append(r.term)
return (credit, len(reqs), evidenced, skills_only, missing)
def validate_resume(jd_text: str, parsed_text: str,
experience_text: str = None,
base_resume_text: str = None) -> ValidationResult:
req = analyze_jd(jd_text) # structure only; deterministic
full_low = (parsed_text or "").lower()
exp_low = (experience_text or parsed_text or "").lower()
buckets = {
"must_have_hard": req.required_hard_skills,
"responsibilities": req.responsibilities,
"tools": req.tools_platforms,
"title_domain": req.domain_terms,
"seniority": req.seniority_signals,
"certs_education": req.certifications + req.education_requirements,
"soft": req.soft_skills,
}
breakdown: Dict[str, dict] = {}
weighted = 0.0
all_evidenced, all_skills_only, all_missing = [], [], []
notes: List[str] = []
# Strict seniority: JD's required years vs the candidate's REAL years. We
# measure from the BASE resume when provided — never the generated text,
# which could have a JD year-phrase ("12+ years") injected into it (that
# would let the system game its own seniority check).
jd_years = _max_years(jd_text)
resume_years = _max_years(base_resume_text if base_resume_text else parsed_text)
seniority_ok = (jd_years == 0) or (resume_years >= jd_years)
if not seniority_ok:
notes.append(f"JD wants {jd_years}y; candidate shows {resume_years}y")
for key, reqs in buckets.items():
credit, tot, ev, so, miss = _coverage(reqs, full_low, exp_low)
all_evidenced += ev; all_skills_only += so; all_missing += miss
if key == "title_domain":
tcov = sum(1 for t in req.target_role_titles if _kw_in_text(t, full_low))
credit += tcov
tot += len(req.target_role_titles)
if key == "seniority":
# STRICT: no fuzzy credit. Full only if the actual year bar is met.
frac = 1.0 if (tot == 0 or seniority_ok) else 0.0
else:
frac = (credit / tot) if tot else 1.0
w = _WEIGHTS[key]
contrib = frac * w
weighted += contrib
breakdown[key] = {"credit": round(credit, 1), "total": tot,
"weight": w, "contribution": round(contrib * 100, 1)}
score = round(max(0, min(100, weighted * 100)))
readability = score_ats_readability(parsed_text).score
return ValidationResult(
independent_jd_match=score,
ats_readability=readability,
breakdown=breakdown,
evidenced_terms=all_evidenced,
skills_only_terms=all_skills_only,
missing_terms=all_missing,
seniority_ok=seniority_ok,
notes=notes,
)