""" External-ATS coverage simulator (Jobalytics/Simplify-style breadth). WHY THIS EXISTS ─────────────── Our internal scorer extracts keywords from a NARROW PM taxonomy and scores the resume against that same narrow set → it easily reports 90%+. External checkers (Jobalytics/Simplify) extract a MUCH broader set (40-46 terms incl. domain words, JD-specific responsibilities, soft skills) → the same resume scores ~54%. The internal number is therefore NOT a valid success signal for the external goal. This module computes a broad, Jobalytics-style EXPECTED keyword set from the JD, and measures coverage against the *re-parsed exported resume text* (the only truth). Used to (a) decide what to physically place in Maximum ATS Mode and (b) gate/report readiness on real external-style coverage, not internal score. It is intentionally broader than `extract_jd_keywords` (which stays skills-only to keep the honest default pipeline clean). The broad set is only used in Maximum ATS Mode / external-feedback repair, where the user has explicitly opted into aggressive coverage. """ from __future__ import annotations import re from typing import Dict, List from .ats_scorer import ( _kw_in_text, extract_jd_keywords, PM_SKILL_TAXONOMY, GENERIC_PROFESSIONAL_VOCAB, ) # Words that are never standalone keywords (checkers penalise/ignore them). _STOP = { "the", "and", "for", "with", "you", "your", "our", "are", "will", "have", "has", "had", "this", "that", "these", "those", "from", "into", "out", "off", "who", "what", "when", "where", "why", "how", "all", "any", "can", "may", "should", "would", "could", "must", "able", "etc", "per", "via", "not", "but", "they", "them", "their", "his", "her", "its", "she", "him", "was", "were", "been", "being", "more", "most", "such", "than", "then", "also", "about", "across", "within", "while", "each", "other", "some", "many", "well", "very", "much", "like", "just", "only", "even", "both", "over", "under", "between", "during", "including", "include", "includes", "looking", "join", "join us", "work", "working", "team", "teams", "role", "job", "company", "candidate", "candidates", "ideal", "great", "good", "strong", "years", "year", "experience", "experiences", "plus", "preferred", "required", "requirement", "responsibilities", "qualifications", "skills", "ability", "knowledge", "understanding", "familiarity", "proficiency", "we", "us", "is", "in", "on", "of", "to", "at", "as", "an", "or", "be", "it", "by", "a", "i", } # Vague filler — never a keyword (mirrors jd_analyzer._BUZZWORDS intent). _FILLER = { "innovation", "innovative", "solution", "solutions", "tools", "tool", "ownership", "synergy", "dynamic", "passionate", "motivated", "self-starter", "results-driven", "detail-oriented", "team player", "track record", "expertise", "best practices", "thought leadership", "fast-paced", "cutting-edge", "world-class", "robust", "seamless", "holistic", "leverage", "excellence", "proven", "successful", "goals", "productivity", "reinvent", "reinvention", "mission", "vision", "culture", "value", "values", "impact", "environment", "opportunity", "opportunities", "responsibility", } # Prose tokens that survive atomic_keywords() decomposition but are NOT genuine # ATS-relevant skills. Dropped from the V2 scoring DENOMINATOR by # skill_relevant_filter(). DESIGN: a BLOCKLIST (not an allowlist) so unfamiliar # tool names and niche skills still survive (avoids the Phase-5 self-grading trap). # Do NOT add "logistics" — it is a legitimate business domain a PM can claim. _PROSE_NOISE_EXTRA: frozenset = frozenset({ # Industry-context nouns, not PM skills "driver-partners", "intra-city", "inter-city", "connecting", "largest", "reliable", "redefine", # JD-prose action-verb atoms (survive splitting but aren't skill names) "synthesise", "synthesize", "refine", "validate", "iterate", "establish", "proactively", "concurrent", "concurrently", "independently", # Marketing scale/superlative descriptors "millions", "billion", "thousands", "smarter", "fastest", "meaningful", # Role/seniority modifiers (not skills) "junior", "independence", "accountable", "operates", # Generic best-practice fragments "coach", }) def skill_relevant_filter(terms: List[str]) -> List[str]: """Drop prose-noise atoms from the atomized expected set (V2 scoring denominator only). BLOCKLIST: drops tokens in _FILLER or _PROSE_NOISE_EXTRA; keeps every other token (including unfamiliar tools). Applied in `_build_report` AFTER `atomic_keywords`, BEFORE `external_coverage`. V2-only; never called from V1.""" combined_noise = _FILLER | _PROSE_NOISE_EXTRA return [t for t in (terms or []) if t.lower() not in combined_noise] # Acronym ↔ expansion doubling: recruiters' Boolean search treats "AWS" and # "Amazon Web Services" as different strings, so both forms should count. _ACRONYM_MAP_RAW: dict = { "aws": "amazon web services", "azure": "microsoft azure", "gcp": "google cloud platform", "api": "application programming interface", "sql": "structured query language", "crm": "customer relationship management", "erp": "enterprise resource planning", "saas": "software as a service", "paas": "platform as a service", "kpi": "key performance indicator", "okr": "objective and key result", "mvp": "minimum viable product", "prd": "product requirements document", "gtm": "go-to-market", "ux": "user experience", "ui": "user interface", "ml": "machine learning", "ai": "artificial intelligence", "nlp": "natural language processing", "llm": "large language model", "b2b": "business to business", "b2c": "business to consumer", "seo": "search engine optimization", "roi": "return on investment", "nps": "net promoter score", "ltv": "lifetime value", "arpu": "average revenue per user", } _ACRONYM_MAP: dict = {} for _short, _full in _ACRONYM_MAP_RAW.items(): _ACRONYM_MAP[_short] = _full _ACRONYM_MAP[_full] = _short def _expand_acronyms(terms: List[str]) -> List[str]: """For each term matching an acronym or its expansion, also append the partner form (so both short and full forms count in coverage / recruiter search). Order-preserving, de-duplicated. V2-only.""" out = list(terms or []) seen = {t.lower() for t in out} for t in list(terms or []): partner = _ACRONYM_MAP.get(t.lower().strip()) if partner and partner not in seen: seen.add(partner) out.append(partner) return out # UI/CTA/marketing phrases injected by checker overlays (Simplify/Jobalytics/…) # — never genuine JD keywords. Exact lowercased matches. _UI_NOISE = frozenset({ "show match details", "people clicked apply", "month free trial", "free trial", "actively reviewing applicants", "message hiring managers", "get personalized cover letter", "get insider access", "members use premium", "recruiting bond", "visit website", "lacs pa", "help me stand", "tailor my resume", "uses advanced ai", "try premium", "easy apply", "see how you compare", "am i a good fit", "save job", "apply now", "show more", "show less", "sign in", "join now", }) # CTA/marketing lead tokens: a multi-word gram starting with one is UI noise. _UI_LEAD = { "show", "get", "try", "visit", "apply", "save", "click", "join", "sign", "message", "see", "tailor", "unlock", "upgrade", "start", } # Marketing substrings that mark a gram as UI noise wherever they appear. _UI_SUBSTR = ( "free trial", "premium", "hiring manager", "cover letter", "clicked apply", "match details", "lacs pa", ) # ── V2-only scraped-noise filter (R27) ─────────────────────────────────────── # These three sets + helpers reject scraped, non-skill tokens (company # geography, executive/recruiter names, role/boilerplate nouns) that the broad # extractor cannot tell apart from real skills. They are applied ONLY in the V2 # allocation path via `filter_scraped_noise()`; V1 structured placement does NOT # call them, so V1 behaviour is unchanged. # Geography / location tokens that are never skills. Word-boundary matching is # used for single-word entries so 'hq' never blocks 'graphql'; multi-word phrases # ('latin america') match as substrings. _GEO_EXTENDED: frozenset = frozenset({ "amsterdam", "netherlands", "europe", "africa", "asia", "asia-pacific", "latin america", "latin", "north america", "south america", "middle east", "apac", "emea", "headquartered", "headquarters", "hq", "worldwide", "global", "offices", "office", # Common India / global locations the job-card header drags in as "keywords". "india", "bengaluru", "bangalore", "karnataka", "hyderabad", "telangana", "mumbai", "maharashtra", "delhi", "new delhi", "ncr", "gurgaon", "gurugram", "noida", "pune", "chennai", "kolkata", "ahmedabad", "boca raton", "florida", "usa", "united states", "uk", "united kingdom", "singapore", "dubai", }) # Corporate-entity / boilerplate suffix words that travel with "About us" copy. _ENTITY_NOISE: frozenset = frozenset({ "corporation", "incorporated", "subsidiary", "subsidiaries", "conglomerate", }) # Job-board posting metadata: employment type, seniority chips, posting age, # work-mode, applicant counts, section labels — never resume keywords. _META_NOISE: frozenset = frozenset({ "full-time", "full time", "fulltime", "part-time", "part time", "parttime", "contract", "contractor", "internship", "intern", "temporary", "permanent", "associate", "senior", "junior", "mid-level", "entry-level", "entry level", "applicants", "applicant", "applied", "alerts", "alert", "get alerts", "posted", "remote", "hybrid", "onsite", "on-site", "on site", "seniority level", "employment type", "job function", "industries", "week ago", "weeks ago", "day ago", "days ago", "month ago", "months ago", "hour ago", "hours ago", "minute ago", "minutes ago", }) # Single tokens that mark a gram as posting metadata wherever they appear. _META_WORDS: frozenset = frozenset({ "ago", "full-time", "fulltime", "part-time", "parttime", "applicants", "applicant", "alerts", "posted", "internship", "hybrid", "onsite", }) # Job-board / LinkedIn page chrome that the DOM scrape drags in (the broad V1 # extractor's _UI_NOISE list doesn't cover these exact fragments). _UI_BLURB: tuple = ( "actively engaged", "actively reviewing", "get alerts", "reach applicants", "top applicant", "easy apply", "be an early applicant", "promoted by", "set alert", "job alert", "people you can reach", ) # Generic role/people nouns that indicate a person or HR context, not a skill. _ROLE_NOISE: frozenset = frozenset({ "recruiter", "recruiters", "applicant", "applicants", "founder", "co-founder", "cofounder", "ceo", "cto", "coo", "cfo", "investor", "investors", "partner", "partners", "member", "members", "shareholder", "shareholders", "colleague", "colleagues", "peer", "peers", "backbasers", "hire", "hiring", "hired", }) # Common sentence/clause-starting words that appear Title Case but are NOT part # of a person name — excluded from the two-word name pattern. _SENTENCE_STARTERS: frozenset = frozenset({ "The", "A", "An", "We", "Our", "You", "This", "That", "These", "Those", "For", "At", "In", "On", "By", "With", "To", "From", "And", "Or", "But", "If", "As", "Is", "Are", "Was", "About", "During", "After", "Before", "When", "Where", "Please", "Must", "Will", "Can", "Has", "Have", "Had", "Be", "Do", "Does", "Did", "Should", "Would", "Could", "May", "Might", "Not", "All", "Some", "Any", "Each", "Also", "Both", "More", "Most", "How", "What", "Who", "Why", "Which", }) # Cue words that appear immediately BEFORE a person name in a JD ("Recruiter: # Mohammed Nayeem", "Founder Jouk Pleiter"). _NAME_CUE_BEFORE = ( r"recruiter|contact|hiring manager|posted by|submitted by|reach out to|" r"founder|co-?founder|ceo|cto|coo|cfo|president|vice president|vp|" r"director|head of [a-z ]+|lead" ) def _contains_geo_token(term: str) -> bool: """True if `term` is or contains a geography/location token. Single-word geo entries use word-boundary matching so the 'hq' substring never blocks 'graphql'; multi-word geo phrases (e.g. 'latin america') match as substrings.""" tl = (term or "").lower() for g in _GEO_EXTENDED: if g == tl: return True if " " in g and g in tl: return True if " " not in g and re.search( r"(? bool: """True if `term_lower` contains a two-word Title-Case sequence that, in the original JD, sits next to a person cue — a label before it ('Recruiter:') or a predicate/appositive after it ('is the CEO', '— Account Manager', ', founder'). Context-gated on purpose: a bare Title-Case bigram is NOT enough (job-title and domain headings like 'Forward Deployed' or 'Banking Integrations' are Title Case too). Requiring a person cue avoids those false positives while still catching real names embedded anywhere in a multi-word gram. """ words = term_lower.split() if len(words) < 2: return False orig = original_jd or "" for w1, w2 in zip(words, words[1:]): cap1, cap2 = w1.capitalize(), w2.capitalize() if cap1 in _SENTENCE_STARTERS or cap2 in _SENTENCE_STARTERS: continue name = re.escape(cap1) + r"\s+" + re.escape(cap2) before = r"(?:" + _NAME_CUE_BEFORE + r")\s*[:\-–—]?\s+" + name after = name + r"\s*(?:\bis\b|\bwas\b|,|[\-–—]|\bthe ceo\b|\bfounder\b)" if re.search(before, orig, flags=re.I) or re.search(after, orig): return True return False def _is_skill_like(term: str, original_jd: str = "", company: str = "") -> bool: """Return False (reject) if `term` is geography, a corporate-entity/role noun, the company's own name, or a person name. This is the sole V2 rejection gate; it does NOT touch the V1 extractor's stopword/UI/filler checks.""" tl = (term or "").strip().lower() if not tl: return True if _is_ui_noise(tl): return False if any(b in tl for b in _UI_BLURB): return False if tl in _META_NOISE: return False if _contains_geo_token(tl): return False parts = tl.split() if any(w in _META_WORDS for w in parts): return False if tl in _ROLE_NOISE or any(w in _ROLE_NOISE for w in parts): return False if any(w in _ENTITY_NOISE for w in parts): return False if company: for ctok in re.findall(r"[a-z0-9]+", company.lower()): if len(ctok) >= 4 and re.search( r"\b" + re.escape(ctok) + r"(?:s|rs|ers)?\b", tl ): return False if original_jd and _contains_person_name(tl, original_jd): return False return True def filter_scraped_noise( terms: List[str], original_jd: str = "", company: str = "" ) -> List[str]: """V2-only: drop scraped non-skill tokens (geography, executive/recruiter names, corporate-entity/boilerplate nouns, the company's own name) from an already-extracted term list, preserving order. V1 does NOT call this — V1's pool is unchanged by design.""" return [t for t in (terms or []) if _is_skill_like(t, original_jd, company)] # Generic JD action-verbs / weak edge words trimmed from the START/END of a # maximal run so an extracted phrase reads like a SKILL, not a sentence fragment # ("lead end-to-end product lifecycle" → "end-to-end product lifecycle"). _EDGE_TRIM = { "lead", "leads", "leading", "led", "drive", "drives", "driving", "drove", "build", "builds", "building", "built", "manage", "manages", "managing", "managed", "monitor", "monitors", "monitoring", "create", "creates", "creating", "created", "develop", "develops", "developing", "developed", "ensure", "ensures", "ensuring", "own", "owns", "owning", "owned", "define", "defines", "defining", "defined", "track", "tracks", "tracking", "tracked", "deliver", "delivers", "delivering", "delivered", "leverage", "leverages", "utilize", "utilizes", "utilizing", "use", "uses", "using", "used", "conduct", "conducts", "support", "supports", "supporting", "collaborate", "grow", "grows", "growing", "measure", "measures", "measuring", "interpret", "watch", "help", "helps", "helping", "based", "through", "across", "drive", "make", "makes", "making", "made", "run", "runs", "running", "set", "sets", "shape", "shapes", "shaping", "foster", "fosters", "integrate", "integrates", "maximize", "optimize", "optimizes", "validate", "validates", "meets", "meet", "need", "needs", "needed", "require", "requires", "required", "seeking", "seek", "want", "wants", "provide", "provides", "providing", "perform", "performs", "performing", "identify", "identifies", "execute", "executes", } def _emit_run(run: List[str], add) -> None: """Trim generic action-verbs from a maximal non-stop run, then emit it (long runs split into non-overlapping <=4-word chunks).""" r = list(run) while r and r[0] in _EDGE_TRIM: r = r[1:] while r and r[-1] in _EDGE_TRIM: r = r[:-1] if not r: return i = 0 while i < len(r): chunk = r[i:i + 4] gram = " ".join(chunk) if len(gram) <= 40: add(gram) i += 4 def _clean(term: str) -> str: t = re.sub(r"[^\w\s/+.\-]", "", term or "").strip().lower() # Drop leading/trailing sentence punctuation so grams that end a sentence # ("initiatives.", "optimization.") aren't kept as distinct dotted tokens. return t.strip(".-/ ").strip() def _is_ui_noise(t: str) -> bool: """True if `t` is a checker-overlay CTA / marketing phrase, not a JD term.""" t = (t or "").strip() if not t: return False if t in _UI_NOISE: return True words = t.split() if len(words) > 1 and words[0] in _UI_LEAD: return True if any(sub in t for sub in _UI_SUBSTR): return True return False def _is_term_like(t: str) -> bool: """A token/phrase that reads like a real skill/responsibility/domain term. Phase 9 (R21): intentionally WIDENED — "if in doubt, pick it up." We keep only genuine-noise guards (stopwords, filler, UI/CTA overlay strings, pure digits, too-short, >4 words). There is NO count cap; the honesty gate lives in decide_includable_terms (specialty/cert/seniority/blocked), not here. """ t = t.strip() if not t or t in _STOP or t in _FILLER: return False if _is_ui_noise(t): return False words = t.split() if len(words) > 4: return False if len(t) < 3: return False if t.isdigit(): return False # All words must be non-stop. for w in words: if w in _STOP: return False return True def extract_external_keywords(jd_text: str, extra: List[str] = None) -> List[str]: """Broad, Jobalytics-style expected keyword set for a JD. Union of: the reliable taxonomy floor (`extract_jd_keywords`), curated Maximum-ATS safe terms present in the JD, and term-like 1-3 word phrases pulled from the JD body (filtered against stopwords/filler). De-duplicated, lowercase. `extra` (e.g. pasted external missing/matched terms) is unioned in. """ jd_text = jd_text or "" jd_low = jd_text.lower() out: List[str] = [] seen = set() def _add(term: str): t = _clean(term) if t and t not in seen and _is_term_like(t): seen.add(t) out.append(t) # 1. Reliable taxonomy floor. for k in extract_jd_keywords(jd_text): _add(k) # 2. Curated safe vocabulary that actually appears in this JD. try: from config import MAXIMUM_ATS_SAFE_TERMS as _SAFE except Exception: _SAFE = set() for t in _SAFE: if t in jd_low: _add(t) # 3. Term-like phrases from the JD body. Instead of a sliding window (which # emits a gram at EVERY offset → many overlapping near-duplicates), extract # MAXIMAL RUNS of consecutive non-stop/non-filler words within each clause. # Stopwords/filler naturally split prose into clean skill phrases. Leading/ # trailing generic action-verbs are trimmed so a phrase reads like a skill # ("lead end-to-end product lifecycle" → "end-to-end product lifecycle"). # Uncapped in COUNT (R21); the subsumption pass removes residual overlaps. for seg in re.split(r"[.;:!?\n•‣●,]", jd_low): seg_words = re.findall(r"[a-zA-Z][a-zA-Z\-/+.]{1,}", seg) run: List[str] = [] for w in seg_words: if w in _STOP or w in _FILLER: _emit_run(run, _add) run = [] else: run.append(w) _emit_run(run, _add) # 4. Pasted external terms (ground truth from a checker). for t in (extra or []): _add(t) # 5. Subsumption dedup (R21): drop any term that is a whole-phrase substring of # a longer kept term, so overlapping grams collapse to the longest unique # phrase ("end-to-end product", "product lifecycle" → "end-to-end product # lifecycle"). Coverage is preserved (the longer phrase contains the shorter, # so it still matches in the resume) and the keyword list has NO duplicates. return _dedupe_subsumed(out) def _dedupe_subsumed(terms: List[str]) -> List[str]: """Remove terms wholly contained (whole-word) in a longer retained term. Preserves first-seen order of the survivors.""" uniq = list(dict.fromkeys(terms)) order = {t: i for i, t in enumerate(uniq)} kept: List[str] = [] for t in sorted(uniq, key=len, reverse=True): # longest first tl = t.lower() subsumed = False for k in kept: if t == k: continue if re.search(r"(? List[str]: """Decompose multi-word run-grams into individual significant terms, preserving known multi-word skills and 2-word units. Order-preserving, de-duplicated.""" out: List[str] = [] seen = set() def _add(tok: str): tok = tok.strip(".-/ ").strip() if len(tok) < 2 or tok in _STOP: return if tok not in seen: seen.add(tok) out.append(tok) for term in (terms or []): tl = (term or "").strip().lower() if not tl: continue # Known multi-word skill → keep as a single semantic unit. if tl in _ATOMIC_VOCAB: if tl not in seen: seen.add(tl) out.append(tl) continue content = [w for w in re.split(r"[\s/]+", tl) if w and w not in _STOP] if not content: continue # all stopwords/noise → drop entirely if len(content) <= 2: # 1–2 content words: an atomic unit already (e.g. "metric definition"). if tl not in seen: seen.add(tl) out.append(tl) else: # 3+ words: a run-gram from a JD list → split into individual atoms. for w in re.split(r"[\s/]+", tl): _add(w) return out def external_coverage(expected: List[str], exported_text: str) -> Dict: """Measure coverage of `expected` terms against the exported resume text. Returns {expected, found, pct, present, missing}. Matching uses the same word-boundary/phrase logic as scoring (`_kw_in_text`). """ text = (exported_text or "").lower() exp = [] seen = set() for t in expected: tl = _clean(t) if tl and tl not in seen: seen.add(tl) exp.append(tl) present = [t for t in exp if _kw_in_text(t, text)] missing = [t for t in exp if t not in present] total = len(exp) return { "expected": total, "found": len(present), "pct": int(round(100 * len(present) / max(1, total))), "present": present, "missing": missing, }