src/ats_scorer.py

"""
ATS Scorer — Resume-Matcher style hybrid.

HOW IT WORKS (plain English):
─────────────────────────────────────────────────────────────
Step 1: Extract keywords FROM the job description
   Method A (regex):  finds PM tools, skills, domain terms in the JD text
   Method B (LLM):    a fast model reads the JD and extracts EXACTLY what
                      the employer wants (required + preferred + general)
                      → This is what Resume-Matcher does; catches synonyms
                        and context that regex misses

Step 2: Check each keyword in YOUR resume
   Uses word-boundary regex: (?<!\\w)keyword(?!\\w)
   Example: "sql" matches "SQL skills" but NOT "visual" or "casual"
   Multi-word: "product roadmap" matches as full phrase

Step 3: Calculate JD Match Score
   matched_keywords / total_jd_keywords × 100
   → 90% means 9 out of 10 JD keywords appear in your resume
   → 40% means the resume is missing most of what the JD asks for

Step 4: Resume Quality Score (independent of JD)
   Checks: sections present, action verbs, quantified metrics,
           skill count, bullet count, formatting
   This measures "is this a good PM resume?" regardless of which job

Step 5: Combined ATS Score
   Final = JD Match × 70%  +  Resume Quality × 30%
   → JD match dominates because that's what ATS systems actually filter on

BEFORE tailoring: maybe 55-70% (missing JD-specific tools/keywords)
AFTER tailoring:  85-95% (LLM added the missing JD keywords naturally)
"""

import re
from typing import Dict, List, Tuple


# ── JD Keyword Categories (extracted from job descriptions) ─────────────────

# PM domain base keywords (always checked against any PM JD)
PM_BASE_KEYWORDS = [
    "product manager", "product roadmap", "product strategy", "product vision",
    "go-to-market", "mvp", "agile", "scrum", "sprint", "backlog",
    "user story", "stakeholder", "cross-functional", "a/b testing",
    "funnel optimization", "conversion rate", "retention", "kpi",
    "user research", "ux", "data-driven", "analytics", "growth",
    "product lifecycle", "feature prioritization", "product discovery",
]

# Tools and platforms commonly required in PM JDs
PM_TOOLS = [
    "jira", "confluence", "notion", "asana", "trello", "linear",
    "figma", "miro", "amplitude", "mixpanel", "segment", "hotjar",
    "tableau", "power bi", "looker", "google analytics",
    "salesforce", "hubspot", "webengage", "clevertap",
    "sql", "python", "api", "crm", "automation",
]

# Action verbs (quality signal for bullet points)
ACTION_VERBS = [
    "achieved", "built", "created", "delivered", "enhanced",
    "generated", "improved", "launched", "managed", "optimized",
    "led", "developed", "designed", "implemented", "analyzed",
    "automated", "scaled", "reduced", "increased", "drove",
    "spearheaded", "pioneered", "transformed", "streamlined",
]

# PM skills for quality score
PM_SKILLS = {
    "tools": [
        "jira", "confluence", "notion", "figma", "miro", "amplitude",
        "mixpanel", "segment", "tableau", "power bi", "looker",
        "google analytics", "salesforce", "hubspot", "webengage",
        "clevertap", "slack", "airtable", "productboard",
    ],
    "frameworks": [
        "agile", "scrum", "kanban", "lean", "okr", "design thinking",
        "sprint planning", "story mapping", "hypothesis testing",
    ],
    "technical": [
        "sql", "python", "api", "crm", "automation", "llm",
        "conversational ai", "ocr", "machine learning", "ai",
    ],
    "soft_skills": [
        "leadership", "communication", "stakeholder management",
        "cross-functional", "mentoring", "prioritization",
        "problem-solving", "strategic thinking", "collaboration",
    ],
}

SECTION_HEADERS = {
    "experience": ["experience", "work experience", "employment", "professional experience"],
    "education":  ["education", "academic", "qualification"],
    "skills":     ["skills", "technical skills", "competencies", "expertise", "core competencies"],
    "summary":    ["summary", "professional summary", "profile", "objective", "about me"],
    "projects":   ["projects", "key projects", "products", "key achievements"],
    "achievements": ["achievements", "key metrics", "highlights", "accomplishments"],
}

IMPACT_KEYWORDS = [
    "improved", "increased", "reduced", "users", "revenue",
    "growth", "efficiency", "conversion", "retention", "leads",
    "cost", "performance", "engagement", "scale",
]


# ─────────────────────────────────────────────────────────────────────────
# PM SKILL TAXONOMY — the allowlist that drives JD keyword extraction.
#
# A JD token/phrase becomes a "keyword" ONLY if it is here (or matches a
# skill regex below). This makes extraction robust across ANY company / JD:
# company names, locations, stock tickers, JD prose, and section headers are
# never in the taxonomy, so they can never pollute the keyword set.
#
# Organized by category for readability. All entries are lowercase.
# ─────────────────────────────────────────────────────────────────────────
PM_SKILL_TAXONOMY = {
    # ── Core PM craft ──
    "product manager", "product owner", "product management", "product strategy",
    "product roadmap", "roadmap", "product vision", "product discovery",
    "product lifecycle", "product development", "feature prioritization",
    "prioritization", "go-to-market", "gtm", "mvp", "product-market fit",
    "0 to 1", "0→1", "zero to one", "product analytics", "product sense",
    # ── Agile / delivery ──
    "agile", "scrum", "kanban", "lean", "sprint", "sprint planning",
    "backlog", "backlog grooming", "backlog management", "story mapping",
    "user story", "user stories", "epics", "epic", "acceptance criteria",
    "release notes", "release management", "iteration", "retrospective",
    "scrum master", "agile methodology", "agile/scrum", "ceremonies",
    # ── Requirements / documentation ──
    "prd", "prds", "fsd", "brd", "wireframes", "wireframing", "mockups",
    "requirements gathering", "requirements elicitation", "elicitation",
    "documentation", "functional specification", "specifications",
    "user acceptance testing", "uat", "test plans", "gap analysis",
    # ── Research / analytics ──
    "user research", "market research", "competitive analysis",
    "competitor analysis", "competitive benchmarking", "ux research",
    "usability testing", "user testing", "a/b testing", "ab testing",
    "experimentation", "hypothesis testing", "cohort analysis",
    "funnel analysis", "funnel optimization", "data analysis",
    "data-driven", "analytics", "kpi", "kpis", "okr", "okrs", "metrics",
    "conversion rate", "conversion rate optimization", "retention",
    "activation", "adoption", "engagement", "churn", "ltv", "arpu",
    # ── Design / UX ──
    "ux", "ui", "user experience", "user-centric", "design thinking",
    "customer journey", "user journey", "journey mapping", "personas",
    "customer empathy", "design systems",
    # ── Technical ──
    "api", "apis", "webhooks", "sql", "python", "rest", "graphql",
    "microservices", "system architecture", "databases", "data pipelines",
    "etl", "cloud", "aws", "azure", "gcp", "saas", "paas",
    "integrations", "automation", "ci/cd", "devops", "machine learning",
    "ml", "ai", "artificial intelligence", "llm", "llms", "generative ai",
    "conversational ai", "nlp", "ocr", "mlops", "model validation",
    "foundation models", "data science", "rca", "observability",
    # ── Security domain (for security PM roles) ──
    "siem", "soar", "xdr", "edr", "threat detection", "threat intelligence",
    "security operations", "secops", "incident response", "vulnerability",
    # ── Domain / business ──
    "b2b", "b2c", "saas", "martech", "fintech", "edtech", "healthtech",
    "ecommerce", "e-commerce", "marketplace", "payments", "lending",
    "credit", "banking", "insurance", "crm", "erp", "supply chain",
    "logistics", "growth", "growth hacking", "user acquisition",
    "monetization", "pricing", "billing", "subscription", "onboarding",
    "campaign management", "personalization", "recommendation",
    "chatbot", "chatbots", "whatsapp business api", "messaging",
    # ── Tools / platforms ──
    "jira", "confluence", "notion", "asana", "trello", "linear", "monday",
    "figma", "sketch", "miro", "mural", "amplitude", "mixpanel", "segment",
    "hotjar", "fullstory", "pendo", "heap", "tableau", "power bi", "looker",
    "metabase", "google analytics", "ga4", "salesforce", "hubspot",
    "webengage", "clevertap", "braze", "moengage", "zendesk", "intercom",
    "slack", "airtable", "productboard", "aha", "google ads", "zoom",
    # ── Leadership / collaboration ──
    "stakeholder management", "stakeholder", "cross-functional",
    "cross functional", "leadership", "team leadership", "mentoring",
    "communication", "collaboration", "strategic thinking", "problem-solving",
    "problem solving", "stakeholder communication", "change management",
    "vendor management", "p&l", "go-to-market strategy",
}

# Multi-word phrases in the taxonomy (matched as phrases, not single tokens)
PM_SKILL_PHRASES = sorted(
    [s for s in PM_SKILL_TAXONOMY if " " in s or "/" in s or "→" in s or "-" in s],
    key=len, reverse=True,  # longest first so "product roadmap" beats "product"
)

# Regex patterns for skills that may appear in many surface forms.
_TAXONOMY_PATTERNS = [
    re.compile(r"\b\d+\s*to\s*\d+\b"),          # "0 to 1"
    re.compile(r"\ba/?b\s*test\w*\b", re.I),     # a/b testing, ab test
]


def _is_taxonomy_skill(token_or_phrase: str) -> bool:
    """True if the token/phrase is a recognized PM skill (allowlist)."""
    t = token_or_phrase.strip().lower()
    if not t:
        return False
    if t in PM_SKILL_TAXONOMY:
        return True
    for pat in _TAXONOMY_PATTERNS:
        if pat.fullmatch(t) or pat.match(t):
            return True
    return False


# ─────────────────────────────────────────────────────────────────────────
# GENERIC PROFESSIONAL VOCABULARY — the terms real ATS checkers (Jobalytics,
# Simplify, JobScan) count that our skill taxonomy deliberately excluded.
#
# These are NOT PM-specific skills, but they ARE legitimate professional
# words that appear in JDs and that real checkers extract as keywords
# (Jobalytics counted "development", "application", "software", "solutions",
# "market" for the Experian JD). Including them is what makes our score
# track real checkers. They are safe to carry in a resume (a PM resume
# naturally says "product development", "software solutions", "go-to-market").
#
# Proper-noun noise (company names, locations, tickers) is STILL excluded
# because it's in neither this set nor the taxonomy.
# ─────────────────────────────────────────────────────────────────────────
GENERIC_PROFESSIONAL_VOCAB = {
    # Work/output nouns
    "development", "design", "engineering", "implementation", "delivery",
    "execution", "deployment", "operations", "maintenance", "support",
    "documentation", "testing", "validation", "monitoring", "reporting",
    "planning", "management", "administration", "coordination", "facilitation",
    # Product/tech nouns
    "software", "application", "applications", "platform", "platforms",
    "system", "systems", "technology", "technologies", "infrastructure",
    "architecture", "solution", "solutions", "product", "products", "feature",
    "features", "module", "modules", "tool", "tools", "service", "services",
    "data", "database", "databases", "dashboard", "dashboards", "interface",
    "integration", "integrations", "pipeline", "pipelines", "workflow",
    "workflows", "framework", "frameworks", "environment", "release",
    # Business nouns
    "market", "business", "strategy", "growth", "revenue", "customer",
    "customers", "user", "users", "stakeholder", "stakeholders", "team",
    "teams", "process", "processes", "quality", "performance", "efficiency",
    "impact", "outcome", "outcomes", "initiative", "initiatives", "project",
    "projects", "program", "programs", "portfolio", "roadmap", "vision",
    "requirements", "specification", "specifications", "scope", "priorities",
    "prioritization", "metrics", "kpis", "analytics", "insights", "research",
    "experimentation", "optimization", "automation", "innovation",
    # Collaboration / methodology nouns
    "collaboration", "collaborate", "collaborative", "communication",
    "communicate", "leadership", "ownership", "mentoring", "mentor",
    "agile", "scrum", "sprint", "iteration", "iterative", "backlog",
    "discovery", "launch", "lifecycle", "feedback", "alignment", "governance",
    "agile methodologies", "end-to-end", "cross-functional",
    # Domain-adjacent (kept generic)
    "cloud", "api", "apis", "frontend", "backend", "fullstack", "mobile",
    "web", "ml", "ai", "ux", "ui",
    # ── Additional terms real checkers flagged on the user's resumes
    # (Meta / Sitetracker / Vuori / Rupeek Jobalytics + Simplify screenshots).
    # These are common JD/PM vocabulary the resume should carry — adding them
    # broadens the denominator (more honest score) AND tells the tailoring to
    # cover them (higher real-checker score).
    "consumer", "consumers", "engineer", "engineers", "analysis",
    "competitive analysis", "customer needs", "data-driven", "data driven",
    "problem-solving", "problem solving", "decision-making", "decision making",
    "go-to-market", "user research", "user experience", "user-centric",
    "wireframes", "wireframing", "prototyping", "prototype", "prototypes",
    "acceptance criteria", "user stories", "user story", "epics", "epic",
    "personas", "journey", "segmentation", "positioning", "messaging",
    "experiments", "experiment", "a/b testing", "hypothesis", "validation",
    "instrumentation", "tracking", "funnel", "conversion", "retention",
    "activation", "adoption", "engagement", "churn", "ltv", "arpu", "nps",
    "scalable", "scalability", "reliability", "availability", "latency",
    "stakeholder management", "vendor", "partners", "partnerships",
    "negotiation", "influence", "presentation", "storytelling",
    "competitor", "competitors", "benchmarking", "market research",
    "gtm", "monetization", "pricing", "billing", "subscription",
    "onboarding", "activation", "personalization", "recommendation",
    "requirements gathering", "documentation", "specs", "prd", "prds",
    "okr", "okrs", "kpi", "kpis", "north star", "metrics-driven",
    "quantitative", "qualitative", "sql", "excel", "spreadsheets",
    "tableau", "looker", "powerbi", "amplitude", "mixpanel", "ga4",
    "jira", "confluence", "figma", "notion", "asana", "miro",
    # ── Common PM/business JD terms real checkers extract (so coverage stays
    # high now that extraction is skills-only). Multi-word forms also live in
    # PM_SKILL_PHRASES via the taxonomy; these single tokens + phrases fill gaps
    # seen on real PM JDs (Ema/Jobalytics): use cases, business objectives, etc.
    "use cases", "use case", "business objectives", "market trends",
    "user personas", "customer support", "product strategy", "product vision",
    "product development", "product management", "product manager",
    "product features", "product requirements",
    "competitor analysis", "competitive", "roadmapping", "gap analysis",
    "performance tracking", "iteration", "go-to-market strategy",
    "cross-functional collaboration", "senior management",
    "user-friendly", "milestones", "timelines",
}


def _is_professional_term(token_or_phrase: str) -> bool:
    """True if the term is a real skill OR generic professional vocabulary."""
    t = token_or_phrase.strip().lower()
    if not t:
        return False
    return _is_taxonomy_skill(t) or t in GENERIC_PROFESSIONAL_VOCAB


# ── Rules-based lemmatizer (no NLTK dependency, deterministic on HF Spaces) ──

# Order matters: longer suffixes first so we don't strip "s" before "ses".
_LEMMA_RULES: List[Tuple[str, str]] = [
    ("ies",  "y"),    # categories → category
    ("ied",  "y"),    # categorized → category-ish; close enough for matching
    ("ying", "y"),    # carrying → carry
    ("sses", "ss"),   # processes → process
    ("ches", "ch"),   # batches → batch
    ("shes", "sh"),   # finishes → finish
    ("oes",  "o"),    # goes → go
    ("ses",  "s"),    # houses → house (acceptable lossy)
    ("ings", ""),     # ratings → rat — only fires if longer than ings+3
    ("ing",  ""),     # running → runn ; close enough — we compare stems
    ("ed",   ""),     # automated → automat ; matches "automation" prefix
    ("er",   ""),     # builder → build
    ("est",  ""),     # fastest → fast
    ("ly",   ""),     # quickly → quick
    ("s",    ""),     # roadmaps → roadmap
]

# Small alias map for cases the rule-based stemmer can't bridge cleanly.
# Keys and values are both lemmatized forms — these become equivalent.
_LEMMA_ALIASES: Dict[str, str] = {
    "automat": "automat",       # auto-canonicalize automate/automated/automation/automating
    "automation": "automat",
    "automate": "automat",
    "implementatio": "implement",
    "implementation": "implement",
    "configuratio": "configur",
    "configuration": "configur",
    "communicatio": "communic",
    "communication": "communic",
    "applicatio": "applic",
    "application": "applic",
    "integration": "integrat",
    "integrations": "integrat",
    "operatio": "operat",
    "operation": "operat",
    "operations": "operat",
    "optimizatio": "optim",
    "optimization": "optim",
    "documentatio": "document",
    "documentation": "document",
    "specificatio": "specif",
    "specification": "specif",
}


def _lemma(word: str) -> str:
    """Reduce a word to a stem so morphological variants compare equal.

    Examples:
      automated → automat      automation → automat       automate → automat
      roadmaps → roadmap       wireframes → wirefram      authoring → author
      API → api                PRDs → prd                 SaaS → saa
    """
    w = word.lower().strip()
    if len(w) <= 3:
        return w
    # Honor aliases first
    if w in _LEMMA_ALIASES:
        return _LEMMA_ALIASES[w]
    for suffix, replacement in _LEMMA_RULES:
        if w.endswith(suffix) and len(w) - len(suffix) >= 3:
            stem = w[: -len(suffix)] + replacement
            return _LEMMA_ALIASES.get(stem, stem)
    return w


_TOKEN_RE = re.compile(r"\w+")
_STOPWORDS = {"the", "a", "an", "of", "and", "or", "to", "in", "on", "for", "with"}


def _tokens(text: str) -> List[str]:
    return _TOKEN_RE.findall(text.lower())


def _lemma_tokens(text: str) -> List[str]:
    return [_lemma(t) for t in _tokens(text)]


def _phrase_in_text(phrase: str, text: str, _cached_lemmas: List[str] = None) -> bool:
    """Match a (possibly multi-word) phrase via lemma + sliding-window.

    - Exact substring match (case-insensitive) returns True immediately
    - Single-word phrase: lemma-equal to any text token
    - Multi-word phrase: all phrase-lemmas appear within a 5-token window
      in the text (allows reordering and intervening words)
    """
    if not phrase:
        return False
    phrase_low = phrase.lower()
    text_low = text.lower()
    if phrase_low in text_low:
        return True

    p_lemmas = [_lemma(t) for t in _tokens(phrase_low) if t not in _STOPWORDS]
    if not p_lemmas:
        return False

    t_lemmas = _cached_lemmas if _cached_lemmas is not None else _lemma_tokens(text_low)

    # Single-word phrase: any text token whose lemma matches
    if len(p_lemmas) == 1:
        return p_lemmas[0] in t_lemmas

    # Multi-word: require the phrase lemmas to appear IN ORDER within a tight
    # window (matches how real ATS checkers score phrases — they want the
    # actual phrase, not its words scattered across the resume). A loose
    # any-order 5-token window over-matched and inflated our score vs real
    # checkers; in-order with a small gap is stricter and calibrated.
    n = len(p_lemmas)
    max_gap = 2  # allow up to 2 filler tokens between phrase words
    for i in range(len(t_lemmas)):
        if t_lemmas[i] != p_lemmas[0]:
            continue
        # Try to match the rest in order, allowing small gaps
        pos = i + 1
        matched = 1
        for target_lemma in p_lemmas[1:]:
            found_at = None
            for j in range(pos, min(pos + max_gap + 1, len(t_lemmas))):
                if t_lemmas[j] == target_lemma:
                    found_at = j
                    break
            if found_at is None:
                break
            matched += 1
            pos = found_at + 1
        if matched == n:
            return True
    return False


# ── JD keyword cleanup: drop company names and marketing noise ───────────────

# Words that surface from JD "about us" / "our clients" sections but aren't
# real skills. They shouldn't be counted as JD requirements.
_JD_NOISE_WORDS = {
    # Company / brand names commonly in "clients include" lists
    "adani", "godrej", "yakult", "wipro", "physicswallah", "physics wallah",
    "asian", "asian paints", "bluelotus", "marsshot", "skullcandy", "vivo",
    "cosco", "aditya", "aditya birla", "delhi", "transport", "corporation",
    "birla", "paints", "physics", "wallah", "aisensy", "navi", "zenda",
    "edgeverve", "ainext", "sumo", "logic", "airtel",
    # Generic prose / marketing
    "businesses", "businesses grow", "revenues", "high revenues",
    "messages", "working", "platform", "mission", "startup", "angel",
    "angel investors", "investors", "crores", "crore", "today",
    "enabling", "group", "about", "high", "team", "teams",
    "billion", "million", "hundred", "thousand",
    # Section labels rather than skills
    "requirements", "responsibilities", "preferred", "background",
    "qualifications", "opportunity", "company", "role", "roles",
    "summary", "overview", "purpose", "context", "challenges",
    # Adjectives describing requirements (not skills)
    "proven", "solid", "basic", "strong", "deep", "advanced", "excellent",
    "extensive", "demonstrated", "fundamental", "good", "great", "passionate",
    "results", "driven", "detail", "oriented", "proactive", "hands",
    "exceptional", "highly", "deeply", "structured", "scalable", "impactful",
    "innovative", "cutting", "intuitive", "powerful", "complex", "critical",
    "fast", "paced", "first", "minimal", "oversight",
    # Modals & generic action words that get extracted as proper nouns
    "will", "must", "can", "should", "would", "shall", "may", "might",
    "has", "have", "had", "able", "ability", "want", "wants",
    # Bullet-starter verbs (not skills)
    "develop", "drive", "drives", "drove", "deliver", "delivers", "delivered",
    "define", "defines", "defined", "ensure", "ensures", "ensured",
    "support", "supports", "supported", "execute", "executes", "executed",
    "engage", "engages", "engaged", "manage", "manages", "managed",
    "lead", "leads", "led", "create", "creates", "created",
    "design", "designs", "designed", "implement", "implements", "implemented",
    "build", "builds", "built", "launch", "launches", "launched",
    "monitor", "monitors", "monitored", "track", "tracks", "tracked",
    "improve", "improves", "improved", "review", "reviews", "reviewed",
    # Generic non-skill nouns
    "level", "year", "years", "candidate", "candidates", "month", "months",
    "position", "positions", "function", "functions", "process", "processes",
    "experience", "experiences", "knowledge", "exposure", "needs", "need",
    "outcomes", "outcome", "value", "values", "voice", "users", "user",
    "customer", "customers", "stakeholder", "stakeholders", "feedback",
    "insight", "insights", "vision", "decision", "decisions", "decisioning",
    # Adverbs
    "continuously", "regularly", "frequently", "occasionally", "primarily",
    "directly", "independently", "effectively", "successfully", "actively",
    # Joining phrases / generic
    "best", "key", "major", "core", "various", "multiple", "several",
    "many", "few", "additional",
    # More process verbs that leak through proper-noun extraction
    "perform", "performs", "performed", "performing",
    "present", "presents", "presented", "presenting",
    "establish", "establishes", "established", "establishing",
    "evangelize", "evangelizes", "evangelized", "evangelizing",
    "stay", "stays", "stayed", "staying",
    "integrate", "integrates", "integrated", "integrating",
    "sign", "signs", "signed", "signing",
    "publish", "publishes", "published", "publishing",
    "handle", "handles", "handled", "handling",
    "moving", "moved", "move",
    "provide", "provides", "provided", "providing",
    "evaluate", "evaluates", "evaluated", "evaluating",
    "meet", "meets", "met", "meeting",
    "gather", "gathers", "gathered", "gathering",
    "champion", "champions", "championed",
    "transform", "transforms", "transformed", "transforming",
    "spearhead", "spearheads", "spearheaded", "spearheading",
    "contribute", "contributes", "contributed", "contributing",
    "represent", "represents", "represented", "representing",
    "navigate", "navigates", "navigated", "navigating",
    # JD section headers + meta words
    "what", "doing", "bring", "join", "located", "location",
    "inc", "ltd", "limited", "llc", "pvt", "private",
    "experience", "experiences", "background", "knowledge",
    "result", "results", "areas", "kra", "kras",
    # Education noise (it's required, not a skill)
    "bachelor", "bachelors", "master", "masters", "degree", "phd",
    "mba", "btech", "bsc", "msc", "diploma", "certificate",
    "computer", "science", "administration",
    # City/region names
    "bangalore", "bengaluru", "pune", "hyderabad", "nellore",
    "mumbai", "delhi", "chennai", "noida", "gurgaon", "gurugram",
    "india", "remote", "worldwide", "us", "uk", "usa",
    # Generic role-context words
    "purpose", "context", "challenges", "summary", "overview",
    "title", "field", "related", "relevant", "responsible",
    "internal", "external", "across", "between", "around",
    "across", "real", "complex", "diverse",
    # Outcome words (not skills)
    "ownership", "mindset", "drive", "passion", "thinking",
    "thinker", "thinkers", "approach", "approaches",
    # JD table-cell boilerplate (Aditya Birla and similar tabular JDs)
    "accountabilities", "accountability",
    "max", "characters", "character",
    "supporting", "supports",
    "kra", "kras",
    "show", "shows", "showing",
    "actions", "action",  # only as a standalone capitalized table column header
    "result", "results", "areas", "area",
    "key", "keys",
    "moving", "handing",
    # JD section / boilerplate words that get extracted as proper nouns
    "job", "jobs", "title", "purpose", "scope", "cost", "time",
    "assistance", "acceptance",  # leak from "Assistance is provided" / "...arrive at"
    "intelligent",  # from "Intelligent Operations Platform" — marketing adjective
    "iterative", "iteration", "iterations",
    "voice", "core", "central", "main", "primary", "secondary",
    "agreed", "appropriate", "applicable",
    # Standalone words from compound JD terms (e.g. "Product Road Mapping" → "Road",
    # "Machine Learning Algorithms" → "Machine" alone). These aren't skills on their own.
    "road", "mapping", "industry", "industries", "field", "fields",
    "talent", "talented", "talents", "candidate", "talent-driven",
    "world", "global", "international", "national", "domestic",
    # NOTE: keeping skill keywords intentionally: ai, ml, saas, api, siem, soar,
    # xdr, elicitation, fsd, uat, mlops, prd — all are legit JD-specific skills
    # the LLM should weave into the resume.
}


def _is_real_jd_keyword(kw: str) -> bool:
    """Return False for company names, marketing prose, and noise words."""
    k = kw.strip().lower()
    if not k or len(k) < 2:
        return False
    if k in _JD_NOISE_WORDS:
        return False
    # Single ALL-CAPS-extracted noun that's just a word like "the" / "you"
    # has already been filtered by extract_jd_keywords' stoplist. But other
    # short verbs like "join", "build", "help" can slip through if used in
    # a sentence — drop if too generic.
    if k in {
        # Modal / generic
        "will", "must", "able", "good", "strong", "great", "make",
        "need", "join", "look", "looking", "help", "build", "work",
        # Generic JD action verbs that get extracted as proper nouns when
        # they start a bullet. None of these are skills.
        "own", "translate", "gather", "produce", "partner", "prioritize",
        "conduct", "collaborate", "improve", "track", "manage", "drive",
        "develop", "support", "ensure", "deliver", "execute", "engage",
        "analyze", "analytical", "review", "lead", "create", "design",
        "implement", "launch", "ship", "validate", "evaluate", "identify",
        "monitor", "report", "communicate", "negotiate", "demonstrate",
        "understand", "convert", "scale", "grow", "test", "research",
        "interview", "advise", "coach", "mentor", "facilitate", "assist",
        # Generic bullet-starter words from JDs
        "own", "owns", "owning", "tracks", "tracking", "tracked",
        "responsible", "expected", "successful", "preferred", "required",
        "experience", "background", "exposure", "knowledge", "ability",
        "level", "senior", "junior", "principal", "associate", "head",
        # Numeric / quantifier
        "many", "several", "various", "multiple", "few",
    }:
        return False
    # Single-word verbs ending in -ing / -ed are usually not skills
    if re.fullmatch(r"[a-z]{4,}(?:ing|ed)", k) and " " not in k:
        # Allow specific skills that end this way
        if k not in {"testing", "coaching", "mentoring", "engineering",
                     "training", "scaling", "marketing", "messaging",
                     "branding", "billing", "onboarding", "fundraising",
                     "consulting", "shipping", "tracking"}:
            return False
    return True


# ── Anti-spam: strip keyword-stuffing sections before scoring ────────────────

def _strip_keyword_spam(resume_text: str) -> str:
    """
    Remove keyword-stuffing sections (e.g. "ADDITIONAL SKILLS & KEYWORDS" with
    raw comma/bullet-separated dumps) so they can't inflate the ATS score.

    Also collapses bullet-only lines containing 15+ words separated by bullets,
    which are a classic keyword-spam pattern regardless of header.
    """
    if not resume_text:
        return resume_text

    # 1) Drop any section literally titled "ADDITIONAL SKILLS & KEYWORDS"
    text = re.sub(
        r"ADDITIONAL\s+SKILLS\s*&\s*KEYWORDS.*?(?=\n[A-Z][A-Z\s&]{2,}\n|\Z)",
        "",
        resume_text,
        flags=re.IGNORECASE | re.DOTALL,
    )

    # 2) Drop lines that look like keyword dumps:
    #    15+ short tokens separated by bullets / pipes / commas, no real sentence
    clean_lines = []
    for line in text.split("\n"):
        stripped = line.strip()
        # Count separators
        sep_count = stripped.count("•") + stripped.count("|") + stripped.count(",")
        if sep_count >= 15 and len(stripped.split()) <= sep_count * 2 + 5:
            # Looks like a keyword dump — drop it
            continue
        clean_lines.append(line)

    return "\n".join(clean_lines)


# ── LLM keyword extraction (Resume-Matcher approach) ─────────────────────────

_LLM_KW_CACHE: dict = {}

def extract_jd_keywords_llm(jd_text: str, fast_model_cfg: dict = None) -> List[str]:
    """
    Use a fast LLM to extract exactly what the employer wants.
    Resume-Matcher approach — catches synonyms + context that regex misses.
    Falls back to regex if LLM unavailable.

    fast_model_cfg: dict with model/api_key/base_url/extra_body keys.
    """
    if not jd_text or len(jd_text.strip()) < 50:
        return []

    cache_key = hash(jd_text[:500])
    if cache_key in _LLM_KW_CACHE:
        return _LLM_KW_CACHE[cache_key]

    if not fast_model_cfg:
        return extract_jd_keywords(jd_text)

    prompt = (
        "Extract keywords from this job description for ATS resume matching.\n"
        "Return ONLY valid JSON (no markdown):\n"
        '{"required_skills":["s1","s2"],"preferred_skills":["t1"],"keywords":["k1","k2"]}\n\n'
        f"Job Description:\n{jd_text[:1500]}"
    )

    try:
        import json as _json, re as _re
        from openai import OpenAI

        client = OpenAI(
            base_url=fast_model_cfg["base_url"],
            api_key=fast_model_cfg["api_key"],
            timeout=25,
        )
        extra = fast_model_cfg.get("extra_body", {})
        kwargs = dict(
            model=fast_model_cfg["model"],
            messages=[
                {"role": "system", "content": "Return ONLY valid JSON. No markdown."},
                {"role": "user",   "content": prompt},
            ],
            temperature=0.1,
            max_tokens=400,
            stream=False,
        )
        if extra:
            kwargs["extra_body"] = extra

        text = client.chat.completions.create(**kwargs).choices[0].message.content or ""
        text = _re.sub(r'^```(?:json)?\s*', '', text.strip())
        text = _re.sub(r'\s*```$', '', text)
        data = _json.loads(text)

        keywords = []
        for field in ("required_skills", "preferred_skills", "keywords"):
            for kw in data.get(field, []):
                if kw and isinstance(kw, str):
                    keywords.append(kw.lower().strip())

        seen = set()
        # Drop noise words / company names; the LLM occasionally picks up
        # client names from "about us" prose.
        unique = [
            k for k in keywords
            if k not in seen and _is_real_jd_keyword(k) and not seen.add(k)
        ]
        _LLM_KW_CACHE[cache_key] = unique[:45]
        return unique[:45]

    except Exception:
        result = extract_jd_keywords(jd_text)
        _LLM_KW_CACHE[cache_key] = result
        return result


# ── Regex keyword extraction (fast fallback) ─────────────────────────────────

# Locations — never skills. Used to exclude city/country tokens from extraction.
_LOCATIONS = {
    "india", "usa", "us", "uk", "uae", "canada", "australia", "germany",
    "france", "ireland", "singapore", "dublin", "london", "bengaluru",
    "bangalore", "hyderabad", "mumbai", "delhi", "pune", "chennai", "noida",
    "gurgaon", "gurugram", "kolkata", "ahmedabad", "remote", "onsite",
    "hybrid", "worldwide", "global", "sunnyvale", "carlsbad", "california",
    "ca", "ny", "york", "francisco", "seattle", "austin", "boston", "chicago",
    "telangana", "karnataka", "maharashtra", "haryana", "tamil", "nadu",
}

# Comprehensive English/JD stopword set — words real ATS checkers do NOT
# count as keywords. Anything NOT here (and not a proper-noun) is fair game.
_CONTENT_STOPWORDS = {
    # articles/conjunctions/prepositions/pronouns
    "the", "a", "an", "and", "or", "but", "nor", "for", "yet", "so", "of",
    "to", "in", "on", "at", "by", "with", "from", "as", "into", "onto",
    "upon", "about", "above", "below", "over", "under", "between", "through",
    "during", "before", "after", "this", "that", "these", "those", "it",
    "its", "they", "them", "their", "you", "your", "yours", "we", "our",
    "ours", "us", "i", "me", "my", "he", "she", "his", "her", "who", "whom",
    "which", "what", "whose", "where", "when", "why", "how", "all", "any",
    "both", "each", "few", "more", "most", "other", "some", "such", "no",
    "not", "only", "own", "same", "than", "too", "very", "can", "will",
    "just", "should", "now", "is", "are", "was", "were", "be", "been",
    "being", "have", "has", "had", "do", "does", "did", "doing", "would",
    "could", "shall", "may", "might", "must", "ought",
    # JD boilerplate / filler
    "job", "role", "roles", "team", "teams", "work", "working", "company",
    "looking", "join", "help", "make", "need", "able", "good", "great",
    "strong", "able", "well", "across", "within", "while", "also", "etc",
    "including", "include", "includes", "ability", "experience", "years",
    "year", "month", "months", "responsibilities", "requirements", "must",
    "haves", "have", "preferred", "qualifications", "candidate", "candidates",
    "opportunity", "about", "us", "you", "your", "we", "are", "seeking",
    "responsible", "expected", "ideal", "plus", "bonus", "nice", "based",
    "level", "senior", "junior", "lead", "minimum", "least", "demonstrated",
    "proven", "track", "record", "deep", "solid", "excellent", "exceptional",
    "highly", "ability", "skills", "skill", "knowledge", "understanding",
    "passion", "passionate", "drive", "driven", "self", "fast", "paced",
    "environment", "culture", "mission", "values", "value", "world", "global",
    "millions", "million", "billion", "thousands", "today", "future", "every",
    "real", "true", "best", "leading", "leader", "leaders", "top", "high",
    "new", "key", "core", "major", "main", "multiple", "various", "several",
    "many", "first", "one", "two", "three", "day", "days", "week", "weeks",
    "time", "times", "way", "ways", "thing", "things", "people", "person",
    "someone", "anyone", "everyone", "something", "anything", "everything",
    "here", "there", "then", "once", "out", "up", "down", "off", "again",
    "further", "because", "until", "against", "per", "via", "like", "want",
    "wants", "wanted", "get", "got", "set", "go", "going", "come", "coming",
    "know", "knowing", "see", "seeing", "use", "using", "used", "made",
    "take", "taking", "give", "giving", "keep", "keeping", "let", "even",
    "ensure", "ensuring", "provide", "providing", "support", "supporting",
}

# Generic English words that look like content but aren't useful resume
# keywords — drop these too even though they're not classic stopwords.
_CONTENT_DROP = {
    "everything", "anyone", "someone", "everyone", "yourself", "themselves",
    "ourselves", "myself", "himself", "herself", "itself", "whatever",
    "whenever", "wherever", "however", "therefore", "moreover", "furthermore",
    "additionally", "essentially", "basically", "literally", "actually",
    "clearly", "simply", "really", "truly", "fully", "quite", "rather",
    "around", "along", "across", "behind", "beyond", "toward", "towards",
}


# Narrative/verb/prose words that slip through (GENERAL English JD prose —
# never tuned to a specific JD; applies to every job description).
_NARRATIVE_NOISE = {
    "night", "calls", "call", "sat", "wrote", "queried", "watched",
    "shipped", "owned", "personally", "familiarity", "yourself",
    "anyone", "everyone", "bar", "line", "code", "clause", "policy",
    "spec", "name", "named", "phase", "stage", "step", "point", "thing",
    "stuff", "lot", "bit", "kind", "sort", "type", "part", "side", "end",
    "place", "area", "areas", "case", "cases", "fact", "idea", "ideas",
    "reason", "result", "results", "example", "examples", "number", "numbers",
    "amount", "rate", "rates", "list", "lists", "group", "groups",
}


def _is_proper_noun_noise(tok: str, lower_seen: set) -> bool:
    """Proper-noun noise (company/person/product name): never appears lowercase
    in the JD AND isn't a known professional term."""
    return (tok not in lower_seen) and (not _is_professional_term(tok))


def _extract_content_terms(jd_text: str, max_terms: int = 0) -> List[str]:
    """
    Comprehensive, UNCAPPED content extraction — capture EVERY meaningful
    term/phrase IN THE JD ITSELF, driven by the JD (not our curated vocab).
    Our vocab only ASSISTS filtering; it never limits what's extracted.

    Returns unigrams AND bigrams:
    - unigrams: content words (nouns/skills) — drops stopwords, locations,
      company/person names (capitalized-only unknowns), narrative verbs.
    - bigrams: consecutive content-word pairs (competitive analysis, customer
      needs, user research, data analysis…) not already captured.

    max_terms=0 → NO CAP. If the JD has N meaningful terms, return all N.
    This is the explicit design: keywords ALWAYS derive from the JD; a JD with
    100 keywords yields 100, a JD with 20 new ones yields those 20.
    """
    lower_seen = set(re.findall(r"\b[a-z][a-z]{2,}\b", jd_text))
    text_low = jd_text.lower()

    def _good(tok: str) -> bool:
        tok = tok.strip(".-/")
        if len(tok) < 3:
            return False
        if tok in _CONTENT_STOPWORDS or tok in _CONTENT_DROP or tok in _NARRATIVE_NOISE:
            return False
        if tok in _LOCATIONS or tok in _JD_NOISE_WORDS:
            return False
        known = _is_professional_term(tok)
        if not known and (tok.endswith("ed") or tok.endswith("ly")):
            return False
        if _is_proper_noun_noise(tok, lower_seen):
            return False
        return True

    # ── Unigrams ──
    freq: dict = {}
    for tok in re.findall(r"\b[a-z][a-z+/.\-]{2,}\b", text_low):
        tok = tok.strip(".-/")
        if tok and _good(tok):
            freq[tok] = freq.get(tok, 0) + 1

    # ── Bigrams ── consecutive content words (captures JD multi-word skills)
    bigram_freq: dict = {}
    seq = re.findall(r"\b[a-z][a-z+/.\-]{1,}\b", text_low)
    for i in range(len(seq) - 1):
        w1 = seq[i].strip(".-/")
        w2 = seq[i + 1].strip(".-/")
        if (len(w1) >= 3 and len(w2) >= 3
                and w1 not in _CONTENT_STOPWORDS and w2 not in _CONTENT_STOPWORDS
                and w1 not in _LOCATIONS and w2 not in _LOCATIONS
                and w1 not in _NARRATIVE_NOISE and w2 not in _NARRATIVE_NOISE
                and not _is_proper_noun_noise(w1, lower_seen)
                and not _is_proper_noun_noise(w2, lower_seen)):
            bg = f"{w1} {w2}"
            bigram_freq[bg] = bigram_freq.get(bg, 0) + 1

    # SKILLS ONLY — match how real ATS checkers (Jobalytics/Resume Worded)
    # actually work: they compare against a curated gazetteer of hard skills,
    # tools, methods, domains, and real soft skills — NOT "any noun in the JD".
    # We therefore keep a discovered unigram ONLY if it is a recognised
    # professional term (in our skill vocab/taxonomy). This is what stops prose
    # nouns like "Goals", "Enterprise", "Authority", "Productivity",
    # "Generation", "Organisation" from ever being treated as keywords. Coverage
    # of genuinely common PM/business terms comes from expanding the vocab, not
    # from blindly grabbing every noun (which produced garbage and lowered the
    # real-checker score).
    meaningful = {t: c for t, c in freq.items() if _is_professional_term(t)}
    # Rank unigrams: known skills first, then frequency
    uni = sorted(meaningful.items(),
                 key=lambda x: (_is_professional_term(x[0]), x[1]), reverse=True)
    # Keep a bigram only if it's a GENUINE skill phrase, not a prose-adjacency
    # artifact. Require BOTH tokens to be real skill/professional terms AND the
    # pair to either recur (≥2×) or be a known curated phrase. This admits
    # "product roadmap"/"data analysis"/"cross-functional teams" while rejecting
    # junk like "shape products"/"gather platform"/"directly impact" that would
    # otherwise flood the keyword set and crater the JD-match ratio.
    _known_phrases = {p.lower() for p in PM_SKILL_PHRASES}
    big = [bg for bg, c in bigram_freq.items()
           if all(_is_professional_term(w) for w in bg.split())
           and (c >= 2 or bg in _known_phrases)]

    result = [t for (t, _c) in uni] + big
    if max_terms and max_terms > 0:
        return result[:max_terms]
    return result


def extract_jd_keywords(jd_text: str) -> List[str]:
    """
    Extract keywords FROM any job description without per-JD blocklist tuning.

    Strategy (in order of confidence):
      1. PM base keywords found in the JD (high signal — known PM terms)
      2. PM tools found in the JD (high signal — known tool names)
      3. Common PM requirement phrases found in the JD (high signal)
      4. Multi-occurrence capitalized terms (≥2 times) — distinguishes
         legitimate skills from one-off proper nouns like company names
         or table-header words

    Step 4 replaces the old "every capitalized word becomes a keyword"
    extraction that was the source of cross-JD noise. Words like
    "Accountabilities", "Bachelor", "Sumo" appear ONCE in their JD;
    real skills like "Jira", "Mixpanel", "PRDs", "MLOps" appear multiple
    times because the JD repeats them in requirements + responsibilities.

    This makes the extractor work on ANY new JD without needing per-JD
    noise additions.
    """
    if not jd_text:
        return []

    text = jd_text.lower()
    keywords: list[str] = []

    # ── CALIBRATED extraction (Phase 5) ──
    # Matches what real ATS checkers (Jobalytics/Simplify) count: PM skills
    # PLUS generic professional vocabulary (development/application/software/
    # solutions/market…). Proper-noun noise (company names, locations,
    # tickers) is still excluded because it's in NEITHER the taxonomy NOR the
    # generic professional vocab.

    # 1. Multi-word skill phrases first (longest-first to avoid double-count)
    consumed_spans: list[tuple] = []
    for phrase in PM_SKILL_PHRASES:
        for m in re.finditer(r"(?<!\w)" + re.escape(phrase) + r"(?!\w)", text):
            span = (m.start(), m.end())
            if any(span[0] < e and s < span[1] for (s, e) in consumed_spans):
                continue
            consumed_spans.append(span)
            keywords.append(phrase)
            break

    # 2. Single-word taxonomy tokens (high-signal PM skills)
    for token in PM_SKILL_TAXONOMY:
        if " " in token or "/" in token or "→" in token or "-" in token:
            continue
        if re.search(r"(?<!\w)" + re.escape(token) + r"(?!\w)", text):
            keywords.append(token)

    # 3. Generic professional vocabulary present in the JD — this is the
    # Phase 5 broadening that makes our score track real checkers. These are
    # the words Jobalytics/Simplify count that our taxonomy alone missed.
    for token in GENERIC_PROFESSIONAL_VOCAB:
        if re.search(r"(?<!\w)" + re.escape(token) + r"(?!\w)", text):
            keywords.append(token)

    # 4. Regex-pattern skills (a/b testing variants, "0 to 1", etc.)
    for pat in _TAXONOMY_PATTERNS:
        for m in pat.finditer(text):
            kw = re.sub(r"\s+", " ", m.group().strip().lower())
            keywords.append(kw)

    # 5. COMPREHENSIVE content terms — capture every meaningful JD word/skill
    # that our curated lists missed (this is what makes coverage match what
    # Jobalytics extracts). Proper-noun noise (company/location/person names)
    # is filtered inside _extract_content_terms.
    keywords.extend(_extract_content_terms(jd_text, max_terms=0))

    # Deduplicate exact repeats
    seen = set()
    unique = []
    for kw in keywords:
        kw = kw.strip()
        if kw and kw not in seen:
            seen.add(kw)
            unique.append(kw)

    # Collapse redundancy so our count tracks real checkers (~32, not 40):
    #  - lemma-equal singular/plural (stakeholder/stakeholders,
    #    application/applications, solution/solutions)
    #  - single-word token subsumed by a multiword phrase already present
    #    (product ⊂ product strategy; backlog ⊂ backlog grooming;
    #     agile ⊂ agile/scrum; discovery ⊂ product discovery)
    unique = _collapse_redundant_keywords(unique)

    # NO CAP — keywords always derive from the JD. A JD with 100 meaningful
    # terms yields 100; a JD with 20 yields 20. We never truncate to a stored
    # ceiling (per design: coverage must match what real ATS checkers extract).
    return unique


def _collapse_redundant_keywords(keywords: List[str]) -> List[str]:
    """Collapse lemma-duplicate and phrase-subsumed keywords."""
    # 1. Lemma-dedup: group by lemma-of-each-word, keep longest surface form
    best_by_key: dict = {}
    order: list = []
    for kw in keywords:
        k = " ".join(_lemma(w) for w in re.split(r"[\s/]+", kw.lower()))
        if k not in best_by_key:
            best_by_key[k] = kw
            order.append(k)
        elif len(kw) > len(best_by_key[k]):
            best_by_key[k] = kw
    deduped = [best_by_key[k] for k in order]

    # 2. Drop a single-word kw if it's a token inside any multiword kw
    multiword_tokens = set()
    for kw in deduped:
        parts = re.split(r"[\s/]+", kw.lower())
        if len(parts) > 1:
            multiword_tokens.update(parts)
    final = []
    for kw in deduped:
        parts = re.split(r"[\s/]+", kw.lower())
        if len(parts) == 1 and parts[0] in multiword_tokens:
            continue  # subsumed by a phrase
        final.append(kw)
    return final


def _kw_in_text(keyword: str, text: str) -> bool:
    """
    Lemma + phrase aware matching.

    - Exact substring (cheapest, catches most matches) — return True
    - Single-word: lemma-equal to any text token (so "automation" matches
      a resume that says "automated"; "roadmap" matches "roadmaps")
    - Multi-word: all component lemmas within a 5-token window

    This is materially more forgiving than the prior word-boundary regex
    and recovers ~15-20pp of false-negative misses observed in production.
    """
    if not keyword:
        return False
    return _phrase_in_text(keyword, text)


# ── JD Match Score (PRIMARY — 70% weight) ────────────────────────────────────

def jd_match_score(resume_text: str, jd_text: str, extra_keywords: List[str] = None) -> Dict:
    """
    PRIMARY ATS metric: what % of JD keywords appear in the resume?

    This is the Resume-Matcher approach:
      1. Extract keywords from JD
      2. Check each in resume using word-boundary regex
      3. Score = matched / total * 100

    Args:
      extra_keywords: keywords already extracted by LLM (from job assessment),
                      merged with regex-extracted keywords for better coverage
    """
    jd_keywords = extract_jd_keywords(jd_text)

    # Merge with LLM-extracted keywords if provided
    if extra_keywords:
        for kw in extra_keywords:
            if kw and kw.lower() not in jd_keywords:
                jd_keywords.append(kw.lower())

    if not jd_keywords:
        return {"score": 0, "matched": [], "missing": [], "total": 0}

    text = resume_text.lower()
    matched = [kw for kw in jd_keywords if _kw_in_text(kw, text)]
    missing = [kw for kw in jd_keywords if not _kw_in_text(kw, text)]

    score = int(len(matched) / len(jd_keywords) * 100)

    return {
        "score":   score,
        "matched": matched[:15],
        "missing": missing[:12],
        "total":   len(jd_keywords),
        "matched_count": len(matched),
    }


# ── Resume Quality Score (SECONDARY — 30% weight) ────────────────────────────

def resume_quality_score(resume_text: str) -> Dict:
    """
    SECONDARY metric: Resume-ATS style quality score.
    Checks structure, formatting, action verbs, skills.
    Independent of JD — measures raw resume quality.
    """
    text = resume_text.lower()
    words = text.split()
    word_count = len(words)
    bullet_count = sum(1 for ch in resume_text if ch in "•▪") + resume_text.count(" - ")

    sections = _detect_sections(resume_text)

    # Keyword quality (PM domain verbs + keywords)
    verb_count = sum(1 for v in ACTION_VERBS if v in text)
    pm_kw_count = sum(1 for kw in PM_BASE_KEYWORDS if kw in text)
    kw_score = min(100, int((pm_kw_count / len(PM_BASE_KEYWORDS)) * 60 + min(1.0, verb_count / 8) * 40))

    # Sections
    # Section score — canonical Phase 4 format has NO Skills section by policy,
    # so we don't count it against the resume. Experience and Education each
    # get 30pts (was 20pts each, with Skills also at 20pts — total budget kept
    # the same at 60pts for required sections).
    sec_score = 0
    for sec in ["experience", "education"]:
        if len(sections.get(sec, "")) > 50:
            sec_score += 30
    if re.search(r'[\w.+-]+@[\w-]+\.\w{2,}', resume_text): sec_score += 10
    if re.search(r'\+?[\d\s\-()]{10,}', resume_text):       sec_score += 10
    for sec in ["summary", "projects", "achievements", "certifications"]:
        if len(sections.get(sec, "")) > 20:
            sec_score += 7
    sec_score = min(100, sec_score)

    # Formatting
    fmt_score = 100
    if word_count < 200:  fmt_score -= 20
    if word_count > 1500: fmt_score -= 10
    if bullet_count < 5:  fmt_score -= 15
    if bullet_count > 50: fmt_score -= 5
    fmt_score = max(0, fmt_score)

    # Skills
    total_skills = sum(1 for cat in PM_SKILLS.values() for s in cat if s in text)
    cat_bonus = sum(15 if any(s in text for s in PM_SKILLS["tools"]) else 0 for _ in [1])
    cat_bonus += sum(15 if any(s in text for s in PM_SKILLS["frameworks"]) else 0 for _ in [1])
    cat_bonus += sum(10 if any(s in text for s in PM_SKILLS["technical"]) else 0 for _ in [1])
    cat_bonus += sum(10 if any(s in text for s in PM_SKILLS["soft_skills"]) else 0 for _ in [1])
    skill_base = 40 if total_skills >= 15 else (30 if total_skills >= 10 else (20 if total_skills >= 5 else 10))
    skl_score = min(100, skill_base + cat_bonus)

    # Experience
    exp_text = sections.get("experience", "")
    positions = max(1, len(re.findall(
        r'(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*[\s,]*\d{4}',
        exp_text.lower()
    )) // 2)
    exp_base = 30 + (20 if positions >= 3 else 15 if positions >= 2 else 10)
    has_metrics = bool(re.search(r'\d+%|₹[\d,]+|\$[\d,]+|\d+x|\d+\s*(?:users|leads|crore|lakh|k\b)', exp_text))
    exp_verbs = sum(1 for v in ACTION_VERBS if v in exp_text.lower())
    exp_quality = min(100, int(min(1.0, exp_verbs / 8) * 70) + (30 if has_metrics else 0))
    exp_score = min(100, exp_base + int(exp_quality * 0.5))

    # Projects
    prj_text = sections.get("projects", "") or sections.get("achievements", "")
    prj_score = 50
    if prj_text:
        prj_score = 50
        if any(t in prj_text.lower() for t in PM_TOOLS): prj_score += 20
        if sum(1 for k in IMPACT_KEYWORDS if k in prj_text.lower()) >= 2: prj_score += 15
        if len(prj_text) > 100: prj_score += 15
    prj_score = min(100, prj_score)

    quality = int(kw_score*0.20 + sec_score*0.20 + fmt_score*0.15 + skl_score*0.20 + exp_score*0.15 + prj_score*0.10)

    return {
        "quality_score":     quality,
        "keyword_score":     kw_score,
        "section_score":     sec_score,
        "formatting_score":  fmt_score,
        "skill_score":       skl_score,
        "experience_score":  exp_score,
        "project_score":     prj_score,
        "word_count":        word_count,
        "bullet_count":      bullet_count,
    }


# ── Combined ATS Score ────────────────────────────────────────────────────────

def score_resume(resume_text: str, jd_text: str = "", extra_kw: List[str] = None,
                 fast_model_cfg: dict = None) -> Dict:
    """
    Full ATS score combining JD match (70%) + resume quality (30%).

    Anti-cheat: strips keyword-spam sections from the resume before scoring so
    raw keyword dumps can't inflate the score. Also applies penalties for
    structurally incomplete resumes (missing education, single role, low word
    count) so an aggressively trimmed resume can't outscore a complete one.

    fast_model_cfg: if provided, uses LLM to extract JD keywords (more accurate).
    extra_kw: additional keywords already extracted by the job assessment LLM.
    """
    # Strip keyword-spam sections so they can't inflate the score
    clean_resume = _strip_keyword_spam(resume_text)

    # Use LLM keyword extraction if a fast model is available
    if fast_model_cfg and jd_text:
        llm_kw = extract_jd_keywords_llm(jd_text, fast_model_cfg)
        regex_kw = extract_jd_keywords(jd_text)
        combined = llm_kw[:]
        for kw in (regex_kw + (extra_kw or [])):
            if kw.lower() not in {k.lower() for k in combined}:
                combined.append(kw)
        extra_kw = combined

    jd_result  = jd_match_score(clean_resume, jd_text, extra_kw)
    qlt_result = resume_quality_score(clean_resume)

    jd_score  = jd_result["score"]
    qlt_score = qlt_result["quality_score"]

    # Combined: JD match weighted 70%, resume quality 30%
    if jd_text:
        final = int(jd_score * 0.70 + qlt_score * 0.30)
    else:
        final = qlt_score  # No JD → quality only

    # ── Structural-integrity penalties ───────────────────────────────────────
    # An ATS-friendly resume needs: a real experience section, education, and
    # enough content. Penalize anything that's structurally hollow so a keyword-
    # stuffed 1-page resume cannot outscore a complete, well-structured one.
    sections = _detect_sections(clean_resume)
    word_count = qlt_result["word_count"]
    penalties: List[str] = []

    # Hard cap only when the resume is essentially empty (<300 words).
    # The canonical Phase 4 format is intentionally tight — 2 pages, 5-7
    # bullets per role. Typical word count is 450-650. Anything ≥350 is fine.
    if word_count < 250:
        final = min(final, 55)
        penalties.append(f"Resume too short ({word_count} words; min 250)")
    elif word_count < 400:
        final = max(0, final - 3)
        penalties.append(f"Resume short ({word_count} words; recommended 400+)")

    if len(sections.get("education", "")) < 30:
        final = max(0, final - 8)
        penalties.append("Missing or empty Education section (-8 pts)")

    # NOTE: No penalty for missing Skills/Core Competencies section.
    # Per project policy R6, the tailored resume intentionally has no skills
    # section — keywords live in the summary and experience bullets instead.
    # Penalizing here would create the opposite incentive.

    # Count distinct role headers (date ranges) in experience — single-role
    # resumes for a 5+ year candidate are a red flag. Match both
    # "Jan 2023 - Present" and "Oct 2021 - Dec 2022" formats.
    exp_text = sections.get("experience", "")
    role_count = len(re.findall(
        r"\d{4}\s*[-–—to]+\s*(?:(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\w*\s+)?(?:\d{4}|Present|Current|Now|Date)",
        exp_text, re.IGNORECASE,
    ))
    if exp_text and role_count <= 1 and word_count < 700:
        final = max(0, final - 6)
        penalties.append("Experience shows only one role (-6 pts)")

    final = max(0, min(100, final))
    label = "Excellent" if final >= 80 else ("Good" if final >= 60 else ("Needs Improvement" if final >= 40 else "Poor"))

    # Identify gaps
    gaps = []
    if jd_score < 85 and jd_result["missing"]:
        gaps.append(f"Add JD keywords to resume: {', '.join(jd_result['missing'][:6])}")
    if qlt_result["section_score"] < 80:
        gaps.append("Add missing sections: Summary, Skills, Projects/Achievements")
    if qlt_result["experience_score"] < 80:
        gaps.append("Add action verbs + quantified metrics (%, numbers) to experience bullets")
    if qlt_result["skill_score"] < 70:
        gaps.append("List 15+ skills: tools (Jira/Figma/Amplitude), frameworks (Agile/Scrum), technical (SQL/API)")

    # Add structural penalties to the gap list so the LLM retry loop sees them
    for p in penalties:
        gaps.append(p)

    return {
        "ats_score":          final,
        "jd_match_score":     jd_score,
        "resume_quality":     qlt_score,
        "matched_kw":         jd_result["matched"][:10],
        "missing_kw":         jd_result["missing"][:10],
        "total_jd_kw":        jd_result["total"],
        "matched_count":      jd_result["matched_count"],
        "word_count":         qlt_result["word_count"],
        "label":              label,
        "gaps":               gaps,
        "penalties":          penalties,
        "quality_breakdown":  qlt_result,
    }


def conservative_display_score(raw: int) -> int:
    """
    Convert our RAW internal keyword-coverage score into a CONSERVATIVE,
    honest estimate that lands near real third-party checkers (Jobalytics et al).

    Why: our raw score measures coverage of OUR keyword set, which a tailored
    resume covers very well (~85-95%). Real checkers use their own (broader,
    proprietary) keyword lists and stricter matching, so they report ~15-25
    points lower. Calibrated against the user's data point (our raw 78 →
    Jobalytics 58) plus a safety margin, we discount by ~0.72 and lean low.

    The RAW score is still used internally by the tailoring loop (so it keeps
    aggressively maximizing real coverage); only the DISPLAYED number is
    discounted so we never overstate to the user.
    """
    if raw <= 0:
        return 0
    # Extraction now comprehensively matches real-checker breadth (Phase 5.x),
    # so raw coverage is a closer proxy. Mild discount keeps us honest/
    # conservative (real checkers still vary), without absurdly understating.
    est = int(round(raw * 0.85))
    # Never claim a perfect score — cap at 92.
    return max(0, min(est, 92))


def score_before_after(original_resume: str, tailored_text: str,
                       jd_text: str = "", extra_kw: List[str] = None) -> Tuple[int, int, int]:
    """Returns (score_before, score_after, improvement) as CONSERVATIVE display
    values calibrated to track real third-party checkers."""
    before_raw = score_resume(original_resume, jd_text, extra_kw)["ats_score"]
    after_raw  = score_resume(tailored_text,   jd_text, extra_kw)["ats_score"]
    before = conservative_display_score(before_raw)
    after  = conservative_display_score(after_raw)
    return before, after, after - before


# Backward-compat alias used by resume_customizer.py
def score_resume_against_jd(resume_text: str, jd_text: str = "") -> Dict:
    """Alias for score_resume — kept for backward compatibility."""
    result = score_resume(resume_text, jd_text)
    # Map to old dict shape that resume_customizer.py expects
    result["ats_score"] = result["ats_score"]  # already present
    return result


def get_gap_report(resume_text: str, jd_text: str = "", extra_kw: List[str] = None) -> str:
    """Human-readable gap report for the LLM to fix."""
    r = score_resume(resume_text, jd_text, extra_kw)
    lines = [
        f"Current ATS Score: {r['ats_score']}/100  (Target: 95+)",
        f"  JD Match Score:    {r['jd_match_score']}/100  (matched {r['matched_count']}/{r['total_jd_kw']} JD keywords) [weight 70%]",
        f"  Resume Quality:    {r['resume_quality']}/100  [weight 30%]",
        f"",
        f"JD keywords MISSING from resume (add these naturally):",
        f"  {', '.join(r['missing_kw'])}",
        f"",
        f"Gaps to fix:",
    ]
    for g in r["gaps"]:
        lines.append(f"  - {g}")
    return "\n".join(lines)


# ── Section detection helper ──────────────────────────────────────────────────

def _detect_sections(text: str) -> Dict[str, str]:
    sections: Dict[str, str] = {}
    lines = text.split("\n")
    current_section = None
    current_lines: List[str] = []

    for line in lines:
        stripped = line.strip().lower()
        found_section = None
        for sec_name, headers in SECTION_HEADERS.items():
            for header in headers:
                if stripped == header or stripped.startswith(header):
                    found_section = sec_name
                    break
            if found_section:
                break

        if found_section:
            if current_section and current_lines:
                sections[current_section] = "\n".join(current_lines).strip()
            current_section = found_section
            current_lines = []
        elif current_section:
            current_lines.append(line)

    if current_section and current_lines:
        sections[current_section] = "\n".join(current_lines).strip()

    return sections