src/external_ats.py

"""
External-ATS coverage simulator (Jobalytics/Simplify-style breadth).

WHY THIS EXISTS
───────────────
Our internal scorer extracts keywords from a NARROW PM taxonomy and scores the
resume against that same narrow set → it easily reports 90%+. External checkers
(Jobalytics/Simplify) extract a MUCH broader set (40-46 terms incl. domain words,
JD-specific responsibilities, soft skills) → the same resume scores ~54%. The
internal number is therefore NOT a valid success signal for the external goal.

This module computes a broad, Jobalytics-style EXPECTED keyword set from the JD,
and measures coverage against the *re-parsed exported resume text* (the only
truth). Used to (a) decide what to physically place in Maximum ATS Mode and
(b) gate/report readiness on real external-style coverage, not internal score.

It is intentionally broader than `extract_jd_keywords` (which stays skills-only
to keep the honest default pipeline clean). The broad set is only used in
Maximum ATS Mode / external-feedback repair, where the user has explicitly opted
into aggressive coverage.
"""
from __future__ import annotations

import re
from typing import Dict, List

from .ats_scorer import (
    _kw_in_text, extract_jd_keywords, PM_SKILL_TAXONOMY, GENERIC_PROFESSIONAL_VOCAB,
)

# Words that are never standalone keywords (checkers penalise/ignore them).
_STOP = {
    "the", "and", "for", "with", "you", "your", "our", "are", "will", "have",
    "has", "had", "this", "that", "these", "those", "from", "into", "out", "off",
    "who", "what", "when", "where", "why", "how", "all", "any", "can", "may",
    "should", "would", "could", "must", "able", "etc", "per", "via", "not",
    "but", "they", "them", "their", "his", "her", "its", "she", "him", "was",
    "were", "been", "being", "more", "most", "such", "than", "then", "also",
    "about", "across", "within", "while", "each", "other", "some", "many",
    "well", "very", "much", "like", "just", "only", "even", "both", "over",
    "under", "between", "during", "including", "include", "includes",
    "looking", "join", "join us", "work", "working", "team", "teams", "role",
    "job", "company", "candidate", "candidates", "ideal", "great", "good",
    "strong", "years", "year", "experience", "experiences", "plus", "preferred",
    "required", "requirement", "responsibilities", "qualifications", "skills",
    "ability", "knowledge", "understanding", "familiarity", "proficiency",
    "we", "us", "is", "in", "on", "of", "to", "at", "as", "an", "or", "be",
    "it", "by", "a", "i",
}
# Vague filler — never a keyword (mirrors jd_analyzer._BUZZWORDS intent).
_FILLER = {
    "innovation", "innovative", "solution", "solutions", "tools", "tool",
    "ownership", "synergy", "dynamic", "passionate", "motivated", "self-starter",
    "results-driven", "detail-oriented", "team player", "track record",
    "expertise", "best practices", "thought leadership", "fast-paced",
    "cutting-edge", "world-class", "robust", "seamless", "holistic", "leverage",
    "excellence", "proven", "successful", "goals", "productivity", "reinvent",
    "reinvention", "mission", "vision", "culture", "value", "values", "impact",
    "environment", "opportunity", "opportunities", "responsibility",
}

# Prose tokens that survive atomic_keywords() decomposition but are NOT genuine
# ATS-relevant skills. Dropped from the V2 scoring DENOMINATOR by
# skill_relevant_filter(). DESIGN: a BLOCKLIST (not an allowlist) so unfamiliar
# tool names and niche skills still survive (avoids the Phase-5 self-grading trap).
# Do NOT add "logistics" — it is a legitimate business domain a PM can claim.
_PROSE_NOISE_EXTRA: frozenset = frozenset({
    # Industry-context nouns, not PM skills
    "driver-partners", "intra-city", "inter-city", "connecting", "largest",
    "reliable", "redefine",
    # JD-prose action-verb atoms (survive splitting but aren't skill names)
    "synthesise", "synthesize", "refine", "validate", "iterate", "establish",
    "proactively", "concurrent", "concurrently", "independently",
    # Marketing scale/superlative descriptors
    "millions", "billion", "thousands", "smarter", "fastest", "meaningful",
    # Role/seniority modifiers (not skills)
    "junior", "independence", "accountable", "operates",
    # Generic best-practice fragments
    "coach",
})


def skill_relevant_filter(terms: List[str]) -> List[str]:
    """Drop prose-noise atoms from the atomized expected set (V2 scoring denominator
    only). BLOCKLIST: drops tokens in _FILLER or _PROSE_NOISE_EXTRA; keeps every
    other token (including unfamiliar tools). Applied in `_build_report` AFTER
    `atomic_keywords`, BEFORE `external_coverage`. V2-only; never called from V1."""
    combined_noise = _FILLER | _PROSE_NOISE_EXTRA
    return [t for t in (terms or []) if t.lower() not in combined_noise]


# Acronym ↔ expansion doubling: recruiters' Boolean search treats "AWS" and
# "Amazon Web Services" as different strings, so both forms should count.
_ACRONYM_MAP_RAW: dict = {
    "aws": "amazon web services", "azure": "microsoft azure",
    "gcp": "google cloud platform", "api": "application programming interface",
    "sql": "structured query language", "crm": "customer relationship management",
    "erp": "enterprise resource planning", "saas": "software as a service",
    "paas": "platform as a service", "kpi": "key performance indicator",
    "okr": "objective and key result", "mvp": "minimum viable product",
    "prd": "product requirements document", "gtm": "go-to-market",
    "ux": "user experience", "ui": "user interface", "ml": "machine learning",
    "ai": "artificial intelligence", "nlp": "natural language processing",
    "llm": "large language model", "b2b": "business to business",
    "b2c": "business to consumer", "seo": "search engine optimization",
    "roi": "return on investment", "nps": "net promoter score",
    "ltv": "lifetime value", "arpu": "average revenue per user",
}
_ACRONYM_MAP: dict = {}
for _short, _full in _ACRONYM_MAP_RAW.items():
    _ACRONYM_MAP[_short] = _full
    _ACRONYM_MAP[_full] = _short


def _expand_acronyms(terms: List[str]) -> List[str]:
    """For each term matching an acronym or its expansion, also append the partner
    form (so both short and full forms count in coverage / recruiter search).
    Order-preserving, de-duplicated. V2-only."""
    out = list(terms or [])
    seen = {t.lower() for t in out}
    for t in list(terms or []):
        partner = _ACRONYM_MAP.get(t.lower().strip())
        if partner and partner not in seen:
            seen.add(partner)
            out.append(partner)
    return out


# UI/CTA/marketing phrases injected by checker overlays (Simplify/Jobalytics/…)
# — never genuine JD keywords. Exact lowercased matches.
_UI_NOISE = frozenset({
    "show match details", "people clicked apply", "month free trial", "free trial",
    "actively reviewing applicants", "message hiring managers",
    "get personalized cover letter", "get insider access", "members use premium",
    "recruiting bond", "visit website", "lacs pa", "help me stand",
    "tailor my resume", "uses advanced ai", "try premium", "easy apply",
    "see how you compare", "am i a good fit", "save job", "apply now",
    "show more", "show less", "sign in", "join now",
})
# CTA/marketing lead tokens: a multi-word gram starting with one is UI noise.
_UI_LEAD = {
    "show", "get", "try", "visit", "apply", "save", "click", "join", "sign",
    "message", "see", "tailor", "unlock", "upgrade", "start",
}
# Marketing substrings that mark a gram as UI noise wherever they appear.
_UI_SUBSTR = (
    "free trial", "premium", "hiring manager", "cover letter", "clicked apply",
    "match details", "lacs pa",
)


# ── V2-only scraped-noise filter (R27) ───────────────────────────────────────
# These three sets + helpers reject scraped, non-skill tokens (company
# geography, executive/recruiter names, role/boilerplate nouns) that the broad
# extractor cannot tell apart from real skills. They are applied ONLY in the V2
# allocation path via `filter_scraped_noise()`; V1 structured placement does NOT
# call them, so V1 behaviour is unchanged.

# Geography / location tokens that are never skills. Word-boundary matching is
# used for single-word entries so 'hq' never blocks 'graphql'; multi-word phrases
# ('latin america') match as substrings.
_GEO_EXTENDED: frozenset = frozenset({
    "amsterdam", "netherlands", "europe", "africa", "asia", "asia-pacific",
    "latin america", "latin", "north america", "south america", "middle east",
    "apac", "emea", "headquartered", "headquarters", "hq", "worldwide",
    "global", "offices", "office",
    # Common India / global locations the job-card header drags in as "keywords".
    "india", "bengaluru", "bangalore", "karnataka", "hyderabad", "telangana",
    "mumbai", "maharashtra", "delhi", "new delhi", "ncr", "gurgaon", "gurugram",
    "noida", "pune", "chennai", "kolkata", "ahmedabad", "boca raton", "florida",
    "usa", "united states", "uk", "united kingdom", "singapore", "dubai",
})

# Corporate-entity / boilerplate suffix words that travel with "About us" copy.
_ENTITY_NOISE: frozenset = frozenset({
    "corporation", "incorporated", "subsidiary", "subsidiaries", "conglomerate",
})

# Job-board posting metadata: employment type, seniority chips, posting age,
# work-mode, applicant counts, section labels — never resume keywords.
_META_NOISE: frozenset = frozenset({
    "full-time", "full time", "fulltime", "part-time", "part time", "parttime",
    "contract", "contractor", "internship", "intern", "temporary", "permanent",
    "associate", "senior", "junior", "mid-level", "entry-level", "entry level",
    "applicants", "applicant", "applied", "alerts", "alert", "get alerts",
    "posted", "remote", "hybrid", "onsite", "on-site", "on site",
    "seniority level", "employment type", "job function", "industries",
    "week ago", "weeks ago", "day ago", "days ago", "month ago", "months ago",
    "hour ago", "hours ago", "minute ago", "minutes ago",
})
# Single tokens that mark a gram as posting metadata wherever they appear.
_META_WORDS: frozenset = frozenset({
    "ago", "full-time", "fulltime", "part-time", "parttime", "applicants",
    "applicant", "alerts", "posted", "internship", "hybrid", "onsite",
})

# Job-board / LinkedIn page chrome that the DOM scrape drags in (the broad V1
# extractor's _UI_NOISE list doesn't cover these exact fragments).
_UI_BLURB: tuple = (
    "actively engaged", "actively reviewing", "get alerts", "reach applicants",
    "top applicant", "easy apply", "be an early applicant", "promoted by",
    "set alert", "job alert", "people you can reach",
)

# Generic role/people nouns that indicate a person or HR context, not a skill.
_ROLE_NOISE: frozenset = frozenset({
    "recruiter", "recruiters", "applicant", "applicants",
    "founder", "co-founder", "cofounder", "ceo", "cto", "coo", "cfo",
    "investor", "investors", "partner", "partners",
    "member", "members", "shareholder", "shareholders",
    "colleague", "colleagues", "peer", "peers", "backbasers",
    "hire", "hiring", "hired",
})

# Common sentence/clause-starting words that appear Title Case but are NOT part
# of a person name — excluded from the two-word name pattern.
_SENTENCE_STARTERS: frozenset = frozenset({
    "The", "A", "An", "We", "Our", "You", "This", "That", "These", "Those",
    "For", "At", "In", "On", "By", "With", "To", "From", "And", "Or", "But",
    "If", "As", "Is", "Are", "Was", "About", "During", "After", "Before",
    "When", "Where", "Please", "Must", "Will", "Can", "Has", "Have", "Had",
    "Be", "Do", "Does", "Did", "Should", "Would", "Could", "May", "Might",
    "Not", "All", "Some", "Any", "Each", "Also", "Both", "More", "Most",
    "How", "What", "Who", "Why", "Which",
})

# Cue words that appear immediately BEFORE a person name in a JD ("Recruiter:
# Mohammed Nayeem", "Founder Jouk Pleiter").
_NAME_CUE_BEFORE = (
    r"recruiter|contact|hiring manager|posted by|submitted by|reach out to|"
    r"founder|co-?founder|ceo|cto|coo|cfo|president|vice president|vp|"
    r"director|head of [a-z ]+|lead"
)


def _contains_geo_token(term: str) -> bool:
    """True if `term` is or contains a geography/location token. Single-word geo
    entries use word-boundary matching so the 'hq' substring never blocks
    'graphql'; multi-word geo phrases (e.g. 'latin america') match as substrings."""
    tl = (term or "").lower()
    for g in _GEO_EXTENDED:
        if g == tl:
            return True
        if " " in g and g in tl:
            return True
        if " " not in g and re.search(
            r"(?<![a-z])" + re.escape(g) + r"(?![a-z])", tl
        ):
            return True
    return False


def _contains_person_name(term_lower: str, original_jd: str) -> bool:
    """True if `term_lower` contains a two-word Title-Case sequence that, in the
    original JD, sits next to a person cue — a label before it ('Recruiter:') or a
    predicate/appositive after it ('is the CEO', '— Account Manager', ', founder').

    Context-gated on purpose: a bare Title-Case bigram is NOT enough (job-title and
    domain headings like 'Forward Deployed' or 'Banking Integrations' are Title
    Case too). Requiring a person cue avoids those false positives while still
    catching real names embedded anywhere in a multi-word gram.
    """
    words = term_lower.split()
    if len(words) < 2:
        return False
    orig = original_jd or ""
    for w1, w2 in zip(words, words[1:]):
        cap1, cap2 = w1.capitalize(), w2.capitalize()
        if cap1 in _SENTENCE_STARTERS or cap2 in _SENTENCE_STARTERS:
            continue
        name = re.escape(cap1) + r"\s+" + re.escape(cap2)
        before = r"(?:" + _NAME_CUE_BEFORE + r")\s*[:\-–—]?\s+" + name
        after = name + r"\s*(?:\bis\b|\bwas\b|,|[\-–—]|\bthe ceo\b|\bfounder\b)"
        if re.search(before, orig, flags=re.I) or re.search(after, orig):
            return True
    return False


def _is_skill_like(term: str, original_jd: str = "", company: str = "") -> bool:
    """Return False (reject) if `term` is geography, a corporate-entity/role noun,
    the company's own name, or a person name. This is the sole V2 rejection gate;
    it does NOT touch the V1 extractor's stopword/UI/filler checks."""
    tl = (term or "").strip().lower()
    if not tl:
        return True
    if _is_ui_noise(tl):
        return False
    if any(b in tl for b in _UI_BLURB):
        return False
    if tl in _META_NOISE:
        return False
    if _contains_geo_token(tl):
        return False
    parts = tl.split()
    if any(w in _META_WORDS for w in parts):
        return False
    if tl in _ROLE_NOISE or any(w in _ROLE_NOISE for w in parts):
        return False
    if any(w in _ENTITY_NOISE for w in parts):
        return False
    if company:
        for ctok in re.findall(r"[a-z0-9]+", company.lower()):
            if len(ctok) >= 4 and re.search(
                r"\b" + re.escape(ctok) + r"(?:s|rs|ers)?\b", tl
            ):
                return False
    if original_jd and _contains_person_name(tl, original_jd):
        return False
    return True


def filter_scraped_noise(
    terms: List[str], original_jd: str = "", company: str = ""
) -> List[str]:
    """V2-only: drop scraped non-skill tokens (geography, executive/recruiter
    names, corporate-entity/boilerplate nouns, the company's own name) from an
    already-extracted term list, preserving order. V1 does NOT call this — V1's
    pool is unchanged by design."""
    return [t for t in (terms or []) if _is_skill_like(t, original_jd, company)]


# Generic JD action-verbs / weak edge words trimmed from the START/END of a
# maximal run so an extracted phrase reads like a SKILL, not a sentence fragment
# ("lead end-to-end product lifecycle" → "end-to-end product lifecycle").
_EDGE_TRIM = {
    "lead", "leads", "leading", "led", "drive", "drives", "driving", "drove",
    "build", "builds", "building", "built", "manage", "manages", "managing",
    "managed", "monitor", "monitors", "monitoring", "create", "creates",
    "creating", "created", "develop", "develops", "developing", "developed",
    "ensure", "ensures", "ensuring", "own", "owns", "owning", "owned", "define",
    "defines", "defining", "defined", "track", "tracks", "tracking", "tracked",
    "deliver", "delivers", "delivering", "delivered", "leverage", "leverages",
    "utilize", "utilizes", "utilizing", "use", "uses", "using", "used",
    "conduct", "conducts", "support", "supports", "supporting", "collaborate",
    "grow", "grows", "growing", "measure", "measures", "measuring", "interpret",
    "watch", "help", "helps", "helping", "based", "through", "across", "drive",
    "make", "makes", "making", "made", "run", "runs", "running", "set", "sets",
    "shape", "shapes", "shaping", "foster", "fosters", "integrate", "integrates",
    "maximize", "optimize", "optimizes", "validate", "validates", "meets", "meet",
    "need", "needs", "needed", "require", "requires", "required", "seeking",
    "seek", "want", "wants", "provide", "provides", "providing", "perform",
    "performs", "performing", "identify", "identifies", "execute", "executes",
}


def _emit_run(run: List[str], add) -> None:
    """Trim generic action-verbs from a maximal non-stop run, then emit it (long
    runs split into non-overlapping <=4-word chunks)."""
    r = list(run)
    while r and r[0] in _EDGE_TRIM:
        r = r[1:]
    while r and r[-1] in _EDGE_TRIM:
        r = r[:-1]
    if not r:
        return
    i = 0
    while i < len(r):
        chunk = r[i:i + 4]
        gram = " ".join(chunk)
        if len(gram) <= 40:
            add(gram)
        i += 4


def _clean(term: str) -> str:
    t = re.sub(r"[^\w\s/+.\-]", "", term or "").strip().lower()
    # Drop leading/trailing sentence punctuation so grams that end a sentence
    # ("initiatives.", "optimization.") aren't kept as distinct dotted tokens.
    return t.strip(".-/ ").strip()


def _is_ui_noise(t: str) -> bool:
    """True if `t` is a checker-overlay CTA / marketing phrase, not a JD term."""
    t = (t or "").strip()
    if not t:
        return False
    if t in _UI_NOISE:
        return True
    words = t.split()
    if len(words) > 1 and words[0] in _UI_LEAD:
        return True
    if any(sub in t for sub in _UI_SUBSTR):
        return True
    return False


def _is_term_like(t: str) -> bool:
    """A token/phrase that reads like a real skill/responsibility/domain term.

    Phase 9 (R21): intentionally WIDENED — "if in doubt, pick it up." We keep only
    genuine-noise guards (stopwords, filler, UI/CTA overlay strings, pure digits,
    too-short, >4 words). There is NO count cap; the honesty gate lives in
    decide_includable_terms (specialty/cert/seniority/blocked), not here.
    """
    t = t.strip()
    if not t or t in _STOP or t in _FILLER:
        return False
    if _is_ui_noise(t):
        return False
    words = t.split()
    if len(words) > 4:
        return False
    if len(t) < 3:
        return False
    if t.isdigit():
        return False
    # All words must be non-stop.
    for w in words:
        if w in _STOP:
            return False
    return True


def extract_external_keywords(jd_text: str, extra: List[str] = None) -> List[str]:
    """Broad, Jobalytics-style expected keyword set for a JD.

    Union of: the reliable taxonomy floor (`extract_jd_keywords`), curated
    Maximum-ATS safe terms present in the JD, and term-like 1-3 word phrases
    pulled from the JD body (filtered against stopwords/filler). De-duplicated,
    lowercase. `extra` (e.g. pasted external missing/matched terms) is unioned in.
    """
    jd_text = jd_text or ""
    jd_low = jd_text.lower()
    out: List[str] = []
    seen = set()

    def _add(term: str):
        t = _clean(term)
        if t and t not in seen and _is_term_like(t):
            seen.add(t)
            out.append(t)

    # 1. Reliable taxonomy floor.
    for k in extract_jd_keywords(jd_text):
        _add(k)

    # 2. Curated safe vocabulary that actually appears in this JD.
    try:
        from config import MAXIMUM_ATS_SAFE_TERMS as _SAFE
    except Exception:
        _SAFE = set()
    for t in _SAFE:
        if t in jd_low:
            _add(t)

    # 3. Term-like phrases from the JD body. Instead of a sliding window (which
    #    emits a gram at EVERY offset → many overlapping near-duplicates), extract
    #    MAXIMAL RUNS of consecutive non-stop/non-filler words within each clause.
    #    Stopwords/filler naturally split prose into clean skill phrases. Leading/
    #    trailing generic action-verbs are trimmed so a phrase reads like a skill
    #    ("lead end-to-end product lifecycle" → "end-to-end product lifecycle").
    #    Uncapped in COUNT (R21); the subsumption pass removes residual overlaps.
    for seg in re.split(r"[.;:!?\n•‣●,]", jd_low):
        seg_words = re.findall(r"[a-zA-Z][a-zA-Z\-/+.]{1,}", seg)
        run: List[str] = []
        for w in seg_words:
            if w in _STOP or w in _FILLER:
                _emit_run(run, _add)
                run = []
            else:
                run.append(w)
        _emit_run(run, _add)

    # 4. Pasted external terms (ground truth from a checker).
    for t in (extra or []):
        _add(t)

    # 5. Subsumption dedup (R21): drop any term that is a whole-phrase substring of
    #    a longer kept term, so overlapping grams collapse to the longest unique
    #    phrase ("end-to-end product", "product lifecycle" → "end-to-end product
    #    lifecycle"). Coverage is preserved (the longer phrase contains the shorter,
    #    so it still matches in the resume) and the keyword list has NO duplicates.
    return _dedupe_subsumed(out)


def _dedupe_subsumed(terms: List[str]) -> List[str]:
    """Remove terms wholly contained (whole-word) in a longer retained term.
    Preserves first-seen order of the survivors."""
    uniq = list(dict.fromkeys(terms))
    order = {t: i for i, t in enumerate(uniq)}
    kept: List[str] = []
    for t in sorted(uniq, key=len, reverse=True):   # longest first
        tl = t.lower()
        subsumed = False
        for k in kept:
            if t == k:
                continue
            if re.search(r"(?<!\w)" + re.escape(tl) + r"(?!\w)", k.lower()):
                subsumed = True
                break
        if not subsumed:
            kept.append(t)
    return sorted(kept, key=lambda t: order[t])


# ── Atomic keyword decomposition (R32) ───────────────────────────────────────
# The extractor emits multi-word run-grams from a JD's comma lists (e.g.
# "saas cloud aws azure"); the matcher then needs them verbatim/in-order, so a
# resume that genuinely has those skills scattered scores them as MISSING. Real
# ATS checkers (Jobalytics/Simplify) score INDIVIDUAL keywords. `atomic_keywords`
# decomposes long grams into single terms for scoring — but keeps genuine 2-word
# semantic units (go-to-market, machine learning, product roadmap) intact so they
# aren't shredded. Used ONLY in V2's `_build_report` (the V1 scorer is untouched).
_ATOMIC_VOCAB: frozenset = frozenset(
    t.lower() for t in (PM_SKILL_TAXONOMY | GENERIC_PROFESSIONAL_VOCAB)
    if " " in t or "-" in t or "/" in t
)


def atomic_keywords(terms: List[str]) -> List[str]:
    """Decompose multi-word run-grams into individual significant terms, preserving
    known multi-word skills and 2-word units. Order-preserving, de-duplicated."""
    out: List[str] = []
    seen = set()

    def _add(tok: str):
        tok = tok.strip(".-/ ").strip()
        if len(tok) < 2 or tok in _STOP:
            return
        if tok not in seen:
            seen.add(tok)
            out.append(tok)

    for term in (terms or []):
        tl = (term or "").strip().lower()
        if not tl:
            continue
        # Known multi-word skill → keep as a single semantic unit.
        if tl in _ATOMIC_VOCAB:
            if tl not in seen:
                seen.add(tl)
                out.append(tl)
            continue
        content = [w for w in re.split(r"[\s/]+", tl) if w and w not in _STOP]
        if not content:
            continue   # all stopwords/noise → drop entirely
        if len(content) <= 2:
            # 1–2 content words: an atomic unit already (e.g. "metric definition").
            if tl not in seen:
                seen.add(tl)
                out.append(tl)
        else:
            # 3+ words: a run-gram from a JD list → split into individual atoms.
            for w in re.split(r"[\s/]+", tl):
                _add(w)
    return out


def external_coverage(expected: List[str], exported_text: str) -> Dict:
    """Measure coverage of `expected` terms against the exported resume text.

    Returns {expected, found, pct, present, missing}. Matching uses the same
    word-boundary/phrase logic as scoring (`_kw_in_text`).
    """
    text = (exported_text or "").lower()
    exp = []
    seen = set()
    for t in expected:
        tl = _clean(t)
        if tl and tl not in seen:
            seen.add(tl)
            exp.append(tl)
    present = [t for t in exp if _kw_in_text(t, text)]
    missing = [t for t in exp if t not in present]
    total = len(exp)
    return {
        "expected": total,
        "found": len(present),
        "pct": int(round(100 * len(present) / max(1, total))),
        "present": present,
        "missing": missing,
    }