src/jd_from_url.py

"""Server-side JD extraction from a job URL.

The Chrome extension reads the JD from the page DOM; a server can't do that, so
this fetches the URL (stealth browser for LinkedIn/Indeed/Glassdoor/Naukri, plain
HTTP otherwise) and extracts the description text from the HTML with BeautifulSoup.

Used by the Telegram bot. Returns a dict:
  {ok: bool, jd_text: str, job_title: str, company: str, reason: str}
`ok` is True only when the extracted text is long enough AND reads like a real JD
(JD-signal phrases). When False, the caller should ask the user to paste the JD.
"""
from __future__ import annotations

import re
import logging
from typing import Dict

log = logging.getLogger("jd_from_url")

# Phrases that indicate the text IS a job description (mirrors the extension).
_JD_SIGNALS = (
    "responsibilit", "requirement", "qualification", "about the job",
    "about the role", "about the company", "what you'll do", "what you will do",
    "who you are", "you will", "we are looking", "role overview", "job description",
    "preferred", "minimum qualifications", "key skills", "what we", "your role",
    "in this role", "nice to have", "must have", "day to day", "day-to-day",
    "experience in", "skills", "we're looking",
)

# Hosts that need a real browser (JS render + anti-bot) rather than plain HTTP.
_BROWSER_HOSTS = ("linkedin.", "indeed.", "glassdoor.", "naukri.", "ziprecruiter.")

# Ordered, MOST-SPECIFIC-FIRST JD containers per platform. We pick the first one
# that yields a real JD (length + JD-signal), so we grab ONLY the description —
# never the whole page (which on LinkedIn includes "similar jobs", the recruiter
# card, footer links, cookie banner, etc.).
_JD_PRIMARY = [
    # LinkedIn (public job-view markup + authed app)
    ".show-more-less-html__markup",
    ".description__text .show-more-less-html__markup",
    ".jobs-description__content .jobs-box__html-content",
    ".jobs-description__content", ".jobs-box__html-content",
    "#job-details", ".description__text",
    # Indeed
    "#jobDescriptionText",
    # Greenhouse
    ".job__description", "#content .body", "#content",
    # Lever
    "[data-qa='job-description']", ".section-wrapper.page-full-width",
    # Workday
    "[data-automation-id='jobPostingDescription']",
    # Ashby / SmartRecruiters / Recruitee / generic ATS
    ".ashby-job-posting-right-pane", ".jobAd", ".job-sections",
    "[class*='jobDescription']", "[class*='job-description']",
    "[class*='JobDescription']",
    # last-resort generic page regions
    "article", "main", "[role='main']",
]

# Whole sub-trees to delete BEFORE extracting — page chrome that pollutes the JD
# (this is what put "cookie policy", recruiter names, and other companies' jobs
# into the resume). Matched on tag, role, or class/id substring.
_NOISE_SELECTOR = (
    "nav, header, footer, aside, script, style, svg, noscript, iframe, form, button, "
    "[role='navigation'], [role='banner'], [role='contentinfo'], [aria-hidden='true'], "
    "[class*='similar'], [class*='related'], [class*='recommend'], [class*='also-viewed'], "
    "[class*='alsoViewed'], [class*='people-also'], [class*='more-jobs'], [class*='moreJobs'], "
    "[class*='footer'], [class*='nav-'], [class*='navbar'], [class*='header'], "
    "[class*='cookie'], [class*='consent'], [class*='banner'], [class*='promo'], "
    "[class*='signup'], [class*='sign-up'], [class*='signin'], [class*='sign-in'], "
    "[class*='login'], [class*='subscribe'], [class*='advert'], [class*='newsletter'], "
    "[id*='similar'], [id*='related'], [id*='footer'], [id*='cookie'], [id*='nav']"
)

# Short label/legal/CTA lines that are page chrome, not JD content. A line is
# dropped if it CONTAINS one of these (case-insensitive).
_LINE_NOISE = (
    "cookie policy", "privacy policy", "user agreement", "terms of service",
    "by clicking", "you agree", "sign in", "join now", "create job alert",
    "set alert", "get notified", "people also viewed", "similar jobs",
    "be an early applicant", "easy apply", "show more", "show less",
    "seniority level", "employment type", "job function", "referrals increase",
    "see who you know", "help center", "explore", "first name", "last name",
    "save job", "apply now", "report this job", "skip to", "follow company",
)


def _scrub_lines(text: str) -> str:
    out = []
    for ln in (text or "").splitlines():
        low = ln.strip().lower()
        if not low:
            out.append(ln)
            continue
        if any(n in low for n in _LINE_NOISE):
            continue
        out.append(ln)
    return "\n".join(out)


def _has_jd_signal(text: str) -> bool:
    low = (text or "").lower()
    return any(s in low for s in _JD_SIGNALS)


def _extract_from_html(html: str) -> tuple[str, str]:
    """Return (page_title, JOB-DESCRIPTION-ONLY text) from raw HTML.

    Strategy: delete page-chrome sub-trees, then pick the FIRST most-specific JD
    container that reads like a real JD — so we capture only the description, not
    the surrounding page (similar jobs, recruiter card, footer, cookie banner)."""
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html or "", "lxml")

    page_title = ""
    if soup.title and soup.title.string:
        page_title = soup.title.string.strip()

    # 1. Delete chrome / noise sub-trees up front.
    try:
        for el in soup.select(_NOISE_SELECTOR):
            el.decompose()
    except Exception:  # noqa: BLE001 - exotic markup
        for tag in soup(["script", "style", "nav", "header", "footer", "aside",
                         "form", "button", "svg", "noscript", "iframe"]):
            tag.decompose()

    def _clean_text(el) -> str:
        return _scrub_lines(
            re.sub(r"\n{3,}", "\n\n", el.get_text(separator="\n", strip=True))
        ).strip()

    # 2. Most-specific-first: return the first container that IS a real JD.
    best = ""
    for sel in _JD_PRIMARY:
        try:
            els = soup.select(sel)
        except Exception:  # noqa: BLE001
            continue
        for el in els:
            txt = _clean_text(el)
            if len(txt) >= 200 and _has_jd_signal(txt):
                return page_title, txt        # scoped, clean JD — done
            if len(txt) > len(best):
                best = txt

    # 3. Fallback: densest div/section block (chrome already removed), bounded.
    if len(best) < 250:
        for el in soup.find_all(["div", "section"]):
            txt = _clean_text(el)
            if 250 <= len(txt) < 20000 and len(txt) > len(best):
                best = txt

    return page_title, best


def fetch_jd_from_url(url: str, *, timeout: int = 35) -> Dict:
    """Fetch a job URL and extract its description. See module docstring."""
    url = (url or "").strip()
    if not re.match(r"^https?://", url, re.I):
        return {"ok": False, "jd_text": "", "job_title": "", "company": "",
                "reason": "not_a_url"}

    host = url.split("/")[2].lower() if "//" in url else url.lower()
    use_browser = any(h in host for h in _BROWSER_HOSTS)

    try:
        from src.scrapers import fetch
    except Exception as exc:  # noqa: BLE001
        log.warning("fetch layer unavailable: %s", exc)
        return {"ok": False, "jd_text": "", "job_title": "", "company": "",
                "reason": "fetch_unavailable"}

    resp = fetch.get(url, use_browser=use_browser,
                     solve_cloudflare=use_browser, timeout=timeout)
    # If the plain-HTTP path returned nothing, try the stealth browser once.
    if (not resp or not resp.text) and not use_browser:
        resp = fetch.get(url, use_browser=True, solve_cloudflare=True, timeout=timeout)
    if not resp or not (resp.text or "").strip():
        return {"ok": False, "jd_text": "", "job_title": "", "company": "",
                "reason": "fetch_failed"}

    title, text = _extract_from_html(resp.text)
    if len(text) > 16000:
        text = text[:16000]

    ok = len(text) >= 200 and _has_jd_signal(text)
    return {
        "ok": ok,
        "jd_text": text,
        "job_title": (title or "").split(" - ")[0].split(" | ")[0].strip()[:120],
        "company": "",
        "reason": "" if ok else "weak_extraction",
    }