"""Unified, stealth-capable fetch layer for all job scrapers.

Goal: scrape without getting blocked. This module routes every HTTP fetch
through Scrapling when it is installed (real-Chrome TLS fingerprint via
`Fetcher`, Cloudflare-Turnstile bypass via `StealthyFetcher`), and transparently
falls back to plain `requests` when Scrapling (or its browsers) are unavailable.

Two kinds of blocking, and what this layer does about each:

  1. Fingerprint blocking (TLS / headers / headless detection / Cloudflare)
     -> Scrapling's `impersonate=` + `StealthyFetcher` defeat this.
  2. IP-reputation blocking (shared datacenter IP, e.g. on HF Spaces)
     -> Only a proxy fixes this. Set the `SCRAPER_PROXIES` env var (comma- or
        newline-separated proxy URLs) and they are rotated round-robin across
        requests. Without proxies we still try our best with fingerprint stealth.

The public surface is intentionally tiny and `requests.Response`-compatible
(`.text`, `.content`, `.json()`, `.status_code`, `.ok`) so existing scrapers do
not need to change how they read responses.
"""
from __future__ import annotations

import os
import time
import json as _json
import logging
import itertools
import threading
from typing import Optional

log = logging.getLogger("scraper.fetch")

# ── Optional Scrapling import (degrade gracefully if missing) ─────────────────
try:  # pragma: no cover - import side effect depends on the environment
    from scrapling.fetchers import Fetcher as _SFetcher  # HTTP, TLS-impersonate
    _HAS_SCRAPLING = True
except Exception:  # noqa: BLE001 - any import error => fall back to requests
    _SFetcher = None
    _HAS_SCRAPLING = False

try:  # StealthyFetcher needs the extra Camoufox browser (`scrapling install`)
    from scrapling.fetchers import StealthyFetcher as _SStealthy
    _HAS_STEALTHY = True
except Exception:  # noqa: BLE001
    _SStealthy = None
    _HAS_STEALTHY = False

# `requests` is always available (a hard dependency of the project).
import requests  # noqa: E402
from fake_useragent import UserAgent  # noqa: E402

_DEFAULT_IMPERSONATE = os.getenv("SCRAPER_IMPERSONATE", "chrome")
_DEFAULT_TIMEOUT = int(os.getenv("SCRAPER_TIMEOUT", "20"))


def scrapling_available() -> bool:
    return _HAS_SCRAPLING


def stealthy_available() -> bool:
    return _HAS_STEALTHY


# ── Proxy rotation (the only real fix for datacenter-IP blocking) ─────────────
def _load_proxies() -> list[str]:
    """Read proxies from env at call time so secrets can be set after import.

    Accepts comma- OR newline-separated proxy URLs in `SCRAPER_PROXIES`
    (e.g. ``http://user:pass@host:port``). A single `SCRAPER_PROXY` also works.
    """
    raw = os.getenv("SCRAPER_PROXIES", "") or os.getenv("SCRAPER_PROXY", "")
    if not raw:
        return []
    parts = [p.strip() for chunk in raw.split("\n") for p in chunk.split(",")]
    return [p for p in parts if p]


_proxy_lock = threading.Lock()
_proxy_cycle: Optional["itertools.cycle"] = None
_proxy_snapshot: tuple[str, ...] = ()


def _next_proxy() -> Optional[str]:
    """Round-robin the configured proxies; rebuild the cycle if env changed."""
    global _proxy_cycle, _proxy_snapshot
    proxies = tuple(_load_proxies())
    if not proxies:
        return None
    with _proxy_lock:
        if proxies != _proxy_snapshot or _proxy_cycle is None:
            _proxy_snapshot = proxies
            _proxy_cycle = itertools.cycle(proxies)
        return next(_proxy_cycle)


# ── requests.Response-compatible shim ─────────────────────────────────────────
class FetchResponse:
    """Minimal response wrapper exposing the bits scrapers actually use."""

    __slots__ = ("status_code", "text", "url", "backend")

    def __init__(self, status_code: int, text: str, url: str = "",
                 backend: str = "requests"):
        self.status_code = int(status_code or 0)
        self.text = text or ""
        self.url = url
        self.backend = backend

    @property
    def ok(self) -> bool:
        return 200 <= self.status_code < 300

    @property
    def content(self) -> bytes:
        return self.text.encode("utf-8", "replace")

    def json(self):
        return _json.loads(self.text)


_ua = UserAgent()


def _scrapling_response_text(page) -> str:
    """Extract the response body text across Scrapling versions, defensively."""
    for attr in ("body", "text", "html_content", "html"):
        try:
            val = getattr(page, attr, None)
        except Exception:  # noqa: BLE001
            val = None
        if isinstance(val, bytes):
            return val.decode("utf-8", "replace")
        if isinstance(val, str) and val:
            return val
    try:
        return str(page)
    except Exception:  # noqa: BLE001
        return ""


def _scrapling_status(page) -> int:
    for attr in ("status", "status_code"):
        val = getattr(page, attr, None)
        if isinstance(val, int):
            return val
    return 200


def _build_url(url: str, params: Optional[dict]) -> str:
    if not params:
        return url
    from urllib.parse import urlencode
    sep = "&" if "?" in url else "?"
    return f"{url}{sep}{urlencode(params)}"


# ── Public API ────────────────────────────────────────────────────────────────
def get(
    url: str,
    *,
    params: Optional[dict] = None,
    headers: Optional[dict] = None,
    timeout: int = _DEFAULT_TIMEOUT,
    retries: int = 3,
    impersonate: str = _DEFAULT_IMPERSONATE,
    use_browser: bool = False,
    solve_cloudflare: bool = False,
) -> Optional[FetchResponse]:
    """Fetch a URL, preferring Scrapling stealth, falling back to requests.

    `use_browser`/`solve_cloudflare` route through `StealthyFetcher` (Camoufox)
    for JS-heavy or Cloudflare-protected pages when it is installed.
    Returns a `FetchResponse` on HTTP 200, else None (mirrors the old `_get`).
    """
    full_url = _build_url(url, params)

    if use_browser or solve_cloudflare:
        r = _get_browser(full_url, timeout=timeout, solve_cloudflare=solve_cloudflare)
        if r is not None:
            return r
        # fall through to HTTP if the browser path is unavailable/failed

    if _HAS_SCRAPLING:
        r = _get_scrapling_http(full_url, headers, timeout, retries, impersonate)
        if r is not None:
            return r
        # fall through to requests on total failure

    return _get_requests(full_url, headers, timeout, retries)


def _get_scrapling_http(url, headers, timeout, retries, impersonate):
    for attempt in range(retries):
        proxy = _next_proxy()
        try:
            kwargs = {
                "timeout": timeout,
                "stealthy_headers": True,
                "impersonate": impersonate,
            }
            if headers:
                kwargs["headers"] = headers
            if proxy:
                kwargs["proxy"] = proxy
            page = _SFetcher.get(url, **kwargs)
            status = _scrapling_status(page)
            if status == 200:
                return FetchResponse(status, _scrapling_response_text(page),
                                     url=url, backend="scrapling-http")
            if status == 429:
                time.sleep((attempt + 1) * 8)
                continue
            if status in (401, 403):
                # try once more via a different proxy; else give up on this backend
                if proxy and attempt < retries - 1:
                    continue
                return None
        except TypeError:
            # Older/newer Scrapling signature mismatch — drop unsupported kwargs.
            try:
                page = _SFetcher.get(url, timeout=timeout)
                if _scrapling_status(page) == 200:
                    return FetchResponse(200, _scrapling_response_text(page),
                                         url=url, backend="scrapling-http")
            except Exception:  # noqa: BLE001
                pass
            return None
        except Exception as exc:  # noqa: BLE001
            log.debug("scrapling http error (%s): %s", url, exc)
            time.sleep(2 ** attempt)
    return None


def _get_browser(url, timeout, solve_cloudflare):
    if not _HAS_STEALTHY:
        return None
    proxy = _next_proxy()
    try:
        kwargs = {
            "headless": True,
            "network_idle": True,
            "timeout": timeout * 1000,  # Scrapling browser timeouts are in ms
        }
        if solve_cloudflare:
            kwargs["solve_cloudflare"] = True
        if proxy:
            kwargs["proxy"] = proxy
        page = _SStealthy.fetch(url, **kwargs)
        status = _scrapling_status(page)
        if status and status != 200:
            return None
        return FetchResponse(status or 200, _scrapling_response_text(page),
                             url=url, backend="scrapling-stealth")
    except TypeError:
        try:
            page = _SStealthy.fetch(url, headless=True)
            return FetchResponse(_scrapling_status(page),
                                 _scrapling_response_text(page),
                                 url=url, backend="scrapling-stealth")
        except Exception:  # noqa: BLE001
            return None
    except Exception as exc:  # noqa: BLE001
        log.debug("scrapling stealth error (%s): %s", url, exc)
        return None


def _get_requests(url, headers, timeout, retries):
    base_headers = {
        "User-Agent": _ua.random,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,"
                  "image/webp,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip, deflate, br",
        "Connection": "keep-alive",
        "Upgrade-Insecure-Requests": "1",
    }
    if headers:
        base_headers.update(headers)

    for attempt in range(retries):
        proxy = _next_proxy()
        proxies = {"http": proxy, "https": proxy} if proxy else None
        try:
            base_headers["User-Agent"] = _ua.random
            resp = requests.get(url, headers=base_headers, timeout=timeout,
                                proxies=proxies)
            if resp.status_code == 200:
                return FetchResponse(200, resp.text, url=resp.url,
                                     backend="requests")
            if resp.status_code == 429:
                time.sleep((attempt + 1) * 10)
            elif resp.status_code in (401, 403):
                break
        except Exception:  # noqa: BLE001
            time.sleep(2 ** attempt)
    return None


def fetch_browser_html(url: str, *, solve_cloudflare: bool = True,
                       timeout: int = 30) -> Optional[str]:
    """Return raw HTML for a JS/Cloudflare page via StealthyFetcher, or None.

    Convenience wrapper for scrapers that already parse HTML with BeautifulSoup
    (e.g. Glassdoor) and want a Camoufox stealth fetch before falling back to
    their own Playwright path.
    """
    r = _get_browser(url, timeout=timeout, solve_cloudflare=solve_cloudflare)
    return r.text if (r and r.text) else None