src/research/search.py

"""
Standalone web search + content fetching for the research engine.
Uses DuckDuckGo (no API key needed) with httpx for content fetching.
"""
import hashlib
import json
import logging
import re
import time
from datetime import datetime, timedelta
from pathlib import Path
from typing import Dict, List, Optional
from urllib.parse import urlparse

import httpx
from bs4 import BeautifulSoup

logger = logging.getLogger(__name__)

# Cache directory
_CACHE_DIR = Path("data/research_cache")
_CACHE_DIR.mkdir(parents=True, exist_ok=True)

# Domains to skip (paywall, login walls, etc.)
_SKIP_DOMAINS = {
    "facebook.com", "twitter.com", "x.com", "instagram.com",
    "linkedin.com/in/", "tiktok.com", "youtube.com",
}

USER_AGENT = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/124.0.0.0 Safari/537.36"
)


def _cache_key(text: str) -> str:
    return hashlib.sha256(text.encode()).hexdigest()[:16]


def _read_cache(key: str, ttl_hours: int = 4) -> Optional[dict]:
    f = _CACHE_DIR / f"{key}.json"
    if not f.exists():
        return None
    try:
        data = json.loads(f.read_text(encoding="utf-8"))
        ts = datetime.fromisoformat(data["ts"])
        if datetime.now() - ts < timedelta(hours=ttl_hours):
            return data["payload"]
    except Exception:
        pass
    return None


def _write_cache(key: str, payload) -> None:
    try:
        f = _CACHE_DIR / f"{key}.json"
        f.write_text(
            json.dumps({"ts": datetime.now().isoformat(), "payload": payload}),
            encoding="utf-8",
        )
    except Exception:
        pass


def web_search(query: str, max_results: int = 8) -> List[Dict]:
    """
    Search the web using DuckDuckGo. Returns list of {url, title, snippet}.
    Falls back to Bing scraping if DDG fails.
    """
    key = _cache_key(f"search:{query}:{max_results}")
    cached = _read_cache(key, ttl_hours=2)
    if cached:
        logger.debug(f"Search cache hit: {query}")
        return cached

    results = _ddg_search(query, max_results)
    if not results:
        results = _bing_scrape(query, max_results)

    _write_cache(key, results)
    return results


def _ddg_search(query: str, max_results: int) -> List[Dict]:
    try:
        try:
            from ddgs import DDGS          # new package name
        except ImportError:
            from duckduckgo_search import DDGS  # old name fallback
        with DDGS(timeout=15) as ddgs:
            raw = list(ddgs.text(query, max_results=max_results))
        results = []
        for r in raw:
            url = r.get("href") or r.get("url", "")
            if not url or _should_skip(url):
                continue
            results.append({
                "url": url,
                "title": r.get("title", ""),
                "snippet": r.get("body", "")[:300],
            })
        logger.info(f"DDG search '{query}': {len(results)} results")
        return results
    except Exception as e:
        logger.warning(f"DDG search failed: {e}")
        return []


def _bing_scrape(query: str, max_results: int) -> List[Dict]:
    """Fallback: scrape Bing search results."""
    try:
        url = f"https://www.bing.com/search?q={query.replace(' ', '+')}&count={max_results}"
        resp = httpx.get(url, headers={"User-Agent": USER_AGENT}, timeout=10, follow_redirects=True)
        soup = BeautifulSoup(resp.text, "lxml")
        results = []
        for li in soup.find_all("li", class_="b_algo")[:max_results]:
            link = li.find("a", href=True)
            snippet_el = li.find("p") or li.find("div", class_="b_caption")
            if not link:
                continue
            href = link["href"]
            if _should_skip(href):
                continue
            results.append({
                "url": href,
                "title": link.get_text(strip=True),
                "snippet": snippet_el.get_text(strip=True)[:300] if snippet_el else "",
            })
        logger.info(f"Bing fallback '{query}': {len(results)} results")
        return results
    except Exception as e:
        logger.warning(f"Bing fallback failed: {e}")
        return []


def _should_skip(url: str) -> bool:
    try:
        domain = urlparse(url).netloc.lower()
        return any(skip in domain for skip in _SKIP_DOMAINS)
    except Exception:
        return False


def fetch_page_content(url: str, timeout: int = 10) -> Dict:
    """
    Fetch a URL and extract its main text content.
    Returns {success, url, title, content, og_image}.
    """
    key = _cache_key(f"content:{url}")
    cached = _read_cache(key, ttl_hours=12)
    if cached:
        return cached

    result = {"success": False, "url": url, "title": "", "content": "", "og_image": ""}
    try:
        resp = httpx.get(
            url,
            headers={"User-Agent": USER_AGENT, "Accept-Language": "en-US,en;q=0.9"},
            timeout=timeout,
            follow_redirects=True,
        )
        if resp.status_code != 200:
            return result

        content_type = resp.headers.get("content-type", "")
        if "text/html" not in content_type and "text/plain" not in content_type:
            return result

        soup = BeautifulSoup(resp.text, "lxml")

        # Title
        title_el = soup.find("title")
        result["title"] = title_el.get_text(strip=True) if title_el else ""

        # OG image
        og = soup.find("meta", property="og:image")
        if og:
            result["og_image"] = og.get("content", "")

        # Remove boilerplate
        for tag in soup(["script", "style", "noscript", "nav", "header", "footer",
                          "aside", "form", "iframe", "advertisement"]):
            tag.decompose()

        # Try article / main / content div first
        main = (
            soup.find("article")
            or soup.find("main")
            or soup.find(id=re.compile(r"content|main|article|post", re.I))
            or soup.find(class_=re.compile(r"content|main|article|post|body", re.I))
        )
        if main:
            text = main.get_text(separator="\n", strip=True)
        else:
            text = soup.get_text(separator="\n", strip=True)

        # Clean up excessive whitespace
        text = re.sub(r"\n{3,}", "\n\n", text).strip()

        if len(text) < 100:
            return result

        result["content"] = text[:20000]
        result["success"] = True
        _write_cache(key, result)
        return result

    except Exception as e:
        logger.warning(f"Failed to fetch {url}: {e}")
        return result