"""Gradio explorer — metehan777/cc-turkish-fulltext-CC-MAIN-2026-21 (index, not fulltext)"""
import gradio as gr
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
import pyarrow.compute as pc
from huggingface_hub import hf_hub_download
from urllib.parse import urlparse
import html as html_mod
import json, re, os

DATASET_ID = os.environ.get("HF_DATASET_ID", "metehan777/cc-turkish-fulltext-CC-MAIN-2026-21")
_cache = {}
_meta_cache = None
_lb_clean = None
_urls_table = None
_browse_tbl = None

_BLOCKED_PATTERN = (
    r"casino|slots?|porn|porno|xxx|bahis|escort|rulet|roulette|jackpot|"
    r"betting|gambl|seks-|sextube|adult|nsfw|hentai|viagra|cialis"
)
_BLOCKED_RE = re.compile(_BLOCKED_PATTERN, re.I)

CSS = """
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap');
:root {
  --bg: #f8fafc; --bg2: #fff; --bg3: #f1f5f9; --border: #e2e8f0;
  --text: #0f172a; --muted: #64748b; --accent: #dc2626; --accent-soft: #fef2f2;
  --row-hover: #fff5f5; --th-bg: #f8fafc; --shadow: 0 1px 3px rgba(15,23,42,.06);
}
@media (prefers-color-scheme: dark) {
  :root {
    --bg: #0c0e14; --bg2: #141824; --bg3: #1c2233; --border: #2a3347;
    --text: #e8edf5; --muted: #8b9ab5; --accent: #f87171; --accent-soft: #2a1a1a;
    --row-hover: #1f2433; --th-bg: #181d2a; --shadow: none;
  }
}
body, .gradio-container {
  background: var(--bg) !important; color: var(--text) !important;
  font-family: 'Inter', system-ui, sans-serif !important; font-size: 13px !important;
}
.gradio-container { max-width: 1280px !important; margin: 0 auto !important; padding: 16px !important; }
.hero {
  background: linear-gradient(135deg, var(--bg2) 0%, var(--bg3) 100%);
  border: 1px solid var(--border); border-radius: 12px; padding: 20px 24px; margin-bottom: 16px;
  box-shadow: var(--shadow);
}
.hero h1 { margin: 0 0 6px; font-size: 1.35rem !important; font-weight: 700 !important; color: var(--text) !important; }
.hero p { margin: 0; color: var(--muted); font-size: 0.85rem; line-height: 1.5; }
.hero a { color: var(--accent) !important; text-decoration: none; font-weight: 500; }
.stats-row { display: flex; flex-wrap: wrap; gap: 10px; margin-top: 14px; }
.stat-card {
  background: var(--bg2); border: 1px solid var(--border); border-radius: 8px;
  padding: 10px 14px; min-width: 120px; flex: 1;
}
.stat-card .lbl { font-size: 0.7rem; text-transform: uppercase; letter-spacing: .04em; color: var(--muted); }
.stat-card .val { font-size: 1.1rem; font-weight: 700; color: var(--text); margin-top: 2px; }
.toolbar {
  background: var(--bg2) !important; border: 1px solid var(--border) !important;
  border-radius: 10px !important; padding: 12px 14px !important; margin-bottom: 10px !important;
  box-shadow: var(--shadow);
}
.toolbar label { font-size: 0.75rem !important; color: var(--muted) !important; font-weight: 600 !important; }
.toolbar input, .toolbar textarea, .toolbar .wrap select {
  background: var(--bg3) !important; border: 1px solid var(--border) !important;
  border-radius: 8px !important; color: var(--text) !important;
}
.toolbar input:focus { border-color: var(--accent) !important; box-shadow: 0 0 0 2px var(--accent-soft) !important; }
.toolbar button {
  border-radius: 8px !important; font-weight: 600 !important; min-height: 38px !important;
}
.toolbar button.primary {
  background: var(--accent) !important; color: #fff !important; border: none !important;
}
.toolbar button.secondary {
  background: var(--bg3) !important; border: 1px solid var(--border) !important; color: var(--text) !important;
}
.page-bar {
  display: flex; align-items: center; justify-content: space-between; gap: 12px;
  padding: 8px 4px; color: var(--muted); font-size: 0.82rem;
}
.page-bar b { color: var(--text); }
.tab-nav { border-bottom: 1px solid var(--border) !important; gap: 4px !important; }
.tab-nav button {
  font-size: 0.85rem !important; font-weight: 500 !important; padding: 10px 16px !important;
  border-radius: 8px 8px 0 0 !important;
}
.tab-nav button.selected { color: var(--accent) !important; border-bottom: 2px solid var(--accent) !important; }
.panel { background: var(--bg2); border: 1px solid var(--border); border-radius: 10px; overflow: hidden; box-shadow: var(--shadow); }
.cc-table-wrap { overflow: auto; max-height: 62vh; border: 1px solid var(--border); border-radius: 10px; background: var(--bg2); }
.cc-table-wrap table { border-collapse: collapse; width: 100%; }
.cc-table-wrap thead th {
  background: var(--th-bg); color: var(--muted); padding: 9px 12px; text-align: left;
  font-size: 0.72rem; font-weight: 600; text-transform: uppercase; letter-spacing: .03em;
  border-bottom: 1px solid var(--border); white-space: nowrap;
}
.cc-table-wrap tbody td {
  padding: 8px 12px; border-bottom: 1px solid var(--border); font-size: 0.82rem;
  vertical-align: middle; line-height: 1.4;
}
.cc-table-wrap tbody tr:hover td { background: var(--row-hover) !important; }
.cc-table-wrap .c-num { white-space: nowrap; width: 1%; }
.cc-table-wrap .c-domain { max-width: 180px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.cc-table-wrap .c-url { max-width: 340px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.cc-table-wrap .c-url a { display: inline-block; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; vertical-align: bottom; }
.badge {
  display: inline-block; background: var(--bg3); border: 1px solid var(--border);
  border-radius: 999px; padding: 2px 8px; font-size: 0.72rem; color: var(--muted);
}
.loading-box {
  padding: 48px 20px; text-align: center; color: var(--muted);
  background: var(--bg2); border: 1px dashed var(--border); border-radius: 10px;
}
.lb-frame .table-wrap { max-height: 62vh !important; overflow: auto !important; border-radius: 10px !important; }
.lb-frame table { font-size: 0.82rem !important; }
.lb-frame td, .lb-frame th { padding: 7px 10px !important; white-space: nowrap !important; }
.lb-frame td:last-child, .lb-frame th:last-child {
  max-width: 360px !important; overflow: hidden !important; text-overflow: ellipsis !important;
}
a { color: var(--accent) !important; text-decoration: none; }
a.url-long { word-break: break-all; white-space: normal; }
a.url-clip { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; display: inline-block; max-width: 100%; }
"""


def _is_blocked(text):
    return bool(text and _BLOCKED_RE.search(str(text)))


def _is_pure_turkish(lang):
    if lang is None or (isinstance(lang, float) and pd.isna(lang)):
        return False
    parts = [p.strip().lower() for p in str(lang).split(",") if p.strip()]
    return parts == ["tur"]


def _load_json(fname):
    if fname not in _cache:
        try:
            path = hf_hub_download(DATASET_ID, fname, repo_type="dataset")
            _cache[fname] = json.load(open(path))
        except Exception as e:
            _cache[fname] = {"_error": str(e)}
    return _cache[fname]


def get_meta():
    global _meta_cache
    if _meta_cache is None:
        m = _load_json("turkish_index_summary.json")
        _meta_cache = m if isinstance(m, dict) and "_error" not in m else {}
    return _meta_cache


def _leaderboard_path():
    if "lb:path" not in _cache:
        for fname in (
            "turkish_CC-MAIN-2026-21_domain_leaderboard_webgraph.parquet",
            "turkish_CC-MAIN-2026-21_domain_leaderboard.parquet",
        ):
            try:
                _cache["lb:path"] = hf_hub_download(DATASET_ID, fname, repo_type="dataset")
                _cache["lb:fname"] = fname
                break
            except Exception:
                continue
        if "lb:path" not in _cache:
            _cache["lb:path"] = None
    return _cache.get("lb:path")


def _prepare_leaderboard():
    """Bir kez yükle + spam filtrele — her istekte tekrar etme."""
    global _lb_clean
    if _lb_clean is not None:
        return _lb_clean
    path = _leaderboard_path()
    if not path:
        return None
    cols = ["rank", "domain", "page_count", "sample_url", "hc_rank", "pr_rank", "tier"]
    try:
        table = pq.read_table(path)
        have = [c for c in cols if c in table.column_names]
        df = table.select(have).to_pandas()
        for c in ("hc_rank", "pr_rank", "page_count", "rank"):
            if c in df.columns:
                df[c] = pd.to_numeric(df[c], errors="coerce").fillna(0).astype(int)
        for c in df.columns:
            if df[c].dtype == object:
                df[c] = df[c].astype(str)
        df = df[~df["domain"].map(_is_blocked)]
        if "sample_url" in df.columns:
            df = df[~df["sample_url"].map(_is_blocked)]
        _lb_clean = df.reset_index(drop=True)
        return _lb_clean
    except Exception:
        return None


def _urls_path():
    if "urls:path" not in _cache:
        try:
            _cache["urls:path"] = hf_hub_download(
                DATASET_ID, "turkish_CC-MAIN-2026-21_urls.parquet", repo_type="dataset"
            )
        except Exception:
            _cache["urls:path"] = None
    return _cache["urls:path"]


def _urls_total():
    if "urls:total" not in _cache:
        path = _urls_path()
        _cache["urls:total"] = pq.read_metadata(path).num_rows if path else 0
    return _cache["urls:total"]


def _url_domain(url):
    try:
        h = urlparse(url or "").netloc.lower()
        return h[4:] if h.startswith("www.") else h
    except Exception:
        return ""


def _clip(s, n=72):
    s = str(s or "")
    return s if len(s) <= n else s[: n - 1] + "…"


def _hero_html():
    meta = get_meta()
    return f"""
    <div class="hero">
      <h1>🇹🇷 Common Crawl Türkçe Explorer</h1>
      <p>CC-MAIN-2026-21 · columnar index · <code>tur</code> · HTTP 200 ·
      <a href="https://huggingface.co/datasets/{DATASET_ID}" target="_blank">Dataset →</a></p>
      <div class="stats-row">
        <div class="stat-card"><div class="lbl">URL</div><div class="val">{meta.get('total_urls', 0):,}</div></div>
        <div class="stat-card"><div class="lbl">Domain</div><div class="val">{meta.get('unique_domains', 0):,}</div></div>
        <div class="stat-card"><div class="lbl">Crawl</div><div class="val">2026-21</div></div>
      </div>
    </div>"""


def _page_bar(start, end, total, pg, total_pages, extra=""):
    return (
        f'<div class="page-bar">'
        f'<span>Gösterilen <b>#{start:,}–{end:,}</b> · toplam <b>{total:,}</b></span>'
        f'<span>Sayfa <b>{pg:,}</b> / <b>{total_pages:,}</b> {extra}</span>'
        f"</div>"
    )


_COL_CLASS = {
    "#": "c-num", "sayfa": "c-num", "tier": "c-num", "hc": "c-num", "pr": "c-num",
    "lang": "c-num", "fetch": "c-num",
    "domain": "c-domain", "örnek": "c-url", "url": "c-url",
}

_LB_COLS = ["Sıra", "Domain", "Sayfa", "Tier", "HC", "PR", "Örnek URL"]


def _rows_to_html(rows, cols, url_col="url", clip_url=False):
    if not rows:
        return '<div class="loading-box">Sonuç bulunamadı.</div>'
    head = "".join(
        f"<th class='{_COL_CLASS.get(c, '')}'>{html_mod.escape(str(c))}</th>" for c in cols
    )
    body = ""
    for r in rows:
        cells = ""
        for c in cols:
            v = r.get(c, "")
            cls = _COL_CLASS.get(c, "")
            if c == url_col and v:
                sv = str(v)
                link_cls = "url-clip" if clip_url else "url-long"
                cells += (
                    f"<td class='{cls}'><a class='{link_cls}' href='{html_mod.escape(sv, quote=True)}' "
                    f"target='_blank' title='{html_mod.escape(sv)}'>{html_mod.escape(_clip(sv) if clip_url else sv)}</a></td>"
                )
            else:
                cells += (
                    f"<td class='{cls}' title='{html_mod.escape(str(v)) if v not in (None, '') else ''}'>"
                    f"{html_mod.escape(str(v)) if v not in (None, '') else '—'}</td>"
                )
        body += f"<tr>{cells}</tr>"
    return f'<div class="cc-table-wrap"><table><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table></div>'


def show_domains(kw, per_page, page, sort_by):
    df = _prepare_leaderboard()
    if df is None or len(df) == 0:
        return [], _page_bar(0, 0, 0, 1, 1), 1

    if kw and str(kw).strip():
        mask = df["domain"].str.contains(re.escape(str(kw).strip()), case=False, na=False)
        df = df[mask].copy()

    if sort_by == "Domain A→Z":
        df = df.sort_values("domain", kind="stable")
    elif sort_by == "HC ↑":
        df = df.assign(_sort=df["hc_rank"].where(df["hc_rank"] > 0, 10**12))
        df = df.sort_values(["_sort", "page_count"], ascending=[True, False], kind="stable").drop(columns="_sort")
    elif sort_by == "PR ↑":
        df = df.assign(_sort=df["pr_rank"].where(df["pr_rank"] > 0, 10**12))
        df = df.sort_values(["_sort", "page_count"], ascending=[True, False], kind="stable").drop(columns="_sort")
    else:
        df = df.sort_values("page_count", ascending=False, kind="stable")

    df = df.reset_index(drop=True)
    pp = max(1, int(per_page))
    total = len(df)
    total_pages = max(1, (total + pp - 1) // pp)
    pg = max(1, min(int(page or 1), total_pages))
    start = (pg - 1) * pp
    chunk = df.iloc[start:start + pp]

    out_rows = []
    for j, (_, r) in enumerate(chunk.iterrows()):
        hc = int(r["hc_rank"]) if "hc_rank" in r.index and pd.notna(r["hc_rank"]) else 0
        pr = int(r["pr_rank"]) if "pr_rank" in r.index and pd.notna(r["pr_rank"]) else 0
        sample = _clip(r.get("sample_url", ""), 90)
        out_rows.append([
            start + j + 1,
            r["domain"],
            f"{int(r['page_count']):,}",
            r.get("tier") or "—",
            f"#{hc:,}" if hc > 0 else "—",
            f"#{pr:,}" if pr > 0 else "—",
            sample,
        ])

    end = start + len(out_rows)
    extra = '<span class="badge">filtreli</span>' if kw and str(kw).strip() else ""
    bar = _page_bar(start + 1 if out_rows else 0, end, total, pg, total_pages, extra)
    return out_rows, bar, pg


def _load_urls_table():
    """Tüm URL parquet'ini belleğe BİR kez yükle (3 sütun)."""
    global _urls_table
    if _urls_table is not None:
        return _urls_table
    path = _urls_path()
    if not path:
        return None
    try:
        t = pq.read_table(path, columns=["url", "content_languages", "fetch_time"])
        new_cols = []
        for name in t.column_names:
            col = t.column(name)
            if pa.types.is_large_string(col.type):
                col = pc.cast(col, pa.string())
            new_cols.append(col)
        _urls_table = pa.table(dict(zip(t.column_names, new_cols)))
        _cache["urls:total"] = _urls_table.num_rows
        return _urls_table
    except Exception as e:
        print(f"URL table load error: {e}", flush=True)
        return None


def _browse_table():
    """content_languages == 'tur' ve spam olmayan satırlar — bir kez filtrele."""
    global _browse_tbl
    if _browse_tbl is not None:
        return _browse_tbl
    t = _load_urls_table()
    if t is None:
        return None
    lang = pc.utf8_trim_whitespace(t.column("content_languages"))
    pure = pc.equal(lang, "tur")
    not_blocked = pc.invert(
        pc.match_substring_regex(t.column("url"), _BLOCKED_PATTERN, ignore_case=True)
    )
    _browse_tbl = t.filter(pc.and_(pure, not_blocked))
    return _browse_tbl


def _search_table(url_kw, domain_kw):
    """Arama sonucu pyarrow tablo — sorgu başına cache'lenir."""
    key = f"search:{url_kw.strip().lower()}|{domain_kw.strip().lower()}"
    if key in _cache:
        return _cache[key]
    t = _load_urls_table()
    if t is None:
        return None
    url_col = t.column("url")
    mask = pc.invert(pc.match_substring_regex(url_col, _BLOCKED_PATTERN, ignore_case=True))
    if domain_kw.strip():
        mask = pc.and_(mask, pc.match_substring(url_col, domain_kw.strip(), ignore_case=True))
    if url_kw.strip():
        mask = pc.and_(mask, pc.match_substring(url_col, url_kw.strip(), ignore_case=True))
    res = t.filter(mask)
    if len(_cache) > 40:
        for k in [k for k in _cache if k.startswith("search:")][:20]:
            _cache.pop(k, None)
    _cache[key] = res
    return res


def _table_rows(table, start, count):
    """Sadece sayfa dilimini Python'a çevir."""
    sl = table.slice(start, count).to_pydict()
    urls = sl.get("url", [])
    langs = sl.get("content_languages", [])
    fetches = sl.get("fetch_time", [])
    rows = []
    for i, u in enumerate(urls):
        rows.append({
            "url": u or "",
            "domain": _url_domain(u),
            "lang": langs[i] if i < len(langs) else "",
            "fetch": str(fetches[i])[:19] if i < len(fetches) and fetches[i] else "",
        })
    return rows


def show_urls(kw, domain_kw, per_page, page):
    pp = max(1, int(per_page))
    has_filter = bool((kw or "").strip()) or bool((domain_kw or "").strip())

    table = _search_table(kw or "", domain_kw or "") if has_filter else _browse_table()
    if table is None:
        return (
            '<div class="loading-box">URL listesi henüz hazır değil.</div>',
            _page_bar(0, 0, 0, 1, 1),
            1,
        )

    total = table.num_rows
    total_pages = max(1, (total + pp - 1) // pp)
    pg = max(1, min(int(page or 1), total_pages))
    start = (pg - 1) * pp
    rows = _table_rows(table, start, pp)

    if has_filter:
        extra = '<span class="badge">arama</span><span class="badge">spam gizli</span>'
    else:
        extra = '<span class="badge">yalnızca tur</span><span class="badge">spam gizli</span>'

    end = start + len(rows)
    html = _rows_to_html(rows, ["url", "domain", "lang", "fetch"], url_col="url", clip_url=True)
    bar = _page_bar(start + 1 if rows else 0, end, total, pg, total_pages, extra)
    return html, bar, pg


# ── Startup: veriyi Space açılmadan önce belleğe yükle ──────────────────────
print("Preloading Turkish explorer data…", flush=True)
try:
    get_meta()
    _prepare_leaderboard()
    print(f"Leaderboard ready: {len(_lb_clean):,} domains", flush=True)
    _browse_table()
    if _browse_tbl is not None:
        print(f"URL table ready: {_urls_table.num_rows:,} rows · "
              f"{_browse_tbl.num_rows:,} pure-tur", flush=True)
except Exception as e:
    print(f"Preload warning: {e}", flush=True)


SORT_OPTS = ["Sayfa ↓", "HC ↑", "PR ↑", "Domain A→Z"]
PAGE_OPTS = ["25", "50", "100"]

with gr.Blocks(title="CC Turkish Explorer", css=CSS, theme=gr.themes.Soft(primary_hue="red")) as demo:
    gr.HTML(_hero_html())

    with gr.Tabs():
        with gr.Tab("🏆 Domainler", id="domains"):
            with gr.Group(elem_classes=["toolbar"]):
                with gr.Row(equal_height=True):
                    kwD = gr.Textbox(
                        label="Ara", placeholder="hurriyet, .tr, cumhuriyet…",
                        scale=5, container=True,
                    )
                    soD = gr.Dropdown(SORT_OPTS, value="Sayfa ↓", label="Sırala", scale=2)
                    ppD = gr.Dropdown(PAGE_OPTS, value="50", label="Sayfa başı", scale=1)
                    srD = gr.Button("Ara", variant="primary", scale=1, elem_classes=["primary"])
                with gr.Row(equal_height=True):
                    prevD = gr.Button("◀ Önceki", scale=1, elem_classes=["secondary"])
                    nextD = gr.Button("Sonraki ▶", scale=1, elem_classes=["secondary"])
            stD = gr.State(1)
            barD = gr.HTML()
            outD = gr.Dataframe(
                headers=_LB_COLS,
                datatype=["number"] + ["str"] * 6,
                interactive=False,
                wrap=False,
                column_widths=["6%", "16%", "10%", "8%", "9%", "9%", "42%"],
                elem_classes=["lb-frame"],
            )

            def _refresh_d(kw, pp, so, page):
                rows, bar, pg = show_domains(kw, pp, page, so)
                return rows, bar, pg

            def _go_d(kw, pp, so):
                return _refresh_d(kw, pp, so, 1)

            def _page_d(kw, pp, so, delta, cur):
                return _refresh_d(kw, pp, so, int(cur or 1) + delta)

            d_in = [kwD, ppD, soD, stD]
            demo.load(lambda: _refresh_d("", "50", "Sayfa ↓", 1), outputs=[outD, barD, stD])
            srD.click(_go_d, [kwD, ppD, soD], [outD, barD, stD])
            kwD.submit(_go_d, [kwD, ppD, soD], [outD, barD, stD])
            soD.change(_go_d, [kwD, ppD, soD], [outD, barD, stD])
            ppD.change(_go_d, [kwD, ppD, soD], [outD, barD, stD])
            prevD.click(lambda k, p, s, n: _page_d(k, p, s, -1, n), d_in, [outD, barD, stD])
            nextD.click(lambda k, p, s, n: _page_d(k, p, s, 1, n), d_in, [outD, barD, stD])

        with gr.Tab("🔗 URL'ler", id="urls"):
            with gr.Group(elem_classes=["toolbar"]):
                with gr.Row(equal_height=True):
                    domU = gr.Textbox(label="Domain", placeholder="cumhuriyet.com.tr", scale=3)
                    kwU = gr.Textbox(label="URL metni", placeholder="opsiyonel", scale=3)
                    ppU = gr.Dropdown(PAGE_OPTS, value="50", label="Sayfa başı", scale=1)
                    srU = gr.Button("Ara", variant="primary", scale=1)
                with gr.Row(equal_height=True):
                    prevU = gr.Button("◀ Önceki", scale=1, elem_classes=["secondary"])
                    nextU = gr.Button("Sonraki ▶", scale=1, elem_classes=["secondary"])
            stU = gr.State(1)
            barU = gr.HTML()
            outU = gr.HTML()

            def _refresh_u(kw, dom, pp, page):
                html, bar, pg = show_urls(kw, dom, pp, page)
                return html, bar, pg

            def _go_u(kw, dom, pp):
                return _refresh_u(kw, dom, pp, 1)

            def _page_u(kw, dom, pp, delta, cur):
                return _refresh_u(kw, dom, pp, int(cur or 1) + delta)

            u_in = [kwU, domU, ppU, stU]
            demo.load(lambda: _refresh_u("", "", "50", 1), outputs=[outU, barU, stU])
            srU.click(_go_u, [kwU, domU, ppU], [outU, barU, stU])
            domU.submit(_go_u, [kwU, domU, ppU], [outU, barU, stU])
            kwU.submit(_go_u, [kwU, domU, ppU], [outU, barU, stU])
            ppU.change(_go_u, [kwU, domU, ppU], [outU, barU, stU])
            prevU.click(lambda k, d, p, n: _page_u(k, d, p, -1, n), u_in, [outU, barU, stU])
            nextU.click(lambda k, d, p, n: _page_u(k, d, p, 1, n), u_in, [outU, barU, stU])

demo.launch()