Spaces:
Running
Running
| """Real World VoiceEQ Benchmark — Gradio Space (Text-to-Speech / Speech-to-Speech / | |
| Speech Understanding / ASR). | |
| One Space: an About tab (about.json prose + one clickable card per panel that | |
| jumps to its tab), then one tab per modality. Each leaderboard tab renders a | |
| heatmap-style ranking | |
| table from a self-describing JSON file that holds a *list* of boards (an Overall | |
| board + one per factor). The boards are pivoted into one wide table: each factor's | |
| `score` becomes a heatmapped column. Rows are ranked by the first factor column by | |
| default; clicking any factor column re-sorts and re-ranks, and rows with no value | |
| in the active column sink to the bottom, unranked (rank/provider/License are not | |
| sortable). The Overall board's composite score and the `coverage` count are not | |
| shown as columns. | |
| Heatmap modes: | |
| * Default (no board-level "heatmap" key): absolute 1–5 rater scale, higher = greener | |
| (all rater-scored tabs). | |
| * "heatmap": {"mode": "absolute", "stops": [...], "unit", "legend"}: fixed value | |
| anchors shared by every column, so a color means the same thing across columns; | |
| each column's "direction" is honoured ("asc" => lower is better) — the ASR | |
| WER tab (and the standalone SLM Judge Space's Pearson-r board). | |
| * "heatmap": {"mode": "normalized"}: each factor column is scaled by its own | |
| min/max across providers (direction honoured too), best value greenest. | |
| Theming: light and dark render from the same markup. All card colors live in | |
| `--lb-*` custom properties on `.ttslb` (light values = the canonical look) with a | |
| `.dark .ttslb` override set; each heatmap pill carries both themes' colors as | |
| inline custom properties. Gradio toggles the `dark` class on <body> from | |
| `?__theme=` / system preference, so the tables restyle live with the page theme. | |
| Data source resolution order (per modality file): | |
| 1. $LEADERBOARD_DATASET (HF dataset holding all three JSONs, set on the Space). | |
| Private datasets need an $HF_TOKEN Space secret with read access. | |
| 2. local ./data/<file> (local development). | |
| Run locally: pip install -r requirements.txt && python app.py | |
| """ | |
| import base64 | |
| import html | |
| import json | |
| import os | |
| import re | |
| from pathlib import Path | |
| import gradio as gr | |
| HERE = Path(__file__).resolve().parent | |
| DATA_DIR = HERE / "data" | |
| DATASET_REPO = os.environ.get("LEADERBOARD_DATASET", "").strip() | |
| # Tab label / manifest key / board JSON. One file per modality; every tab renders | |
| # the same pivoted table from its own file. The key is what samples.json records | |
| # use in their "modality" field. | |
| MODALITIES = [ | |
| ("Text-to-Speech", "tts", "tts_leaderboard.json"), | |
| ("Speech-to-Speech", "sts", "sts_leaderboard.json"), | |
| ("Speech Understanding", "stt", "stt_leaderboard.json"), | |
| ("ASR", "asr", "asr_leaderboard.json"), | |
| ] | |
| # Data that exists but is deliberately not rendered here (kept in the dataset): | |
| # voice_creation_leaderboard.json (the old "Voice Controllability" tab, retired | |
| # outright) and slm_judge_leaderboard.json (shown by the standalone SLM Judge | |
| # Space in ../slm-judge instead), plus their samples.json records and about.json | |
| # panels — render_about skips panels whose tab isn't listed above. | |
| # Board ids kept in the data files but not rendered as columns (the SLM Judge | |
| # sibling hides its "Overall" mean column this way; nothing is hidden here). | |
| HIDDEN_BOARDS: frozenset = frozenset() | |
| # Curated audio samples (optional): a samples.json manifest next to the board | |
| # JSONs, each record {modality, factor, label?, group?, verdict?, model?, | |
| # text?, transcript?, audio} where "audio" is a repo-relative path (e.g. | |
| # "samples/foo.mp3") resolved against the dataset (or ./data locally). A | |
| # record's modality routes it to that tab, which appends a samples section | |
| # (per-factor category tabs) below the board table. "text" (generation prompt) | |
| # is kept but never shown; "transcript" (reference transcript, ASR golden | |
| # samples) is rendered next to the player. Consecutive records sharing a | |
| # "group" (e.g. the Accents tab's Native / 2nd language / Foreign buckets) | |
| # render under one shared subheading instead of per-row labels; a group whose | |
| # records carry a "scenario" renders a bold "Scenario:" blurb as its lead-in | |
| # instead of the heading. | |
| # A flat record with "verdict" (better|worse) is one half of a curated TTS | |
| # contrast pair: it renders as a colored chip + player row under its group | |
| # (the eval's display name), whose shared prompt "transcript" prints once at | |
| # the lead-in. A pair's shared "reference_transcript" (the enrollment passage | |
| # audible at the head of voice-identity clips) prints italicized above that — | |
| # without it the clip seems to open with the wrong text. "model"/"score"/ | |
| # "instructions"/"tag"/"eval" ride along as provenance, never displayed. | |
| # A record with "turns" is a conversation (STS samples) rendered as a card: | |
| # header = model (or "label" for stimulus cards) + optional "score" | |
| # ({value, max, label}) badge + "verdict"/"verdict_label" chip + "caption"; | |
| # each turn is {role: user|agent, speaker?, transcript, audio?, tag?, note?}. | |
| SAMPLES_NAME = "samples.json" | |
| # Each board in the data carries a single primary `score` column plus a `coverage` | |
| # count. Coverage is kept in the merged metrics but not shown as a column (were a | |
| # board to emit one, it would render as a plain count, not heatmapped). | |
| COVERAGE_FIELD = "coverage" | |
| # Heatmap palette, red (worst) -> green (best), one row per theme. The dark row | |
| # keeps the light row's hues but deepened and luminance-matched to the dark card, | |
| # so the light ink stays >= 4.5:1 along the whole interpolated ramp. | |
| _PALETTE = [(192, 86, 74), (217, 140, 95), (232, 192, 106), (230, 224, 138), | |
| (197, 217, 107), (156, 204, 101), (139, 195, 74)] | |
| _PALETTE_DARK = [(130, 62, 54), (131, 74, 41), (121, 91, 28), (108, 103, 31), | |
| (91, 104, 34), (74, 103, 41), (73, 100, 40)] | |
| # Absolute 1–5 anchors: (score, palette colour) for the default rater-scale mode. | |
| _STOPS = (2.20, 2.50, 2.80, 3.00, 3.20, 3.50, 3.90) | |
| _ANCHORS = list(zip(_STOPS, _PALETTE)) | |
| _ANCHORS_DARK = list(zip(_STOPS, _PALETTE_DARK)) | |
| _DARK_FG = "#e9edf2" | |
| def _num(x): | |
| try: | |
| return float(x) | |
| except (TypeError, ValueError): | |
| return None | |
| def _hex(rgb): | |
| return "#%02x%02x%02x" % tuple(rgb) | |
| def _lerp(a, b, t): | |
| return [round(x + (y - x) * t) for x, y in zip(a, b)] | |
| def _ramp(anchors, v): | |
| """Interpolate v along (value, rgb) anchor stops -> hex.""" | |
| lo, hi = anchors[0][0], anchors[-1][0] | |
| v = max(lo, min(hi, v)) | |
| rgb = anchors[-1][1] | |
| for (av, argb), (bv, brgb) in zip(anchors, anchors[1:]): | |
| if v <= bv: | |
| t = 0.0 if bv == av else (v - av) / (bv - av) | |
| rgb = _lerp(argb, brgb, t) | |
| break | |
| return _hex(rgb) | |
| def _heat_abs(v): | |
| """Absolute 1–5 rater scale: higher is greener. Returns the two themes' | |
| (bg, fg) pairs: ((light bg, light fg), (dark bg, dark fg)).""" | |
| light = (_ramp(_ANCHORS, v), "#ffffff" if v <= 2.30 else "#33352f") | |
| return light, (_ramp(_ANCHORS_DARK, v), _DARK_FG) | |
| def _heat_t(t): | |
| """Normalised position t in [0,1] (0 = worst/red, 1 = best/green). | |
| Same ((bg, fg), (bg, fg)) light/dark shape as _heat_abs.""" | |
| t = max(0.0, min(1.0, t)) | |
| x = t * (len(_PALETTE) - 1) | |
| i = int(x) | |
| def pick(pal): | |
| return _hex(pal[-1] if i >= len(pal) - 1 else _lerp(pal[i], pal[i + 1], x - i)) | |
| light = (pick(_PALETTE), "#ffffff" if t <= 0.12 else "#33352f") | |
| return light, (pick(_PALETTE_DARK), _DARK_FG) | |
| def _make_color_fn(board: dict, factor_cols: list): | |
| """Return color_fn(col, value) -> (bg, fg), per the board's heatmap mode.""" | |
| hm = board.get("heatmap") | |
| if not hm: | |
| return lambda col, v: _heat_abs(v) | |
| if hm.get("mode") == "absolute" and hm.get("stops"): | |
| # Fixed value anchors shared by every column (board-supplied stops, e.g. | |
| # WER percentages), so a color means the same thing in every column — | |
| # the absolute-scale analogue of the rater tabs' 1–5 anchors. A column's | |
| # "asc" direction flips the ramp (low WER = green). | |
| stops = [float(s) for s in hm["stops"]] | |
| def fn_abs(col, v): | |
| x = max(stops[0], min(stops[-1], float(v))) | |
| t = 1.0 | |
| for i, (a, b) in enumerate(zip(stops, stops[1:])): | |
| if x <= b: | |
| f = 0.0 if b == a else (x - a) / (b - a) | |
| t = (i + f) / (len(stops) - 1) | |
| break | |
| if col.get("direction") == "asc": | |
| t = 1.0 - t | |
| return _heat_t(t) | |
| return fn_abs | |
| parts = board.get("participants", []) | |
| stats = {} | |
| for c in factor_cols: | |
| vals = [_num((p.get("metrics") or {}).get(c["field"])) for p in parts] | |
| vals = [x for x in vals if x is not None] | |
| stats[c["field"]] = (min(vals), max(vals)) if vals else (0.0, 1.0) | |
| def fn(col, v): | |
| lo, hi = stats[col["field"]] | |
| t = 0.5 if hi == lo else (v - lo) / (hi - lo) | |
| if col.get("direction") == "asc": # lower is better (e.g. WER) | |
| t = 1.0 - t | |
| return _heat_t(t) | |
| return fn | |
| def _data_path(data_name: str) -> Path: | |
| if DATASET_REPO: | |
| from huggingface_hub import hf_hub_download | |
| return Path(hf_hub_download( | |
| DATASET_REPO, data_name, repo_type="dataset", | |
| token=os.environ.get("HF_TOKEN"), | |
| )) | |
| local = DATA_DIR / data_name | |
| if local.exists(): | |
| return local | |
| raise FileNotFoundError(f"No data found. Set $LEADERBOARD_DATASET or add {local}.") | |
| def load_boards(data_name: str) -> list: | |
| """The full list of boards in a modality's data file, minus any HIDDEN_BOARDS. | |
| A single-object file (legacy shape) is wrapped so callers always get a list.""" | |
| data = json.loads(_data_path(data_name).read_text()) | |
| boards = data if isinstance(data, list) else [data] | |
| return [b for b in boards if b.get("id") not in HIDDEN_BOARDS] | |
| def load_samples() -> list: | |
| """samples.json from the dataset (or ./data), [] when absent.""" | |
| try: | |
| return json.loads(_data_path(SAMPLES_NAME).read_text()) | |
| except Exception: | |
| return [] | |
| def _audio_src(rel_path: str) -> str: | |
| """Servable URL for a repo-relative audio path. Files are pulled into the HF | |
| cache (dataset mode) or read from ./data, then exposed through gradio's | |
| static file route — both roots are passed to launch(allowed_paths=...).""" | |
| if DATASET_REPO: | |
| from huggingface_hub import hf_hub_download | |
| try: | |
| local = hf_hub_download( | |
| DATASET_REPO, rel_path, repo_type="dataset", | |
| token=os.environ.get("HF_TOKEN"), | |
| ) | |
| return f"/gradio_api/file={local}" | |
| except Exception: | |
| return "" | |
| local = DATA_DIR / rel_path | |
| return f"/gradio_api/file={local}" if local.exists() else "" | |
| def _factor_label(board: dict) -> str: | |
| """Column label for a factor board: its title without the modality prefix | |
| ("TTS — Acting / Role-fit" → "Acting / Role-fit").""" | |
| t = board.get("title", "") | |
| return t.split(" — ", 1)[1] if " — " in t else t | |
| def merge_boards(boards: list) -> dict: | |
| """Pivot the per-factor boards into one wide board for a single table. | |
| The Overall board (id "overall") supplies per-model license/size and any | |
| provider-level `note` (rendered as a marker on the model name); its rank, | |
| composite score and `coverage` count are kept on the participant but not | |
| displayed — ranking is dynamic, by whichever column the table is sorted on. | |
| Every other board contributes one factor column keyed by its id, whose value | |
| is that board's `score` for the model. The participant roster is shared, so a | |
| model's row gathers its score from each board. A file with no Overall board | |
| (a single-factor file) treats every board as a | |
| factor board; license/size then come from the factor boards' participants.""" | |
| if not boards: | |
| return {"title": "", "description": "", "keyMetric": {}, | |
| "metricColumns": [], "participants": []} | |
| overall = next((b for b in boards if b.get("id") == "overall"), None) | |
| factor_boards = [b for b in boards if b is not overall] | |
| metric_columns = [] | |
| for b in factor_boards: | |
| # The factor board's description doubles as the column-header tooltip; | |
| # its keyMetric supplies the column's unit and sort direction (WER-style | |
| # boards declare "asc" = lower is better). | |
| km = b.get("keyMetric") or {} | |
| metric_columns.append({"field": b["id"], "label": _factor_label(b), | |
| "unit": km.get("unit", ""), | |
| "direction": km.get("direction", "desc"), | |
| "description": (b.get("description") or "").strip(), | |
| # a note on the board itself footnotes the header | |
| "note": (b.get("note") or "").strip()}) | |
| by_model: dict = {} | |
| order: list = [] | |
| def slot(p): | |
| model = p["model"] | |
| e = by_model.get(model) | |
| if e is None: | |
| e = {"rank": None, "model": model, "license": p.get("license"), | |
| "sizeB": p.get("sizeB"), "note": None, "notes": {}, | |
| "metrics": {"composite": None, "coverage": None}} | |
| by_model[model] = e | |
| order.append(model) | |
| return e | |
| for p in (overall.get("participants", []) if overall else []): | |
| e = slot(p) | |
| e["rank"] = p.get("rank") | |
| e["license"] = p.get("license") | |
| e["sizeB"] = p.get("sizeB") | |
| e["note"] = p.get("note") # provider-level caveat -> marker on the name | |
| e["metrics"]["composite"] = (p.get("metrics") or {}).get("score") | |
| e["metrics"]["coverage"] = (p.get("metrics") or {}).get("coverage") | |
| for b in factor_boards: | |
| for p in b.get("participants", []): | |
| e = slot(p) | |
| e["metrics"][b["id"]] = (p.get("metrics") or {}).get("score") | |
| if p.get("note"): # why this factor's score is missing -> footnote | |
| e["notes"][b["id"]] = p["note"] | |
| if e["metrics"].get("coverage") is None: | |
| e["metrics"]["coverage"] = (p.get("metrics") or {}).get("coverage") | |
| meta = overall or boards[0] | |
| return { | |
| "id": "merged", | |
| "title": meta.get("title", ""), | |
| "description": meta.get("description", ""), | |
| "keyMetric": {}, | |
| # any board declaring a heatmap mode (normalized WER-style tabs) sets it | |
| # for the whole merged table | |
| "heatmap": next((b.get("heatmap") for b in boards if b.get("heatmap")), None), | |
| "metricColumns": metric_columns, | |
| "participants": [by_model[m] for m in order], | |
| } | |
| # ── About tab ─────────────────────────────────────────────────────────────── | |
| # Content lives in about.json next to the board JSONs ({intro: [...], | |
| # panels: [{tab, title, paragraphs}], outro: [...]}). Each panel renders as | |
| # one clickable card that jumps to that leaderboard tab (wired in APP_JS); | |
| # "tab" must match the MODALITIES label exactly. | |
| ABOUT_NAME = "about.json" | |
| def load_about() -> dict: | |
| """about.json from the dataset (or ./data), {} when absent — the About | |
| tab is simply skipped without it.""" | |
| try: | |
| return json.loads(_data_path(ABOUT_NAME).read_text()) | |
| except Exception: | |
| return {} | |
| def _about_p(p: str) -> str: | |
| """An intro/outro paragraph: plain text, except markdown-style | |
| [label](https://...) spans, which become links opening in a new tab | |
| (e.g. the outro's technical-report PDF).""" | |
| return "<p class='abt-p'>" + re.sub( | |
| r"\[([^\]]+)\]\((https?://[^)\s]+)\)", | |
| r"<a href='\2' target='_blank' rel='noopener'>\1</a>", | |
| html.escape(p)) + "</p>" | |
| def render_about(about: dict) -> str: | |
| """The About tab: intro paragraphs, one clickable card per leaderboard | |
| panel (title + description, click = jump to that tab), outro. Panels whose | |
| tab isn't currently in MODALITIES (e.g. Voice Controllability while its | |
| tab is retired) are kept in the data but not rendered.""" | |
| tabs = {label for label, _key, _fname in MODALITIES} | |
| parts = ["<div class='ttslb abt'>"] | |
| for p in about.get("intro", []): | |
| parts.append(_about_p(p)) | |
| cards = [] | |
| for panel in about.get("panels", []): | |
| if panel.get("tab") not in tabs: | |
| continue | |
| body = "".join(f"<p>{html.escape(q)}</p>" for q in panel.get("paragraphs", [])) | |
| cards.append( | |
| f"<div class='abt-card' role='button' tabindex='0' " | |
| f"data-tab='{html.escape(panel['tab'], quote=True)}'>" | |
| f"<div class='abt-card-t'>{html.escape(panel.get('title') or panel['tab'])}" | |
| f"<span class='abt-go'>→</span></div>{body}</div>") | |
| parts.append("<div class='abt-cards'>" + "".join(cards) + "</div>") | |
| for p in about.get("outro", []): | |
| parts.append(_about_p(p)) | |
| parts.append("</div>") | |
| return "".join(parts) | |
| def _logo_data_uri(suffix: str = "") -> str: | |
| mimes = {".avif": "image/avif", ".svg": "image/svg+xml", | |
| ".png": "image/png", ".webp": "image/webp"} | |
| for stem in ("hume-logo", "hume_logo"): | |
| for ext, mime in mimes.items(): | |
| p = HERE / f"{stem}{suffix}{ext}" | |
| if p.exists(): | |
| b64 = base64.b64encode(p.read_bytes()).decode("ascii") | |
| return f"data:{mime};base64,{b64}" | |
| return "" | |
| _LOGO_URI = _logo_data_uri() | |
| # Dark-surface wordmark (light ink, same pastel dots); when the asset is missing | |
| # the light logo is used alone, un-swapped. | |
| _LOGO_DARK_URI = _logo_data_uri("-dark") | |
| def license_choices(board: dict) -> list: | |
| seen = [] | |
| for p in board.get("participants", []): | |
| lic = (p.get("license") or "").strip() | |
| if lic and lic not in seen: | |
| seen.append(lic) | |
| return ["all"] + sorted(seen) | |
| # The card's colors all live in `--lb-*` custom properties on `.ttslb`. The light | |
| # values ARE the original look, verbatim; `.dark .ttslb` (gradio toggles `dark` on | |
| # <body> from ?__theme= / system preference) swaps in a dark set whose grays are | |
| # contrast-matched to the light hierarchy against the dark card. `color-scheme` | |
| # follows, so native audio controls and scrollbars flip too. Heatmap pills read | |
| # `--hm-*` properties that each cell sets inline for both themes. Injected once | |
| # into <head> alongside APP_JS via launch(head=...) and shared by all tabs, so | |
| # the card is a class (`.ttslb`), not an id. | |
| TABLE_CSS = """ | |
| <style> | |
| .ttslb { --lb-surface:#ffffff; --lb-border:#e5e7eb; --lb-ink:#1f2328; --lb-ink-2:#6b7280; | |
| --lb-ink-3:#9ca3af; --lb-ink-4:#c3c8d0; --lb-empty:#cbd0d6; --lb-accent:#7b61c9; | |
| --lb-row-line:#f1f2f4; --lb-row-alt:#fafafa; --lb-tip-bg:#1f2328; --lb-tip-ink:#f5f6f7; | |
| --lb-tip-border:#1f2328; --lb-shadow:rgba(0,0,0,.18); --lb-ramp:@RAMP_LIGHT@; | |
| --lb-dim-bg:#eef0f2; --lb-dim-ink:#9aa1ab; | |
| --lb-good-bg:#eaf3df; --lb-good-ink:#567d2e; --lb-bad-bg:#f9e9e6; --lb-bad-ink:#b04a3e; | |
| font-family: ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, Arial, sans-serif; | |
| color:var(--lb-ink); max-width:100%; margin:0; color-scheme:light; | |
| background:var(--lb-surface); border:1px solid var(--lb-border); border-radius:12px; padding:16px 18px; } | |
| .dark .ttslb { --lb-surface:#151a23; --lb-border:#2e3238; --lb-ink:#e7eaef; --lb-ink-2:#b0b5be; | |
| --lb-ink-3:#9098a4; --lb-ink-4:#67707e; --lb-empty:#3f444d; --lb-accent:#9c83d6; | |
| --lb-row-line:#26292e; --lb-row-alt:#202327; --lb-tip-bg:#262e3a; --lb-tip-ink:#e7eaef; | |
| --lb-tip-border:#39424f; --lb-shadow:rgba(0,0,0,.55); --lb-ramp:@RAMP_DARK@; | |
| --lb-dim-bg:#22262d; --lb-dim-ink:#79818d; | |
| --lb-good-bg:#2c3f1e; --lb-good-ink:#a3c585; --lb-bad-bg:#48261f; --lb-bad-ink:#d98a80; | |
| color-scheme:dark; } | |
| .ttslb .tbl-wrap { overflow-x:auto; } | |
| /* Natural table width + pinned widths keep the rank/provider/License columns the | |
| same on every tab; factor columns size to their single-line label (min 120px). | |
| Sparse tabs simply leave card whitespace to the right instead of stretching. */ | |
| .ttslb table.lb { border-collapse:separate; border-spacing:0; font-size:13px; } | |
| .ttslb table.lb th { color:var(--lb-ink-2); font-weight:600; font-size:12px; text-align:center; | |
| padding:8px 10px; border-bottom:1px solid var(--lb-border); white-space:nowrap; vertical-align:bottom; } | |
| .ttslb table.lb th.prov { width:300px; min-width:300px; } | |
| .ttslb table.lb th.lic { width:100px; min-width:100px; } | |
| .ttslb table.lb th.num { min-width:120px; } | |
| .ttslb table.lb th[data-ci] { cursor:pointer; user-select:none; } | |
| .ttslb table.lb th[data-ci]:hover { color:var(--lb-ink); } | |
| .ttslb table.lb th[data-sort] { color:var(--lb-ink); } | |
| .ttslb .arr { font-size:10px; color:var(--lb-ink-4); } | |
| .ttslb table.lb th[data-sort] .arr { color:var(--lb-accent); } | |
| /* CSS-only tooltip bubbles reading from data-tip: ⓘ in factor headers (drops | |
| below) and footnote markers in empty cells (rises above — marked cells often | |
| sit in low rows, where a downward bubble would poke past the table). */ | |
| .ttslb .info { position:relative; margin-left:5px; color:var(--lb-ink-4); cursor:help; font-weight:400; } | |
| .ttslb .info:hover { color:var(--lb-ink-2); } | |
| .ttslb .info::after, .ttslb .notemark[data-tip]::after { | |
| content:attr(data-tip); position:absolute; display:none; width:260px; | |
| background:var(--lb-tip-bg); color:var(--lb-tip-ink); font-size:11.5px; | |
| font-weight:400; letter-spacing:normal; line-height:1.45; text-align:left; white-space:normal; | |
| padding:8px 10px; border-radius:8px; border:1px solid var(--lb-tip-border); | |
| box-shadow:0 4px 12px var(--lb-shadow); z-index:20; pointer-events:none; } | |
| .ttslb .info::after { top:calc(100% + 6px); right:-12px; } | |
| .ttslb .notemark[data-tip]::after { bottom:calc(100% + 8px); left:50%; transform:translateX(-50%); width:240px; } | |
| .ttslb .info:hover::after { display:block; } | |
| .ttslb td.notecell { cursor:help; } | |
| .ttslb td.notecell:hover .notemark[data-tip]::after { display:block; } | |
| .ttslb table.lb th.rank { width:34px; text-align:right; } | |
| .ttslb table.lb th.prov, .ttslb table.lb th.lic { text-align:left; } | |
| .ttslb table.lb td { padding:6px 10px; border-bottom:1px solid var(--lb-row-line); text-align:center; vertical-align:middle; } | |
| .ttslb table.lb tr.alt td { background:var(--lb-row-alt); } | |
| .ttslb td.rank { color:var(--lb-ink-3); text-align:right; font-variant-numeric:tabular-nums; width:34px; } | |
| .ttslb td.prov { text-align:left; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; | |
| font-size:12.5px; white-space:nowrap; color:var(--lb-ink); } | |
| .ttslb td.lic { text-align:left; color:var(--lb-ink-2); font-size:12px; white-space:nowrap; } | |
| .ttslb .pill { display:inline-block; min-width:40px; padding:3px 8px; border-radius:6px; | |
| font-variant-numeric:tabular-nums; font-weight:500; | |
| background:var(--hm-bg); color:var(--hm-fg); } | |
| .dark .ttslb .pill { background:var(--hm-bg-dark); color:var(--hm-fg-dark); } | |
| /* focus mode (default): only the active sort column keeps its heatmap — pills | |
| in other columns grey out with dimmer numbers. The sort JS moves td.dim as | |
| the active column changes; the toggle-all button puts .allcolors on the | |
| table, which disables this rule and restores every column's colors. */ | |
| .ttslb table.lb:not(.allcolors) td.dim .pill { background:var(--lb-dim-bg); color:var(--lb-dim-ink); } | |
| .ttslb .pill.primary { min-width:46px; font-weight:700; } | |
| .ttslb .cov-count { color:var(--lb-ink-2); font-size:12px; font-variant-numeric:tabular-nums; } | |
| .ttslb .empty { color:var(--lb-empty); } | |
| /* footnote markers are content (they explain a hole or caveat a value), so they | |
| wear the secondary ink, one tier above the decorative empty-cell dots */ | |
| .ttslb .notemark { position:relative; color:var(--lb-ink-2); font-weight:600; letter-spacing:0.5px; } | |
| /* a marker after a pill hangs beside it out of flow (absolute, no offsets = | |
| static position), so starred pills stay centered in line with the column */ | |
| .ttslb .pill + .notemark { position:absolute; margin-left:3px; } | |
| /* provider-level and column-header markers ride superscript after their label | |
| (.notemark is already position:relative for its tooltip, so a top offset | |
| lifts it) */ | |
| .ttslb td.prov .notemark, .ttslb th .notemark { margin-left:2px; font-size:9.5px; top:-0.45em; } | |
| .ttslb .footnotes { margin-top:10px; color:var(--lb-ink-2); font-size:12.5px; line-height:1.7; text-align:left; } | |
| /* the symbol leading each footnote line gets a slight bump so a lone "*" reads */ | |
| .ttslb .footnotes .notemark { font-size:1.15em; margin-right:2px; } | |
| .ttslb .nomatch { padding:24px 8px; color:var(--lb-ink-3); } | |
| /* toolbar above each table: score-scale legend (left) + CSV download (right). | |
| .tbl-box shrink-wraps to the table's width (capped at the card), so both | |
| toolbar ends align with the table edges, not the card's. */ | |
| .ttslb .tbl-box { width:max-content; max-width:100%; } | |
| .ttslb .lb-toolbar { display:flex; align-items:center; justify-content:space-between; gap:12px; margin:0 0 10px; } | |
| .ttslb .legend { display:flex; align-items:center; gap:7px; font-size:11.5px; color:var(--lb-ink-2); } | |
| .ttslb .legend-bar { width:160px; height:10px; border-radius:5px; background:var(--lb-ramp); } | |
| .ttslb .legend-end { color:var(--lb-ink-3); font-variant-numeric:tabular-nums; } | |
| /* toolbar controls (right side): a vertical stack — the "Toggle colors" | |
| switch on top, "Download CSV" + bordered glyph button below, labels left / | |
| controls right-aligned (track and box are both 26px wide, so their right | |
| edges line up). The labels are display-styled — muted ink, no hover cue — | |
| but stay inside the buttons, so they still enlarge the hit target. Buttons | |
| inside gr.HTML get gradio's own button skin, so every contested property | |
| here (and on .smp-tab) is element-qualified and !important. */ | |
| .ttslb .lb-toolbar .btns { display:flex; flex-direction:column; align-items:flex-end; gap:7px; } | |
| .ttslb button.hm-btn, .ttslb button.csv-btn { display:inline-flex; align-items:center; gap:7px; | |
| margin:0; appearance:none; cursor:pointer; font-family:inherit; line-height:1; | |
| border:0 !important; box-shadow:none !important; background:transparent !important; | |
| padding:0 !important; color:var(--lb-ink-2) !important; | |
| font-size:11.5px !important; font-weight:600 !important; } | |
| /* the switch: off = focus mode (only the sorted column colored); on (.on, | |
| accent track, knob right) = every column's heatmap shown */ | |
| .ttslb .hm-track { flex:none; position:relative; width:26px; height:14px; | |
| border-radius:999px; background:var(--lb-ink-4); transition:background .15s; } | |
| .ttslb button.hm-btn:hover .hm-track { background:var(--lb-ink-3); } | |
| .ttslb .hm-knob { position:absolute; top:2px; left:2px; width:10px; height:10px; | |
| border-radius:50%; background:#ffffff; transition:left .15s; } | |
| .ttslb button.hm-btn.on .hm-track, .ttslb button.hm-btn.on:hover .hm-track { background:var(--lb-accent); } | |
| .ttslb button.hm-btn.on .hm-knob { left:14px; } | |
| /* the download glyph in its bordered box (the affordance lives on the box, | |
| not the label): a data-URI SVG background (an inline <svg> in gr.HTML would | |
| risk the sanitizer). Stroke colors are baked per state/theme below — url() | |
| can't read tokens — tracking --lb-ink-2 (idle) / --lb-ink (hover). */ | |
| .ttslb .csv-ico { flex:none; width:26px; height:26px; border:1px solid var(--lb-border); | |
| border-radius:6px; background-image:@CSV_ICON_L@; | |
| background-position:center; background-size:14px 14px; background-repeat:no-repeat; } | |
| .ttslb button.csv-btn:hover .csv-ico { border-color:var(--lb-ink-3); background-image:@CSV_ICON_LH@; } | |
| .dark .ttslb .csv-ico { background-image:@CSV_ICON_D@; } | |
| .dark .ttslb button.csv-btn:hover .csv-ico { background-image:@CSV_ICON_DH@; } | |
| /* per-tab Sample Generations section: factor category tabs + player rows */ | |
| .ttslb .smp-h { margin:2px 2px 10px; font-size:15px; } | |
| .ttslb .smp-tabs { display:flex; gap:6px; flex-wrap:wrap; margin-bottom:8px; } | |
| .ttslb button.smp-tab { display:inline-block; margin:0; appearance:none; box-shadow:none; | |
| cursor:pointer; font-family:inherit; font-weight:600; line-height:1.4; | |
| border:1px solid var(--lb-border) !important; border-radius:999px !important; | |
| background:transparent !important; color:var(--lb-ink-2) !important; | |
| font-size:12px !important; padding:4px 11px !important; } | |
| .ttslb button.smp-tab:hover { color:var(--lb-ink) !important; border-color:var(--lb-ink-3) !important; } | |
| /* the active category wears a 2px accent ring (1px border + 1px inset) */ | |
| .ttslb button.smp-tab.on { color:var(--lb-ink) !important; border-color:var(--lb-accent) !important; | |
| box-shadow:inset 0 0 0 1px var(--lb-accent); background:var(--lb-row-alt) !important; } | |
| .ttslb .smp-n { font-weight:400; color:var(--lb-ink-3); } | |
| .ttslb .smp-row { display:flex; gap:14px; align-items:center; padding:8px 0; border-bottom:1px solid var(--lb-row-line); } | |
| .ttslb .smp-row:last-child { border-bottom:0; } | |
| .ttslb .smp-head { width:300px; flex:none; } | |
| .ttslb .smp-label { font-weight:600; font-size:12.5px; } | |
| /* tag-bucket subheadings (ASR golden samples): one heading per group, its rows | |
| indented beneath it with the head column dropped */ | |
| .ttslb .smp-group { margin:16px 0 2px; text-align:left; font-size:12.5px; font-weight:700; } | |
| .ttslb .smp-panel > .smp-group:first-child { margin-top:4px; } | |
| .ttslb .smp-row.grp { padding-left:14px; } | |
| .ttslb .smp-gnote { margin:2px 0 8px; max-width:760px; text-align:left; | |
| font-size:12px; line-height:1.5; color:var(--lb-ink-2); } | |
| .ttslb .smp-gnote strong { color:var(--lb-ink); } | |
| /* Better/Worse contrast pairs (curated TTS samples): fixed-width verdict | |
| chips keep the pair's two players aligned; inks match the STS verdict | |
| dots below. The pair's shared prompt text (.smp-gtx) sits under its group | |
| heading — .long (the multi-minute long-form evals) caps it into a scroll | |
| box instead of a wall of prose. */ | |
| .ttslb .smp-chip { flex:none; width:60px; text-align:center; font-size:11px; font-weight:700; | |
| letter-spacing:.2px; padding:3px 0; border-radius:999px; } | |
| .ttslb .smp-chip.better { background:var(--lb-good-bg); color:var(--lb-good-ink); } | |
| .ttslb .smp-chip.worse { background:var(--lb-bad-bg); color:var(--lb-bad-ink); } | |
| .ttslb .smp-gtx { margin:2px 0 4px; padding-left:14px; max-width:760px; text-align:left; | |
| font-size:12.5px; line-height:1.5; color:var(--lb-ink-2); } | |
| .ttslb .smp-gtx.long { max-height:76px; overflow-y:auto; margin-left:14px; padding:8px 10px; | |
| border:1px solid var(--lb-row-line); border-radius:8px; } | |
| /* the voice-identity pairs' enrollment passage, heard before the prompt text */ | |
| .ttslb .smp-gtx.ref { font-style:italic; margin-bottom:0; } | |
| /* a Scenario blurb starting the next group gets breathing room after the | |
| previous group's cards */ | |
| .ttslb .smp-convo + .smp-gnote { margin-top:20px; } | |
| /* conversation cards (STS samples): header = model / score badge / verdict | |
| chip, then role-guttered turns. User turns wear the secondary ink; agent | |
| turns the primary ink with the accent on the role tag. */ | |
| .ttslb .smp-convo { border:1px solid var(--lb-border); border-radius:10px; | |
| padding:12px 14px; margin:10px 0; text-align:left; } | |
| .ttslb .smp-cv-head { display:flex; align-items:center; gap:10px; flex-wrap:wrap; } | |
| .ttslb .smp-cv-model { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; | |
| font-size:12.5px; font-weight:600; } | |
| .ttslb .smp-cv-label { font-size:12.5px; font-weight:700; } | |
| /* outcome titles carry their verdict as a leading colored dot */ | |
| .ttslb .smp-cv-label.good::before, .ttslb .smp-cv-label.poor::before { | |
| content:'●'; margin-right:6px; font-size:9px; vertical-align:1.5px; } | |
| .ttslb .smp-cv-label.good::before { color:#567d2e; } | |
| .dark .ttslb .smp-cv-label.good::before { color:#a3c585; } | |
| .ttslb .smp-cv-label.poor::before { color:#b04a3e; } | |
| .dark .ttslb .smp-cv-label.poor::before { color:#d98a80; } | |
| .ttslb .smp-score { border:1px solid var(--lb-border); border-radius:999px; | |
| padding:2px 9px; font-size:11.5px; color:var(--lb-ink-2); | |
| font-variant-numeric:tabular-nums; white-space:nowrap; } | |
| .ttslb .smp-verdict { font-size:11.5px; font-weight:600; color:var(--lb-ink-2); } | |
| .ttslb .smp-verdict::before { content:'●'; margin-right:5px; font-size:9px; vertical-align:1px; } | |
| .ttslb .smp-verdict.good { color:#567d2e; } | |
| .dark .ttslb .smp-verdict.good { color:#a3c585; } | |
| .ttslb .smp-verdict.poor { color:#b04a3e; } | |
| .dark .ttslb .smp-verdict.poor { color:#d98a80; } | |
| .ttslb .smp-cv-cap { margin:5px 0 2px; font-size:12px; line-height:1.5; color:var(--lb-ink-2); } | |
| .ttslb .smp-cv-turns { margin-top:6px; } | |
| .ttslb .smp-cv-turn { display:flex; gap:12px; padding:7px 0; } | |
| .ttslb .smp-cv-turn + .smp-cv-turn { border-top:1px solid var(--lb-row-line); } | |
| .ttslb .smp-role { width:90px; flex:none; padding-top:7px; font-size:10.5px; | |
| font-weight:700; letter-spacing:.5px; text-transform:uppercase; | |
| color:var(--lb-ink-3); } | |
| .ttslb .smp-cv-turn.agent .smp-role { color:var(--lb-accent); } | |
| .ttslb .smp-cv-body { flex:1; min-width:0; } | |
| .ttslb .smp-cv-body audio { display:block; width:300px; max-width:100%; | |
| height:32px; margin-bottom:4px; } | |
| .ttslb .smp-cv-text { font-size:12.5px; line-height:1.5; } | |
| .ttslb .smp-cv-turn.user .smp-cv-text { color:var(--lb-ink-2); } | |
| .ttslb .smp-tag { display:inline-block; border:1px solid var(--lb-border); | |
| border-radius:999px; padding:0 7px; margin-left:4px; | |
| font-size:10.5px; color:var(--lb-ink-3); white-space:nowrap; } | |
| .ttslb .smp-cv-note { margin-top:3px; font-size:11.5px; font-style:italic; | |
| color:var(--lb-ink-3); } | |
| .ttslb .smp-row audio { flex:1; min-width:260px; height:32px; } | |
| /* transcript rows (ASR golden samples): the category tag is short, so the head | |
| narrows, the player takes a fixed slot, and the reference transcript fills | |
| the rest of the row */ | |
| .ttslb .smp-row.txt .smp-head { width:150px; } | |
| .ttslb .smp-row.txt audio { flex:0 0 300px; min-width:300px; } | |
| .ttslb .smp-text { flex:1; min-width:220px; text-align:left; font-size:12.5px; | |
| line-height:1.45; color:var(--lb-ink-2); } | |
| .ttslb .smp-pending { color:var(--lb-ink-3); font-size:12px; } | |
| /* About tab: intro/outro prose + one clickable card per leaderboard panel | |
| (title + description; the whole card jumps to that tab via APP_JS). */ | |
| .ttslb.abt { max-width:920px; } | |
| .ttslb .abt-p { margin:8px 2px; font-size:13px; line-height:1.55; color:var(--lb-ink-2); text-align:left; } | |
| /* gradio 6 skins every anchor like a button (2px 8px padding + press | |
| transform) — flatten that back to a plain inline text link */ | |
| .ttslb .abt-p a { color:var(--lb-accent); padding:0 !important; } | |
| .ttslb .abt-p a:active { transform:none !important; } | |
| .ttslb .abt-cards { display:grid; grid-template-columns:repeat(auto-fill, minmax(360px, 1fr)); | |
| gap:12px; margin:14px 0 12px; } | |
| .ttslb .abt-card { border:1px solid var(--lb-border); border-radius:10px; padding:13px 16px; | |
| cursor:pointer; text-align:left; } | |
| .ttslb .abt-card:hover { border-color:var(--lb-accent); box-shadow:inset 0 0 0 1px var(--lb-accent); } | |
| .ttslb .abt-card-t { display:flex; align-items:baseline; justify-content:space-between; gap:10px; | |
| font-size:13.5px; font-weight:700; } | |
| .ttslb .abt-go { color:var(--lb-ink-4); font-weight:400; } | |
| .ttslb .abt-card:hover .abt-go { color:var(--lb-accent); } | |
| .ttslb .abt-card p { margin:7px 0 0; font-size:12.5px; line-height:1.5; color:var(--lb-ink-2); } | |
| /* Header logo swap: each img bakes its default visibility inline as | |
| display:var(--lb-logo-*, ...) — gradio strips class-based styling from | |
| gr.HTML imgs, but inline styles and inherited custom properties survive. | |
| This one rule flips both under the dark theme. */ | |
| .dark { --lb-logo-light:none; --lb-logo-dark:inline; } | |
| /* hide the gradio chrome footer (Use via API · Built with Gradio · Settings) */ | |
| footer { display:none !important; } | |
| </style> | |
| """ | |
| def _ramp_css(palette) -> str: | |
| """linear-gradient for the legend's 1–5 bar; colors clamp outside the anchor | |
| span exactly like the pill ramp does.""" | |
| stops = ", ".join(f"{_hex(rgb)} {(v - 1) / 4 * 100:.1f}%" for v, rgb in zip(_STOPS, palette)) | |
| return f"linear-gradient(90deg, {_hex(palette[0])} 0%, {stops}, {_hex(palette[-1])} 100%)" | |
| def _csv_icon_css(stroke: str) -> str: | |
| """Fully percent-encoded data-URI url(...) of the download glyph | |
| (arrow into tray), stroked in the given color.""" | |
| from urllib.parse import quote | |
| svg = ("<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'>" | |
| f"<path fill='none' stroke='{stroke}' stroke-width='1.8' " | |
| "stroke-linecap='round' stroke-linejoin='round' " | |
| "d='M8 2.5v7.5M4.5 6.5 8 10l3.5-3.5M3 13.5h10'/></svg>") | |
| return f"url(\"data:image/svg+xml,{quote(svg, safe='')}\")" | |
| TABLE_CSS = (TABLE_CSS.replace("@RAMP_LIGHT@", _ramp_css(_PALETTE)) | |
| .replace("@RAMP_DARK@", _ramp_css(_PALETTE_DARK)) | |
| # idle/hover strokes mirror --lb-ink-2/--lb-ink per theme | |
| .replace("@CSV_ICON_L@", _csv_icon_css("#6b7280")) | |
| .replace("@CSV_ICON_LH@", _csv_icon_css("#1f2328")) | |
| .replace("@CSV_ICON_D@", _csv_icon_css("#b0b5be")) | |
| .replace("@CSV_ICON_DH@", _csv_icon_css("#e7eaef"))) | |
| # Behaviour, injected into <head> (a <script> inside gr.HTML would not execute); | |
| # delegated listeners survive filter re-renders. Concerns: | |
| # * click-to-sort — every row stays visible: rows with a value in the active | |
| # column are sorted and renumbered in the `#` column; rows without one sink | |
| # to the bottom, unranked ("—"). Striping is reapplied in display order. Idle | |
| # sortable headers keep a faint ↕ glyph; the active column shows ▲/▼ instead. | |
| # Focus mode follows: the active column's pills keep their heatmap, all | |
| # other pill cells are marked .dim (greyed unless .allcolors is on). | |
| # * toggle-all-colors — the toolbar's grid button flips .allcolors on the | |
| # table, restoring (or re-hiding) every column's heatmap at once. | |
| # * CSV download — serialises the button's own table exactly as displayed | |
| # (current sort order, current license filter, raw data-v values). | |
| # * About-tab panel cards — click (or Enter/Space) jumps to that tab. | |
| # * sample category tabs — one factor panel visible at a time per section. | |
| APP_JS = """ | |
| <script> | |
| (function () { | |
| function csvField(s) { | |
| s = String(s == null ? '' : s); | |
| return /[",\\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s; | |
| } | |
| document.addEventListener('click', function (e) { | |
| if (!e.target.closest) return; | |
| var chip = e.target.closest('.ttslb .abt-card'); | |
| if (chip) { // About-tab panel card: jump to that panel's tab | |
| var want = chip.getAttribute('data-tab'); | |
| var tabs = document.querySelectorAll("button[role='tab']"); | |
| for (var i = 0; i < tabs.length; i++) { | |
| if (tabs[i].textContent.trim() === want) { tabs[i].click(); window.scrollTo(0, 0); break; } | |
| } | |
| return; | |
| } | |
| var btn = e.target.closest('.ttslb .csv-btn'); | |
| if (btn) { | |
| var card = btn.closest('.ttslb'); | |
| var table = card && card.querySelector('table.lb'); | |
| if (!table || !table.tHead) return; | |
| var heads = Array.prototype.map.call(table.tHead.rows[0].cells, function (th) { | |
| var c = th.cloneNode(true); | |
| c.querySelectorAll('.arr,.info').forEach(function (n) { n.remove(); }); | |
| return c.textContent.trim(); | |
| }); | |
| var lines = [heads.map(csvField).join(',')]; | |
| Array.prototype.forEach.call(table.tBodies[0].rows, function (r) { | |
| lines.push(Array.prototype.map.call(r.cells, function (td) { | |
| if (td.hasAttribute('data-v')) return csvField(td.getAttribute('data-v')); | |
| var c = td.cloneNode(true); // footnote markers stay out of the CSV | |
| c.querySelectorAll('.notemark').forEach(function (n) { n.remove(); }); | |
| return csvField(c.textContent.trim()); | |
| }).join(',')); | |
| }); | |
| var a = document.createElement('a'); | |
| a.href = URL.createObjectURL(new Blob([lines.join('\\n') + '\\n'], { type: 'text/csv' })); | |
| a.download = 'rw-voice-eq-' + (btn.getAttribute('data-fname') || 'board') + '.csv'; | |
| document.body.appendChild(a); a.click(); a.remove(); | |
| setTimeout(function () { URL.revokeObjectURL(a.href); }, 1000); | |
| return; | |
| } | |
| var hmb = e.target.closest('.ttslb .hm-btn'); | |
| if (hmb) { // "Toggle colors" switch: all-colors mode for this card's table | |
| var hcard = hmb.closest('.ttslb'); | |
| var htable = hcard && hcard.querySelector('table.lb'); | |
| if (htable) { | |
| htable.classList.toggle('allcolors'); | |
| var on = htable.classList.contains('allcolors'); | |
| hmb.classList.toggle('on', on); | |
| hmb.setAttribute('aria-checked', on ? 'true' : 'false'); | |
| } | |
| return; | |
| } | |
| var stab = e.target.closest('.ttslb .smp-tab'); | |
| if (stab) { | |
| var wrap = stab.closest('.smp'); | |
| wrap.querySelectorAll('.smp-tab').forEach(function (b) { b.classList.toggle('on', b === stab); }); | |
| wrap.querySelectorAll('.smp-panel').forEach(function (p) { | |
| p.hidden = p.getAttribute('data-p') !== stab.getAttribute('data-p'); | |
| }); | |
| return; | |
| } | |
| }); | |
| function cellVal(td, numeric) { | |
| var v = td ? (td.getAttribute('data-v') || '') : ''; | |
| if (v === '') return { empty: true }; | |
| if (numeric) { var f = parseFloat(v); return isNaN(f) ? { empty: true } : { n: f }; } | |
| return { s: v }; | |
| } | |
| document.addEventListener('keydown', function (e) { | |
| // About-tab panel cards are div[role=button]: Enter/Space activates | |
| if ((e.key === 'Enter' || e.key === ' ') && e.target.closest) { | |
| var card = e.target.closest('.ttslb .abt-card'); | |
| if (card) { e.preventDefault(); card.click(); } | |
| } | |
| }); | |
| document.addEventListener('click', function (e) { | |
| if (!e.target.closest) return; | |
| if (e.target.closest('.info')) return; // ⓘ tooltip icon: not a sort click | |
| var th = e.target.closest('.ttslb table.lb th[data-ci]'); | |
| if (!th) return; | |
| var table = th.closest('table.lb'); | |
| var tbody = table && table.tBodies[0]; | |
| if (!tbody) return; | |
| var ci = parseInt(th.getAttribute('data-ci'), 10); | |
| var numeric = th.getAttribute('data-num') === '1'; | |
| var cur = th.getAttribute('data-sort'); | |
| var dir = cur === 'asc' ? 'desc' : cur === 'desc' ? 'asc' | |
| : (th.getAttribute('data-desc') === '1' ? 'desc' : 'asc'); | |
| table.querySelectorAll('th').forEach(function (h) { | |
| if (h !== th) { h.removeAttribute('data-sort'); var a = h.querySelector('.arr'); if (a) a.textContent = ' ↕'; } | |
| }); | |
| th.setAttribute('data-sort', dir); | |
| var arr = th.querySelector('.arr'); if (arr) arr.textContent = dir === 'asc' ? ' ▲' : ' ▼'; | |
| var rows = Array.prototype.slice.call(tbody.rows); | |
| var ranked = [], unranked = []; | |
| rows.forEach(function (r) { | |
| if (cellVal(r.children[ci], numeric).empty) unranked.push(r); else ranked.push(r); | |
| }); | |
| var sign = dir === 'asc' ? 1 : -1; | |
| ranked.sort(function (ra, rb) { | |
| var a = cellVal(ra.children[ci], numeric), b = cellVal(rb.children[ci], numeric); | |
| var c = numeric ? (a.n - b.n) : (a.s < b.s ? -1 : a.s > b.s ? 1 : 0); | |
| return sign * c; | |
| }); | |
| ranked.concat(unranked).forEach(function (r, i) { | |
| r.className = (i + 1) % 2 === 0 ? 'alt' : ''; | |
| var rk = r.querySelector('td.rank'); | |
| if (rk) rk.textContent = i < ranked.length ? i + 1 : '—'; | |
| tbody.appendChild(r); | |
| }); | |
| // focus mode follows the sort: only the active column's pills stay | |
| // colored; every other pill cell is marked dim (a no-op visually while | |
| // the table is in .allcolors mode) | |
| Array.prototype.forEach.call(tbody.rows, function (r) { | |
| Array.prototype.forEach.call(r.cells, function (td, i) { | |
| if (td.querySelector('.pill')) td.classList.toggle('dim', i !== ci); | |
| }); | |
| }); | |
| }); | |
| })(); | |
| </script> | |
| """ | |
| def _row_html(p, heat_cols, primary_col, cov_col, rank_label, color_fn, marker_of=None, | |
| active_col=None) -> str: | |
| m = p.get("metrics", {}) or {} | |
| model_raw = str(p.get("model", "")) | |
| lic_raw = (p.get("license") or "").strip() or "—" | |
| # a provider-level note (a caveat about the model itself, e.g. "this name is | |
| # a cascade alias") marks the model name rather than any one factor cell | |
| pnote = p.get("note") | |
| pmark = (marker_of or {}).get(pnote) if pnote else None | |
| if pmark: | |
| ptip = html.escape(pnote, quote=True) | |
| prov_cell = (f"<td class='prov notecell'>{html.escape(model_raw)}" | |
| f"<span class='notemark' data-tip=\"{ptip}\">{pmark}</span></td>") | |
| else: | |
| prov_cell = f"<td class='prov'>{html.escape(model_raw)}</td>" | |
| cells = [ | |
| f"<td class='rank'>{rank_label}</td>", | |
| prov_cell, | |
| f"<td class='lic'>{html.escape(lic_raw)}</td>", | |
| ] | |
| # primary score + any extra factor columns: heatmapped pills. A participant | |
| # "note" renders as a footnote marker — alone in the cell when the value is | |
| # missing (it explains the gap), after the pill when a value is present (it | |
| # caveats the number). Marked cells also carry the note as a hover tooltip. | |
| for c in heat_cols: | |
| v = _num(m.get(c["field"])) | |
| note = (p.get("notes") or {}).get(c["field"]) | |
| mark = (marker_of or {}).get(note) | |
| tip = html.escape(note, quote=True) if mark else "" | |
| marker = f"<span class='notemark' data-tip=\"{tip}\">{mark}</span>" if mark else "" | |
| if v is None: | |
| if mark: | |
| cells.append(f"<td data-v='' class='notecell'>{marker}</td>") | |
| else: | |
| cells.append("<td data-v=''><span class='empty'>·</span></td>") | |
| else: | |
| (bg, fg), (bgd, fgd) = color_fn(c, v) | |
| cls = "pill primary" if c is primary_col else "pill" | |
| # dim = outside the initially-sorted column (focus mode); the sort | |
| # JS re-marks these as the active column changes | |
| td_classes = ((["notecell"] if mark else []) | |
| + (["dim"] if active_col is not None and c is not active_col else [])) | |
| td_cls = f" class='{' '.join(td_classes)}'" if td_classes else "" | |
| # unit suffix ("%", "x") rides along in the pill; data-v stays the | |
| # bare number so sorting and CSV export keep working | |
| txt = f"{v:.2f}{html.escape(c.get('unit') or '')}" | |
| cells.append( | |
| f"<td data-v='{v}'{td_cls}>" | |
| f"<span class='{cls}' style='--hm-bg:{bg};--hm-fg:{fg};" | |
| f"--hm-bg-dark:{bgd};--hm-fg-dark:{fgd}'>{txt}</span>{marker}</td>" | |
| ) | |
| # coverage: plain count (e.g. "9/9"), not heatmapped. | |
| if cov_col is not None: | |
| cov = _num(m.get(COVERAGE_FIELD)) | |
| if cov is None: | |
| cells.append("<td data-v=''><span class='empty'>·</span></td>") | |
| else: | |
| cov_txt = f"{int(cov)}{cov_col.get('unit', '')}" | |
| cells.append(f"<td data-v='{int(cov)}'><span class='cov-count'>{html.escape(cov_txt)}</span></td>") | |
| return "".join(cells) | |
| def render_board_html(board: dict, lic: str = "all", fname: str = "board") -> str: | |
| cols = board.get("metricColumns", []) | |
| by_field = {c["field"]: c for c in cols} | |
| key_field = (board.get("keyMetric") or {}).get("field") | |
| primary_col = (next((c for c in cols if c.get("primary")), None) | |
| or by_field.get(key_field)) | |
| cov_col = by_field.get(COVERAGE_FIELD) | |
| # Heatmapped columns: the primary score first, then any extra factor columns | |
| # (coverage is excluded — it's a count, rendered plain and shown last). | |
| factor_cols = [c for c in cols if c is not primary_col and c["field"] != COVERAGE_FIELD] | |
| heat_cols = ([primary_col] if primary_col else []) + factor_cols | |
| color_fn = _make_color_fn(board, heat_cols) | |
| # Display order after rank/provider/License: primary, extra factors, coverage. | |
| value_cols = heat_cols + ([cov_col] if cov_col else []) | |
| parts = board.get("participants", []) | |
| if lic != "all": | |
| parts = [p for p in parts if (p.get("license") or "") == lic] | |
| # Default ranking: the first value column, best first. Every row is always | |
| # shown; rows with no value in the active sort column sink to the bottom, | |
| # unranked (rank "—"). | |
| first_col = value_cols[0] if value_cols else None | |
| def _cell(p, c): | |
| return _num((p.get("metrics") or {}).get(c["field"])) | |
| if first_col is not None: | |
| ranked = [p for p in parts if _cell(p, first_col) is not None] | |
| unranked = [p for p in parts if _cell(p, first_col) is None] | |
| ranked.sort(key=lambda p: _cell(p, first_col), | |
| reverse=first_col.get("direction") != "asc") | |
| else: | |
| ranked, unranked = list(parts), [] | |
| # Footnote markers: one symbol per distinct note among the visible cells — a | |
| # column-level note marks the header (a caveat about the whole metric, shown | |
| # without a hover since the header already carries the ⓘ description tip), a | |
| # provider-level note marks the model name (a caveat about the model itself), | |
| # a note on an empty cell explains the gap, and a note on a valued cell flags | |
| # a caveat (e.g. r reported as 0 where zero variance left it undefined). | |
| # Collection order tracks visual position: headers, then rows, then cells. | |
| _MARKS = ("*", "††", "‡‡", "§§", "‖‖", "¶¶") | |
| marker_of = {} | |
| for c in value_cols: | |
| note = c.get("note") | |
| if note and note not in marker_of: | |
| marker_of[note] = _MARKS[len(marker_of) % len(_MARKS)] | |
| for p in parts: | |
| note = p.get("note") | |
| if note and note not in marker_of: | |
| marker_of[note] = _MARKS[len(marker_of) % len(_MARKS)] | |
| for c in value_cols: | |
| for p in parts: | |
| note = (p.get("notes") or {}).get(c["field"]) | |
| if note and note not in marker_of: | |
| marker_of[note] = _MARKS[len(marker_of) % len(_MARKS)] | |
| # Sortable headers (factor columns only): data-ci = column index, data-num = | |
| # numeric, data-desc = first click sorts descending. (The sort behaviour lives | |
| # in SORT_JS.) `#` is display position, renumbered on every sort; the rank, | |
| # provider, and License columns are not sortable. | |
| ths = [ | |
| "<th class='rank'>#</th>", | |
| "<th class='prov'>provider</th>", | |
| "<th class='lic'>License</th>", | |
| ] | |
| for j, c in enumerate(value_cols): | |
| d = "asc" if c.get("direction") == "asc" else "desc" | |
| init = f" data-sort='{d}'" if c is first_col else "" | |
| arrow = (" ▲" if d == "asc" else " ▼") if c is first_col else " ↕" | |
| tip = (c.get("description") or "").strip() | |
| info = (f"<span class='info' data-tip=\"{html.escape(tip, quote=True)}\">ⓘ</span>" | |
| if tip else "") | |
| hmark = marker_of.get(c.get("note")) | |
| hmarker = f"<span class='notemark'>{hmark}</span>" if hmark else "" | |
| ths.append( | |
| f"<th class='num' data-ci='{3 + j}' data-num='1' data-desc='{'1' if d == 'desc' else '0'}'{init}>" | |
| f"{html.escape(c['label'])}{hmarker}<span class='arr'>{arrow}</span>{info}</th>" | |
| ) | |
| header = "".join(ths) | |
| body = [] | |
| for i, p in enumerate(ranked + unranked): | |
| alt = "alt" if (i + 1) % 2 == 0 else "" | |
| label = i + 1 if i < len(ranked) else "—" | |
| body.append(f"<tr class='{alt}'>{_row_html(p, heat_cols, primary_col, cov_col, label, color_fn, marker_of, first_col)}</tr>") | |
| if not body: | |
| return "<div class='ttslb'><p class='nomatch'>No providers match this filter.</p></div>" | |
| footnotes = "" | |
| if marker_of: | |
| footnotes = ("<div class='footnotes'>" + "".join( | |
| f"<div><span class='notemark'>{mark}</span> {html.escape(note)}</div>" | |
| for note, mark in marker_of.items()) + "</div>") | |
| # Toolbar: the score-scale legend (left) and a vertical stack of labelled | |
| # controls (right): the "Toggle colors" switch over "Download CSV", both | |
| # handled in APP_JS. All sit inside .tbl-box, so they align with the | |
| # table's edges. The legend follows the heatmap mode: | |
| # 1–5 rater scale by default, the board's own value anchors for "absolute" | |
| # boards (ends flipped for asc columns so red stays on the worst end), and a | |
| # generic worst→best for "normalized" boards. | |
| hm = board.get("heatmap") or {} | |
| if hm.get("mode") == "absolute" and hm.get("stops"): | |
| unit, cap = hm.get("unit", ""), (hm.get("legend") or "").strip() | |
| ends = [f"{float(s):g}{unit}" for s in (hm["stops"][0], hm["stops"][-1])] | |
| if value_cols and value_cols[0].get("direction") == "asc": | |
| ends.reverse() | |
| legend = (f"<div class='legend'><span class='legend-end'>{html.escape(ends[0])}</span>" | |
| f"<span class='legend-bar'></span><span class='legend-end'>{html.escape(ends[1])}</span>" | |
| + (f"<span>{html.escape(cap)}</span>" if cap else "") + "</div>") | |
| elif hm: | |
| # normalized boards default to explaining the per-column scaling; a | |
| # board-supplied "legend" caption overrides it | |
| cap = (hm.get("legend") or "scaled per column").strip() | |
| legend = ("<div class='legend'><span class='legend-end'>worst</span>" | |
| "<span class='legend-bar'></span><span class='legend-end'>best</span>" | |
| f"<span>{html.escape(cap)}</span></div>") | |
| else: | |
| legend = ("<div class='legend'><span class='legend-end'>1</span>" | |
| "<span class='legend-bar'></span><span class='legend-end'>5</span>" | |
| "<span>mean human rating</span></div>") | |
| toolbar = (f"<div class='lb-toolbar'>{legend}" | |
| "<div class='btns'>" | |
| "<button class='hm-btn' role='switch' aria-checked='false'>" | |
| "<span class='hm-lbl'>Toggle colors</span>" | |
| "<span class='hm-track'><span class='hm-knob'></span></span></button>" | |
| f"<button class='csv-btn' data-fname='{html.escape(fname, quote=True)}'>" | |
| "<span class='csv-lbl'>Download CSV</span><span class='csv-ico'></span></button>" | |
| "</div></div>") | |
| return ( | |
| "<div class='ttslb'><div class='tbl-box'>" | |
| + toolbar | |
| + "<div class='tbl-wrap'><table class='lb'><thead><tr>" | |
| + header | |
| + "</tr></thead><tbody>" | |
| + "".join(body) | |
| + "</tbody></table></div></div>" | |
| + footnotes | |
| + "</div>" | |
| ) | |
| def _convo_html(s: dict) -> str: | |
| """A conversation sample ({"turns": [...]}) as a bordered card: header row | |
| (model or label, score badge, verdict chip), optional caption, then the | |
| turns — role gutter, per-turn player when audio exists, transcript, and | |
| optional tag pill / muted note (curator commentary).""" | |
| title = s.get("model") or s.get("label") or "" | |
| title_cls = "smp-cv-model" if s.get("model") else "smp-cv-label" | |
| # an anonymized outcome title ("Poorly rated conversation") wears its | |
| # verdict as a colored dot; chips only render when verdict_label is set | |
| tone = s.get("verdict") if s.get("verdict") in ("good", "poor") else "" | |
| if tone and not s.get("model"): | |
| title_cls += f" {tone}" | |
| head = [f"<span class='{title_cls}'>{html.escape(title)}</span>"] | |
| sc = s.get("score") | |
| if sc: | |
| head.append(f"<span class='smp-score'>{sc['value']:g} / {sc['max']:g}" | |
| f" {html.escape(sc.get('label', ''))}</span>") | |
| if s.get("verdict_label"): | |
| head.append(f"<span class='smp-verdict {tone}'>" | |
| f"{html.escape(s['verdict_label'])}</span>") | |
| parts = [f"<div class='smp-cv-head'>{''.join(head)}</div>"] | |
| if s.get("caption"): | |
| parts.append(f"<div class='smp-cv-cap'>{html.escape(s['caption'])}</div>") | |
| turns = [] | |
| for t in s.get("turns", []): | |
| role = "agent" if t.get("role") == "agent" else "user" | |
| speaker = t.get("speaker") or ("Agent" if role == "agent" else "User") | |
| src = _audio_src(t["audio"]) if t.get("audio") else "" | |
| player = f"<audio controls preload='none' src='{src}'></audio>" if src else "" | |
| txt = html.escape(t.get("transcript") or "").replace("\n", "<br>") | |
| tag = (f" <span class='smp-tag'>{html.escape(t['tag'])}</span>" | |
| if t.get("tag") else "") | |
| note = (f"<div class='smp-cv-note'>{html.escape(t['note'])}</div>" | |
| if t.get("note") else "") | |
| turns.append(f"<div class='smp-cv-turn {role}'>" | |
| f"<span class='smp-role'>{html.escape(speaker)}</span>" | |
| f"<div class='smp-cv-body'>{player}" | |
| f"<div class='smp-cv-text'>{txt}{tag}</div>{note}</div></div>") | |
| parts.append(f"<div class='smp-cv-turns'>{''.join(turns)}</div>") | |
| return f"<div class='smp-convo'>{''.join(parts)}</div>" | |
| def render_samples_section(samples: list, factor_labels: dict, | |
| heading: str = "Sample Generations") -> str: | |
| """A modality tab's samples card ("Sample Generations"; "Golden Samples" | |
| for ASR's human-curated references; "Sample Conversations" for STS's | |
| curated multi-turn contrasts), rendered below the board | |
| table: one category tab per factor (first-seen manifest order, switching in | |
| APP_JS), each a list of label + audio player rows. A single-category | |
| modality gets no tab row — just the rows. A sample's | |
| "text" (generation prompt) and "model" attribution are kept in the manifest | |
| but not displayed; a "transcript" (the human reference for ASR golden | |
| samples) is rendered next to the player. Rows carrying a "group" cluster | |
| under one shared subheading (with a sample count) instead of repeating a | |
| per-row label — the manifest is expected to keep a group's rows adjacent. | |
| Flat rows carrying a "verdict" (better|worse) are curated contrast pairs: | |
| each renders as a colored chip + player, and the pair's shared prompt | |
| transcript prints once under the group heading (whose count is dropped — | |
| "(2)" on every pair says nothing).""" | |
| if not samples: | |
| return "" | |
| factors = [] | |
| for s in samples: # first-seen factor order | |
| if s.get("factor") not in factors: | |
| factors.append(s.get("factor")) | |
| tabs, panels = [], [] | |
| for j, f in enumerate(factors): | |
| rows = [s for s in samples if s.get("factor") == f] | |
| f_label = factor_labels.get(f) or (f or "—").replace("_", " ").title() | |
| tabs.append(f"<button class='smp-tab{' on' if j == 0 else ''}' data-p='{j}'>" | |
| f"{html.escape(f_label)} <span class='smp-n'>({len(rows)})</span></button>") | |
| rows_html = [] | |
| cur_group = None | |
| shared_tx = False # the group lead-in already printed the pair's prompt | |
| for s in rows: | |
| group = (s.get("group") or "").strip() | |
| # each run of same-group records opens with a lead-in: a bold | |
| # "Scenario:" blurb when the record carries one (STS conversation | |
| # groups), else the group name subheading — with a row count for | |
| # ASR's tag buckets, but without one for Better/Worse pairs, where | |
| # "(2)" says nothing. A verdict pair is a same-text contrast, so | |
| # its shared "transcript" prints once here; the multi-minute | |
| # long-form evals' text gets a capped scroll box (.long) instead | |
| # of a wall of prose. | |
| if group and group != cur_group: | |
| grp_rows = [r for r in rows if (r.get("group") or "").strip() == group] | |
| pair = all(r.get("verdict") and not r.get("turns") for r in grp_rows) | |
| if s.get("scenario"): | |
| rows_html.append(f"<div class='smp-gnote'><strong>Scenario:" | |
| f"</strong> {html.escape(s['scenario'])}</div>") | |
| elif group != f_label: | |
| # a bucket named exactly like its tab would just repeat the | |
| # tab label — skip the subheading, keep the grouped rows | |
| count = ("" if pair else | |
| f" <span class='smp-n'>({len(grp_rows)})</span>") | |
| rows_html.append(f"<div class='smp-group'>{html.escape(group)}{count}</div>") | |
| shared_tx = False | |
| if pair: | |
| # the enrollment passage heard at the head of each clip | |
| # (voice-identity evals) — shown so the opening audio | |
| # doesn't read as a mismatch with the prompt text | |
| refs = {(r.get("reference_transcript") or "").strip() for r in grp_rows} | |
| ref = refs.pop() if len(refs) == 1 else "" | |
| if ref: | |
| rows_html.append(f"<div class='smp-gtx ref'>Reference: " | |
| f"“{html.escape(ref)}”</div>") | |
| txs = {(r.get("transcript") or "").strip() for r in grp_rows} | |
| tx = txs.pop() if len(txs) == 1 else "" | |
| if tx: | |
| shared_tx = True | |
| rows_html.append(f"<div class='smp-gtx{' long' if len(tx) > 400 else ''}'>" | |
| f"{html.escape(tx)}</div>") | |
| elif not group: | |
| shared_tx = False | |
| cur_group = group or None | |
| if s.get("turns"): | |
| rows_html.append(_convo_html(s)) | |
| continue | |
| src = _audio_src(s.get("audio", "")) | |
| player = (f"<audio controls preload='none' src='{src}'></audio>" | |
| if src else "<span class='smp-pending'>audio pending</span>") | |
| transcript = (s.get("transcript") or "").strip() | |
| if s.get("verdict"): | |
| # a contrast-pair half: Better/Worse chip + player; the | |
| # transcript rides in the row only when the lead-in didn't | |
| # already print it for the whole pair | |
| v = "better" if s["verdict"] == "better" else "worse" | |
| tx = "" if shared_tx else transcript | |
| body = (f"<span class='smp-chip {v}'>{v.capitalize()}</span>{player}" | |
| + (f"<div class='smp-text'>{html.escape(tx)}</div>" if tx else "")) | |
| rows_html.append(f"<div class='smp-row grp{' txt' if tx else ''}'>{body}</div>") | |
| continue | |
| if group: | |
| body = (f"{player}<div class='smp-text'>{html.escape(transcript)}</div>" | |
| if transcript else player) | |
| rows_html.append(f"<div class='smp-row grp{' txt' if transcript else ''}'>{body}</div>") | |
| continue | |
| head = (f"<span class='smp-label'>{html.escape(s['label'])}</span>" | |
| if s.get("label") else "") | |
| # a reference transcript (ASR golden samples) shares the row with the | |
| # player; the row's .txt class narrows the head and pins the player | |
| if transcript: | |
| rows_html.append( | |
| f"<div class='smp-row txt'><div class='smp-head'>{head}</div>{player}" | |
| f"<div class='smp-text'>{html.escape(transcript)}</div></div>") | |
| else: | |
| rows_html.append(f"<div class='smp-row'><div class='smp-head'>{head}</div>{player}</div>") | |
| panels.append(f"<div class='smp-panel' data-p='{j}'{'' if j == 0 else ' hidden'}>" | |
| + "".join(rows_html) + "</div>") | |
| tab_row = ("<div class='smp-tabs'>" + "".join(tabs) + "</div>") if len(factors) > 1 else "" | |
| return (f"<div class='ttslb smp'><h3 class='smp-h'>{html.escape(heading)}</h3>" | |
| + tab_row + "".join(panels) + "</div>") | |
| def build_demo() -> gr.Blocks: | |
| title = "Real World VoiceEQ Benchmark" | |
| # NB: the CSS/JS goes to launch(head=...) — gradio 6 moved `head` off Blocks. | |
| with gr.Blocks(title=title) as demo: | |
| if _LOGO_URI: | |
| # 26px: the current wordmark asset is cropped to its ink (the original | |
| # had padding filling ~61% of a 48px box), sized down a touch further | |
| if _LOGO_DARK_URI: | |
| logo = (f"<img src='{_LOGO_URI}' alt='Hume AI'" | |
| " style='height:26px;display:var(--lb-logo-light,inline)'/>" | |
| f"<img src='{_LOGO_DARK_URI}' alt='Hume AI'" | |
| " style='height:26px;display:var(--lb-logo-dark,none)'/>") | |
| else: | |
| logo = f"<img src='{_LOGO_URI}' alt='Hume AI' style='height:26px'/>" | |
| gr.HTML( | |
| "<div style='display:flex;align-items:center;gap:14px'>" | |
| + logo | |
| + f"<h1 style='margin:0;font-size:1.875em;font-weight:600'>{html.escape(title)}</h1>" | |
| "</div>" | |
| ) | |
| else: | |
| gr.Markdown(f"# {title}") | |
| samples = load_samples() | |
| about = load_about() | |
| with gr.Tabs(): | |
| if about: | |
| with gr.Tab("About"): | |
| gr.HTML(render_about(about)) | |
| for label, key, data_name in MODALITIES: | |
| board = merge_boards(load_boards(data_name)) | |
| factor_labels = {c["field"]: c["label"] for c in board["metricColumns"]} | |
| # CSV filenames follow the visible tab label, e.g. | |
| # rw-voice-eq-speech-understanding.csv | |
| slug = label.lower().replace(" ", "-") | |
| with gr.Tab(label): | |
| gr.Markdown(board.get("description", "")) | |
| with gr.Row(): | |
| license_filter = gr.Radio( | |
| choices=license_choices(board), | |
| value="all", | |
| label="License", | |
| ) | |
| table = gr.HTML(render_board_html(board, "all", slug)) | |
| license_filter.change( | |
| lambda lic, b=board, s=slug: render_board_html(b, lic, s), | |
| inputs=license_filter, | |
| outputs=table, | |
| ) | |
| # this modality's curated samples, below the board | |
| mod_samples = [s for s in samples if s.get("modality") == key] | |
| if mod_samples: | |
| heading = {"asr": "Golden Samples", | |
| "sts": "Sample Conversations"}.get( | |
| key, "Sample Generations") | |
| gr.HTML(render_samples_section(mod_samples, factor_labels, | |
| heading)) | |
| return demo | |
| demo = build_demo() | |
| if __name__ == "__main__": | |
| from huggingface_hub.constants import HF_HUB_CACHE | |
| # SSR mode (gradio 6 default) needs to reach the app over localhost, which | |
| # HF Spaces blocks — disable it so the Space serves normally. allowed_paths | |
| # lets the /gradio_api/file= route serve sample audio from the HF cache | |
| # (dataset mode) or ./data (local dev). head= injects the table CSS + JS | |
| # (gradio 6 moved it here from the Blocks constructor). | |
| demo.launch(ssr_mode=False, head=TABLE_CSS + APP_JS, | |
| allowed_paths=[HF_HUB_CACHE, str(DATA_DIR)]) | |