Add Speech Recognition + LLM Judge tabs (absolute heatmap mode, grouped samples with transcripts)

app.py

@@ -1,7 +1,8 @@
 """Real World VoiceEQ Benchmark — Gradio Space (Text-to-Speech / Speech-to-Speech /
-Voice Controllability / Speech Understanding).
+Voice Controllability / Speech Understanding / Speech Recognition / LLM Judge).
 
-One Space, four tabs — one per modality. Each tab renders a heatmap-style ranking
+One Space, six tabs — one per modality (plus the LLM Judge meta-leaderboard of
+judge-vs-human agreement). Each tab renders a heatmap-style ranking
 table from a self-describing JSON file that holds a *list* of boards (an Overall
 board + one per factor). The boards are pivoted into one wide table: each factor's
 `score` becomes a heatmapped column. Rows are ranked by the first factor column by
@@ -12,10 +13,13 @@ shown as columns.
 
 Heatmap modes:
   * Default (no board-level "heatmap" key): absolute 1–5 rater scale, higher = greener
-    (all current tabs).
-  * Board with "heatmap": {"mode": "normalized"}: each factor column is scaled by its
-    own min/max across providers and the column's "direction" is honoured
-    ("asc" => lower is better, e.g. WER), so the best value is greenest.
+    (all rater-scored tabs).
+  * "heatmap": {"mode": "absolute", "stops": [...], "unit", "legend"}: fixed value
+    anchors shared by every column, so a color means the same thing across columns;
+    each column's "direction" is honoured ("asc" => lower is better) — Speech
+    Recognition's WER and LLM Judge's Pearson-r tabs.
+  * "heatmap": {"mode": "normalized"}: each factor column is scaled by its own
+    min/max across providers (direction honoured too), best value greenest.
 
 Theming: light and dark render from the same markup. All card colors live in
 `--lb-*` custom properties on `.ttslb` (light values = the canonical look) with a
@@ -51,14 +55,20 @@ MODALITIES = [
     ("Voice Controllability", "voice_creation", "voice_creation_leaderboard.json"),
     ("Speech-to-Speech", "sts", "sts_leaderboard.json"),
     ("Speech Understanding", "stt", "stt_leaderboard.json"),
+    ("Speech Recognition", "asr", "asr_leaderboard.json"),
+    ("LLM Judge", "llm_judge", "llm_judge_leaderboard.json"),
 ]
 
 # Curated audio samples (optional): a samples.json manifest next to the board
-# JSONs, each record {modality, factor, label?, model?, text?, audio} where
-# "audio" is a repo-relative path (e.g. "samples/foo.mp3") resolved against the
-# dataset (or ./data locally). A record's modality routes it to that tab, which
-# appends a "Sample Generations" section (per-factor category tabs) below the
-# board table.
+# JSONs, each record {modality, factor, label?, group?, model?, text?,
+# transcript?, audio} where "audio" is a repo-relative path (e.g.
+# "samples/foo.mp3") resolved against the dataset (or ./data locally). A
+# record's modality routes it to that tab, which appends a samples section
+# (per-factor category tabs) below the board table. "text" (generation prompt)
+# is kept but never shown; "transcript" (reference transcript, ASR golden
+# samples) is rendered next to the player. Consecutive records sharing a
+# "group" (e.g. the Accents tab's Native / 2nd language / Foreign buckets)
+# render under one shared subheading instead of per-row labels.
 SAMPLES_NAME = "samples.json"
 
 # Each board in the data carries a single primary `score` column plus a `coverage`
@@ -131,8 +141,29 @@ def _heat_t(t):
 
 def _make_color_fn(board: dict, factor_cols: list):
     """Return color_fn(col, value) -> (bg, fg), per the board's heatmap mode."""
-    if not board.get("heatmap"):
+    hm = board.get("heatmap")
+    if not hm:
         return lambda col, v: _heat_abs(v)
+    if hm.get("mode") == "absolute" and hm.get("stops"):
+        # Fixed value anchors shared by every column (board-supplied stops, e.g.
+        # WER percentages), so a color means the same thing in every column —
+        # the absolute-scale analogue of the rater tabs' 1–5 anchors. A column's
+        # "asc" direction flips the ramp (low WER = green).
+        stops = [float(s) for s in hm["stops"]]
+
+        def fn_abs(col, v):
+            x = max(stops[0], min(stops[-1], float(v)))
+            t = 1.0
+            for i, (a, b) in enumerate(zip(stops, stops[1:])):
+                if x <= b:
+                    f = 0.0 if b == a else (x - a) / (b - a)
+                    t = (i + f) / (len(stops) - 1)
+                    break
+            if col.get("direction") == "asc":
+                t = 1.0 - t
+            return _heat_t(t)
+
+        return fn_abs
     parts = board.get("participants", [])
     stats = {}
     for c in factor_cols:
@@ -224,9 +255,13 @@ def merge_boards(boards: list) -> dict:
 
     metric_columns = []
     for b in factor_boards:
-        # The factor board's description doubles as the column-header tooltip.
+        # The factor board's description doubles as the column-header tooltip;
+        # its keyMetric supplies the column's unit and sort direction (WER-style
+        # boards declare "asc" = lower is better).
+        km = b.get("keyMetric") or {}
         metric_columns.append({"field": b["id"], "label": _factor_label(b),
-                               "unit": "", "direction": "desc",
+                               "unit": km.get("unit", ""),
+                               "direction": km.get("direction", "desc"),
                                "description": (b.get("description") or "").strip()})
 
     by_model: dict = {}
@@ -265,6 +300,9 @@ def merge_boards(boards: list) -> dict:
         "title": meta.get("title", ""),
         "description": meta.get("description", ""),
         "keyMetric": {},
+        # any board declaring a heatmap mode (normalized WER-style tabs) sets it
+        # for the whole merged table
+        "heatmap": next((b.get("heatmap") for b in boards if b.get("heatmap")), None),
         "metricColumns": metric_columns,
         "participants": [by_model[m] for m in order],
     }
@@ -413,7 +451,19 @@ TABLE_CSS = """
 .ttslb .smp-row:last-child { border-bottom:0; }
 .ttslb .smp-head { width:300px; flex:none; }
 .ttslb .smp-label { font-weight:600; font-size:12.5px; }
+/* tag-bucket subheadings (ASR golden samples): one heading per group, its rows
+   indented beneath it with the head column dropped */
+.ttslb .smp-group { margin:16px 0 2px; text-align:left; font-size:12.5px; font-weight:700; }
+.ttslb .smp-panel > .smp-group:first-child { margin-top:4px; }
+.ttslb .smp-row.grp { padding-left:14px; }
 .ttslb .smp-row audio { flex:1; min-width:260px; height:32px; }
+/* transcript rows (ASR golden samples): the category tag is short, so the head
+   narrows, the player takes a fixed slot, and the reference transcript fills
+   the rest of the row */
+.ttslb .smp-row.txt .smp-head { width:150px; }
+.ttslb .smp-row.txt audio { flex:0 0 300px; min-width:300px; }
+.ttslb .smp-text { flex:1; min-width:220px; text-align:left; font-size:12.5px;
+                   line-height:1.45; color:var(--lb-ink-2); }
 .ttslb .smp-pending { color:var(--lb-ink-3); font-size:12px; }
 /* Header logo swap: each img bakes its default visibility inline as
    display:var(--lb-logo-*, ...) — gradio strips class-based styling from
@@ -579,9 +629,12 @@ def _row_html(p, heat_cols, primary_col, cov_col, rank_label, color_fn, marker_o
         else:
             (bg, fg), (bgd, fgd) = color_fn(c, v)
             cls = "pill primary" if c is primary_col else "pill"
+            # unit suffix ("%", "x") rides along in the pill; data-v stays the
+            # bare number so sorting and CSV export keep working
+            txt = f"{v:.2f}{html.escape(c.get('unit') or '')}"
             cells.append(
                 f"<td data-v='{v}'><span class='{cls}' style='--hm-bg:{bg};--hm-fg:{fg};"
-                f"--hm-bg-dark:{bgd};--hm-fg-dark:{fgd}'>{v:.2f}</span></td>"
+                f"--hm-bg-dark:{bgd};--hm-fg-dark:{fgd}'>{txt}</span></td>"
             )
     # coverage: plain count (e.g. "9/9"), not heatmapped.
     if cov_col is not None:
@@ -675,13 +728,29 @@ def render_board_html(board: dict, lic: str = "all", fname: str = "board") -> st
             f"<div><span class='notemark'>{mark}</span> {html.escape(note)}</div>"
             for note, mark in marker_of.items()) + "</div>")
 
-    # Toolbar: the 1–5 score-scale legend (absolute rater-scale boards only) and
-    # an icon button downloading the table as CSV, exactly as displayed (handled
-    # in APP_JS). Both sit inside .tbl-box, so they align with the table's edges.
-    legend = ("" if board.get("heatmap") else
-              "<div class='legend'><span class='legend-end'>1</span>"
-              "<span class='legend-bar'></span><span class='legend-end'>5</span>"
-              "<span>mean human rating</span></div>")
+    # Toolbar: the score-scale legend and an icon button downloading the table as
+    # CSV, exactly as displayed (handled in APP_JS). Both sit inside .tbl-box, so
+    # they align with the table's edges. The legend follows the heatmap mode:
+    # 1–5 rater scale by default, the board's own value anchors for "absolute"
+    # boards (ends flipped for asc columns so red stays on the worst end), and a
+    # generic worst→best for "normalized" boards.
+    hm = board.get("heatmap") or {}
+    if hm.get("mode") == "absolute" and hm.get("stops"):
+        unit, cap = hm.get("unit", ""), (hm.get("legend") or "").strip()
+        ends = [f"{float(s):g}{unit}" for s in (hm["stops"][0], hm["stops"][-1])]
+        if value_cols and value_cols[0].get("direction") == "asc":
+            ends.reverse()
+        legend = (f"<div class='legend'><span class='legend-end'>{html.escape(ends[0])}</span>"
+                  f"<span class='legend-bar'></span><span class='legend-end'>{html.escape(ends[1])}</span>"
+                  + (f"<span>{html.escape(cap)}</span>" if cap else "") + "</div>")
+    elif hm:
+        legend = ("<div class='legend'><span class='legend-end'>worst</span>"
+                  "<span class='legend-bar'></span><span class='legend-end'>best</span>"
+                  "<span>scaled per column</span></div>")
+    else:
+        legend = ("<div class='legend'><span class='legend-end'>1</span>"
+                  "<span class='legend-bar'></span><span class='legend-end'>5</span>"
+                  "<span>mean human rating</span></div>")
     toolbar = (f"<div class='lb-toolbar'>{legend}"
                f"<button class='csv-btn' data-fname='{html.escape(fname, quote=True)}'"
                " data-tip='Download CSV' aria-label='Download CSV'></button></div>")
@@ -699,13 +768,18 @@ def render_board_html(board: dict, lic: str = "all", fname: str = "board") -> st
     )
 
 
-def render_samples_section(samples: list, factor_labels: dict) -> str:
-    """A modality tab's "Sample Generations" card, rendered below the board
+def render_samples_section(samples: list, factor_labels: dict,
+                           heading: str = "Sample Generations") -> str:
+    """A modality tab's samples card ("Sample Generations", or "Golden Samples"
+    for ASR's human-curated references), rendered below the board
     table: one category tab per factor (first-seen manifest order, switching in
     APP_JS), each a list of label + audio player rows. A single-category
     modality (e.g. Voice Controllability) gets no tab row — just the rows. A sample's
-    "text" (transcript/prompt) and "model" attribution are kept in the manifest
-    but not displayed."""
+    "text" (generation prompt) and "model" attribution are kept in the manifest
+    but not displayed; a "transcript" (the human reference for ASR golden
+    samples) is rendered next to the player. Rows carrying a "group" cluster
+    under one shared subheading (with a sample count) instead of repeating a
+    per-row label — the manifest is expected to keep a group's rows adjacent."""
     if not samples:
         return ""
     factors = []
@@ -720,18 +794,41 @@ def render_samples_section(samples: list, factor_labels: dict) -> str:
         tabs.append(f"<button class='smp-tab{' on' if j == 0 else ''}' data-p='{j}'>"
                     f"{html.escape(f_label)} <span class='smp-n'>({len(rows)})</span></button>")
         rows_html = []
+        cur_group = None
         for s in rows:
-            head = (f"<span class='smp-label'>{html.escape(s['label'])}</span>"
-                    if s.get("label") else "")
             src = _audio_src(s.get("audio", ""))
             player = (f"<audio controls preload='none' src='{src}'></audio>"
                       if src else "<span class='smp-pending'>audio pending</span>")
-            rows_html.append(f"<div class='smp-row'><div class='smp-head'>{head}</div>{player}</div>")
+            transcript = (s.get("transcript") or "").strip()
+            group = (s.get("group") or "").strip()
+            if group:
+                # one shared subheading per run of same-group rows; the rows
+                # below it drop the head column and indent under the heading
+                if group != cur_group:
+                    n = sum(1 for r in rows if (r.get("group") or "").strip() == group)
+                    rows_html.append(f"<div class='smp-group'>{html.escape(group)} "
+                                     f"<span class='smp-n'>({n})</span></div>")
+                    cur_group = group
+                body = (f"{player}<div class='smp-text'>{html.escape(transcript)}</div>"
+                        if transcript else player)
+                rows_html.append(f"<div class='smp-row grp{' txt' if transcript else ''}'>{body}</div>")
+                continue
+            cur_group = None
+            head = (f"<span class='smp-label'>{html.escape(s['label'])}</span>"
+                    if s.get("label") else "")
+            # a reference transcript (ASR golden samples) shares the row with the
+            # player; the row's .txt class narrows the head and pins the player
+            if transcript:
+                rows_html.append(
+                    f"<div class='smp-row txt'><div class='smp-head'>{head}</div>{player}"
+                    f"<div class='smp-text'>{html.escape(transcript)}</div></div>")
+            else:
+                rows_html.append(f"<div class='smp-row'><div class='smp-head'>{head}</div>{player}</div>")
         panels.append(f"<div class='smp-panel' data-p='{j}'{'' if j == 0 else ' hidden'}>"
                       + "".join(rows_html) + "</div>")
 
     tab_row = ("<div class='smp-tabs'>" + "".join(tabs) + "</div>") if len(factors) > 1 else ""
-    return ("<div class='ttslb smp'><h3 class='smp-h'>Sample Generations</h3>"
+    return (f"<div class='ttslb smp'><h3 class='smp-h'>{html.escape(heading)}</h3>"
             + tab_row + "".join(panels) + "</div>")
 
 
@@ -783,7 +880,10 @@ def build_demo() -> gr.Blocks:
                     # this modality's curated samples, below the board
                     mod_samples = [s for s in samples if s.get("modality") == key]
                     if mod_samples:
-                        gr.HTML(render_samples_section(mod_samples, factor_labels))
+                        heading = ("Golden Samples" if key == "asr"
+                                   else "Sample Generations")
+                        gr.HTML(render_samples_section(mod_samples, factor_labels,
+                                                       heading))
 
     return demo