feat: active filter banner, scope/filter tab rename, refactor docs

- Rename '⚙️ Settings' tab to '🔬 Scope / Filter'
- Add active filter status banner shown across all tabs when filters reduce the row count
- Add _filter_status_banner helper wired to both filter buttons
- Add docs/dashboard-questions.md: 6-question variant-first manifest
- Add docs/refactor-plan.md: full visualization refactor design doc

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

app.py

@@ -32,9 +32,7 @@ except Exception as exc:
 _METRIC_CHOICES = list(CLASSIFICATION_METRICS.values())
 _METRIC_KEYS = list(CLASSIFICATION_METRICS.keys())
 
-_CATEGORY_CHOICES = (
-    sorted(_RAW_DF["task_category"].unique().tolist()) if not _RAW_DF.empty else []
-)
+_CATEGORY_CHOICES = sorted(_RAW_DF["task_category"].unique().tolist()) if not _RAW_DF.empty else []
 
 _DATASET_CHOICES = (
     sorted(_RAW_DF["dataset_name"].dropna().unique().tolist())
@@ -62,6 +60,7 @@ def _numeric_range(df: pd.DataFrame, col: str) -> tuple[float, float]:
 # Render helpers
 # ---------------------------------------------------------------------------
 
+
 def render_leaderboard(df: pd.DataFrame, metric_label: str):
     key = _metric_key(metric_label)
     return ranking_table(df, key), violin_plot(df, key)
@@ -94,7 +93,7 @@ def render_context(df: pd.DataFrame, metric_label: str):
 def render_variant_heatmap(
     df: pd.DataFrame,
     metric_label: str,
-    model_alias: str,
+    model_alias: list[str] | str,
     aggregate_by_dataset: bool = False,
 ):
     key = _metric_key(metric_label)
@@ -118,6 +117,7 @@ def apply_and_preview(
     dim_max: float,
     ctx_min: float,
     ctx_max: float,
+    pooling_strategies: list[str] | None = None,
 ) -> tuple[pd.DataFrame, pd.DataFrame, str]:
     filters = {
         "subsample_train": "full" if prod_only else None,
@@ -128,6 +128,7 @@ def apply_and_preview(
         "embedding_dim_max": dim_max,
         "max_context_size_min": ctx_min,
         "max_context_size_max": ctx_max,
+        "pooling_strategies": pooling_strategies or [],
     }
     result = apply_filters(df, filters)
     preview = result.head(50) if not result.empty else pd.DataFrame()
@@ -136,6 +137,7 @@ def apply_and_preview(
 
 def _hash_label(row: pd.Series) -> str:
     """Human-readable label for an embedding config hash dropdown entry."""
+
     def _s(val) -> str:
         return "" if val is None or (isinstance(val, float) and pd.isna(val)) else str(val)
 
@@ -175,11 +177,17 @@ def get_head_types_for_hash(loaded_df: pd.DataFrame, model: str, key: str) -> li
 _DEFAULT_INSP_COLS = ["kept", "run_id", "run_at", "subsample_train", "task_name", "mcc_test", "accuracy_test"]
 
 
-def build_inspector_table(loaded_df: pd.DataFrame, raw_df: pd.DataFrame,
-                          model: str, key: str, head_type: str,
-                          selected_cols: list[str] | None = None,
-                          task_cats: list[str] | None = None,
-                          tasks: list[str] | None = None) -> pd.DataFrame:
+def build_inspector_table(
+    loaded_df: pd.DataFrame,
+    raw_df: pd.DataFrame,
+    model: str,
+    key: str,
+    head_type: str,
+    selected_cols: list[str] | None = None,
+    task_cats: list[str] | None = None,
+    tasks: list[str] | None = None,
+    pooling_strategies: list[str] | None = None,
+) -> pd.DataFrame:
     if loaded_df.empty:
         return pd.DataFrame()
     masks = []
@@ -193,6 +201,8 @@ def build_inspector_table(loaded_df: pd.DataFrame, raw_df: pd.DataFrame,
         masks.append(loaded_df["task_category"].isin(task_cats))
     if tasks and "task_name" in loaded_df.columns:
         masks.append(loaded_df["task_name"].isin(tasks))
+    if pooling_strategies and "pooling_strategy" in loaded_df.columns:
+        masks.append(loaded_df["pooling_strategy"].isin(pooling_strategies))
     if not masks:
         return pd.DataFrame()
     combined = masks[0]
@@ -205,7 +215,6 @@ def build_inspector_table(loaded_df: pd.DataFrame, raw_df: pd.DataFrame,
     return sub[[c for c in cols if c in sub.columns or c == "kept"]].reset_index(drop=True)
 
 
-
 def exclude_run_ids(df: pd.DataFrame, run_ids_text: str) -> tuple[pd.DataFrame, pd.DataFrame, str]:
     ids = [r.strip() for r in run_ids_text.split(",") if r.strip()]
     result = apply_filters(df, {"exclude_run_ids": ids})
@@ -213,6 +222,17 @@ def exclude_run_ids(df: pd.DataFrame, run_ids_text: str) -> tuple[pd.DataFrame,
     return result, preview, f"{len(result)} rows active"
 
 
+def _filter_status_banner(filtered_df: pd.DataFrame, raw_df: pd.DataFrame) -> str:
+    if filtered_df.empty or raw_df.empty:
+        return ""
+    if len(filtered_df) < len(raw_df):
+        return (
+            f"⚠️ **Active scope filters** — {len(filtered_df):,} / {len(raw_df):,} rows visible. "
+            "Go to 🔬 Scope / Filter to review."
+        )
+    return ""
+
+
 # ---------------------------------------------------------------------------
 # Build UI
 # ---------------------------------------------------------------------------
@@ -226,21 +246,33 @@ _insp_default_model = all_models[0] if all_models else None
 _insp_default_keys = get_hash_choices_for_model(_LOADED_DF, _insp_default_model)
 _insp_default_heads = get_head_types_for_hash(_LOADED_DF, _insp_default_model, None)
 _ALL_INSP_COLS = ["kept"] + _LOADED_DF.columns.tolist() if not _LOADED_DF.empty else _DEFAULT_INSP_COLS
-_all_task_cats = sorted(_LOADED_DF["task_category"].dropna().unique().tolist()) if not _LOADED_DF.empty and "task_category" in _LOADED_DF.columns else []
-_all_tasks = sorted(_LOADED_DF["task_name"].dropna().unique().tolist()) if not _LOADED_DF.empty and "task_name" in _LOADED_DF.columns else []
+_all_task_cats = (
+    sorted(_LOADED_DF["task_category"].dropna().unique().tolist())
+    if not _LOADED_DF.empty and "task_category" in _LOADED_DF.columns
+    else []
+)
+_all_tasks = (
+    sorted(_LOADED_DF["task_name"].dropna().unique().tolist())
+    if not _LOADED_DF.empty and "task_name" in _LOADED_DF.columns
+    else []
+)
+_all_pooling_strategies = (
+    sorted(_LOADED_DF["pooling_strategy"].dropna().unique().tolist())
+    if not _LOADED_DF.empty and "pooling_strategy" in _LOADED_DF.columns
+    else []
+)
 
 with gr.Blocks(title="DNA Benchmark Leaderboard", theme=gr.themes.Soft()) as demo:
-
     gr.Markdown(
         "# 🧬 DNA Foundation Model Benchmark Leaderboard\n"
         "Compare DNA language models across genomics classification tasks."
     )
 
+    filter_banner = gr.Markdown("")
+
     # ── Global controls ───────────────────────────────────────────────────────
     with gr.Row():
-        metric_dd = gr.Dropdown(
-            choices=_METRIC_CHOICES, value=_METRIC_CHOICES[0], label="Metric", scale=2
-        )
+        metric_dd = gr.Dropdown(choices=_METRIC_CHOICES, value=_METRIC_CHOICES[0], label="Metric", scale=2)
 
     # ── Shared state ──────────────────────────────────────────────────────────
     loaded_df_state = gr.State(_LOADED_DF)
@@ -249,19 +281,17 @@ with gr.Blocks(title="DNA Benchmark Leaderboard", theme=gr.themes.Soft()) as dem
 
     # ── Tabs ──────────────────────────────────────────────────────────────────
     with gr.Tabs() as tabs:
-
         # ── Tab 1: Leaderboard ────────────────────────────────────────────────
         with gr.Tab("🏅 Leaderboard"):
-            leaderboard_table = gr.Dataframe(
-                label="Rankings (mean metric per category)", interactive=False, wrap=True
-            )
+            leaderboard_table = gr.Dataframe(label="Rankings (mean metric per category)", interactive=False, wrap=True)
             violin_fig = gr.Plot(label="Metric distribution per model")
 
             gr.Markdown("---\n### Drill into model variants")
             model_dd = gr.Dropdown(
                 choices=all_models,
-                value=all_models[0] if all_models else None,
-                label="Select model",
+                value=[all_models[0]] if all_models else [],
+                multiselect=True,
+                label="Select model(s)  — single model shows config variants; multiple shows one row per model",
             )
             agg_dataset_cb = gr.Checkbox(
                 label="Aggregate by dataset (show mean per benchmark dataset instead of per task)",
@@ -295,63 +325,69 @@ with gr.Blocks(title="DNA Benchmark Leaderboard", theme=gr.themes.Soft()) as dem
         with gr.Tab("🔬 Context Length"):
             context_fig = gr.Plot(label="Performance vs task sequence length")
 
-        # ── Tab 5: Settings ───────────────────────────────────────────────────
-        with gr.Tab("⚙️ Settings"):
+        # ── Tab 5: Scope / Filter ─────────────────────────────────────────────
+        with gr.Tab("🔬 Scope / Filter"):
             prod_only_cb = gr.Checkbox(
                 label="Production runs only (subsample_train IS NULL)",
                 value=False,
                 info="Currently returns no data — all runs use subsample_train=0.05.",
             )
-            exclude_models_ms = gr.Dropdown(
-                choices=all_models, multiselect=True, label="Exclude models", value=[]
+            exclude_models_ms = gr.Dropdown(choices=all_models, multiselect=True, label="Exclude models", value=[])
+            pooling_filter_ms = gr.Dropdown(
+                choices=_all_pooling_strategies,
+                multiselect=True,
+                label="Pooling strategy",
+                value=[],
+                info="Leave empty to include all pooling strategies.",
             )
             with gr.Row():
                 params_min_sl = gr.Slider(
-                    minimum=params_min_v, maximum=params_max_v,
-                    value=params_min_v, label="Model params (min)"
+                    minimum=params_min_v, maximum=params_max_v, value=params_min_v, label="Model params (min)"
                 )
                 params_max_sl = gr.Slider(
-                    minimum=params_min_v, maximum=params_max_v,
-                    value=params_max_v, label="Model params (max)"
+                    minimum=params_min_v, maximum=params_max_v, value=params_max_v, label="Model params (max)"
                 )
             with gr.Row():
                 dim_min_sl = gr.Slider(
-                    minimum=dim_min_v, maximum=dim_max_v,
-                    value=dim_min_v, label="Embedding dim (min)"
+                    minimum=dim_min_v, maximum=dim_max_v, value=dim_min_v, label="Embedding dim (min)"
                 )
                 dim_max_sl = gr.Slider(
-                    minimum=dim_min_v, maximum=dim_max_v,
-                    value=dim_max_v, label="Embedding dim (max)"
+                    minimum=dim_min_v, maximum=dim_max_v, value=dim_max_v, label="Embedding dim (max)"
                 )
             with gr.Row():
                 ctx_min_sl = gr.Slider(
-                    minimum=ctx_min_v, maximum=ctx_max_v,
-                    value=ctx_min_v, label="Max context size (min bp)"
+                    minimum=ctx_min_v, maximum=ctx_max_v, value=ctx_min_v, label="Max context size (min bp)"
                 )
                 ctx_max_sl = gr.Slider(
-                    minimum=ctx_min_v, maximum=ctx_max_v,
-                    value=ctx_max_v, label="Max context size (max bp)"
+                    minimum=ctx_min_v, maximum=ctx_max_v, value=ctx_max_v, label="Max context size (max bp)"
                 )
             apply_btn = gr.Button("Apply Filters", variant="primary")
 
             gr.Markdown("---")
-            run_ids_tb = gr.Textbox(
-                label="Exclude run_ids (comma-separated)", placeholder="run_id_1, run_id_2, …"
-            )
+            run_ids_tb = gr.Textbox(label="Exclude run_ids (comma-separated)", placeholder="run_id_1, run_id_2, …")
             exclude_runs_btn = gr.Button("Exclude run_ids")
 
             gr.Markdown("---")
             settings_count = gr.Label(label="Active rows")
             settings_preview = gr.Dataframe(label="Preview (first 50 rows)", interactive=False)
 
-            gr.Markdown("---\n### Raw Run Inspector\nSelect a model and embedding config to audit which runs were kept or dropped by dedup.")
+            gr.Markdown(
+                "---\n### Raw Run Inspector\nSelect a model and embedding config to audit which runs were kept or dropped by dedup."
+            )
             with gr.Row():
                 insp_model_dd = gr.Dropdown(choices=all_models, value=_insp_default_model, label="Model", scale=2)
-                insp_cache_dd = gr.Dropdown(choices=_insp_default_keys, value=None, label="Embedding config hash", scale=3)
+                insp_cache_dd = gr.Dropdown(
+                    choices=_insp_default_keys, value=None, label="Embedding config hash", scale=3
+                )
                 insp_head_dd = gr.Dropdown(choices=_insp_default_heads, value=None, label="Head type", scale=2)
             with gr.Row():
-                insp_task_cat_ms = gr.Dropdown(choices=_all_task_cats, value=None, multiselect=True, label="Task category", scale=2)
+                insp_task_cat_ms = gr.Dropdown(
+                    choices=_all_task_cats, value=None, multiselect=True, label="Task category", scale=2
+                )
                 insp_task_ms = gr.Dropdown(choices=_all_tasks, value=None, multiselect=True, label="Task", scale=3)
+                insp_pooling_ms = gr.Dropdown(
+                    choices=_all_pooling_strategies, value=None, multiselect=True, label="Pooling strategy", scale=2
+                )
             insp_cols_dd = gr.Dropdown(
                 choices=_ALL_INSP_COLS,
                 value=_DEFAULT_INSP_COLS,
@@ -395,12 +431,22 @@ with gr.Blocks(title="DNA Benchmark Leaderboard", theme=gr.themes.Soft()) as dem
     apply_btn.click(
         apply_and_preview,
         inputs=[
-            raw_df_state, prod_only_cb, exclude_models_ms,
-            params_min_sl, params_max_sl,
-            dim_min_sl, dim_max_sl,
-            ctx_min_sl, ctx_max_sl,
+            raw_df_state,
+            prod_only_cb,
+            exclude_models_ms,
+            params_min_sl,
+            params_max_sl,
+            dim_min_sl,
+            dim_max_sl,
+            ctx_min_sl,
+            ctx_max_sl,
+            pooling_filter_ms,
         ],
         outputs=[filtered_df_state, settings_preview, settings_count],
+    ).then(
+        _filter_status_banner,
+        inputs=[filtered_df_state, raw_df_state],
+        outputs=filter_banner,
     )
 
     # Settings — Exclude run_ids
@@ -408,6 +454,10 @@ with gr.Blocks(title="DNA Benchmark Leaderboard", theme=gr.themes.Soft()) as dem
         exclude_run_ids,
         inputs=[filtered_df_state, run_ids_tb],
         outputs=[filtered_df_state, settings_preview, settings_count],
+    ).then(
+        _filter_status_banner,
+        inputs=[filtered_df_state, raw_df_state],
+        outputs=filter_banner,
     )
 
     # Inspector — model updates hash choices; model+hash each update head_type choices independently
@@ -426,8 +476,26 @@ with gr.Blocks(title="DNA Benchmark Leaderboard", theme=gr.themes.Soft()) as dem
         [loaded_df_state, insp_model_dd, insp_cache_dd],
         insp_head_dd,
     )
-    _insp_inputs = [loaded_df_state, raw_df_state, insp_model_dd, insp_cache_dd, insp_head_dd, insp_cols_dd, insp_task_cat_ms, insp_task_ms]
-    for trigger in [insp_model_dd, insp_cache_dd, insp_head_dd, insp_cols_dd, insp_task_cat_ms, insp_task_ms]:
+    _insp_inputs = [
+        loaded_df_state,
+        raw_df_state,
+        insp_model_dd,
+        insp_cache_dd,
+        insp_head_dd,
+        insp_cols_dd,
+        insp_task_cat_ms,
+        insp_task_ms,
+        insp_pooling_ms,
+    ]
+    for trigger in [
+        insp_model_dd,
+        insp_cache_dd,
+        insp_head_dd,
+        insp_cols_dd,
+        insp_task_cat_ms,
+        insp_task_ms,
+        insp_pooling_ms,
+    ]:
         trigger.change(build_inspector_table, _insp_inputs, insp_table)
 
     # Trigger resize on tab switch so Plotly reflows to correct container width

docs/dashboard-questions.md

@@ -0,0 +1,15 @@
+# Dashboard Questions
+
+The ranking unit is a **variant** — a specific tuple of `(model_hf_id, pooling, layer, head)`.
+Model family (`model_alias`) is a grouping/filter dimension, not a ranking unit.
+
+| # | Question | Scope |
+|---|---|---|
+| Q1 | Which specific variant achieves the highest overall performance (best mean across all tasks)? | Leaderboard |
+| Q2 | Which specific variant wins each individual task? | Leaderboard |
+| Q3 | For a given model family, which config axis matters most (pooling / layer / head)? | Config ablation |
+| Q4 | Does the best config generalize across tasks, or is it task-specific? | Config ablation |
+| Q5 | What is the performance vs. speed / memory tradeoff across variants? | Efficiency |
+| Q6 | Does performance degrade as task sequence length approaches context size? | Generalization |
+
+> **Note on violin plot:** it shows distribution across all experiments per model, masking the best variant. Replace with a strip/dot plot showing per-task scores of the best variant only.

docs/refactor-plan.md

@@ -0,0 +1,133 @@
+# Dashboard Refactor Plan
+
+> See `docs/dashboard-questions.md` for the question manifest this plan targets.
+
+## Guiding principle
+
+The ranking unit is a **variant** — a concrete tuple of `(model_hf_id, pooling_strategy, layer_selection, head_type)`.  
+`model_alias` is a grouping/filter label, not a ranking unit.  
+Every visualization should default to showing the **best variant** per model family, not the mean across all experiments.
+
+---
+
+## Root gap
+
+Two things were planned but never built:
+
+1. **`get_best_variants(df, metric) → pd.DataFrame`** in `src/data.py`  
+   Should return one row per `(model_alias, task_name)` — the row belonging to the variant with the highest mean metric across all tasks for that family. This is the "best consistent config" view (Q1).
+
+2. **A "best config / all configs" toggle** in `app.py`  
+   A global Radio or Checkbox that switches `filtered_df` between:
+   - `get_best_variants(filtered_df, metric)` — one variant per family
+   - `filtered_df` as-is — all config variants (for ablation exploration)
+
+Without these, every chart is permanently in "all configs" mode and Q1/Q2 cannot be answered.
+
+---
+
+## Data layer changes (`src/data.py`)
+
+### New: `get_best_variants(df, metric) → pd.DataFrame`
+
+**Logic:**
+1. Build a task-agnostic variant key: `(model_hf_id, pooling_strategy, layer_fmt, head_type)` where `layer_fmt = _fmt_layer(layer_selection_params)` (same normalisation used in `heatmap_variants`)
+2. For each `(model_alias, variant_key)`, compute mean of `metric` across all tasks
+3. Per `model_alias`, keep only the variant_key with the highest mean
+4. Return the subset of `df` rows belonging to those winning variants
+
+**Notes:**
+- The variant key must be task-agnostic (same approach as `heatmap_variants._embed_key`)
+- `embedding_config_hash` cannot be used here — it is task-specific
+- Result still has one row per task (not one row per variant), so all per-task charts work unchanged
+- When `metric` changes, the winning variant may change — the function must be called with the currently selected metric
+
+### No changes needed to `deduplicate` or `apply_filters`
+
+---
+
+## App layer changes (`src/app.py`)
+
+### New: best-config toggle
+
+Add a global `gr.Radio` with choices `["Best config per model", "All configs"]`, default `"Best config per model"`.
+
+Add a new `gr.State: display_df` that is derived from `filtered_df` + the toggle:
+- `"Best config per model"` → `display_df = get_best_variants(filtered_df, metric_key)`
+- `"All configs"` → `display_df = filtered_df`
+
+All visualization render functions should consume `display_df`, not `filtered_df` directly — **except** the variant `heatmap_variants` and the Inspector, which always need the full `filtered_df`.
+
+The toggle and the metric dropdown both need to trigger recomputation of `display_df`.
+
+---
+
+## Visualization changes
+
+### `ranking_table` — Q1
+
+**Current:** rows = `model_alias`, values = mean metric per task category across all variants.  
+**Target:** rows = variant label (e.g. `"nt · mean · last (3) · linear"`), grouped under `model_alias`. Values = mean metric per task category, computed only over that variant's rows.  
+**Change:** consume `display_df`; when in "best config" mode the row is already one variant per family, so the table naturally becomes variant-ranked.  
+Add `model_alias` as the first column so family membership is still visible.
+
+### `violin_plot` — Q1
+
+**Current:** distribution across all experiments (configs × tasks) per `model_alias` — masks the best variant.  
+**Target:** replace with a **strip/dot plot** where each dot = one task score of the best variant for that family. The "distribution" then reflects performance spread across tasks, not across experiments.  
+**Change:** rename or replace `violin_plot` with a dot/strip plot; consume `display_df`.  
+Keep the violin shape optional (switchable) but make the default the dot plot.
+
+### `bar_plot_per_category` and `bar_plot_per_task` — Q2
+
+**Current:** X = category/task, bars = `model_alias`, height = mean across all variants.  
+**Target:** bars = best variant per family (labeled with variant name, colored by family).  
+**Change:** consume `display_df`; no logic change needed in the plot functions themselves — the fix is upstream in `display_df`.
+
+### `heatmap_variants` — Q3, Q4
+
+**Current (single model):** rows = all config variants, cols = tasks. ✅ Correct for Q3/Q4.  
+**Current (multi-model):** rows = `model_alias`, mean across all variants. This view is less useful now.  
+**Target for multi-model:** rows = best variant per family (one row per family, using the same task-agnostic key). This lets you compare families at their best without the multi-model averaging obscuring things.  
+**Change:** for multi-model mode, pre-filter to best variant rows before building the heatmap.
+
+**Missing — Q3 single-axis ablation:**  
+There is no view that isolates one config dimension while holding others fixed. This would require a new chart or a pivot within `heatmap_variants`. Defer to a future iteration — the current heatmap already partially answers Q3.
+
+### `scatter_speed` — Q5
+
+**Current:** one point per row — all configs mixed, no labeling of which points are best variants.  
+**Target:** in "best config" mode, one point per family (the best variant), clearly labeled. In "all configs" mode, all points visible with variant label in hover.  
+**Change:** consume `display_df`; add variant label to hover text.  
+**Missing: VRAM chart.** `vram_model_mb` and `peak_vram_extraction_mb` are in the data but unused. Add a third subplot (or a second tab within Speed) for VRAM vs metric.
+
+### `bubble_context_length` — Q6
+
+**Current:** one bubble per row — all configs mixed, signal diluted.  
+**Target:** in "best config" mode, one bubble per `(model_alias, task)` using the best variant.  
+**Change:** consume `display_df`; no logic change needed in the plot function.
+
+---
+
+## Tab / layout changes
+
+| Tab | Change needed |
+|---|---|
+| Leaderboard | Add best-config toggle (global); replace violin with dot/strip plot |
+| Per-Category | Consume `display_df` — no structural change |
+| Speed vs Performance | Add VRAM subplot; consume `display_df` |
+| Context Length | Consume `display_df` — no structural change |
+| Settings | No change |
+
+---
+
+## Implementation order
+
+1. `get_best_variants` in `data.py` — foundation for everything else
+2. `display_df` state + toggle in `app.py`
+3. Swap `violin_plot` for dot/strip plot in Leaderboard
+4. Update `ranking_table` to show variant label
+5. `scatter_speed` VRAM subplot
+6. `heatmap_variants` multi-model best-variant mode
+
+Steps 2–6 are mostly wiring changes once step 1 exists; the individual plot functions require minimal logic changes.

src/data.py

@@ -103,6 +103,10 @@ def apply_filters(df: pd.DataFrame, filters: dict) -> pd.DataFrame:
     if excluded_models:
         result = result[~result["model_alias"].isin(excluded_models)]
 
+    pooling_strategies = filters.get("pooling_strategies") or []
+    if pooling_strategies and "pooling_strategy" in result.columns:
+        result = result[result["pooling_strategy"].isin(pooling_strategies)]
+
     excluded_runs = filters.get("exclude_run_ids") or []
     if excluded_runs and "run_id" in result.columns:
         result = result[~result["run_id"].isin(excluded_runs)]

src/plots.py

@@ -265,21 +265,34 @@ def violin_plot(df: pd.DataFrame, metric: str) -> go.Figure:
 def heatmap_variants(
     df: pd.DataFrame,
     metric: str,
-    model_alias: str,
+    model_alias: str | list[str],
     aggregate_by_dataset: bool = False,
 ) -> go.Figure:
     """Heatmap: rows = config variants (or single row if only one), cols = tasks (or datasets) + Overall.
 
+    When model_alias is a list with >1 entry: 1 row per model (mean over all variants).
+    When a single model: rows = embedding config variants.
     By default columns are individual task_names ordered by category then alphabetically.
     When aggregate_by_dataset=True columns collapse to dataset_key values (mean per dataset).
-    Rows sorted best→worst by Overall when multiple variants exist.
+    Rows sorted best→worst by Overall when multiple rows exist.
     """
     metric_label = CLASSIFICATION_METRICS.get(metric, metric)
 
-    if not model_alias or df.empty or metric not in df.columns:
+    # Normalise to list
+    if isinstance(model_alias, str):
+        model_aliases = [model_alias] if model_alias else []
+    else:
+        model_aliases = [m for m in (model_alias or []) if m]
+
+    if not model_aliases or df.empty or metric not in df.columns:
         return go.Figure()
 
-    mdf = df[df["model_alias"] == model_alias]
+    multi_model = len(model_aliases) > 1
+
+    if multi_model:
+        mdf = df[df["model_alias"].isin(model_aliases)]
+    else:
+        mdf = df[df["model_alias"] == model_aliases[0]]
     if mdf.empty:
         return go.Figure()
 
@@ -292,21 +305,26 @@ def heatmap_variants(
         return str(val)
 
     mdf = mdf.copy()
-    mdf["_embed_key"] = list(zip(
-        mdf.get("model_hf_id", pd.Series("", index=mdf.index)).map(_norm),
-        mdf.get("pooling_strategy", pd.Series("", index=mdf.index)).map(_norm),
-        mdf.get("layer_selection_params", pd.Series("", index=mdf.index)).map(_fmt_layer),
-        mdf.get("head_type", pd.Series("", index=mdf.index)).map(_norm),
-    ))
-    configs = sorted(mdf["_embed_key"].unique())
-
-    if len(configs) > 1:
-        rows_iter = [
-            (_variant_label(mdf[mdf["_embed_key"] == c].iloc[0]), mdf[mdf["_embed_key"] == c])
-            for c in configs
-        ]
+
+    if multi_model:
+        # One row per model; no variant breakdown
+        rows_iter = [(m, mdf[mdf["model_alias"] == m]) for m in model_aliases]
     else:
-        rows_iter = [(_variant_label(mdf.iloc[0]), mdf)]
+        # Build embedding-config key to split into variant rows
+        mdf["_embed_key"] = list(zip(
+            mdf.get("model_hf_id", pd.Series("", index=mdf.index)).map(_norm),
+            mdf.get("pooling_strategy", pd.Series("", index=mdf.index)).map(_norm),
+            mdf.get("layer_selection_params", pd.Series("", index=mdf.index)).map(_fmt_layer),
+            mdf.get("head_type", pd.Series("", index=mdf.index)).map(_norm),
+        ))
+        configs = sorted(mdf["_embed_key"].unique())
+        if len(configs) > 1:
+            rows_iter = [
+                (_variant_label(mdf[mdf["_embed_key"] == c].iloc[0]), mdf[mdf["_embed_key"] == c])
+                for c in configs
+            ]
+        else:
+            rows_iter = [(_variant_label(mdf.iloc[0]), mdf)]
 
     # Build column list depending on aggregation mode
     if aggregate_by_dataset:
@@ -332,7 +350,7 @@ def heatmap_variants(
     rows_z, rows_text, rows_hover, row_labels = [], [], [], []
     for label, vdf in rows_iter:
         z_row, t_row, h_row = [], [], []
-        config_lines = _variant_hover_lines(vdf)
+        config_lines = "" if multi_model else _variant_hover_lines(vdf)
         for col_key, col_name in zip(col_keys, col_names[:-1]):
             vals = _cell_vals(vdf, col_key)
             v = round(float(vals.mean()), 3) if not vals.empty else None
@@ -400,7 +418,7 @@ def heatmap_variants(
     v_overhead = t_margin + bottom_margin
     height = max(v_overhead + 80, n_rows * 48 + v_overhead + 20)
     fig.update_layout(
-        title=f"{metric_label} — {model_alias}",
+        title=f"{metric_label} — {', '.join(model_aliases)}",
         height=height,
         template="plotly_white",
         xaxis=dict(side="top", tickfont=dict(size=9 if not aggregate_by_dataset else 11), tickangle=tick_angle),