Major refactor - V2 (#7)

- refactored the whole webapp (a486179cf332b50a0585aff1550b344815676d91)
- added raw input inspector (a011d958b616b93695e594aefaa0f9ae6bc7e336)

Makefile

@@ -1,8 +1,11 @@
-.PHONY: style run
+.PHONY: style app seq-lengths
 
 style:
 	python -m black --line-length 119 .
 	ruff check --fix .
 
-run:
+app:
 	gradio app.py
+
+seq-lengths:
+	python tools/compute_seq_lengths.py

README.md

@@ -13,31 +13,8 @@ sdk_version: 5.19.0
 
 # DNA Benchmark Leaderboard
 
-A simple leaderboard for evaluating DNA foundational models on genomics tasks.
-
-## Configuration
-
-- `src/envs.py` - Configure repository paths and environment variables
-- `src/about.py` - Define tasks and leaderboard text
-- `src/display/utils.py` - Configure table columns and metrics
-
-## Results Format
-
-Results files should be stored as JSON with the following structure:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16",
-        "model_name": "org/model",
-        "model_sha": "revision"
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score
-        }
-    }
-}
-```
+Interactive leaderboard comparing DNA foundation models across genomics classification tasks.
+Data is loaded from [`lokahq/genomic-benchmark-metrics`](https://huggingface.co/datasets/lokahq/genomic-benchmark-metrics) on startup.
 
 ## Development
 
@@ -46,8 +23,29 @@ Results files should be stored as JSON with the following structure:
 pip install -r requirements.txt
 
 # Run locally
-gradio app.py
+make app
 
 # Format code
 make style
 ```
+
+## Context length bubble chart
+
+The bubble chart requires mean sequence lengths per task to be pre-computed once:
+
+```bash
+make seq-lengths        # computes lengths and writes tools/task_seq_lengths.json
+git add tools/task_seq_lengths.json && git commit -m "populate task seq lengths"
+```
+
+The chart is a no-op until `tools/task_seq_lengths.json` is populated and committed.
+
+## Configuration
+
+| File | Purpose |
+|---|---|
+| `src/constants.py` | Task categories, metric names, seq length lookup |
+| `src/data.py` | HF Hub loading, deduplication, filtering |
+| `src/plots.py` | All Plotly figure factories |
+| `tools/task_seq_lengths.json` | Mean sequence lengths per task (committed) |
+| `.env` | `HF_TOKEN` and `LEADERBOARD_DATASET` |

app.py

@@ -1,163 +1,414 @@
+"""DNA Foundation Model Benchmark Leaderboard — Gradio Blocks app."""
+
 import gradio as gr
-from apscheduler.schedulers.background import BackgroundScheduler
-from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
-from huggingface_hub import snapshot_download
+import pandas as pd
 
-from src.about import INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE
-from src.display.css_html_js import custom_css
-from src.display.utils import BenchRawColumn, fields
-from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_leaderboard_df_from_hf_dataset, summarize_model_task_type_performance
-from src.display.plotting import METRICS_FOR_PLOTS, extract_mean_std, prepare_leaderboard_df, make_plot_wrapper, make_model_all_datasets_wrapper, plot_metric_bar
+from src.constants import CLASSIFICATION_METRICS
+from src.data import apply_filters, deduplicate, load_data
+from src.plots import (
+    bar_plot_per_category,
+    bar_plot_per_task,
+    bubble_context_length,
+    heatmap_variants,
+    ranking_table,
+    scatter_speed,
+    violin_plot,
+)
 
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
+# ---------------------------------------------------------------------------
+# Startup: load & deduplicate once
+# ---------------------------------------------------------------------------
 
+print("Loading benchmark data from HF Hub …")
 try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO,
-        local_dir=EVAL_RESULTS_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-    )
+    _LOADED_DF = load_data()
+    _RAW_DF = deduplicate(_LOADED_DF)
+    print(f"  Loaded {len(_RAW_DF)} rows, {_RAW_DF['model_alias'].nunique()} models.")
+except Exception as exc:
+    print(f"  WARNING: Could not load data — {exc}")
+    _LOADED_DF = pd.DataFrame()
+    _RAW_DF = pd.DataFrame()
 
-    LEADERBOARD_DF = get_leaderboard_df_from_hf_dataset(EVAL_RESULTS_PATH)
-    LEADERBOARD_DF = summarize_model_task_type_performance(LEADERBOARD_DF)
-    LEADERBOARD_DF = prepare_leaderboard_df(LEADERBOARD_DF)
-
-    wrapped_task_plot = make_plot_wrapper(leaderboard_df=LEADERBOARD_DF, group_by="Model", filter_col="Task", orientation="h")
-    wrapped_model_all_datasets_plot = make_model_all_datasets_wrapper(leaderboard_df=LEADERBOARD_DF)
-
-except Exception as e:
-    print(e)
-    restart_space()
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(BenchRawColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(BenchRawColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(BenchRawColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        filter_columns=[
-            ColumnFilter(BenchRawColumn.task.name, type="checkboxgroup", label=BenchRawColumn.task.name),
-            # ColumnFilter(BenchRawColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                BenchRawColumn.model_params.name,
-                type="slider",
-                min=100,
-                max=10000,
-                label="Select the number of parameters (M)",
-            ),
-            ColumnFilter(
-                BenchRawColumn.embds_dim.name,
-                type="slider",
-                min=10,
-                max=10000,
-                label="Select the Embeddings Size",
-            ),
-            # ColumnFilter(BenchRawColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
-            ColumnFilter(
-                BenchRawColumn.max_context_len.name,
-                type="slider",
-                min=10,
-                max=10000000,
-                label="Select the Max Context Length (bp)",
-            ),
-        ],
-        search_columns=[BenchRawColumn.model.name],
-    )
+_METRIC_CHOICES = list(CLASSIFICATION_METRICS.values())
+_METRIC_KEYS = list(CLASSIFICATION_METRICS.keys())
 
-# Create an elegant dark theme with custom colors for checkboxes and sliders
-theme = gr.themes.Soft(
-    primary_hue="violet",
-    secondary_hue="purple",
-    neutral_hue="slate",
-    spacing_size="md",
-    radius_size="lg",
-    text_size="md",
-    font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
-).set(
-    checkbox_background_color_selected="*primary_600",
-    checkbox_border_color_selected="*primary_600",
-    checkbox_background_color_selected_dark="*primary_500",
-    checkbox_border_color_selected_dark="*primary_500",
-    slider_color="*primary_600",
-    slider_color_dark="*primary_500",
-    button_primary_background_fill="*primary_600",
-    button_primary_background_fill_hover="*primary_700",
-    button_primary_background_fill_dark="*primary_500",
-    button_primary_background_fill_hover_dark="*primary_600",
+_CATEGORY_CHOICES = (
+    sorted(_RAW_DF["task_category"].unique().tolist()) if not _RAW_DF.empty else []
 )
 
-demo = gr.Blocks(
-    css=custom_css,
-    theme=theme,
-    js="""
-    () => {
-        const theme = localStorage.getItem('theme');
-        if (theme === null) {
-            localStorage.setItem('theme', 'dark');
-            document.body.classList.add('dark');
-        }
-    }
-    """,
-    title="DNA Benchmark",
-    head="<link rel='icon' href='data:image/svg+xml,<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 100 100\"><text y=\".9em\" font-size=\"90\">🧬</text></svg>'>"
+_DATASET_CHOICES = (
+    sorted(_RAW_DF["dataset_name"].dropna().unique().tolist())
+    if not _RAW_DF.empty and "dataset_name" in _RAW_DF.columns
+    else []
 )
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("📊 Performance Plots", elem_id="llm-benchmark-performance-plots", id=1):
-                with gr.Accordion("🧬 Overview of Performance for a Specific Task", open=False):
-                    gr.Markdown("Visualize model performance for a specific task and metric")
+def _metric_key(label: str) -> str:
+    if not label or label not in _METRIC_CHOICES:
+        return _METRIC_KEYS[0]
+    return _METRIC_KEYS[_METRIC_CHOICES.index(label)]
+
+
+def _numeric_range(df: pd.DataFrame, col: str) -> tuple[float, float]:
+    if col not in df.columns or df.empty:
+        return 0.0, 1.0
+    lo, hi = float(df[col].min()), float(df[col].max())
+    if lo == hi:
+        return lo, lo + 1.0
+    return lo, hi
+
+
+# ---------------------------------------------------------------------------
+# Render helpers
+# ---------------------------------------------------------------------------
+
+def render_leaderboard(df: pd.DataFrame, metric_label: str):
+    key = _metric_key(metric_label)
+    return ranking_table(df, key), violin_plot(df, key)
+
+
+def render_per_category(df: pd.DataFrame, metric_label: str, datasets: list[str] | None = None):
+    if datasets and "dataset_name" in df.columns:
+        df = df[df["dataset_name"].isin(datasets)]
+    key = _metric_key(metric_label)
+    return bar_plot_per_category(df, key)
+
+
+def render_per_task(df: pd.DataFrame, metric_label: str, category: str, datasets: list[str] | None = None):
+    if datasets and "dataset_name" in df.columns:
+        df = df[df["dataset_name"].isin(datasets)]
+    key = _metric_key(metric_label)
+    return bar_plot_per_task(df, key, category)
+
+
+def render_speed(df: pd.DataFrame, metric_label: str):
+    key = _metric_key(metric_label)
+    return scatter_speed(df, key)
+
+
+def render_context(df: pd.DataFrame, metric_label: str):
+    key = _metric_key(metric_label)
+    return bubble_context_length(df, key)
+
+
+def render_variant_heatmap(
+    df: pd.DataFrame,
+    metric_label: str,
+    model_alias: str,
+    aggregate_by_dataset: bool = False,
+):
+    key = _metric_key(metric_label)
+    return heatmap_variants(df, key, model_alias, aggregate_by_dataset)
+
+
+def update_category_choices(df: pd.DataFrame, datasets: list[str] | None = None):
+    if datasets and "dataset_name" in df.columns:
+        df = df[df["dataset_name"].isin(datasets)]
+    cats = sorted(df["task_category"].unique().tolist()) if not df.empty else []
+    return gr.update(choices=cats, value=cats[0] if cats else None)
+
+
+def apply_and_preview(
+    df: pd.DataFrame,
+    prod_only: bool,
+    exclude_models: list[str],
+    params_min: float,
+    params_max: float,
+    dim_min: float,
+    dim_max: float,
+    ctx_min: float,
+    ctx_max: float,
+) -> tuple[pd.DataFrame, pd.DataFrame, str]:
+    filters = {
+        "subsample_train": "full" if prod_only else None,
+        "exclude_models": exclude_models or [],
+        "model_params_min": params_min,
+        "model_params_max": params_max,
+        "embedding_dim_min": dim_min,
+        "embedding_dim_max": dim_max,
+        "max_context_size_min": ctx_min,
+        "max_context_size_max": ctx_max,
+    }
+    result = apply_filters(df, filters)
+    preview = result.head(50) if not result.empty else pd.DataFrame()
+    return result, preview, f"{len(result)} rows active"
+
+
+def _hash_label(row: pd.Series) -> str:
+    """Human-readable label for an embedding config hash dropdown entry."""
+    def _s(val) -> str:
+        return "" if val is None or (isinstance(val, float) and pd.isna(val)) else str(val)
+
+    category = _s(row.get("task_category"))
+    task = _s(row.get("task_name"))
+    pool = _s(row.get("pooling_strategy"))
+    strategy = _s(row.get("layer_selection_strategy"))
+    idx = row.get("layer_index")
+    layer = strategy + (f"({int(idx)})" if idx is not None and not (isinstance(idx, float) and pd.isna(idx)) else "")
+    h = _s(row.get("embedding_config_hash"))[:8]
+
+    parts = [f"{category}/{task}" if category else task, pool, layer]
+    return "  ·  ".join(p for p in parts if p) + f"  ({h})"
+
+
+def get_hash_choices_for_model(loaded_df: pd.DataFrame, model: str) -> list[tuple[str, str]]:
+    if not model or loaded_df.empty or "embedding_config_hash" not in loaded_df.columns:
+        return []
+    sub = loaded_df[loaded_df["model_alias"] == model].drop_duplicates("embedding_config_hash")
+    return sorted(
+        [(_hash_label(row), row["embedding_config_hash"]) for _, row in sub.iterrows()],
+        key=lambda x: x[0],
+    )
+
+
+def get_head_types_for_hash(loaded_df: pd.DataFrame, model: str, key: str) -> list[str]:
+    if not model or not key or loaded_df.empty or "embedding_config_hash" not in loaded_df.columns:
+        return []
+    sub = loaded_df[(loaded_df["model_alias"] == model) & (loaded_df["embedding_config_hash"] == key)]
+    return sorted(sub["head_type"].dropna().unique().tolist()) if "head_type" in sub.columns else []
+
+
+_DEFAULT_INSP_COLS = ["kept", "run_id", "run_at", "subsample_train", "task_name", "mcc_test", "accuracy_test"]
+
+
+def build_inspector_table(loaded_df: pd.DataFrame, raw_df: pd.DataFrame,
+                          model: str, key: str, head_type: str,
+                          selected_cols: list[str] | None = None) -> pd.DataFrame:
+    if not model or not key or not head_type or loaded_df.empty or "embedding_config_hash" not in loaded_df.columns:
+        return pd.DataFrame()
+    mask = (
+        (loaded_df["model_alias"] == model)
+        & (loaded_df["embedding_config_hash"] == key)
+        & (loaded_df["head_type"] == head_type)
+    )
+    sub = loaded_df[mask].copy()
+    kept_ids = set(raw_df["run_id"]) if "run_id" in raw_df.columns else set()
+    sub["kept"] = sub["run_id"].isin(kept_ids).map({True: "✓", False: "✗"})
+    cols = selected_cols if selected_cols else _DEFAULT_INSP_COLS
+    return sub[[c for c in cols if c in sub.columns or c == "kept"]].reset_index(drop=True)
+
+
+def exclude_run_ids(df: pd.DataFrame, run_ids_text: str) -> tuple[pd.DataFrame, pd.DataFrame, str]:
+    ids = [r.strip() for r in run_ids_text.split(",") if r.strip()]
+    result = apply_filters(df, {"exclude_run_ids": ids})
+    preview = result.head(50) if not result.empty else pd.DataFrame()
+    return result, preview, f"{len(result)} rows active"
+
+
+# ---------------------------------------------------------------------------
+# Build UI
+# ---------------------------------------------------------------------------
+
+all_models = sorted(_RAW_DF["model_alias"].unique().tolist()) if not _RAW_DF.empty else []
+params_min_v, params_max_v = _numeric_range(_RAW_DF, "model_params")
+dim_min_v, dim_max_v = _numeric_range(_RAW_DF, "embedding_dim")
+ctx_min_v, ctx_max_v = _numeric_range(_RAW_DF, "max_context_size")
+
+_insp_default_model = all_models[0] if all_models else None
+_insp_default_keys = get_hash_choices_for_model(_LOADED_DF, _insp_default_model)
+_ALL_INSP_COLS = ["kept"] + _LOADED_DF.columns.tolist() if not _LOADED_DF.empty else _DEFAULT_INSP_COLS
+
+with gr.Blocks(title="DNA Benchmark Leaderboard") as demo:
+
+    gr.Markdown(
+        "# 🧬 DNA Foundation Model Benchmark Leaderboard\n"
+        "Compare DNA language models across genomics classification tasks."
+    )
+
+    # ── Global controls ───────────────────────────────────────────────────────
+    with gr.Row():
+        metric_dd = gr.Dropdown(
+            choices=_METRIC_CHOICES, value=_METRIC_CHOICES[0], label="Metric", scale=2
+        )
+
+    # ── Shared state ──────────────────────────────────────────────────────────
+    loaded_df_state = gr.State(_LOADED_DF)
+    raw_df_state = gr.State(_RAW_DF)
+    filtered_df_state = gr.State(_RAW_DF)
 
-                    task_choices = sorted(LEADERBOARD_DF["Task"].dropna().unique())
-                    dataset_choices = sorted(LEADERBOARD_DF["Dataset Name"].dropna().unique())
+    # ── Tabs ──────────────────────────────────────────────────────────────────
+    with gr.Tabs():
 
-                    with gr.Row():
-                        task_dropdown = gr.Dropdown(choices=task_choices, label="Select Task")
-                        metric_dropdown = gr.Dropdown(choices=METRICS_FOR_PLOTS, value="Accuracy", label="Select Metric")
-                        dataset_dropdown = gr.Dropdown(choices=dataset_choices, value="InstaDeepAI/nucleotide_transformer_downstream_tasks_revised", label="Select Dataset")
+        # ── Tab 1: Leaderboard ────────────────────────────────────────────────
+        with gr.Tab("🏅 Leaderboard"):
+            leaderboard_table = gr.Dataframe(
+                label="Rankings (mean metric per category)", interactive=False, wrap=True
+            )
+            violin_fig = gr.Plot(label="Metric distribution per model")
 
-                    performance_plot = gr.Plot()
+            gr.Markdown("---\n### Drill into model variants")
+            model_dd = gr.Dropdown(
+                choices=all_models,
+                value=all_models[0] if all_models else None,
+                label="Select model",
+            )
+            agg_dataset_cb = gr.Checkbox(
+                label="Aggregate by dataset (show mean per benchmark dataset instead of per task)",
+                value=False,
+            )
+            variant_violin_fig = gr.Plot(label="Variant comparison (metric per task)")
 
-                    task_dropdown.change(fn=wrapped_task_plot, inputs=[task_dropdown, metric_dropdown, dataset_dropdown], outputs=performance_plot)
-                    metric_dropdown.change(fn=wrapped_task_plot, inputs=[task_dropdown, metric_dropdown, dataset_dropdown], outputs=performance_plot)
-                    dataset_dropdown.change(fn=wrapped_task_plot, inputs=[task_dropdown, metric_dropdown, dataset_dropdown], outputs=performance_plot)
+        # ── Tab 2: Per-Category Performance ───────────────────────────────────
+        with gr.Tab("📊 Per-Category Performance"):
+            category_fig = gr.Plot(label="Mean performance per task category")
 
-                with gr.Accordion("🤖 Overview of Performance for a Specific Model", open=False):
-                    gr.Markdown("Visualize model performance across all tasks from all datasets")
+            gr.Markdown("---\n### Drill into individual tasks")
+            category_dd = gr.Dropdown(
+                choices=_CATEGORY_CHOICES,
+                value=_CATEGORY_CHOICES[0] if _CATEGORY_CHOICES else None,
+                label="Select category",
+            )
+            dataset_dd = gr.Dropdown(
+                choices=_DATASET_CHOICES,
+                value=[],
+                multiselect=True,
+                label="Filter by dataset",
+            )
+            per_task_fig = gr.Plot(label="Performance per task within selected category")
 
-                    model_choices = sorted(LEADERBOARD_DF["Model"].dropna().unique())
+        # ── Tab 3: Speed vs Performance ───────────────────────────────────────
+        with gr.Tab("⚡ Speed vs Performance"):
+            speed_fig = gr.Plot(label="Throughput & embedding time vs performance")
 
-                    with gr.Row():
-                        model_dropdown = gr.Dropdown(choices=model_choices, label="Select Model")
-                        metric2_dropdown = gr.Dropdown(choices=METRICS_FOR_PLOTS, value="Accuracy", label="Select Metric")
+        # ── Tab 4: Context Length ─────────────────────────────────────────────
+        with gr.Tab("🔬 Context Length"):
+            context_fig = gr.Plot(label="Performance vs task sequence length")
 
-                    model_plot = gr.Plot()
+        # ── Tab 5: Settings ───────────────────────────────────────────────────
+        with gr.Tab("⚙️ Settings"):
+            prod_only_cb = gr.Checkbox(
+                label="Production runs only (subsample_train IS NULL)",
+                value=False,
+                info="Currently returns no data — all runs use subsample_train=0.05.",
+            )
+            exclude_models_ms = gr.Dropdown(
+                choices=all_models, multiselect=True, label="Exclude models", value=[]
+            )
+            with gr.Row():
+                params_min_sl = gr.Slider(
+                    minimum=params_min_v, maximum=params_max_v,
+                    value=params_min_v, label="Model params (min)"
+                )
+                params_max_sl = gr.Slider(
+                    minimum=params_min_v, maximum=params_max_v,
+                    value=params_max_v, label="Model params (max)"
+                )
+            with gr.Row():
+                dim_min_sl = gr.Slider(
+                    minimum=dim_min_v, maximum=dim_max_v,
+                    value=dim_min_v, label="Embedding dim (min)"
+                )
+                dim_max_sl = gr.Slider(
+                    minimum=dim_min_v, maximum=dim_max_v,
+                    value=dim_max_v, label="Embedding dim (max)"
+                )
+            with gr.Row():
+                ctx_min_sl = gr.Slider(
+                    minimum=ctx_min_v, maximum=ctx_max_v,
+                    value=ctx_min_v, label="Max context size (min bp)"
+                )
+                ctx_max_sl = gr.Slider(
+                    minimum=ctx_min_v, maximum=ctx_max_v,
+                    value=ctx_max_v, label="Max context size (max bp)"
+                )
+            apply_btn = gr.Button("Apply Filters", variant="primary")
 
-                    model_dropdown.change(fn=wrapped_model_all_datasets_plot, inputs=[model_dropdown, metric2_dropdown], outputs=model_plot)
-                    metric2_dropdown.change(fn=wrapped_model_all_datasets_plot, inputs=[model_dropdown, metric2_dropdown], outputs=model_plot)
+            gr.Markdown("---")
+            run_ids_tb = gr.Textbox(
+                label="Exclude run_ids (comma-separated)", placeholder="run_id_1, run_id_2, …"
+            )
+            exclude_runs_btn = gr.Button("Exclude run_ids")
+
+            gr.Markdown("---")
+            settings_count = gr.Label(label="Active rows")
+            settings_preview = gr.Dataframe(label="Preview (first 50 rows)", interactive=False)
+
+            gr.Markdown("---\n### Raw Run Inspector\nSelect a model and embedding config to audit which runs were kept or dropped by dedup.")
+            with gr.Row():
+                insp_model_dd = gr.Dropdown(choices=all_models, value=_insp_default_model, label="Model", scale=2)
+                insp_cache_dd = gr.Dropdown(choices=_insp_default_keys, value=None, label="Embedding config hash", scale=3)
+                insp_head_dd = gr.Dropdown(choices=[], value=None, label="Head type", scale=2)
+            insp_cols_dd = gr.Dropdown(
+                choices=_ALL_INSP_COLS,
+                value=_DEFAULT_INSP_COLS,
+                multiselect=True,
+                label="Columns to display",
+            )
+            insp_table = gr.Dataframe(
+                label="Runs for selection  (✓ = kept by dedup, ✗ = dropped)",
+                interactive=False,
+            )
+
+    # ── Event wiring ──────────────────────────────────────────────────────────
+
+    _plot_triggers = [filtered_df_state, metric_dd]
+
+    for trigger in [filtered_df_state, metric_dd]:
+        trigger.change(render_leaderboard, _plot_triggers, [leaderboard_table, violin_fig])
+        trigger.change(render_speed, _plot_triggers, speed_fig)
+        trigger.change(render_context, _plot_triggers, context_fig)
+
+    # Category bar chart: reacts to df, metric, AND dataset filter
+    _cat_triggers = [filtered_df_state, metric_dd, dataset_dd]
+    for trigger in [filtered_df_state, metric_dd, dataset_dd]:
+        trigger.change(render_per_category, _cat_triggers, category_fig)
+
+    # Per-task drill-down: reacts to category/dataset dropdowns OR global controls
+    _per_task_triggers = [filtered_df_state, metric_dd, category_dd, dataset_dd]
+    for trigger in [filtered_df_state, metric_dd, category_dd, dataset_dd]:
+        trigger.change(render_per_task, _per_task_triggers, per_task_fig)
+
+    # Variant heatmap: reacts to model selector, metric, filtered data, or aggregation toggle
+    _variant_triggers = [filtered_df_state, metric_dd, model_dd, agg_dataset_cb]
+    for trigger in [filtered_df_state, metric_dd, model_dd, agg_dataset_cb]:
+        trigger.change(render_variant_heatmap, _variant_triggers, variant_violin_fig)
+
+    # Update category dropdown choices when global filter OR dataset filter changes
+    for trigger in [filtered_df_state, dataset_dd]:
+        trigger.change(update_category_choices, [filtered_df_state, dataset_dd], category_dd)
+
+    # Settings — Apply Filters
+    apply_btn.click(
+        apply_and_preview,
+        inputs=[
+            raw_df_state, prod_only_cb, exclude_models_ms,
+            params_min_sl, params_max_sl,
+            dim_min_sl, dim_max_sl,
+            ctx_min_sl, ctx_max_sl,
+        ],
+        outputs=[filtered_df_state, settings_preview, settings_count],
+    )
+
+    # Settings — Exclude run_ids
+    exclude_runs_btn.click(
+        exclude_run_ids,
+        inputs=[filtered_df_state, run_ids_tb],
+        outputs=[filtered_df_state, settings_preview, settings_count],
+    )
+
+    # Inspector — cascade: model → hash → head_type → table
+    insp_model_dd.change(
+        lambda df, m: gr.update(choices=get_hash_choices_for_model(df, m), value=None),
+        [loaded_df_state, insp_model_dd],
+        insp_cache_dd,
+    )
+    insp_cache_dd.change(
+        lambda df, m, k: gr.update(choices=get_head_types_for_hash(df, m, k), value=None),
+        [loaded_df_state, insp_model_dd, insp_cache_dd],
+        insp_head_dd,
+    )
+    _insp_inputs = [loaded_df_state, raw_df_state, insp_model_dd, insp_cache_dd, insp_head_dd, insp_cols_dd]
+    insp_head_dd.change(build_inspector_table, _insp_inputs, insp_table)
+    insp_cols_dd.change(build_inspector_table, _insp_inputs, insp_table)
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-about", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+    # Initial render on load
+    demo.load(render_leaderboard, inputs=_plot_triggers, outputs=[leaderboard_table, violin_fig])
+    demo.load(render_per_category, inputs=_cat_triggers, outputs=category_fig)
+    demo.load(render_per_task, inputs=_per_task_triggers, outputs=per_task_fig)
+    demo.load(render_speed, inputs=_plot_triggers, outputs=speed_fig)
+    demo.load(render_context, inputs=_plot_triggers, outputs=context_fig)
+    demo.load(render_variant_heatmap, inputs=_variant_triggers, outputs=variant_violin_fig)
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
 
-demo.queue(default_concurrency_limit=40).launch(share=True)
+if __name__ == "__main__":
+    demo.launch(theme=gr.themes.Soft(), share=True)

docs/context-specs-plan.md

@@ -0,0 +1,245 @@
+# bench-dna Leaderboard Webapp — Plan
+
+> **Portability note**: This doc lives in the git repo and is the canonical reference.
+> When starting a new session on a different machine, paste this file as context to Claude Code.
+
+---
+
+## Overview
+
+A Gradio webapp deployed on HuggingFace Spaces that loads benchmark results from the HF Hub
+dataset (`lokahq/genomic-benchmark-metrics`) and provides interactive visualizations to compare
+DNA foundation models across tasks, metrics, and efficiency dimensions.
+
+**Stack**: Gradio (Blocks), Plotly, Pandas, HF datasets library
+**Location**: repo root (`app.py` + `src/`) — HF Spaces expects root-level `app.py`
+**Status**: Implemented (March 2026)
+
+---
+
+## Project Context (for new sessions)
+
+### Data pipeline
+- Benchmark runs are produced by `bDNA/run_benchmark.py` (Hydra-based)
+- Each run generates one `BenchmarkRecord` (see `bDNA/io/results.py`)
+- Records are pushed to HF Hub dataset (`lokahq/genomic-benchmark-metrics`), one split per model alias
+- Parquet files also saved locally at `artifacts/results/<model_alias>.parquet`
+
+### Current data state (as of 2026-03)
+- Models with data: `dnabert2`, `caduceus`, `nt`, `omniDNA`
+- All runs use `subsample_train=0.05` (5% of training data — hardware constraint)
+- All runs are binary classification (short-range tasks only)
+- `subsample_train IS NULL` would mean "full dataset run" — none exist yet
+- Long-range tasks (regression, variant effect) not benchmarked yet
+
+### Key BenchmarkRecord fields for the webapp
+| Field | Use |
+|---|---|
+| `model_alias` | Primary grouping key |
+| `task_name` | Task identifier (18 short-range tasks) |
+| `task_type` | `"binary"`, `"multiclass"`, `"regression"` |
+| `embedding_config_hash` | Deduplication key per config variant |
+| `run_at` | ISO timestamp — used to pick latest run |
+| `subsample_train` | Filter for "production" vs test runs (currently all 0.05) |
+| `model_params` | Filter by model size |
+| `embedding_dim` | Filter by embedding size |
+| `max_context_size` | Model's max sequence length |
+| `mcc_test`, `accuracy_test`, `weighted_f1_test`, `macro_f1_test` | Classification metrics |
+| `test_throughput_seq_s`, `test_embedding_time_s` | Speed metrics |
+| `vram_model_mb`, `peak_vram_extraction_mb` | Memory metrics |
+
+---
+
+## File Structure
+
+```
+app.py                        # main Gradio Blocks app
+requirements.txt
+src/
+  constants.py                # task categories, seq length lookup (from JSON), metric names
+  data.py                     # HF Hub loading, dedup, filtering, best-variant logic
+  plots.py                    # all Plotly figure factories
+docs/
+  context-specs-plan.md       # this file
+tools/
+  compute_seq_lengths.py      # one-time script to populate task_seq_lengths.json
+  task_seq_lengths.json       # committed JSON — populated by compute_seq_lengths.py
+README.md                     # HF Spaces config (title, sdk: gradio, etc.)
+```
+
+---
+
+## Data Layer (`src/data.py`)
+
+### `load_data() -> pd.DataFrame`
+- Reads `LEADERBOARD_DATASET` env var (set to `lokahq/genomic-benchmark-metrics`)
+- Uses `datasets.get_dataset_split_names()` to list all splits (one per model alias)
+- Loads each split and concatenates into one DataFrame
+- Adds derived columns:
+  - `task_category`: mapped from `task_name` via `TASK_TO_CATEGORY`
+  - `task_seq_length`: mapped from `task_name` via `TASK_SEQ_LENGTHS` (from JSON)
+- Called once on app startup
+
+### `deduplicate(df) -> pd.DataFrame`
+- Group by `(model_alias, task_name, embedding_config_hash)`
+- Keep the row with the latest `run_at` per group
+- Eliminates accidental duplicate reruns of the exact same config
+
+### `get_best_variants(df, metric) -> pd.DataFrame`
+- For each `(model_alias, embedding_config_hash)`, compute mean of `metric` across all tasks
+- Per `model_alias`, keep only the config_hash with the highest mean
+- Used when user selects "Best config per model" view
+
+### `apply_filters(df, filters: dict) -> pd.DataFrame`
+- `subsample_train`: None=all, "full"=production runs only (IS NULL / ==0), "subsampled"=test runs
+- `model_params` range (min/max)
+- `embedding_dim` range
+- `max_context_size` range
+- `exclude_models` list
+- `exclude_run_ids` list (row-level, user-managed)
+
+> **Important**: the Settings tab default shows ALL data. "Production only"
+> (subsample_train IS NULL) currently returns nothing — all records have subsample=0.05.
+> The filter is there for the future when full-dataset runs exist.
+
+---
+
+## Constants (`src/constants.py`)
+
+```python
+TASK_CATEGORIES = {
+    "histone_marks": [
+        "H2AFZ", "H3K27ac", "H3K27me3", "H3K36me3",
+        "H3K4me1", "H3K4me2", "H3K4me3", "H3K9ac", "H3K9me3", "H4K20me1",
+    ],
+    "promoter":      ["promoter_all", "promoter_tata", "promoter_no_tata"],
+    "enhancer":      ["enhancers", "enhancers_types"],
+    "splice_site":   ["splice_sites_all", "splice_sites_donors", "splice_sites_acceptors"],
+    "variant_effect":["variant_effect_pathogenic_clinvar", "variant_effect_pathogenic_omim"],
+    "rna_expression":["bulk_rna_expression"],
+}
+
+# Loaded from tools/task_seq_lengths.json at runtime
+TASK_SEQ_LENGTHS: dict[str, int] = json.loads(...)
+
+CLASSIFICATION_METRICS = {
+    "mcc_test":         "MCC",
+    "accuracy_test":    "Accuracy",
+    "weighted_f1_test": "Weighted F1",
+    "macro_f1_test":    "Macro F1",
+}
+```
+
+---
+
+## Visualizations (`src/plots.py`)
+
+| Function | Chart type | Description |
+|---|---|---|
+| `ranking_table` | DataFrame | Rows=models, cols=task categories (mean metric), sorted by overall mean |
+| `violin_plot` | Violin + points | Distribution of metric per model across all tasks |
+| `bar_plot_per_category` | Grouped bar | X=task_category, groups=model, mean metric |
+| `scatter_speed` | 2-subplot scatter | Left: throughput (seq/s) vs metric; Right: embedding time (s) vs metric |
+| `bubble_context_length` | Bubble chart | X=task_seq_length, Y=metric, color=model, size=max_context_size |
+
+### Context length bubble chart — Option B (implemented)
+X=task_seq_length, Y=metric, color=model, size=max_context_size.
+More directly answers "does performance degrade as task length increases, and does context size help?"
+Requires `tools/task_seq_lengths.json` to be populated.
+
+---
+
+## App Layout (`app.py`)
+
+```
+gr.Blocks(theme=gr.themes.Soft())
+│
+├── Global Controls (always visible, affect all tabs)
+│   ├── Metric selector (Dropdown: MCC, Accuracy, Weighted F1, Macro F1)
+│   └── View toggle: "Best config per model" vs "All configs" (Radio)
+│
+├── gr.State: raw_df       ← loaded & deduplicated at startup
+├── gr.State: filtered_df  ← written by Settings Apply
+│
+└── gr.Tabs
+    ├── "Leaderboard"
+    │   ├── gr.Dataframe (ranking table, sortable)
+    │   └── gr.Plot (violin plot)
+    │
+    ├── "Per-Category Performance"
+    │   └── gr.Plot (grouped bar chart by task category)
+    │
+    ├── "Speed vs Performance"
+    │   └── gr.Plot (2-subplot: throughput + embedding time)
+    │
+    ├── "Context Length"
+    │   └── gr.Plot (bubble chart — Option B)
+    │
+    └── "Settings"
+        ├── Checkbox: "Production runs only" (subsample_train IS NULL)
+        ├── Multiselect: exclude specific models
+        ├── Slider: model_params range
+        ├── Slider: embedding_dim range
+        ├── Slider: max_context_size range
+        ├── Button: "Apply Filters" → writes to filtered_df
+        ├── Textbox: "Paste run_ids to exclude (comma-separated)"
+        ├── Button: "Exclude run_ids"
+        └── gr.Dataframe (preview of active rows) + row count label
+```
+
+### State flow
+1. On load: `load_data()` → `deduplicate()` → store in `raw_df` + `filtered_df` state
+2. Settings "Apply Filters" → `apply_filters()` → update `filtered_df`
+3. All tabs: listen to `filtered_df` + global controls → regenerate plots on change
+
+---
+
+## Pre-computation (one-time, run locally before deployment)
+
+`tools/compute_seq_lengths.py`:
+- Loads each benchmark HF dataset (`InstaDeepAI/nucleotide_transformer_downstream_tasks_revised`)
+- Samples ~100 sequences per task_name
+- Computes mean sequence length (bp) per task_name
+- **Writes result to `tools/task_seq_lengths.json`** (commit this file to the repo)
+- `constants.py` reads this JSON at runtime
+- **Bubble chart is a no-op until this is run and `task_seq_lengths.json` is committed**
+
+---
+
+## Environment Variables (`.env`)
+
+```
+HF_TOKEN=<your huggingface token>
+LEADERBOARD_DATASET=lokahq/genomic-benchmark-metrics
+```
+
+---
+
+## Key Design Decisions
+
+| Decision | Choice | Rationale |
+|---|---|---|
+| Framework | Gradio Blocks | Native HF Spaces support |
+| App placement | Root (`app.py`) | HF Spaces expects root-level app; `spaces/leaderboard/` not used |
+| Data source | HF Hub on startup | Auto-updates as new runs are pushed |
+| Dataset env var | `LEADERBOARD_DATASET` in `.env` | Easy to change without code edits |
+| Deduplication key | `(model, task, config_hash)` latest | No prod/test flag needed |
+| Best variant | Best mean-metric config per model | Consistent cross-task comparison |
+| Subsample filter | Use `subsample_train` field | `run_tag` not yet implemented in BenchmarkRecord |
+| Settings default | Show ALL data | All current records have subsample=0.05; "prod only" would show nothing |
+| TASK_SEQ_LENGTHS | JSON file (`tools/task_seq_lengths.json`) | Committed to repo, loaded at runtime — no copy-paste into constants |
+| Context length axes | Option B (X=seq_len, Y=metric, size=ctx) | More directly answers the key question |
+| Regression | Deferred | No data yet |
+| Row-level exclusion | Paste run_ids | Gradio DataFrames lack native per-row checkboxes |
+| Submission workflow | Dropped | Not relevant to new BenchmarkRecord data pipeline |
+
+---
+
+## Out of Scope (v1)
+
+- Regression task visualizations (long-range tasks not yet benchmarked)
+- Real-time data refresh (loads on startup; user refreshes page)
+- Per-subtask bar chart (18 bars is too much; use category aggregation)
+- Custom embedding analysis (PCA/UMAP of raw embedding vectors)
+- `run_tag` field on BenchmarkRecord (discussed, not yet implemented in benchmark runner)
+- Model submission workflow (dropped in v1 remake)

pyproject.toml

@@ -1,3 +1,16 @@
+[project]
+name = "dna-benchmark"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+    "datasets>=4.0.0",
+    "gradio>=6.0.0",
+    "huggingface-hub>=0.18.0",
+    "plotly>=5.0.0",
+    "pandas",
+    "python-dotenv>=1.0.0",
+]
+
 [tool.ruff]
 select = ["E", "F"]
 ignore = ["E501"]

requirements.txt

@@ -1,9 +0,0 @@
-APScheduler
-datasets
-gradio==5.41.0
-gradio[oauth]
-gradio_leaderboard==0.0.13
-huggingface-hub>=0.18.0
-plotly>=5.0.0
-pandas
-python-dotenv==1.1.0

src/about.py

@@ -1,57 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-class Tasks(Enum):
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-
-NUM_FEWSHOT = 0
-
-TITLE = """
-<div style="text-align: center; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); padding: 3rem 2rem; border-radius: 15px; margin-bottom: 2rem; box-shadow: 0 10px 30px rgba(0,0,0,0.2);">
-    <h1 style="color: white; font-size: 3.5rem; margin: 0; font-weight: 800; text-shadow: 2px 2px 4px rgba(0,0,0,0.3);">
-        🧬 DNA Benchmark
-    </h1>
-    <p style="color: rgba(255,255,255,0.95); font-size: 1.3rem; margin-top: 1rem; font-weight: 300;">
-        Evaluating DNA Foundational Models Performance
-    </p>
-</div>
-"""
-
-INTRODUCTION_TEXT = """
-<div style="background: linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%); padding: 1.5rem; border-radius: 10px; border-left: 5px solid #667eea; margin-bottom: 1.5rem;">
-    <p style="font-size: 1.1rem; color: #2d3748; margin: 0; line-height: 1.6;">
-        Compare and analyze the performance of state-of-the-art DNA foundational models across various genomics tasks including histone modification, splicing, promoter identification, enhancer detection, and SNP classification.
-    </p>
-</div>
-"""
-
-LLM_BENCHMARKS_TEXT = """
-<div style="background: var(--background-fill-primary, #fff);">
-
-## 📊 About This Leaderboard
-
-This leaderboard provides comprehensive benchmarking of DNA foundational models across multiple genomics tasks:
-
-- **🧬 Histone Modifications**: H2AFZ, H3K27ac, H3K27me3, H3K36me3, H3K4me1/2/3, H3K9ac, H3K9me3, H4K20me1
-- **✂️ Splicing**: Donor sites, Acceptor sites, All splice sites
-- **📍 Promoter Identification**: TATA-containing, Non-TATA, All promoters
-- **🎯 Enhancer Detection**: Enhancer identification and classification
-- **🔬 SNP Classification**: eQTL causality, ClinVar pathogenic variants, OMIM pathogenic variants
-
-### 📈 Key Metrics
-- **Accuracy**: Overall prediction accuracy
-- **MCC**: Matthews Correlation Coefficient
-- **Weighted F1**: F1-score weighted by class support
-
-</div>
-"""

src/constants.py

@@ -0,0 +1,53 @@
+import json
+import pathlib
+
+TASK_CATEGORIES: dict[str, list[str]] = {
+    "histone_marks": [
+        "h2afz", "h3k27ac", "h3k27me3", "h3k36me3",
+        "h3k4me1", "h3k4me2", "h3k4me3", "h3k9ac", "h3k9me3", "h4k20me1",
+    ],
+    "promoter": ["promoter_all", "promoter_tata", "promoter_no_tata"],
+    "enhancer": ["enhancers", "enhancers_types"],
+    "splice_site": ["splice_sites_all", "splice_sites_donors", "splice_sites_acceptors"],
+    "variant_effect": ["variant_effect_pathogenic_clinvar", "variant_effect_pathogenic_omim"],
+    "rna_expression": ["bulk_rna_expression"],
+}
+
+# Inverse map: task_name -> category label
+TASK_TO_CATEGORY: dict[str, str] = {
+    task: cat for cat, tasks in TASK_CATEGORIES.items() for task in tasks
+}
+
+# Mean sequence lengths in bp per task, populated by tools/compute_seq_lengths.py
+_SEQ_LEN_PATH = pathlib.Path(__file__).parent.parent / "tools" / "task_seq_lengths.json"
+TASK_SEQ_LENGTHS: dict[str, int] = (
+    json.loads(_SEQ_LEN_PATH.read_text()) if _SEQ_LEN_PATH.exists() else {}
+)
+
+CLASSIFICATION_METRICS: dict[str, str] = {
+    "mcc_test": "MCC",
+    "accuracy_test": "Accuracy",
+    "weighted_f1_test": "Weighted F1",
+    "macro_f1_test": "Macro F1",
+}
+
+CATEGORY_DISPLAY_NAMES: dict[str, str] = {
+    "histone_marks": "Histone Marks",
+    "promoter": "Promoter",
+    "enhancer": "Enhancer",
+    "splice_site": "Splice Site",
+    "variant_effect": "Variant Effect",
+    "rna_expression": "RNA Expression",
+    "other": "Other",
+}
+
+# One color per task category — pastel, full-opacity friendly, distinct from _MODEL_PALETTE
+CATEGORY_COLORS: dict[str, str] = {
+    "histone_marks":  "#8EB4E8",
+    "promoter":       "#F5956C",
+    "enhancer":       "#60C4A8",
+    "splice_site":    "#5AC8D8",
+    "variant_effect": "#C49AE0",
+    "rna_expression": "#F075A0",
+    "other":          "#BCC4CC",
+}

src/data.py

@@ -0,0 +1,103 @@
+import os
+
+import pandas as pd
+from datasets import get_dataset_split_names, load_dataset
+from dotenv import load_dotenv
+
+from src.constants import TASK_SEQ_LENGTHS, TASK_TO_CATEGORY
+
+load_dotenv()
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+LEADERBOARD_DATASET = os.environ.get("LEADERBOARD_DATASET", "lokahq/genomic-benchmark-metrics")
+
+
+def load_data() -> pd.DataFrame:
+    """Load all model splits from the HF Hub dataset and return a concatenated DataFrame.
+
+    Each split corresponds to one model alias. Derived columns task_category and
+    task_seq_length are added from the constants maps.
+    """
+    splits = get_dataset_split_names(LEADERBOARD_DATASET, token=HF_TOKEN)
+    frames = []
+    for split in splits:
+        ds = load_dataset(LEADERBOARD_DATASET, split=split, token=HF_TOKEN)
+        frames.append(ds.to_pandas())
+    df = pd.concat(frames, ignore_index=True)
+
+    # Normalize task names to lowercase so TASK_TO_CATEGORY mapping works regardless
+    # of how the benchmark runner capitalizes them (e.g. "H3K27ac" → "h3k27ac")
+    df["task_name"] = df["task_name"].str.lower()
+
+    df["task_category"] = df["task_name"].map(TASK_TO_CATEGORY).fillna("other")
+    df["task_seq_length"] = df["task_name"].map(TASK_SEQ_LENGTHS)
+
+    if "run_at" in df.columns:
+        df["run_at"] = pd.to_datetime(df["run_at"], utc=True, errors="coerce")
+
+    return df
+
+
+def _subsample_priority(val) -> float:
+    """Map subsample_train value to sort priority. NULL/0/1 = production = highest priority."""
+    if val is None or (isinstance(val, float) and pd.isna(val)) or val == 0 or val == 1:
+        return float("inf")
+    return float(val)
+
+
+def deduplicate(df: pd.DataFrame) -> pd.DataFrame:
+    """Keep best run per (embedding_config_hash, head_type).
+
+    embedding_config_hash encodes model, pooling, layer selection (+ task via the cache key it hashes).
+    'Best' = production run preferred (subsample_train NULL/0/1 → inf priority),
+    then highest subsample fraction, then most recent run_at.
+    """
+    if "run_at" not in df.columns:
+        return df
+    dedup_cols = []
+    if "embedding_config_hash" in df.columns:
+        dedup_cols.append("embedding_config_hash")
+    if "head_type" in df.columns:
+        dedup_cols.append("head_type")
+    if not dedup_cols:
+        return df
+    df = df.copy()
+    df["_sub_priority"] = df.get("subsample_train", pd.Series(dtype=float)).map(_subsample_priority)
+    df = df.sort_values(["_sub_priority", "run_at"], ascending=[False, False])
+    return df.drop_duplicates(subset=dedup_cols).drop(columns=["_sub_priority"]).reset_index(drop=True)
+
+
+
+def apply_filters(df: pd.DataFrame, filters: dict) -> pd.DataFrame:
+    """Apply user-selected filters and return the filtered DataFrame."""
+    result = df.copy()
+
+    subsample_mode = filters.get("subsample_train")
+    if subsample_mode == "full" and "subsample_train" in result.columns:
+        result = result[result["subsample_train"].isna() | (result["subsample_train"] == 0)]
+    elif subsample_mode == "subsampled" and "subsample_train" in result.columns:
+        result = result[result["subsample_train"].notna() & (result["subsample_train"] > 0)]
+
+    for col, min_key, max_key in [
+        ("model_params", "model_params_min", "model_params_max"),
+        ("embedding_dim", "embedding_dim_min", "embedding_dim_max"),
+        ("max_context_size", "max_context_size_min", "max_context_size_max"),
+    ]:
+        if col not in result.columns:
+            continue
+        lo = filters.get(min_key)
+        hi = filters.get(max_key)
+        if lo is not None:
+            result = result[result[col] >= lo]
+        if hi is not None:
+            result = result[result[col] <= hi]
+
+    excluded_models = filters.get("exclude_models") or []
+    if excluded_models:
+        result = result[~result["model_alias"].isin(excluded_models)]
+
+    excluded_runs = filters.get("exclude_run_ids") or []
+    if excluded_runs and "run_id" in result.columns:
+        result = result[~result["run_id"].isin(excluded_runs)]
+
+    return result.reset_index(drop=True)

src/display/css_html_js.py

@@ -1,64 +0,0 @@
-custom_css = """
-.gradio-container {
-  max-width: 1400px !important;
-  margin: 0 auto !important;
-  font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important;
-}
-
-.markdown-text {
-  font-size: 16px !important;
-  line-height: 1.7 !important;
-}
-
-body:not(.dark) .markdown-text {
-  color: #2d3748 !important;
-}
-
-body:not(.dark) .markdown-text h2 {
-  color: #667eea !important;
-  font-weight: 700 !important;
-  margin-top: 1.5rem !important;
-  margin-bottom: 1rem !important;
-}
-
-/* Table: only font weights */
-table tbody td,
-table tbody th {
-  font-weight: 400 !important;
-}
-table thead th {
-  font-weight: 600 !important;
-}
-"""
-custom_css = """
-.gradio-container {
-  max-width: 1400px !important;
-  margin: 0 auto !important;
-  font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important;
-}
-
-.markdown-text {
-  font-size: 16px !important;
-  line-height: 1.7 !important;
-}
-
-body:not(.dark) .markdown-text {
-  color: #2d3748 !important;
-}
-
-body:not(.dark) .markdown-text h2 {
-  color: #667eea !important;
-  font-weight: 700 !important;
-  margin-top: 1.5rem !important;
-  margin-bottom: 1rem !important;
-}
-
-/* Table: only font weights */
-table tbody td,
-table tbody th {
-  font-weight: 400 !important;
-}
-table thead th {
-  font-weight: 600 !important;
-}
-"""

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/plotting.py

@@ -1,205 +0,0 @@
-import plotly.express as px
-import plotly.graph_objects as go
-import pandas as pd
-
-METRICS_FOR_PLOTS = ["Accuracy", "MCC", "Weighted F1"]
-
-# Color scheme matching the UI theme
-COLORS = {
-    'primary': '#667eea',
-    'secondary': '#764ba2',
-    'accent': '#F97316',
-    'gradient': ['#667eea', '#764ba2', '#F97316', '#06b6d4', '#10b981']
-}
-
-def extract_mean_std(col):
-    if isinstance(col, str) and '±' in col:
-        try:
-            mean, std = col.split('±')
-            return float(mean.strip()), float(std.strip())
-        except:
-            return None, None
-    elif isinstance(col, (int, float)):
-        return col, 0
-    return None, None
-
-def prepare_leaderboard_df(df):
-    for metric in METRICS_FOR_PLOTS:
-        means, stds = zip(*df[metric].apply(extract_mean_std))
-        df[f"{metric}_mean"] = means
-        df[f"{metric}_std"] = stds
-    return df
-
-def make_plot_wrapper(leaderboard_df, group_by, filter_col, orientation):
-    def plot_fn(filter_val, metric, dataset):
-        return plot_metric_bar(
-            df=leaderboard_df,
-            group_by=group_by,
-            filter_col=filter_col,
-            dataset=str(dataset),
-            filter_val=filter_val,
-            metric=metric,
-            orientation=orientation
-        )
-    return plot_fn
-
-def make_model_all_datasets_wrapper(leaderboard_df):
-    def plot_fn(model, metric):
-        return plot_metric_bar_all_datasets(
-            df=leaderboard_df,
-            model=model,
-            metric=metric
-        )
-    return plot_fn
-
-def plot_metric_bar_all_datasets(df, model, metric, orientation="v"):
-    df = df.copy()
-    df = df[df['Model'] == model]
-
-    y_col = f"{metric}_mean"
-    std_col = f"{metric}_std"
-
-    if df.empty or y_col not in df.columns:
-        return px.bar(title=f"No data found for model: {model}")
-
-    df = df.sort_values(by=y_col, ascending=False)
-
-    x_axis, y_axis = (y_col, "Task") if orientation == "h" else ("Task", y_col)
-    height = 20 * len(df) + 200 if orientation == "h" else None
-
-    fig = px.bar(
-        df,
-        x=x_axis,
-        y=y_axis,
-        color='Dataset Name',
-        orientation=orientation,
-        text_auto=".2f",
-        labels={y_col: metric, "Task": "Task"},
-        title=f"{metric} for {model} across all datasets",
-        height=height,
-        color_discrete_sequence=COLORS['gradient']
-    )
-
-    if orientation == "h":
-        hovertemplate = (
-            "<b>%{y}</b><br>"
-            f"Mean {metric}: %{{x:.2f}}<br>"
-            "Std Dev: %{customdata[0]:.3f}"
-            "<extra></extra>"
-        )
-    else:
-        hovertemplate = (
-            "<b>%{x}</b><br>"
-            f"Mean {metric}: %{{y:.2f}}<br>"
-            "Std Dev: %{customdata[0]:.3f}"
-            "<extra></extra>"
-        )
-
-    fig.update_traces(
-        hovertemplate=hovertemplate,
-        customdata=df[[std_col]].values
-    )
-
-    layout_args = dict(
-        title={
-            'text': f"{metric} for {model} across all tasks",
-            'x': 0.5,
-            'xanchor': 'center',
-            'yanchor': 'top',
-            'font': dict(size=22, color='#2d3748', family='Inter, sans-serif'),
-            'pad': dict(t=20, b=10),
-        },
-        margin=dict(t=80, b=100, l=150, r=10),
-        plot_bgcolor='rgba(0,0,0,0)',
-        paper_bgcolor='white',
-        font=dict(family='Inter, sans-serif', color='#2d3748'),
-    )
-
-    if orientation == "h":
-        layout_args["xaxis"] = dict(range=[0, 1], gridcolor='#e2e8f0')
-        layout_args["yaxis_tickfont_size"] = 10
-        layout_args["yaxis"] = dict(gridcolor='#e2e8f0')
-    else:
-        layout_args["yaxis"] = dict(range=[0, 1], gridcolor='#e2e8f0')
-        layout_args["xaxis"] = dict(gridcolor='#e2e8f0')
-
-    fig.update_layout(**layout_args)
-
-    return fig
-
-def plot_metric_bar(df, group_by, filter_col, filter_val, dataset, metric, orientation="v"):
-    df = df.copy()
-    df = df[df['Dataset Name'] == dataset]
-    df = df[df[filter_col] == filter_val]
-
-    y_col = f"{metric}_mean"
-    std_col = f"{metric}_std"
-
-    if df.empty or y_col not in df.columns:
-        return px.bar(title="No data found.")
-
-    df = df.sort_values(by=y_col, ascending=False)
-
-    x_axis, y_axis = (y_col, group_by) if orientation == "h" else (group_by, y_col)
-    height = 20 * len(df) + 200 if orientation == "h" else None
-
-    fig = px.bar(
-        df,
-        x=x_axis,
-        y=y_axis,
-        orientation=orientation,
-        text_auto=".2f",
-        labels={y_col: metric, group_by: group_by},
-        title=f"{metric} for {filter_val} - {dataset}",
-        height=height,
-    )
-
-    if orientation == "h":
-        hovertemplate = (
-            "<b>%{y}</b><br>"
-            f"Mean {metric}: %{{x:.2f}}<br>"
-            "Std Dev: %{customdata[0]:.3f}"
-            "<extra></extra>"
-        )
-    else:
-        hovertemplate = (
-            "<b>%{x}</b><br>"
-            f"Mean {metric}: %{{y:.2f}}<br>"
-            "Std Dev: %{customdata[0]:.3f}"
-            "<extra></extra>"
-        )
-
-    fig.update_traces(
-        marker_color=COLORS['primary'],
-        marker_line_color=COLORS['secondary'],
-        marker_line_width=1.5,
-        hovertemplate=hovertemplate,
-        customdata=df[[std_col]].values
-    )
-
-    layout_args = dict(
-        title={
-            'text': f"{metric} for {filter_val} - {dataset}",
-            'x': 0.5,
-            'xanchor': 'center',
-            'yanchor': 'top',
-            'font': dict(size=22, color='#2d3748', family='Inter, sans-serif'),
-            'pad': dict(t=20, b=10),
-        },
-        margin=dict(t=80, b=100, l=150, r=10),
-        plot_bgcolor='rgba(0,0,0,0)',
-        paper_bgcolor='white',
-        font=dict(family='Inter, sans-serif', color='#2d3748'),
-    )
-
-    if orientation == "h":
-        layout_args["xaxis"] = dict(range=[0, 1], gridcolor='#e2e8f0')
-        layout_args["yaxis_tickfont_size"] = 10
-        layout_args["yaxis"] = dict(gridcolor='#e2e8f0')
-    else:
-        layout_args["yaxis"] = dict(range=[0, 1], gridcolor='#e2e8f0')
-        layout_args["xaxis"] = dict(gridcolor='#e2e8f0')
-
-    fig.update_layout(**layout_args)
-
-    return fig

src/display/utils.py

@@ -1,65 +0,0 @@
-from dataclasses import dataclass
-
-from src.about import Tasks
-
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    raw_name: str
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class BenchRawColumn:  # Queue column
-    model = ColumnContent("hf_model", "Model", "str", True)
-    task = ColumnContent("task", "Task", "markdown", False)
-    train_len = ColumnContent("train_len", "Train Samples", "number", False)
-    test_len = ColumnContent("test_len", "Test Samples", "number", False)
-    train_proc_time = ColumnContent("train_proc_time", "Train Processing (s)", "number", False)
-    test_proc_time = ColumnContent("test_proc_time", "Test Processing (s)", "number", False)
-    input_length = ColumnContent("input_length", "Number of Samples", "number", False)
-    acc = ColumnContent("accuracy", "Accuracy", "number", True)
-    mcc = ColumnContent("mcc", "MCC", "number", True)
-    infer_time = ColumnContent("inference_time", "Inference Time (s)", "number", False)
-    embds_dim = ColumnContent("embeddings_dim", "Embds Dim", "number", False)
-    model_params = ColumnContent("model_params", "Model Params (M)", "number", True)
-    vram_model = ColumnContent("vram_model", "VRAM Model (MB)", "number", False)
-    classification_report = ColumnContent("classification_report", "Classification Report", "str", False, hidden=True)
-    config = ColumnContent("config", "Config", "str", False, hidden=True)
-    max_context_len = ColumnContent("max_context_size", "Max Context Length (bp)", "number", False)
-    dataset_name = ColumnContent("dataset", "Dataset Name", "str", False)
-    weighted_f1 = ColumnContent("weighted f1", "Weighted F1", "number", True)
-
-
-BenchTableModel = BenchRawColumn()
-
-# Column selection
-COLS = [c.name for c in fields(BenchRawColumn) if not c.hidden]
-METRICS_COLS = [BenchRawColumn.acc.name, BenchRawColumn.mcc.name, BenchRawColumn.weighted_f1.name]
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-
-# For summarizing model performance per task type
-COLS_TO_AVERAGE = ['Accuracy', 'MCC', 'Weighted F1', 'Train Processing (s)', 'Test Processing (s)', 'Inference Time (s)', 
-                   'Number of Samples', 'Train Samples', 'Test Samples']
-COLS_DEPEND_ON_MODEL = ['Model Params (M)', 'Embds Dim', 'VRAM Model (MB)', "Max Context Length (bp)", "Dataset Name"]
-TASK_TYPE_MAP = {
-    "H2AFZ": "Histone", "H3K27ac": "Histone", "H3K27me3": "Histone", "H3K36me3": "Histone",
-    "H3K4me1": "Histone", "H3K4me2": "Histone", "H3K4me3": "Histone", "H3K9ac": "Histone",
-    "H3K9me3": "Histone", "H4K20me1": "Histone",
-    "splice_sites_donors": "Splicing", "splice_sites_acceptors": "Splicing", "splice_sites_all": "Splicing",
-    "promoter_no_tata": "Promoter", "promoter_tata": "Promoter", "promoter_all": "Promoter",
-    "enhancers": "Enhancer", "enhancers_types": "Enhancer", "variant_effect_causal_eqtl": "SNP Classification",
-    "variant_effect_pathogenic_clinvar": "SNP Classification", "variant_effect_pathogenic_omim": "SNP Classification"
-}

src/envs.py

@@ -1,29 +0,0 @@
-import os
-
-from dotenv import load_dotenv
-from huggingface_hub import HfApi
-
-# Load environment variables from .env file
-load_dotenv()
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
-
-OWNER = "lokahq"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/dna-benchmark"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/bench-dna-results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH = os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/plots.py

@@ -0,0 +1,598 @@
+"""Plotly figure factories for the DNA benchmark leaderboard."""
+
+import json
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+
+from src.constants import (
+    CATEGORY_COLORS,
+    CATEGORY_DISPLAY_NAMES,
+    CLASSIFICATION_METRICS,
+    TASK_SEQ_LENGTHS,
+)
+
+# Model colour palette — pastel, full-opacity friendly, distinct from CATEGORY_COLORS
+_MODEL_PALETTE = ["#F08E9E", "#64BDAD", "#F7C96A", "#7FAACC", "#B09AD8", "#80C89A", "#F5A978"]
+
+# Shared horizontal legend layout used across multiple figures.
+# Explicit defaults: single-click toggles a trace, double-click isolates it.
+_LEGEND_H = dict(
+    orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1,
+    itemclick="toggle", itemdoubleclick="toggleothers",
+)
+
+
+def _model_color_map(models: list[str]) -> dict[str, str]:
+    return {m: _MODEL_PALETTE[i % len(_MODEL_PALETTE)] for i, m in enumerate(sorted(models))}
+
+
+def _fmt_layer(params_str) -> str:
+    """Short label for layer selection from layer_selection_params JSON string."""
+    if params_str is None or (isinstance(params_str, float) and pd.isna(params_str)):
+        return "default"
+    try:
+        params = json.loads(params_str) if isinstance(params_str, str) else params_str
+        if not params:
+            return "default"
+        idx = params.get("layer_index")
+        strategy = params.get("strategy") or params.get("layer_selection_strategy")
+        parts = []
+        if strategy:
+            parts.append(str(strategy))
+        if idx is not None:
+            parts.append(f"({int(idx)})")
+        return " ".join(parts) if parts else "default"
+    except (json.JSONDecodeError, TypeError):
+        return "default"
+
+
+def _variant_label(row: pd.Series) -> str:
+    """Human-readable config label from a data row: 'short-hf-id · pooling · layer · head'."""
+    parts = []
+    hf = row.get("model_hf_id")
+    if hf is not None and not (isinstance(hf, float) and pd.isna(hf)):
+        parts.append(str(hf).rsplit("/", 1)[-1])
+    pool = row.get("pooling_strategy")
+    if pool is not None and not (isinstance(pool, float) and pd.isna(pool)):
+        parts.append(str(pool))
+    parts.append(_fmt_layer(row.get("layer_selection_params")))
+    head = row.get("head_type")
+    if head is not None and not (isinstance(head, float) and pd.isna(head)):
+        parts.append(str(head))
+    return " · ".join(parts)
+
+
+def _variant_hover_lines(vdf: pd.DataFrame) -> str:
+    """Multi-line config block for heatmap hover, sourced from first row of variant group."""
+    row = vdf.iloc[0]
+    lines = []
+    hf = row.get("model_hf_id")
+    if hf is not None and not (isinstance(hf, float) and pd.isna(hf)):
+        lines.append(f"HF model: {hf}")
+    pool = row.get("pooling_strategy")
+    if pool is not None and not (isinstance(pool, float) and pd.isna(pool)):
+        lines.append(f"Pooling: {pool}")
+    lines.append(f"Layer: {_fmt_layer(row.get('layer_selection_params'))}")
+    head = row.get("head_type")
+    if head is not None and not (isinstance(head, float) and pd.isna(head)):
+        lines.append(f"Head: {head}")
+    return "<br>".join(lines)
+
+
+# Hover label for config customdata columns
+_CONFIG_HOVER_LABELS: dict[str, str] = {
+    "model_hf_id": "HF model",
+    "pooling_strategy": "Pooling",
+    "_layer_info": "Layer",
+}
+
+
+
+# ---------------------------------------------------------------------------
+# 1. Ranking table
+# ---------------------------------------------------------------------------
+
+def ranking_table(df: pd.DataFrame, metric: str) -> pd.DataFrame:
+    """Return a summary DataFrame: rows=models, cols=task categories + Overall."""
+    if df.empty or metric not in df.columns:
+        return pd.DataFrame()
+
+    records = []
+    for model, group in df.groupby("model_alias"):
+        row: dict = {"Model": model}
+        for cat_key, cat_label in CATEGORY_DISPLAY_NAMES.items():
+            if cat_key == "other":
+                continue
+            cat_data = group[group["task_category"] == cat_key][metric]
+            row[cat_label] = round(cat_data.mean(), 3) if not cat_data.empty else None
+        row["Overall"] = round(group[metric].mean(), 3)
+        records.append(row)
+
+    table = pd.DataFrame(records)
+    # Drop columns where all values are None (no data for that category yet)
+    cat_cols = [col for col in table.columns if col not in ("Model", "Overall")]
+    table = table.drop(columns=[col for col in cat_cols if table[col].isna().all()])
+    if "Overall" in table.columns:
+        table = table.sort_values("Overall", ascending=False).reset_index(drop=True)
+    return table
+
+
+# ---------------------------------------------------------------------------
+# Internal helper: violin shape + category-coloured scatter overlay
+# ---------------------------------------------------------------------------
+
+def _violin_with_category_points(
+    fig: go.Figure,
+    x_pos: int,
+    subset: pd.DataFrame,
+    metric: str,
+    metric_label: str,
+    violin_color: str,
+    shown_cats: set[str],
+    legend_name: str,
+    customdata_cols: list[str] | None = None,
+) -> None:
+    """Add one violin (no built-in points) + jittered scatter overlay coloured by task category.
+
+    Uses numeric x positions so scatter points can be spread horizontally with jitter.
+    Callers must set xaxis tickvals/ticktext after all traces are added.
+    """
+    # Violin shape only (no built-in points)
+    fig.add_trace(
+        go.Violin(
+            y=subset[metric],
+            x=[x_pos] * len(subset),
+            name=legend_name,
+            box_visible=True,
+            meanline=dict(visible=True, color="white", width=3),
+            points=False,
+            line_color=violin_color,
+            fillcolor=violin_color,
+            opacity=0.5,
+            showlegend=True,
+        )
+    )
+
+    rng = np.random.default_rng(seed=x_pos)
+
+    # Category-coloured scatter points with horizontal jitter
+    for cat, cat_df in subset.groupby("task_category"):
+        color = CATEGORY_COLORS.get(cat, CATEGORY_COLORS["other"])
+        cat_label = CATEGORY_DISPLAY_NAMES.get(cat, cat)
+        show_legend = cat not in shown_cats
+        shown_cats.add(cat)
+
+        n = len(cat_df)
+        x_jitter = rng.uniform(x_pos - 0.18, x_pos + 0.18, n)
+
+        hover = f"<b>%{{text}}</b><br>{metric_label}: %{{y:.3f}}"
+        cd = None
+        if customdata_cols:
+            available = [c for c in customdata_cols if c in cat_df.columns]
+            if available:
+                cd = cat_df[available].to_numpy(dtype=object)
+                for i, col in enumerate(available):
+                    label = _CONFIG_HOVER_LABELS.get(col, col)
+                    hover += f"<br>{label}: %{{customdata[{i}]}}"
+        hover += "<extra></extra>"
+
+        fig.add_trace(
+            go.Scatter(
+                x=x_jitter,
+                y=cat_df[metric],
+                mode="markers",
+                name=cat_label,
+                marker=dict(color=color, size=9, opacity=0.9, line=dict(width=1, color="white")),
+                text=cat_df["task_name"],
+                hovertemplate=hover,
+                customdata=cd,
+                legendgroup="cat_" + cat,
+                showlegend=show_legend,
+            )
+        )
+
+
+# ---------------------------------------------------------------------------
+# 2. Violin plot — all models, points coloured by task category
+# ---------------------------------------------------------------------------
+
+def violin_plot(df: pd.DataFrame, metric: str) -> go.Figure:
+    """Violin plot per model with points coloured by task category and task hover text."""
+    if df.empty or metric not in df.columns:
+        return go.Figure()
+
+    models = sorted(df["model_alias"].unique())
+    color_map = _model_color_map(models)
+    metric_label = CLASSIFICATION_METRICS.get(metric, metric)
+
+    # Config columns to expose in hover
+    _CONFIG_RAW = ["model_hf_id", "pooling_strategy", "layer_selection_params"]
+    config_raw = [c for c in _CONFIG_RAW if c in df.columns]
+
+    fig = go.Figure()
+    shown_cats: set[str] = set()
+
+    for i, model in enumerate(models):
+        cols = ["task_name", "task_category", metric] + config_raw
+        subset = (
+            df[df["model_alias"] == model][cols]
+            .dropna(subset=[metric])
+            .reset_index(drop=True)
+            .copy()
+        )
+
+        customdata_cols: list[str] = []
+        if "model_hf_id" in subset.columns:
+            customdata_cols.append("model_hf_id")
+        if "pooling_strategy" in subset.columns:
+            customdata_cols.append("pooling_strategy")
+        if "layer_selection_params" in subset.columns:
+            subset["_layer_info"] = subset["layer_selection_params"].apply(_fmt_layer)
+            customdata_cols.append("_layer_info")
+
+        _violin_with_category_points(
+            fig, x_pos=i, subset=subset, metric=metric,
+            metric_label=metric_label, violin_color=color_map[model],
+            shown_cats=shown_cats, legend_name=model,
+            customdata_cols=customdata_cols or None,
+        )
+
+    fig.update_layout(
+        title=f"{metric_label} distribution per model (points coloured by task category)",
+        yaxis_title=metric_label,
+        height=500,
+        template="plotly_white",
+        legend=_LEGEND_H,
+    )
+    fig.update_xaxes(
+        tickmode="array",
+        tickvals=list(range(len(models))),
+        ticktext=models,
+    )
+    return fig
+
+
+# ---------------------------------------------------------------------------
+# 3. Heatmap — single model, rows = config variants, cols = task categories
+# ---------------------------------------------------------------------------
+
+def heatmap_variants(
+    df: pd.DataFrame,
+    metric: str,
+    model_alias: str,
+    aggregate_by_dataset: bool = False,
+) -> go.Figure:
+    """Heatmap: rows = config variants (or single row if only one), cols = tasks (or datasets) + Overall.
+
+    By default columns are individual task_names ordered by category then alphabetically.
+    When aggregate_by_dataset=True columns collapse to dataset_key values (mean per dataset).
+    Rows sorted best→worst by Overall when multiple variants exist.
+    """
+    metric_label = CLASSIFICATION_METRICS.get(metric, metric)
+
+    if not model_alias or df.empty or metric not in df.columns:
+        return go.Figure()
+
+    mdf = df[df["model_alias"] == model_alias]
+    if mdf.empty:
+        return go.Figure()
+
+    # Group by config fields that are task-agnostic (embedding_cache_key includes task name)
+    _KEY_COLS = ["pooling_strategy", "layer_selection_params", "head_type"]
+    key_cols = [c for c in _KEY_COLS if c in mdf.columns]
+    mdf = mdf.copy()
+    mdf["_embed_key"] = mdf[key_cols].apply(lambda r: str(tuple(r.values)), axis=1)
+    configs = sorted(mdf["_embed_key"].unique())
+
+    if len(configs) > 1:
+        rows_iter = [
+            (_variant_label(mdf[mdf["_embed_key"] == c].iloc[0]), mdf[mdf["_embed_key"] == c])
+            for c in configs
+        ]
+    else:
+        rows_iter = [(_variant_label(mdf.iloc[0]), mdf)]
+
+    # Build column list depending on aggregation mode
+    if aggregate_by_dataset:
+        # Collapse to dataset_name (full HF path, globally unique); one column per dataset
+        col_keys = sorted(mdf["dataset_name"].dropna().unique()) if "dataset_name" in mdf.columns else []
+        col_names = col_keys
+
+        def _cell_vals(vdf: pd.DataFrame, col_key: str) -> pd.Series:
+            return vdf[vdf["dataset_name"] == col_key][metric].dropna()
+    else:
+        # Individual task_names ordered by category then alphabetically
+        col_keys, col_names = [], []
+        for cat in CATEGORY_DISPLAY_NAMES:
+            for task in sorted(mdf[mdf["task_category"] == cat]["task_name"].unique()):
+                col_keys.append(task)
+                col_names.append(task)
+
+        def _cell_vals(vdf: pd.DataFrame, col_key: str) -> pd.Series:
+            return vdf[vdf["task_name"] == col_key][metric].dropna()
+
+    col_names = col_names + ["Overall"]
+
+    rows_z, rows_text, rows_hover, row_labels = [], [], [], []
+    for label, vdf in rows_iter:
+        z_row, t_row, h_row = [], [], []
+        config_lines = _variant_hover_lines(vdf)
+        for col_key, col_name in zip(col_keys, col_names[:-1]):
+            vals = _cell_vals(vdf, col_key)
+            v = round(float(vals.mean()), 3) if not vals.empty else None
+            z_row.append(v)
+            t_row.append(f"{v:.3f}" if v is not None else "—")
+            v_str = f"{v:.3f}" if v is not None else "N/A"
+            h_row.append(f"<b>{col_name}</b><br>{metric_label}: {v_str}<br>{config_lines}")
+        overall_vals = vdf[metric].dropna()
+        overall = round(float(overall_vals.mean()), 3) if not overall_vals.empty else None
+        z_row.append(overall)
+        t_row.append(f"{overall:.3f}" if overall is not None else "—")
+        ov_str = f"{overall:.3f}" if overall is not None else "N/A"
+        h_row.append(f"<b>Overall</b><br>{metric_label}: {ov_str}<br>{config_lines}")
+        rows_z.append(z_row)
+        rows_text.append(t_row)
+        rows_hover.append(h_row)
+        row_labels.append(label)
+
+    # Sort best→worst by Overall; heatmap y is bottom-up so reverse after sort
+    if len(rows_z) > 1:
+        sort_idx = sorted(range(len(rows_z)), key=lambda i: rows_z[i][-1] or -1, reverse=True)
+        rows_z = [rows_z[i] for i in sort_idx][::-1]
+        rows_text = [rows_text[i] for i in sort_idx][::-1]
+        rows_hover = [rows_hover[i] for i in sort_idx][::-1]
+        row_labels = [row_labels[i] for i in sort_idx][::-1]
+
+    zmin = -1 if metric == "mcc_test" else 0
+    n_data_cols = len(col_keys)
+    # Smaller font when many columns; angled labels in task mode for readability
+    cell_font_size = 9 if n_data_cols > 10 else 12
+    tick_angle = -45 if not aggregate_by_dataset else 0
+
+    fig = go.Figure(
+        go.Heatmap(
+            z=rows_z,
+            x=col_names,
+            y=row_labels,
+            text=rows_text,
+            texttemplate="%{text}",
+            textfont=dict(size=cell_font_size, color="black"),
+            hovertext=rows_hover,
+            hovertemplate="%{hovertext}<extra></extra>",
+            colorscale="RdYlGn",
+            zmin=zmin,
+            zmax=1,
+            colorbar=dict(title=metric_label, thickness=14, nticks=5, tickformat=".2f", lenmode="pixels", len=220),
+            hoverongaps=False,
+        )
+    )
+
+    # Highlight the Overall column with a border
+    fig.add_shape(
+        type="rect",
+        x0=n_data_cols - 0.5, x1=n_data_cols + 0.5,
+        y0=-0.5, y1=len(rows_z) - 0.5,
+        line=dict(color="#333", width=2),
+        fillcolor="rgba(0,0,0,0)",
+    )
+
+    n_rows = len(rows_z)
+    left_margin = min(max(len(lbl) for lbl in row_labels) * 7, 320)
+    # Extra bottom margin for angled tick labels in task mode
+    bottom_margin = 120 if not aggregate_by_dataset else 20
+    t_margin = 120 if not aggregate_by_dataset else 80
+    v_overhead = t_margin + bottom_margin
+    height = max(v_overhead + 80, n_rows * 48 + v_overhead + 20)
+    fig.update_layout(
+        title=f"{metric_label} — {model_alias}",
+        height=height,
+        template="plotly_white",
+        xaxis=dict(side="top", tickfont=dict(size=9 if not aggregate_by_dataset else 11), tickangle=tick_angle),
+        yaxis=dict(tickfont=dict(size=10), automargin=True),
+        margin=dict(l=left_margin, r=20, t=t_margin, b=bottom_margin),
+    )
+    return fig
+
+
+# ---------------------------------------------------------------------------
+# 4 & 5. Grouped bar chart (shared helper + two public wrappers)
+# ---------------------------------------------------------------------------
+
+def _grouped_bar(
+    df: pd.DataFrame,
+    metric: str,
+    x_items: list,
+    x_labels: list[str],
+    get_vals,  # Callable[[model_df, item], pd.Series]
+    title: str,
+) -> go.Figure:
+    """Build a grouped bar chart shared by bar_plot_per_category and bar_plot_per_task."""
+    models = sorted(df["model_alias"].unique())
+    color_map = _model_color_map(models)
+    metric_label = CLASSIFICATION_METRICS.get(metric, metric)
+    fig = go.Figure()
+    for model in models:
+        model_df = df[df["model_alias"] == model]
+        y_vals = [
+            round(float(vals.mean()), 3) if not (vals := get_vals(model_df, item).dropna()).empty else None
+            for item in x_items
+        ]
+        fig.add_trace(go.Bar(name=model, x=x_labels, y=y_vals, marker_color=color_map[model]))
+    fig.update_layout(
+        title=title, yaxis_title=metric_label, barmode="group",
+        height=440, template="plotly_white", legend=_LEGEND_H,
+    )
+    return fig
+
+
+def bar_plot_per_category(df: pd.DataFrame, metric: str) -> go.Figure:
+    """Grouped bar: X = task category, bars = models, height = mean metric."""
+    if df.empty or metric not in df.columns:
+        return go.Figure()
+    active_cats = df["task_category"].unique()
+    categories = [k for k in CATEGORY_DISPLAY_NAMES if k != "other" and k in active_cats]
+    metric_label = CLASSIFICATION_METRICS.get(metric, metric)
+    return _grouped_bar(
+        df, metric,
+        x_items=categories,
+        x_labels=[CATEGORY_DISPLAY_NAMES[c] for c in categories],
+        get_vals=lambda mdf, cat: mdf[mdf["task_category"] == cat][metric],
+        title=f"{metric_label} per task category",
+    )
+
+
+def bar_plot_per_task(df: pd.DataFrame, metric: str, category: str) -> go.Figure:
+    """Grouped bar: X = individual tasks within category, bars = models."""
+    if df.empty or metric not in df.columns or not category:
+        return go.Figure()
+    task_df = df[df["task_category"] == category]
+    if task_df.empty:
+        fig = go.Figure()
+        fig.add_annotation(text=f"No data for category '{category}'", showarrow=False,
+                           xref="paper", yref="paper", x=0.5, y=0.5)
+        return fig
+    tasks = sorted(task_df["task_name"].unique())
+    metric_label = CLASSIFICATION_METRICS.get(metric, metric)
+    cat_label = CATEGORY_DISPLAY_NAMES.get(category, category)
+    return _grouped_bar(
+        task_df, metric,
+        x_items=tasks,
+        x_labels=tasks,
+        get_vals=lambda mdf, task: mdf[mdf["task_name"] == task][metric],
+        title=f"{metric_label} per task — {cat_label}",
+    )
+
+
+# ---------------------------------------------------------------------------
+# 6. Speed vs Performance scatter (2-subplot)
+# ---------------------------------------------------------------------------
+
+def scatter_speed(df: pd.DataFrame, metric: str) -> go.Figure:
+    """Two side-by-side subplots: throughput vs metric, embedding time vs metric."""
+    if df.empty or metric not in df.columns:
+        return go.Figure()
+
+    has_throughput = "test_throughput_seq_s" in df.columns
+    has_embed_time = "test_embedding_time_s" in df.columns
+
+    if not has_throughput and not has_embed_time:
+        fig = go.Figure()
+        fig.add_annotation(text="Speed metrics not available in data", showarrow=False)
+        return fig
+
+    fig = make_subplots(
+        rows=1, cols=2,
+        subplot_titles=("Throughput (seq/s) vs Performance", "Embedding Time (s) vs Performance"),
+    )
+    models = sorted(df["model_alias"].unique())
+    color_map = _model_color_map(models)
+    metric_label = CLASSIFICATION_METRICS.get(metric, metric)
+
+    for model in models:
+        mdf = df[df["model_alias"] == model]
+        color = color_map[model]
+
+        if has_throughput:
+            sub = mdf[["test_throughput_seq_s", metric]].dropna()
+            fig.add_trace(
+                go.Scatter(x=sub["test_throughput_seq_s"], y=sub[metric], mode="markers",
+                           name=model, marker=dict(color=color, size=10, opacity=0.75),
+                           legendgroup=model, showlegend=True),
+                row=1, col=1,
+            )
+
+        if has_embed_time:
+            sub = mdf[["test_embedding_time_s", metric]].dropna()
+            fig.add_trace(
+                go.Scatter(x=sub["test_embedding_time_s"], y=sub[metric], mode="markers",
+                           name=model, marker=dict(color=color, size=10, opacity=0.75),
+                           legendgroup=model, showlegend=not has_throughput),
+                row=1, col=2,
+            )
+
+    fig.update_xaxes(title_text="Throughput (seq/s)", row=1, col=1)
+    fig.update_xaxes(title_text="Embedding Time (s)", row=1, col=2)
+    fig.update_yaxes(title_text=metric_label, row=1, col=1)
+    fig.update_yaxes(title_text=metric_label, row=1, col=2)
+    fig.update_layout(height=480, template="plotly_white")
+    return fig
+
+
+# ---------------------------------------------------------------------------
+# 7. Context length bubble chart
+# ---------------------------------------------------------------------------
+
+def bubble_context_length(df: pd.DataFrame, metric: str) -> go.Figure:
+    """Bubble chart: X=task_seq_length, Y=metric, color=model, size=max_context_size."""
+    if not TASK_SEQ_LENGTHS:
+        fig = go.Figure()
+        fig.add_annotation(
+            text=(
+                "Sequence length data not available.<br>"
+                "Run <b>python tools/compute_seq_lengths.py</b> to populate it."
+            ),
+            showarrow=False, font=dict(size=14), xref="paper", yref="paper", x=0.5, y=0.5,
+        )
+        fig.update_layout(height=480, template="plotly_white")
+        return fig
+
+    if df.empty or metric not in df.columns or "task_seq_length" not in df.columns:
+        return go.Figure()
+
+    _CONFIG_COLS = ["model_hf_id", "pooling_strategy", "layer_selection_params", "head_type"]
+    extra_cols = [c for c in _CONFIG_COLS if c in df.columns]
+    base_cols = ["model_alias", "task_name", "task_seq_length", metric, "max_context_size"]
+    plot_df = df[base_cols + extra_cols].dropna(subset=["task_seq_length", metric]).copy()
+    if "layer_selection_params" in plot_df.columns:
+        plot_df["_layer_info"] = plot_df["layer_selection_params"].apply(_fmt_layer)
+
+    def _config_str(row: pd.Series) -> str:
+        parts = []
+        for col, label in [("model_hf_id", "HF model"), ("pooling_strategy", "Pooling"),
+                           ("_layer_info", "Layer"), ("head_type", "Head")]:
+            if col not in row.index:
+                continue
+            v = row[col]
+            if v is not None and not (isinstance(v, float) and pd.isna(v)):
+                parts.append(f"{label}: {v}")
+        return "<br>".join(parts)
+
+    plot_df["_config_str"] = plot_df.apply(_config_str, axis=1)
+
+    models = sorted(plot_df["model_alias"].unique())
+    color_map = _model_color_map(models)
+    max_ctx = plot_df["max_context_size"].max() or 1
+    metric_label = CLASSIFICATION_METRICS.get(metric, metric)
+    fig = go.Figure()
+
+    for model in models:
+        mdf = plot_df[plot_df["model_alias"] == model]
+        sizes = (mdf["max_context_size"].fillna(max_ctx) / max_ctx * 40 + 8).clip(8, 50)
+        cd = mdf[["max_context_size", "_config_str"]].to_numpy(dtype=object)
+        fig.add_trace(
+            go.Scatter(
+                x=mdf["task_seq_length"], y=mdf[metric], mode="markers", name=model,
+                marker=dict(size=sizes, color=color_map[model], opacity=0.7,
+                            line=dict(width=1, color="white")),
+                text=mdf["task_name"],
+                hovertemplate=(
+                    "<b>%{text}</b><br>Seq length: %{x} bp<br>"
+                    f"{metric_label}: %{{y:.3f}}<br>"
+                    "Max context: %{customdata[0]} bp<br>"
+                    "%{customdata[1]}<extra></extra>"
+                ),
+                customdata=cd,
+            )
+        )
+
+    fig.update_layout(
+        title=f"{metric_label} vs task sequence length (bubble size = model max context)",
+        xaxis_title="Task sequence length (bp)",
+        yaxis_title=metric_label,
+        height=520,
+        template="plotly_white",
+        legend=_LEGEND_H,
+    )
+    return fig

src/populate.py

@@ -1,53 +0,0 @@
-import pandas as pd
-from datasets import load_dataset
-
-from src.display.utils import COLS, METRICS_COLS, BenchRawColumn, BenchTableModel, fields
-from src.display.utils import COLS_TO_AVERAGE, COLS_DEPEND_ON_MODEL, TASK_TYPE_MAP
-
-
-def get_leaderboard_df_from_hf_dataset(path: str) -> pd.DataFrame:
-    dataset = load_dataset(path, split="train")
-    df = pd.DataFrame(dataset)
-    headers = {e.raw_name: e.name for e in fields(BenchRawColumn)}
-    df = df.rename(columns=headers)
-    cparams = BenchRawColumn.model_params.name
-    df[cparams] = (df[cparams] / 1000000).round()  # mandatory
-    cinfer = BenchRawColumn.infer_time.name
-    df[cinfer] = (df[cinfer] * 1000).round(2)  # infer time per 1k samples
-    BenchTableModel.infer_time.name = "Infer Time (s) - 1k samples"
-    df = df[COLS]
-    df = df.round(5)
-    df[METRICS_COLS] = df[METRICS_COLS].round(2)
-
-    return df
-
-def summarize_model_task_type_performance(df):
-    """Summarizes model performance across task types"""
-    df = df.copy()
-    df['Task'] = df['Task'].map(TASK_TYPE_MAP)
-    grouped = df.groupby(['Model', 'Task'])
-
-    avg_aggs = {col: ['mean', 'std'] for col in COLS_TO_AVERAGE}
-    model_aggs = {col: 'first' for col in COLS_DEPEND_ON_MODEL}
-    agg_dict = {**avg_aggs, **model_aggs}
-    agg_df = grouped.agg(agg_dict).reset_index()
-
-    agg_df.columns = [
-        f"{col[0]}_{col[1]}" if isinstance(col, tuple) and col[1] in ['mean', 'std']
-        else col[0] if isinstance(col, tuple)
-        else col
-        for col in agg_df.columns
-    ]
-
-    for col in COLS_TO_AVERAGE:
-        mean_col = f"{col}_mean"
-        std_col = f"{col}_std"
-
-        agg_df[col] = agg_df[mean_col].round(3).astype(str) + ' ± ' + agg_df[std_col].round(3).astype(str)
-        agg_df.drop(columns=[mean_col, std_col], inplace=True)
-
-
-    final_cols = ['Model', 'Task'] + COLS_TO_AVERAGE + COLS_DEPEND_ON_MODEL
-    agg_df = agg_df[final_cols]
-
-    return agg_df

src/submission/check_validity.py

@@ -1,109 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-
-def is_model_on_hub(
-    model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False
-) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(
-            model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
-        )
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(
-                    model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
-                )
-            except ValueError as e:
-                return (False, f"uses a tokenizer which is not in a transformers release: {e}", None)
-            except Exception as e:
-                return (
-                    False,
-                    "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?",
-                    None,
-                )
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None,
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,117 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, TOKEN
-from src.submission.check_validity import already_submitted_models, check_model_card, get_model_size, is_model_on_hub
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(
-            model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True
-        )
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )

tools/compute_seq_lengths.py

@@ -0,0 +1,91 @@
+"""One-time script to compute mean sequence lengths (bp) per genomics benchmark task.
+
+Loads each task from the InstaDeepAI/nucleotide_transformer_downstream_tasks HF dataset,
+samples up to SAMPLE_SIZE sequences, measures their lengths, and writes the result to
+tools/task_seq_lengths.json.
+
+Usage:
+    python tools/compute_seq_lengths.py
+
+After running, commit tools/task_seq_lengths.json to the repository.
+The bubble chart in the leaderboard becomes functional once this file is populated.
+"""
+
+import json
+import os
+import pathlib
+from typing import Any
+
+from datasets import DatasetDict, load_dataset
+from dotenv import load_dotenv
+
+load_dotenv()
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+HF_DATASET = "InstaDeepAI/nucleotide_transformer_downstream_tasks_revised"
+SAMPLE_SIZE = 100
+
+TASKS = [
+    "H2AFZ", "H3K27ac", "H3K27me3", "H3K36me3",
+    "H3K4me1", "H3K4me2", "H3K4me3", "H3K9ac", "H3K9me3", "H4K20me1",
+    "promoter_all", "promoter_tata", "promoter_no_tata",
+    "enhancers", "enhancers_types",
+    "splice_sites_all", "splice_sites_donors", "splice_sites_acceptors",
+    "variant_effect_pathogenic_clinvar", "variant_effect_pathogenic_omim",
+    "bulk_rna_expression",
+]
+
+OUTPUT_PATH = pathlib.Path(__file__).parent / "task_seq_lengths.json"
+
+
+def load_dataset_for_task(hf_dataset: str, task_name: str) -> DatasetDict:
+    """Load HuggingFace dataset for specific task.
+    Directly from the bdna package
+    """
+    kwargs: dict[str, Any] = {}
+    if HF_TOKEN:
+        kwargs["token"] = HF_TOKEN
+
+    if hf_dataset == "InstaDeepAI/genomics-long-range-benchmark":
+        return load_dataset(hf_dataset, task_name=task_name, **kwargs)
+
+    return load_dataset(hf_dataset, data_dir=task_name, **kwargs)
+
+
+def compute_mean_length(task_name: str) -> int | None:
+    try:
+        ds_dict = load_dataset_for_task(HF_DATASET, task_name)
+        ds = ds_dict["train"]
+        # Sample a subset to keep it fast
+        sample = ds.select(range(min(SAMPLE_SIZE, len(ds))))
+        seq_col = next(
+            (c for c in sample.column_names if "sequence" in c.lower() or "seq" in c.lower()),
+            None,
+        )
+        if seq_col is None:
+            print(f"  [{task_name}] No sequence column found — skipping.")
+            return None
+        lengths = [len(row[seq_col]) for row in sample]
+        mean_len = int(sum(lengths) / len(lengths))
+        print(f"  [{task_name}] mean length = {mean_len} bp  (n={len(lengths)})")
+        return mean_len
+    except Exception as exc:
+        print(f"  [{task_name}] ERROR: {exc}")
+        return None
+
+
+def main():
+    print(f"Computing sequence lengths from {HF_DATASET} …\n")
+    result: dict[str, int] = {}
+    for task in TASKS:
+        length = compute_mean_length(task)
+        if length is not None:
+            result[task] = length
+
+    OUTPUT_PATH.write_text(json.dumps(result, indent=2))
+    print(f"\nWrote {len(result)} entries to {OUTPUT_PATH}")
+    print("Commit tools/task_seq_lengths.json to enable the Context Length bubble chart.")
+
+
+if __name__ == "__main__":
+    main()

tools/task_seq_lengths.json

@@ -0,0 +1,20 @@
+{
+  "H2AFZ": 1000,
+  "H3K27ac": 1000,
+  "H3K27me3": 1000,
+  "H3K36me3": 1000,
+  "H3K4me1": 1000,
+  "H3K4me2": 1000,
+  "H3K4me3": 1000,
+  "H3K9ac": 1000,
+  "H3K9me3": 1000,
+  "H4K20me1": 1000,
+  "promoter_all": 300,
+  "promoter_tata": 300,
+  "promoter_no_tata": 300,
+  "enhancers": 400,
+  "enhancers_types": 400,
+  "splice_sites_all": 600,
+  "splice_sites_donors": 600,
+  "splice_sites_acceptors": 600
+}