Spaces:
Running
Running
Filterable leaderboard: facets (family/supervision/reranker), pollution selector, NR toggle, search, tabs
Browse files- .gitattributes +1 -0
- README.md +9 -6
- app.js +207 -0
- data.js +27 -0
- index.html +102 -146
- radar.png +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
radar.png filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -3,16 +3,19 @@ title: READ-Bench Leaderboard
|
|
| 3 |
emoji: 🩺
|
| 4 |
colorFrom: indigo
|
| 5 |
colorTo: green
|
| 6 |
-
sdk:
|
|
|
|
|
|
|
| 7 |
pinned: false
|
| 8 |
license: cc-by-nc-sa-4.0
|
| 9 |
---
|
| 10 |
|
| 11 |
# READ-Bench Leaderboard
|
| 12 |
|
| 13 |
-
Live leaderboard for [READ-Bench](https://huggingface.co/datasets/c3aiia3c/read-bench)
|
| 14 |
-
|
| 15 |
-
[`c3aiia3c/read-bench-results`](https://huggingface.co/datasets/c3aiia3c/read-bench-results)
|
| 16 |
-
client-side (no server), ranked by NDCG@10 at pollution levels ρ = 0 / 10 / 20%.
|
| 17 |
|
| 18 |
-
|
|
|
|
|
|
|
|
|
| 3 |
emoji: 🩺
|
| 4 |
colorFrom: indigo
|
| 5 |
colorTo: green
|
| 6 |
+
sdk: gradio
|
| 7 |
+
sdk_version: "4.44.0"
|
| 8 |
+
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
license: cc-by-nc-sa-4.0
|
| 11 |
---
|
| 12 |
|
| 13 |
# READ-Bench Leaderboard
|
| 14 |
|
| 15 |
+
Live leaderboard for [READ-Bench](https://huggingface.co/datasets/c3aiia3c/read-bench),
|
| 16 |
+
rendered from community submissions in
|
| 17 |
+
[`c3aiia3c/read-bench-results`](https://huggingface.co/datasets/c3aiia3c/read-bench-results).
|
|
|
|
| 18 |
|
| 19 |
+
Methods are ranked by **NDCG@10**, macro-averaged over the 12 datasets, at three
|
| 20 |
+
normal-pollution levels (ρ = 0 / 10 / 20%). To appear here, open a PR on the
|
| 21 |
+
results repo — see its README for the submission protocol.
|
app.js
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"use strict";
|
| 2 |
+
const RESULTS_REPO = "c3aiia3c/read-bench-results";
|
| 3 |
+
const RHO_IDX = { rho0: 5, rho10: 6, rho20: 7 }; // index into a row tuple
|
| 4 |
+
const KEY_IDX = { name:0, family:1, supervision:2, reranker:3, nr:4,
|
| 5 |
+
rho0:5, rho10:6, rho20:7, source:8, link:9 };
|
| 6 |
+
|
| 7 |
+
const state = {
|
| 8 |
+
view: "overall",
|
| 9 |
+
rho: "rho0",
|
| 10 |
+
sortKey: "rho0",
|
| 11 |
+
sortDir: -1,
|
| 12 |
+
nrOnly: false,
|
| 13 |
+
search: "",
|
| 14 |
+
active: { family: new Set(), supervision: new Set(), reranker: new Set() },
|
| 15 |
+
rows: [],
|
| 16 |
+
};
|
| 17 |
+
|
| 18 |
+
// ---- filter chips ----
|
| 19 |
+
function buildChips() {
|
| 20 |
+
for (const facet of ["family", "supervision", "reranker"]) {
|
| 21 |
+
const host = document.getElementById("f-" + facet);
|
| 22 |
+
host.innerHTML = "";
|
| 23 |
+
for (const val of window.FACETS[facet]) {
|
| 24 |
+
const chip = document.createElement("span");
|
| 25 |
+
chip.className = "chip";
|
| 26 |
+
chip.textContent = val;
|
| 27 |
+
chip.onclick = () => {
|
| 28 |
+
const s = state.active[facet];
|
| 29 |
+
s.has(val) ? s.delete(val) : s.add(val);
|
| 30 |
+
chip.classList.toggle("on");
|
| 31 |
+
render();
|
| 32 |
+
};
|
| 33 |
+
host.appendChild(chip);
|
| 34 |
+
}
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
// ---- submissions (fetched client-side from the results repo) ----
|
| 39 |
+
async function loadSubmissions() {
|
| 40 |
+
const api = `https://huggingface.co/api/datasets/${RESULTS_REPO}/tree/main/submissions?recursive=1`;
|
| 41 |
+
let tree;
|
| 42 |
+
try { tree = await (await fetch(api)).json(); } catch (e) { return []; }
|
| 43 |
+
if (!Array.isArray(tree)) return [];
|
| 44 |
+
const slugs = new Set();
|
| 45 |
+
for (const f of tree) {
|
| 46 |
+
const m = f.path && f.path.match(/^submissions\/([^/]+)\/results\.json$/);
|
| 47 |
+
if (m) slugs.add(m[1]);
|
| 48 |
+
}
|
| 49 |
+
const rows = [];
|
| 50 |
+
for (const slug of slugs) {
|
| 51 |
+
const base = `https://huggingface.co/datasets/${RESULTS_REPO}/resolve/main/submissions/${slug}`;
|
| 52 |
+
try {
|
| 53 |
+
const res = await (await fetch(`${base}/results.json`)).json();
|
| 54 |
+
let meta = {};
|
| 55 |
+
try { meta = parseYaml(await (await fetch(`${base}/metadata.yaml`)).text()); } catch (e) {}
|
| 56 |
+
const h = res.headline || {};
|
| 57 |
+
rows.push([
|
| 58 |
+
meta.name || slug,
|
| 59 |
+
meta.family || "Other",
|
| 60 |
+
meta.supervision || "label-free",
|
| 61 |
+
meta.reranker || "none",
|
| 62 |
+
/nr|normal-residual/i.test(meta.name || "") || meta.nr === "true",
|
| 63 |
+
h.rho0, h.rho10, h.rho20, "submission", meta.code || meta.paper || "",
|
| 64 |
+
]);
|
| 65 |
+
} catch (e) { /* skip malformed */ }
|
| 66 |
+
}
|
| 67 |
+
return rows;
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
function parseYaml(txt) {
|
| 71 |
+
const o = {};
|
| 72 |
+
for (const line of txt.split("\n")) {
|
| 73 |
+
const m = line.match(/^([a-zA-Z_]+):\s*(.*)$/);
|
| 74 |
+
if (!m) continue;
|
| 75 |
+
let v = m[2].trim();
|
| 76 |
+
if (!/^["']/.test(v)) v = v.replace(/\s+#.*$/, "").trim();
|
| 77 |
+
v = v.replace(/^["']|["']$/g, "");
|
| 78 |
+
if (v) o[m[1]] = v;
|
| 79 |
+
}
|
| 80 |
+
return o;
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
// ---- rendering ----
|
| 84 |
+
const COLS = [
|
| 85 |
+
{ key: "rank", label: "#", cls: "rank" },
|
| 86 |
+
{ key: "name", label: "Method" },
|
| 87 |
+
{ key: "family", label: "Family" },
|
| 88 |
+
{ key: "supervision", label: "Supervision" },
|
| 89 |
+
{ key: "reranker", label: "Reranker" },
|
| 90 |
+
{ key: "rho0", label: "NDCG@10 ρ=0%", cls: "num" },
|
| 91 |
+
{ key: "rho10", label: "ρ=10%", cls: "num" },
|
| 92 |
+
{ key: "rho20", label: "ρ=20%", cls: "num" },
|
| 93 |
+
{ key: "source", label: "Source" },
|
| 94 |
+
];
|
| 95 |
+
|
| 96 |
+
function passesFilters(r) {
|
| 97 |
+
for (const facet of ["family", "supervision", "reranker"]) {
|
| 98 |
+
const s = state.active[facet];
|
| 99 |
+
if (s.size && !s.has(r[KEY_IDX[facet]])) return false;
|
| 100 |
+
}
|
| 101 |
+
if (state.nrOnly && !r[KEY_IDX.nr]) return false;
|
| 102 |
+
if (state.search && !String(r[0]).toLowerCase().includes(state.search)) return false;
|
| 103 |
+
return true;
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
function render() {
|
| 107 |
+
const rhoIdx = RHO_IDX[state.rho];
|
| 108 |
+
const sortIdx = KEY_IDX[state.sortKey];
|
| 109 |
+
let rows = state.rows.filter(r => typeof r[rhoIdx] === "number").filter(passesFilters);
|
| 110 |
+
rows.sort((a, b) => {
|
| 111 |
+
const x = a[sortIdx], y = b[sortIdx];
|
| 112 |
+
if (typeof x === "number") return (x - y) * state.sortDir;
|
| 113 |
+
return String(x).localeCompare(String(y)) * state.sortDir;
|
| 114 |
+
});
|
| 115 |
+
const best = rows.length ? Math.max(...rows.map(r => r[rhoIdx] || 0)) : -1;
|
| 116 |
+
|
| 117 |
+
// header
|
| 118 |
+
const head = document.getElementById("head");
|
| 119 |
+
head.innerHTML = "<tr>" + COLS.map(c => {
|
| 120 |
+
const active = c.key === state.sortKey;
|
| 121 |
+
const arrow = active ? (state.sortDir < 0 ? " ▼" : " ▲") : "";
|
| 122 |
+
const sel = (c.key === "rho0" && state.rho === "rho0") ||
|
| 123 |
+
(c.key === "rho10" && state.rho === "rho10") ||
|
| 124 |
+
(c.key === "rho20" && state.rho === "rho20");
|
| 125 |
+
const label = sel ? `<b>${c.label}</b>` : c.label;
|
| 126 |
+
return `<th class="${c.cls || ""}" data-k="${c.key}">${label}<span class="arrow">${arrow}</span></th>`;
|
| 127 |
+
}).join("") + "</tr>";
|
| 128 |
+
head.querySelectorAll("th[data-k]").forEach(th => {
|
| 129 |
+
const k = th.dataset.k;
|
| 130 |
+
if (k === "rank") return;
|
| 131 |
+
th.onclick = () => {
|
| 132 |
+
if (state.sortKey === k) state.sortDir *= -1;
|
| 133 |
+
else { state.sortKey = k; state.sortDir = k.startsWith("rho") ? -1 : 1; }
|
| 134 |
+
render();
|
| 135 |
+
};
|
| 136 |
+
});
|
| 137 |
+
|
| 138 |
+
// body
|
| 139 |
+
const tb = document.getElementById("rows");
|
| 140 |
+
tb.innerHTML = "";
|
| 141 |
+
rows.forEach((r, i) => {
|
| 142 |
+
const tr = document.createElement("tr");
|
| 143 |
+
if (r[KEY_IDX.source] === "paper") tr.className = "paper";
|
| 144 |
+
const nm = r[KEY_IDX.link] ? `<a href="${r[KEY_IDX.link]}" target="_blank" rel="noopener">${r[0]}</a>` : r[0];
|
| 145 |
+
const f = v => (typeof v === "number" ? v.toFixed(3) : "—");
|
| 146 |
+
const bestCls = c => (c === best ? "num best" : "num");
|
| 147 |
+
tr.innerHTML =
|
| 148 |
+
`<td class="rank">${i + 1}</td>` +
|
| 149 |
+
`<td>${nm}</td>` +
|
| 150 |
+
`<td><span class="pill">${r[1]}</span></td>` +
|
| 151 |
+
`<td>${r[2]}</td>` +
|
| 152 |
+
`<td>${r[3] === "none" ? "—" : r[3]}</td>` +
|
| 153 |
+
`<td class="${state.rho === "rho0" ? bestCls(r[5]) : "num"}">${f(r[5])}</td>` +
|
| 154 |
+
`<td class="${state.rho === "rho10" ? bestCls(r[6]) : "num"}">${f(r[6])}</td>` +
|
| 155 |
+
`<td class="${state.rho === "rho20" ? bestCls(r[7]) : "num"}">${f(r[7])}</td>` +
|
| 156 |
+
`<td><span class="src ${r[8]}">${r[8]}</span></td>`;
|
| 157 |
+
tb.appendChild(tr);
|
| 158 |
+
});
|
| 159 |
+
|
| 160 |
+
document.getElementById("count").textContent =
|
| 161 |
+
`${rows.length} methods · sorted by ${state.sortKey.startsWith("rho") ? "NDCG@10 " + state.sortKey.replace("rho","ρ=") + "%" : state.sortKey}`;
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
// ---- view switching (Overall table vs By-Domain radar) ----
|
| 165 |
+
function setView(view) {
|
| 166 |
+
state.view = view;
|
| 167 |
+
document.querySelectorAll(".tab").forEach(t => t.classList.toggle("active", t.dataset.view === view));
|
| 168 |
+
const table = document.getElementById("board");
|
| 169 |
+
const count = document.getElementById("count");
|
| 170 |
+
let domain = document.getElementById("domain-view");
|
| 171 |
+
if (view === "domain") {
|
| 172 |
+
table.style.display = "none"; count.style.display = "none";
|
| 173 |
+
if (!domain) {
|
| 174 |
+
domain = document.createElement("div");
|
| 175 |
+
domain.id = "domain-view";
|
| 176 |
+
domain.innerHTML =
|
| 177 |
+
`<p style="color:#8b93a4;margin:0 0 12px">Per-dataset NDCG@10 across the five diagnostic
|
| 178 |
+
domains. Base retrievers (coloured by family) form a tight inner band; the composed
|
| 179 |
+
system (CHARM + NR | DTW-I + GPC) expands over most datasets.</p>
|
| 180 |
+
<p align="center"><img src="radar.png" style="max-width:640px;width:100%;
|
| 181 |
+
background:#fff;border-radius:12px;padding:8px"/></p>`;
|
| 182 |
+
table.parentNode.insertBefore(domain, table.nextSibling);
|
| 183 |
+
}
|
| 184 |
+
domain.style.display = "block";
|
| 185 |
+
} else {
|
| 186 |
+
table.style.display = ""; count.style.display = "";
|
| 187 |
+
if (domain) domain.style.display = "none";
|
| 188 |
+
}
|
| 189 |
+
}
|
| 190 |
+
|
| 191 |
+
// ---- wire controls ----
|
| 192 |
+
document.getElementById("rho").onchange = e => {
|
| 193 |
+
state.rho = e.target.value;
|
| 194 |
+
state.sortKey = state.rho; // sort by the selected pollution column
|
| 195 |
+
state.sortDir = -1;
|
| 196 |
+
render();
|
| 197 |
+
};
|
| 198 |
+
document.getElementById("nr-only").onchange = e => { state.nrOnly = e.target.checked; render(); };
|
| 199 |
+
document.getElementById("search").oninput = e => { state.search = e.target.value.toLowerCase(); render(); };
|
| 200 |
+
document.querySelectorAll(".tab").forEach(t => t.onclick = () => setView(t.dataset.view));
|
| 201 |
+
|
| 202 |
+
(async () => {
|
| 203 |
+
buildChips();
|
| 204 |
+
const subs = await loadSubmissions();
|
| 205 |
+
state.rows = window.PAPER_ROWS.concat(subs);
|
| 206 |
+
render();
|
| 207 |
+
})();
|
data.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// READ-Bench leaderboard data: paper reference rows (arXiv:2609.32123, Table 1),
|
| 2 |
+
// tagged for filtering. Community submissions are fetched at runtime from the
|
| 3 |
+
// results repo and appended. NDCG@10 macro-averaged over 12 datasets.
|
| 4 |
+
//
|
| 5 |
+
// Fields: name, family, supervision, reranker, nr(bool), rho0, rho10, rho20, source, link
|
| 6 |
+
window.PAPER_ROWS = [
|
| 7 |
+
["MR-Hydra", "Supervised", "supervised", "none", false, 0.776, 0.672, 0.658, "paper", ""],
|
| 8 |
+
["RDST", "Supervised", "supervised", "none", false, 0.711, 0.634, 0.590, "paper", ""],
|
| 9 |
+
["CHARM + NR | DTW-I + GPC", "Fusion + reranker", "neighbour-labels","GPC", true, 0.687, 0.640, 0.619, "paper", ""],
|
| 10 |
+
["MantisV2", "FM embedder", "label-free", "none", false, 0.510, 0.474, 0.454, "paper", ""],
|
| 11 |
+
["SAX-BM25", "Symbolic", "label-free", "none", false, 0.489, 0.457, 0.431, "paper", ""],
|
| 12 |
+
["CHARM", "FM embedder", "label-free", "none", false, 0.486, 0.461, 0.437, "paper", ""],
|
| 13 |
+
["TiRex", "FM embedder", "label-free", "none", false, 0.483, 0.449, 0.433, "paper", ""],
|
| 14 |
+
["DTW-I", "Raw-window distance", "label-free", "none", false, 0.480, 0.414, 0.395, "paper", ""],
|
| 15 |
+
["Chronos-2", "FM embedder", "label-free", "none", false, 0.471, 0.455, 0.432, "paper", ""],
|
| 16 |
+
["SFA-BM25", "Symbolic", "label-free", "none", false, 0.409, 0.386, 0.376, "paper", ""],
|
| 17 |
+
["SBD-D", "Raw-window distance", "label-free", "none", false, 0.328, 0.271, 0.245, "paper", ""],
|
| 18 |
+
["Majority", "Trivial floor", "label-free", "none", false, 0.246, 0.242, 0.166, "paper", ""],
|
| 19 |
+
["Random", "Trivial floor", "label-free", "none", false, 0.189, 0.168, 0.155, "paper", ""],
|
| 20 |
+
];
|
| 21 |
+
|
| 22 |
+
// Facets the filter chips are built from (order controls chip order).
|
| 23 |
+
window.FACETS = {
|
| 24 |
+
family: ["FM embedder", "Raw-window distance", "Symbolic", "Fusion + reranker", "Supervised", "Trivial floor", "Other"],
|
| 25 |
+
supervision: ["label-free", "neighbour-labels", "supervised"],
|
| 26 |
+
reranker: ["none", "GPC", "PRF", "Purity", "MajVote", "QPurity", "LM"],
|
| 27 |
+
};
|
index.html
CHANGED
|
@@ -5,163 +5,119 @@
|
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
| 6 |
<title>READ-Bench Leaderboard</title>
|
| 7 |
<style>
|
| 8 |
-
:root
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
.
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
.
|
| 28 |
-
.
|
| 29 |
-
|
| 30 |
-
.
|
| 31 |
-
.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
</style>
|
| 33 |
</head>
|
| 34 |
<body>
|
| 35 |
<div class="wrap">
|
| 36 |
-
<
|
| 37 |
-
|
| 38 |
-
Historical-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
<table id="board">
|
| 47 |
-
<thead><
|
| 48 |
-
<th class="rank">#</th>
|
| 49 |
-
<th data-k="method">Method</th>
|
| 50 |
-
<th data-k="family">Family</th>
|
| 51 |
-
<th data-k="supervision">Supervision</th>
|
| 52 |
-
<th class="num" data-k="rho0">NDCG@10 (ρ=0%)</th>
|
| 53 |
-
<th class="num" data-k="rho10">ρ=10%</th>
|
| 54 |
-
<th class="num" data-k="rho20">ρ=20%</th>
|
| 55 |
-
<th data-k="source">Source</th>
|
| 56 |
-
</tr></thead>
|
| 57 |
<tbody id="rows"></tbody>
|
| 58 |
</table>
|
|
|
|
| 59 |
<p class="foot">
|
| 60 |
-
<
|
| 61 |
-
<
|
| 62 |
-
official harness
|
|
|
|
| 63 |
<a href="https://huggingface.co/datasets/c3aiia3c/read-bench-results">results repo</a>.
|
| 64 |
</p>
|
| 65 |
</div>
|
| 66 |
|
| 67 |
-
<script>
|
| 68 |
-
|
| 69 |
-
const PAPER_ROWS = [
|
| 70 |
-
["MR-Hydra","Supervised reference","supervised",0.776,0.672,0.658,"paper"],
|
| 71 |
-
["CHARM + NR | DTW-I + GPC","Our best system","neighbour-labels",0.687,0.640,0.619,"paper"],
|
| 72 |
-
["MantisV2","FM embedder","label-free",0.510,0.474,0.454,"paper"],
|
| 73 |
-
["SAX-BM25","Symbolic","label-free",0.489,0.457,0.431,"paper"],
|
| 74 |
-
["CHARM","FM embedder","label-free",0.486,0.461,0.437,"paper"],
|
| 75 |
-
["TiRex","FM embedder","label-free",0.483,0.449,0.433,"paper"],
|
| 76 |
-
["DTW-I","Raw-window distance","label-free",0.480,0.414,0.395,"paper"],
|
| 77 |
-
["Chronos-2","FM embedder","label-free",0.471,0.455,0.432,"paper"],
|
| 78 |
-
["SFA-BM25","Symbolic","label-free",0.409,0.386,0.376,"paper"],
|
| 79 |
-
["SBD-D","Raw-window distance","label-free",0.328,0.271,0.245,"paper"],
|
| 80 |
-
];
|
| 81 |
-
let SORT_KEY = "rho0", SORT_DIR = -1;
|
| 82 |
-
|
| 83 |
-
async function loadSubmissions() {
|
| 84 |
-
// List the results repo tree, find submissions/<slug>/{metadata.yaml,results.json}.
|
| 85 |
-
const api = `https://huggingface.co/api/datasets/${RESULTS_REPO}/tree/main/submissions?recursive=1`;
|
| 86 |
-
let tree;
|
| 87 |
-
try { tree = await (await fetch(api)).json(); } catch (e) { return []; }
|
| 88 |
-
if (!Array.isArray(tree)) return [];
|
| 89 |
-
const slugs = new Set();
|
| 90 |
-
for (const f of tree) {
|
| 91 |
-
const m = f.path && f.path.match(/^submissions\/([^/]+)\/results\.json$/);
|
| 92 |
-
if (m) slugs.add(m[1]);
|
| 93 |
-
}
|
| 94 |
-
const rows = [];
|
| 95 |
-
for (const slug of slugs) {
|
| 96 |
-
const base = `https://huggingface.co/datasets/${RESULTS_REPO}/resolve/main/submissions/${slug}`;
|
| 97 |
-
try {
|
| 98 |
-
const res = await (await fetch(`${base}/results.json`)).json();
|
| 99 |
-
let meta = {};
|
| 100 |
-
try { meta = parseYaml(await (await fetch(`${base}/metadata.yaml`)).text()); } catch (e) {}
|
| 101 |
-
const h = res.headline || {};
|
| 102 |
-
let name = meta.name || slug;
|
| 103 |
-
const link = meta.code || meta.paper;
|
| 104 |
-
rows.push([name, meta.family || "?", meta.supervision || "?",
|
| 105 |
-
h.rho0, h.rho10, h.rho20, "submission", link]);
|
| 106 |
-
} catch (e) { /* skip malformed */ }
|
| 107 |
-
}
|
| 108 |
-
return rows;
|
| 109 |
-
}
|
| 110 |
-
|
| 111 |
-
// Minimal YAML (flat key: value) parser — submissions use a flat metadata.yaml.
|
| 112 |
-
function parseYaml(txt) {
|
| 113 |
-
const o = {};
|
| 114 |
-
for (const line of txt.split("\n")) {
|
| 115 |
-
const m = line.match(/^([a-zA-Z_]+):\s*(.*)$/);
|
| 116 |
-
if (!m) continue;
|
| 117 |
-
let v = m[2].trim();
|
| 118 |
-
// strip an inline comment that isn't inside quotes
|
| 119 |
-
if (!/^["']/.test(v)) v = v.replace(/\s+#.*$/, "").trim();
|
| 120 |
-
v = v.replace(/^["']|["']$/g, "");
|
| 121 |
-
if (v) o[m[1]] = v;
|
| 122 |
-
}
|
| 123 |
-
return o;
|
| 124 |
-
}
|
| 125 |
-
|
| 126 |
-
function render(rows) {
|
| 127 |
-
const KEYS = {method:0, family:1, supervision:2, rho0:3, rho10:4, rho20:5, source:6};
|
| 128 |
-
const idx = KEYS[SORT_KEY];
|
| 129 |
-
rows = rows.filter(r => typeof r[3] === "number");
|
| 130 |
-
rows.sort((a,b) => {
|
| 131 |
-
const x=a[idx], y=b[idx];
|
| 132 |
-
if (typeof x === "number") return (x-y)*SORT_DIR;
|
| 133 |
-
return String(x).localeCompare(String(y))*SORT_DIR;
|
| 134 |
-
});
|
| 135 |
-
const tb = document.getElementById("rows");
|
| 136 |
-
tb.innerHTML = "";
|
| 137 |
-
rows.forEach((r,i) => {
|
| 138 |
-
const tr = document.createElement("tr");
|
| 139 |
-
if (r[6] === "paper") tr.className = "paper";
|
| 140 |
-
const nameCell = r[7] ? `<a href="${r[7]}" target="_blank" rel="noopener">${r[0]}</a>` : r[0];
|
| 141 |
-
const f = v => (typeof v === "number" ? v.toFixed(3) : "—");
|
| 142 |
-
tr.innerHTML = `<td class="rank">${i+1}</td><td>${nameCell}</td><td>${r[1]}</td>`
|
| 143 |
-
+ `<td>${r[2]}</td><td class="num">${f(r[3])}</td><td class="num">${f(r[4])}</td>`
|
| 144 |
-
+ `<td class="num">${f(r[5])}</td><td><span class="src">${r[6]}</span></td>`;
|
| 145 |
-
tb.appendChild(tr);
|
| 146 |
-
});
|
| 147 |
-
}
|
| 148 |
-
|
| 149 |
-
document.querySelectorAll("th[data-k]").forEach(th => {
|
| 150 |
-
th.addEventListener("click", () => {
|
| 151 |
-
const k = th.dataset.k;
|
| 152 |
-
if (SORT_KEY === k) SORT_DIR *= -1; else { SORT_KEY = k; SORT_DIR = (k.startsWith("rho")?-1:1); }
|
| 153 |
-
render(window._ROWS);
|
| 154 |
-
});
|
| 155 |
-
});
|
| 156 |
-
|
| 157 |
-
(async () => {
|
| 158 |
-
const subs = await loadSubmissions();
|
| 159 |
-
window._ROWS = PAPER_ROWS.concat(subs);
|
| 160 |
-
const n = subs.length;
|
| 161 |
-
document.getElementById("status").textContent =
|
| 162 |
-
`${n} community submission${n===1?"":"s"} + ${PAPER_ROWS.length} paper reference rows. Click a column to sort.`;
|
| 163 |
-
render(window._ROWS);
|
| 164 |
-
})();
|
| 165 |
-
</script>
|
| 166 |
</body>
|
| 167 |
</html>
|
|
|
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
| 6 |
<title>READ-Bench Leaderboard</title>
|
| 7 |
<style>
|
| 8 |
+
:root{
|
| 9 |
+
--bg:#0b0d12; --panel:#141821; --panel2:#1b212c; --line:#272e3b;
|
| 10 |
+
--fg:#eef1f6; --mut:#8b93a4; --accent:#8aa0ff; --accent2:#59d6a8;
|
| 11 |
+
--pill:#1f2838; --pillfg:#aebbff; --paperbg:#20242e;
|
| 12 |
+
}
|
| 13 |
+
*{box-sizing:border-box;}
|
| 14 |
+
body{margin:0;background:var(--bg);color:var(--fg);
|
| 15 |
+
font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Inter,Roboto,Helvetica,Arial,sans-serif;}
|
| 16 |
+
a{color:var(--accent);text-decoration:none;} a:hover{text-decoration:underline;}
|
| 17 |
+
.wrap{max-width:1180px;margin:0 auto;padding:28px 20px 90px;}
|
| 18 |
+
header h1{font-size:1.7rem;margin:0 0 2px;letter-spacing:-.01em;}
|
| 19 |
+
header .tag{color:var(--accent2);font-weight:600;}
|
| 20 |
+
.lead{color:var(--mut);margin:6px 0 2px;max-width:760px;}
|
| 21 |
+
.links{margin:12px 0 20px;font-size:.9rem;}
|
| 22 |
+
.links a{margin-right:16px;}
|
| 23 |
+
.panel{background:var(--panel);border:1px solid var(--line);border-radius:14px;padding:16px 18px;margin-bottom:18px;}
|
| 24 |
+
.tabs{display:flex;gap:6px;margin-bottom:14px;flex-wrap:wrap;}
|
| 25 |
+
.tab{padding:8px 16px;border-radius:10px;background:var(--panel);border:1px solid var(--line);
|
| 26 |
+
color:var(--mut);cursor:pointer;font-size:.9rem;font-weight:600;}
|
| 27 |
+
.tab.active{background:var(--panel2);color:var(--fg);border-color:#3a475f;}
|
| 28 |
+
.filters{display:flex;gap:22px;flex-wrap:wrap;align-items:flex-start;}
|
| 29 |
+
.fgroup{display:flex;flex-direction:column;gap:6px;}
|
| 30 |
+
.fgroup .ftitle{font-size:.72rem;text-transform:uppercase;letter-spacing:.06em;color:var(--mut);}
|
| 31 |
+
.chips{display:flex;gap:6px;flex-wrap:wrap;max-width:520px;}
|
| 32 |
+
.chip{font-size:.8rem;padding:5px 11px;border-radius:999px;border:1px solid var(--line);
|
| 33 |
+
background:var(--panel2);color:var(--mut);cursor:pointer;user-select:none;}
|
| 34 |
+
.chip.on{background:var(--pill);color:var(--pillfg);border-color:#39507f;}
|
| 35 |
+
.ctrls{display:flex;gap:18px;align-items:center;flex-wrap:wrap;margin-top:4px;}
|
| 36 |
+
select,input[type=search]{background:var(--panel2);color:var(--fg);border:1px solid var(--line);
|
| 37 |
+
border-radius:9px;padding:7px 10px;font-size:.88rem;}
|
| 38 |
+
input[type=search]{min-width:220px;}
|
| 39 |
+
.count{color:var(--mut);font-size:.85rem;margin:4px 2px 10px;}
|
| 40 |
+
table{width:100%;border-collapse:collapse;background:var(--panel);
|
| 41 |
+
border:1px solid var(--line);border-radius:14px;overflow:hidden;}
|
| 42 |
+
th,td{padding:11px 13px;text-align:left;border-bottom:1px solid var(--line);font-size:.9rem;}
|
| 43 |
+
th{background:#161b24;color:var(--mut);cursor:pointer;user-select:none;white-space:nowrap;position:sticky;top:0;}
|
| 44 |
+
th.num,td.num{text-align:right;font-variant-numeric:tabular-nums;}
|
| 45 |
+
th .arrow{opacity:.5;font-size:.75rem;}
|
| 46 |
+
tr:last-child td{border-bottom:none;}
|
| 47 |
+
tr.paper{color:var(--mut);}
|
| 48 |
+
tbody tr:hover{background:#171d27;}
|
| 49 |
+
.rank{color:var(--mut);width:38px;text-align:right;}
|
| 50 |
+
.pill{font-size:.72rem;padding:2px 8px;border-radius:999px;background:var(--pill);color:var(--pillfg);white-space:nowrap;}
|
| 51 |
+
.src{font-size:.7rem;padding:2px 8px;border-radius:999px;}
|
| 52 |
+
.src.paper{background:var(--paperbg);color:var(--mut);}
|
| 53 |
+
.src.submission{background:#13351f;color:#7ee6a6;}
|
| 54 |
+
.best{color:#fff;font-weight:700;}
|
| 55 |
+
.foot{margin-top:20px;font-size:.82rem;color:var(--mut);}
|
| 56 |
+
.foot code{background:#20242e;padding:1px 6px;border-radius:5px;}
|
| 57 |
</style>
|
| 58 |
</head>
|
| 59 |
<body>
|
| 60 |
<div class="wrap">
|
| 61 |
+
<header>
|
| 62 |
+
<h1>🩺 READ-Bench <span class="tag">Leaderboard</span></h1>
|
| 63 |
+
<p class="lead">Historical-case <b>retrieval for time-series diagnosis</b> — relevance is shared
|
| 64 |
+
<b>fault type</b>, not signal shape. Ranked by <b>NDCG@10</b>, macro-averaged over 12 datasets.</p>
|
| 65 |
+
<div class="links">
|
| 66 |
+
<a href="https://arxiv.org/abs/2609.32123">📄 Paper</a>
|
| 67 |
+
<a href="https://huggingface.co/datasets/c3aiia3c/read-bench">💾 Data</a>
|
| 68 |
+
<a href="https://huggingface.co/datasets/c3aiia3c/read-bench-results">📥 Submit a method</a>
|
| 69 |
+
</div>
|
| 70 |
+
</header>
|
| 71 |
+
|
| 72 |
+
<div class="tabs" id="tabs">
|
| 73 |
+
<div class="tab active" data-view="overall">🏅 Overall</div>
|
| 74 |
+
<div class="tab" data-view="domain">🗂 By Domain</div>
|
| 75 |
+
</div>
|
| 76 |
+
|
| 77 |
+
<div class="panel">
|
| 78 |
+
<div class="filters">
|
| 79 |
+
<div class="fgroup">
|
| 80 |
+
<span class="ftitle">Family</span>
|
| 81 |
+
<div class="chips" id="f-family"></div>
|
| 82 |
+
</div>
|
| 83 |
+
<div class="fgroup">
|
| 84 |
+
<span class="ftitle">Supervision</span>
|
| 85 |
+
<div class="chips" id="f-supervision"></div>
|
| 86 |
+
</div>
|
| 87 |
+
<div class="fgroup">
|
| 88 |
+
<span class="ftitle">Reranker</span>
|
| 89 |
+
<div class="chips" id="f-reranker"></div>
|
| 90 |
+
</div>
|
| 91 |
+
</div>
|
| 92 |
+
<div class="ctrls">
|
| 93 |
+
<label>Pollution
|
| 94 |
+
<select id="rho">
|
| 95 |
+
<option value="rho0">ρ = 0%</option>
|
| 96 |
+
<option value="rho10">ρ = 10%</option>
|
| 97 |
+
<option value="rho20">ρ = 20%</option>
|
| 98 |
+
</select>
|
| 99 |
+
</label>
|
| 100 |
+
<label><input type="checkbox" id="nr-only"/> NR scoring only</label>
|
| 101 |
+
<input type="search" id="search" placeholder="search method…"/>
|
| 102 |
+
</div>
|
| 103 |
+
</div>
|
| 104 |
+
|
| 105 |
+
<div class="count" id="count"></div>
|
| 106 |
<table id="board">
|
| 107 |
+
<thead id="head"></thead>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
<tbody id="rows"></tbody>
|
| 109 |
</table>
|
| 110 |
+
|
| 111 |
<p class="foot">
|
| 112 |
+
<span class="src paper">paper</span> rows are reference numbers from arXiv:2609.32123;
|
| 113 |
+
<span class="src submission">submission</span> rows are community entries verified by re-running the
|
| 114 |
+
official harness. Click a column header to sort; use the chips to filter.
|
| 115 |
+
Submit via a Pull Request on the
|
| 116 |
<a href="https://huggingface.co/datasets/c3aiia3c/read-bench-results">results repo</a>.
|
| 117 |
</p>
|
| 118 |
</div>
|
| 119 |
|
| 120 |
+
<script src="data.js"></script>
|
| 121 |
+
<script src="app.js"></script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
</body>
|
| 123 |
</html>
|
radar.png
ADDED
|
Git LFS Details
|