Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>DecideBench Leaderboard</title> | |
| <meta name="description" content="Accuracy, cost per task and latency of decision models (JEV and its open alternatives) and general LLMs on 400 contrastive decisions."> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600&family=Geist+Mono:wght@400;500;600&display=swap" rel="stylesheet"> | |
| <style> | |
| :root { | |
| --paper: #f5f5f5; --card: #ffffff; --ink: #2d3142; --muted: #4f5d75; --soft: #7a8399; | |
| --rule: rgba(45,49,66,0.12); --accent: #eb6c36; --accent-tint: rgba(235,108,54,0.10); --llm: #9aa3b5; | |
| --bar-track: rgba(45,49,66,0.07); | |
| } | |
| @media (prefers-color-scheme: dark) { | |
| :root:not([data-theme="light"]) { | |
| --paper: #16181f; --card: #1e2129; --ink: #e7e9ef; --muted: #b3b9c7; --soft: #8a92a3; | |
| --rule: rgba(231,233,239,0.12); --accent: #f07a47; --accent-tint: rgba(240,122,71,0.14); --llm: #7d869a; | |
| --bar-track: rgba(231,233,239,0.08); | |
| } | |
| } | |
| :root[data-theme="dark"] { | |
| --paper: #16181f; --card: #1e2129; --ink: #e7e9ef; --muted: #b3b9c7; --soft: #8a92a3; | |
| --rule: rgba(231,233,239,0.12); --accent: #f07a47; --accent-tint: rgba(240,122,71,0.14); --llm: #7d869a; | |
| --bar-track: rgba(231,233,239,0.08); | |
| } | |
| * { box-sizing: border-box; margin: 0; padding: 0; } | |
| body { background: var(--paper); color: var(--ink); font: 16px/1.5 'Geist', system-ui, sans-serif; } | |
| .wrap { max-width: 1180px; margin: 0 auto; padding: 40px 16px 64px; } | |
| .eyebrow { font: 600 13px/1 'Geist Mono', monospace; letter-spacing: .16em; color: var(--accent); text-transform: uppercase; } | |
| h1 { font-size: clamp(28px, 5vw, 44px); line-height: 1.12; font-weight: 600; margin: 14px 0 12px; letter-spacing: -0.01em; } | |
| .lede { color: var(--muted); max-width: 760px; font-size: 17px; } | |
| .links { margin-top: 16px; display: flex; gap: 18px; flex-wrap: wrap; font: 500 14px 'Geist Mono', monospace; } | |
| .links a { color: var(--ink); text-decoration: none; border-bottom: 1px solid var(--rule); } | |
| .links a:hover { border-color: var(--accent); } | |
| .controls { display: flex; flex-wrap: wrap; gap: 10px 22px; align-items: center; margin: 32px 0 14px; } | |
| .group { display: flex; gap: 6px; align-items: center; flex-wrap: wrap; } | |
| .group-label { font: 500 12px 'Geist Mono', monospace; letter-spacing: .1em; color: var(--soft); text-transform: uppercase; margin-right: 4px; } | |
| .chip { font: 500 14px 'Geist', sans-serif; color: var(--muted); background: var(--card); border: 1px solid var(--rule); | |
| border-radius: 999px; padding: 5px 12px; cursor: pointer; } | |
| .chip[aria-pressed="true"] { color: var(--ink); border-color: var(--accent); background: var(--accent-tint); } | |
| .chip:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } | |
| .table-card { background: var(--card); border: 1px solid var(--rule); border-radius: 14px; overflow: hidden; } | |
| .scroll { overflow-x: auto; } | |
| table { width: 100%; border-collapse: collapse; font-size: 15px; } | |
| th, td { padding: 11px 14px; text-align: right; white-space: nowrap; border-bottom: 1px solid var(--rule); } | |
| tr:last-child td { border-bottom: 0; } | |
| th { font: 500 12px 'Geist Mono', monospace; letter-spacing: .08em; text-transform: uppercase; color: var(--soft); | |
| cursor: pointer; user-select: none; background: var(--card); position: sticky; top: 0; } | |
| th[aria-sort="descending"]::after { content: " ↓"; color: var(--accent); } | |
| th[aria-sort="ascending"]::after { content: " ↑"; color: var(--accent); } | |
| th.nosort { cursor: default; } | |
| th:first-child, td:first-child { text-align: left; position: sticky; left: 0; background: var(--card); z-index: 1; } | |
| td.num { font-family: 'Geist Mono', monospace; } | |
| .name { font-weight: 600; } | |
| .sub { display: block; font: 400 12px 'Geist Mono', monospace; color: var(--soft); margin-top: 2px; } | |
| .acc { display: flex; align-items: center; gap: 10px; justify-content: flex-end; } | |
| .bar { width: 120px; height: 8px; border-radius: 4px; background: var(--bar-track); overflow: hidden; flex: none; } | |
| .bar i { display: block; height: 100%; border-radius: 4px; background: var(--accent); } | |
| tr.llm .bar i { background: var(--llm); } | |
| tr.baseline .bar i { background: var(--soft); } | |
| .badge { display: inline-block; font: 600 11px 'Geist Mono', monospace; letter-spacing: .06em; text-transform: uppercase; | |
| color: var(--accent); border: 1px solid var(--accent); border-radius: 4px; padding: 1px 5px; margin-left: 4px; } | |
| .dim { color: var(--soft); } | |
| .fam { display: none; } | |
| .show-fam .fam { display: table-cell; } | |
| .notes { margin-top: 16px; color: var(--muted); font-size: 14px; max-width: 900px; } | |
| .notes p + p { margin-top: 6px; } | |
| h2 { font-size: 24px; font-weight: 600; margin: 48px 0 16px; } | |
| .charts { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 16px; } | |
| .charts img { width: 100%; border-radius: 12px; border: 1px solid var(--rule); background: #f5f5f5; } | |
| .empty { padding: 24px; color: var(--soft); text-align: center; } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="wrap"> | |
| <div class="eyebrow" id="version">DecideBench</div> | |
| <h1>DecideBench leaderboard</h1> | |
| <p class="lede">Accuracy, cost per task and latency of decision models (JEV and its open alternatives) and general | |
| LLMs on <span id="n-items">400</span> contrastive decisions.</p> | |
| <div class="links"> | |
| <a href="https://huggingface.co/datasets/choyiny/decidebench">Dataset</a> | |
| <a href="https://github.com/choyiny/decidebench">Code & method</a> | |
| <a href="https://github.com/choyiny/decidebench#the-benchmark">How it's measured</a> | |
| </div> | |
| <div class="controls" role="toolbar" aria-label="Filters"> | |
| <div class="group" data-filter="kind"> | |
| <span class="group-label">Show</span> | |
| <button class="chip" data-value="all" aria-pressed="true">All</button> | |
| <button class="chip" data-value="decision model" aria-pressed="false">Decision models</button> | |
| <button class="chip" data-value="chat LLM" aria-pressed="false">Chat LLMs</button> | |
| </div> | |
| <div class="group" data-filter="hosting"> | |
| <span class="group-label">Hosting</span> | |
| <button class="chip" data-value="all" aria-pressed="true">All</button> | |
| <button class="chip" data-value="API" aria-pressed="false">API</button> | |
| <button class="chip" data-value="self-hosted" aria-pressed="false">Self-hosted</button> | |
| </div> | |
| <div class="group"> | |
| <button class="chip" id="fam-toggle" aria-pressed="false">Per-family accuracy</button> | |
| </div> | |
| </div> | |
| <div class="table-card"><div class="scroll"><table id="board"> | |
| <thead><tr id="head"></tr></thead><tbody id="body"></tbody> | |
| </table></div></div> | |
| <div class="notes"> | |
| <p><strong>Accuracy</strong> over 400 items; <strong>pair accuracy</strong> counts contrastive pairs with both | |
| halves right. Every model sees one worked example per option, except Laya, CLM and Julia-1, which run | |
| zero-shot; TEV and JEV were also run zero-shot.</p> | |
| <p><strong>Cost</strong>: API models at list price per billed token. Self-hosted models are priced by GPU time at an NVIDIA | |
| L4's $0.81/h: the decision models ran on an L4, Qwen3-8B on an NVIDIA DGX Spark (GB10). <strong>Latency</strong>: median per call; API | |
| models include the network, self-hosted ones were measured on the box.</p> | |
| <p><span class="badge">cost</span> <span class="badge">latency</span> mark the Pareto frontiers: no other model is | |
| at least as good on both axes and better on one.</p> | |
| </div> | |
| <h2>Frontiers</h2> | |
| <div class="charts"> | |
| <img src="img/cost-vs-accuracy.png" alt="Accuracy against cost per million tasks, log scale"> | |
| <img src="img/latency-vs-accuracy.png" alt="Accuracy against median latency per call, log scale"> | |
| </div> | |
| </div> | |
| <script> | |
| const money = x => x == null ? "–" : "$" + x.toLocaleString("en-US", {maximumFractionDigits: x < 10 ? 2 : 0}); | |
| const ms = x => x == null ? "–" : Math.round(x).toLocaleString("en-US") + " ms"; | |
| const state = {kind: "all", hosting: "all", key: "accuracy", dir: -1, fam: false}; | |
| let data; | |
| const cols = () => [ | |
| {key: "name", label: "Model", sort: r => r.name, dir: 1}, | |
| {key: "accuracy", label: "Accuracy", sort: r => r.accuracy, | |
| cell: r => `<div class="acc"><span class="bar"><i style="width:${(100 * r.accuracy).toFixed(1)}%"></i></span>${r.shown.accuracy}</div>`}, | |
| {key: "pair_accuracy", label: "Pair acc.", sort: r => r.pair_accuracy, cell: r => r.shown.pair_accuracy}, | |
| {key: "zero_shot", label: "Zero-shot", sort: r => r.zero_shot ?? -1, | |
| cell: r => r.zero_shot == null ? '<span class="dim">–</span>' : r.shown.zero_shot + (r.runs_zero_shot ? ' <span class="dim">(only)</span>' : "")}, | |
| {key: "cost_per_1m", label: "$ / 1M tasks", sort: r => r.cost_per_1m ?? Infinity, dir: 1, cell: r => money(r.cost_per_1m)}, | |
| {key: "latency_p50_ms", label: "Latency p50", sort: r => r.latency_p50_ms ?? Infinity, dir: 1, | |
| cell: r => ms(r.latency_p50_ms) + (r.hosting === "self-hosted" ? ' <span class="dim">(on box)</span>' : "")}, | |
| {key: "frontier", label: "Frontier", nosort: true, cell: r => r.frontier.map(f => `<span class="badge">${f}</span>`).join("") || '<span class="dim">–</span>'}, | |
| ...data.families.map(f => ({key: "fam:" + f, label: f.replace("_", " "), fam: true, | |
| sort: r => r.families[f], cell: r => r.shown.families[f]})), | |
| ]; | |
| function render() { | |
| const cs = cols(); | |
| document.getElementById("head").innerHTML = cs.map(c => | |
| `<th class="${c.fam ? "fam" : ""} ${c.nosort ? "nosort" : ""}" data-key="${c.key}" scope="col" | |
| ${c.nosort ? "" : `tabindex="0" aria-sort="${state.key === c.key ? (state.dir < 0 ? "descending" : "ascending") : "none"}"`}>${c.label}</th>`).join(""); | |
| const col = cs.find(c => c.key === state.key) || cs[1]; | |
| const rows = data.entries | |
| .filter(r => r.kind === "baseline" || state.kind === "all" || r.kind === state.kind) | |
| .filter(r => r.kind === "baseline" || state.hosting === "all" || r.hosting === state.hosting) | |
| .sort((a, b) => { | |
| if ((a.kind === "baseline") !== (b.kind === "baseline")) return a.kind === "baseline" ? 1 : -1; | |
| const x = col.sort(a), y = col.sort(b); | |
| return (x < y ? -1 : x > y ? 1 : 0) * state.dir; | |
| }); | |
| document.getElementById("body").innerHTML = rows.length ? rows.map(r => { | |
| const cls = r.kind === "chat LLM" ? "llm" : r.kind === "baseline" ? "baseline" : ""; | |
| return `<tr class="${cls}">` + cs.map(c => c.key === "name" | |
| ? `<td><span class="name">${r.name.replace(/ \((self-hosted|Together|AI Space)\)$/, "")}</span><span class="sub">${r.kind} · ${r.hosting === "—" ? "no calls" : r.hosting === "API" ? (r.name.match(/\((.+)\)$/) || [, "API"])[1] : "self-hosted"}</span></td>` | |
| : `<td class="num ${c.fam ? "fam" : ""}">${c.cell(r)}</td>`).join("") + "</tr>"; | |
| }).join("") : `<tr><td class="empty" colspan="${cs.length}">No models match these filters.</td></tr>`; | |
| document.getElementById("board").classList.toggle("show-fam", state.fam); | |
| } | |
| function sortBy(key) { | |
| const c = cols().find(c => c.key === key); | |
| if (!c || c.nosort) return; | |
| state.dir = state.key === key ? -state.dir : (c.dir ?? -1); | |
| state.key = key; | |
| render(); | |
| } | |
| document.getElementById("head").addEventListener("click", e => { const th = e.target.closest("th"); if (th) sortBy(th.dataset.key); }); | |
| document.getElementById("head").addEventListener("keydown", e => { | |
| if (e.key === "Enter" || e.key === " ") { const th = e.target.closest("th"); if (th) { e.preventDefault(); sortBy(th.dataset.key); } } | |
| }); | |
| document.querySelectorAll(".group[data-filter]").forEach(g => g.addEventListener("click", e => { | |
| const b = e.target.closest(".chip"); if (!b) return; | |
| g.querySelectorAll(".chip").forEach(x => x.setAttribute("aria-pressed", x === b)); | |
| state[g.dataset.filter] = b.dataset.value; render(); | |
| })); | |
| document.getElementById("fam-toggle").addEventListener("click", e => { | |
| state.fam = !state.fam; e.currentTarget.setAttribute("aria-pressed", state.fam); render(); | |
| }); | |
| fetch("leaderboard.json").then(r => r.json()).then(d => { | |
| data = d; | |
| document.getElementById("version").textContent = "DecideBench " + d.version; | |
| document.getElementById("n-items").textContent = d.items; | |
| render(); | |
| }).catch(() => { | |
| document.getElementById("body").innerHTML = '<tr><td class="empty">Could not load leaderboard.json.</td></tr>'; | |
| }); | |
| </script> | |
| </body> | |
| </html> | |