| { |
| "engine": "blink-4b", |
| "edition": "Decision Index 0.2 (local run, descriptive)", |
| "generated_utc": "2026-09-25T23:11:52+00:00", |
| "suite": { |
| "edition": "release-v2", |
| "requests": 121057, |
| "scoreable": 120615, |
| "excluded": 442, |
| "added_requests": 30419, |
| "benchmarks": 44, |
| "rows_sha256": "b2b56d6fb636837ca469e689087bdbf373dda8de7638aa2da6793e6eda0792d5", |
| "added_sha256": "7429f3c9cdddb772c1cfc42bb2a45e8516b0032152b746e6929f1c8b52f4ce89" |
| }, |
| "completed": 151034, |
| "complete": true, |
| "counts": { |
| "ok": 151034 |
| }, |
| "latency_ms": { |
| "median": 30.6, |
| "p95": 717.1, |
| "mean": 144.9 |
| }, |
| "decision_index": 37.85, |
| "raw_index": 53.33, |
| "scores": { |
| "balanced_skill": 37.85, |
| "balanced_raw": 53.33, |
| "breadth_skill": 36.78 |
| }, |
| "areas": [ |
| { |
| "id": "knowledge", |
| "label": "Knowledge & Reasoning", |
| "raw": 0.431, |
| "skill": 0.2642, |
| "coverage": 1.0, |
| "n": 10, |
| "benchmarks": [ |
| 25, |
| 30, |
| 31, |
| 32, |
| 33, |
| 43, |
| 44, |
| 45, |
| 57, |
| 58 |
| ] |
| }, |
| { |
| "id": "language", |
| "label": "Language Understanding", |
| "raw": 0.6271, |
| "skill": 0.4735, |
| "coverage": 1.0, |
| "n": 10, |
| "benchmarks": [ |
| 11, |
| 12, |
| 28, |
| 29, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 59 |
| ] |
| }, |
| { |
| "id": "retrieval", |
| "label": "Retrieval & Classification", |
| "raw": 0.5468, |
| "skill": 0.3676, |
| "coverage": 1.0, |
| "n": 7, |
| "benchmarks": [ |
| 4, |
| 5, |
| 10, |
| 36, |
| 37, |
| 56, |
| 61 |
| ] |
| }, |
| { |
| "id": "tools", |
| "label": "Tools & Automation", |
| "raw": 0.6001, |
| "skill": 0.5155, |
| "coverage": 1.0, |
| "n": 6, |
| "benchmarks": [ |
| 1, |
| 2, |
| 3, |
| 6, |
| 9, |
| 62 |
| ] |
| }, |
| { |
| "id": "arts", |
| "label": "Arts & Human Taste", |
| "raw": 0.4614, |
| "skill": 0.2715, |
| "coverage": 1.0, |
| "n": 7, |
| "benchmarks": [ |
| 20, |
| 21, |
| 22, |
| 23, |
| 48, |
| 50, |
| 64 |
| ] |
| } |
| ], |
| "index_benchmarks": { |
| "1": { |
| "raw": 0.9032, |
| "skill": 0.8693, |
| "coverage": 1.0, |
| "random": 0.2592, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "2": { |
| "raw": 0.4126, |
| "skill": 0.3532, |
| "coverage": 1.0, |
| "random": 0.0918, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "3": { |
| "raw": 0.748, |
| "skill": 0.7431, |
| "coverage": 1.0, |
| "random": 0.0189, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "4": { |
| "raw": 0.7149, |
| "skill": 0.7112, |
| "coverage": 1.0, |
| "random": 0.0127, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "5": { |
| "raw": 0.7391, |
| "skill": 0.7375, |
| "coverage": 1.0, |
| "random": 0.006, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "6": { |
| "raw": 0.7905, |
| "skill": 0.5052, |
| "coverage": 1.0, |
| "random": 0.5245, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [ |
| { |
| "track": "RouterBench-0shot", |
| "score": 0.7797, |
| "headline": false |
| }, |
| { |
| "track": "RouterBench-5shot", |
| "score": 0.8014, |
| "headline": false |
| } |
| ] |
| }, |
| "9": { |
| "raw": 0.1187, |
| "skill": 0.1187, |
| "coverage": 1.0, |
| "random": 0.0, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "10": { |
| "raw": 0.5004, |
| "skill": 0.1687, |
| "coverage": 1.0, |
| "random": 0.399, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "11": { |
| "raw": 0.7612, |
| "skill": 0.6547, |
| "coverage": 1.0, |
| "random": 0.3085, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "12": { |
| "raw": 0.6012, |
| "skill": 0.4026, |
| "coverage": 1.0, |
| "random": 0.3324, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "20": { |
| "raw": 0.8467, |
| "skill": 0.6933, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "21": { |
| "raw": 0.6054, |
| "skill": 0.2108, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "22": { |
| "raw": 0.0338, |
| "skill": 0.0262, |
| "coverage": 1.0, |
| "random": 0.0078, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "23": { |
| "raw": 0.5808, |
| "skill": 0.1616, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "25": { |
| "raw": 0.3724, |
| "skill": 0.1633, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "28": { |
| "raw": 0.7032, |
| "skill": 0.4064, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "29": { |
| "raw": 0.8194, |
| "skill": 0.7592, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "30": { |
| "raw": 0.5788, |
| "skill": 0.4872, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "score": 0.5967, |
| "headline": false |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "score": 0.561, |
| "headline": false |
| } |
| ] |
| }, |
| "31": { |
| "raw": 0.1348, |
| "skill": 0.0577, |
| "coverage": 1.0, |
| "random": 0.0819, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "32": { |
| "raw": 0.5678, |
| "skill": 0.3129, |
| "coverage": 1.0, |
| "random": 0.371, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "33": { |
| "raw": 0.2594, |
| "skill": 0.2496, |
| "coverage": 1.0, |
| "random": 0.0131, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "36": { |
| "raw": 0.1751, |
| "skill": 0.135, |
| "coverage": 1.0, |
| "random": 0.0464, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "37": { |
| "raw": 0.4309, |
| "skill": 0.2862, |
| "coverage": 1.0, |
| "random": 0.2027, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "38": { |
| "raw": 0.03, |
| "skill": 0.03, |
| "coverage": 1.0, |
| "random": 0.0, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "39": { |
| "raw": 0.8603, |
| "skill": 0.7945, |
| "coverage": 1.0, |
| "random": 0.3201, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "40": { |
| "raw": 0.4523, |
| "skill": 0.2953, |
| "coverage": 1.0, |
| "random": 0.2227, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [ |
| { |
| "track": "A \u00b7 Arabic", |
| "score": 0.313, |
| "headline": false |
| }, |
| { |
| "track": "A \u00b7 English", |
| "score": 0.4523, |
| "headline": true |
| }, |
| { |
| "track": "C \u00b7 Arabic pairs", |
| "score": 0.645, |
| "headline": false |
| }, |
| { |
| "track": "C \u00b7 English pairs", |
| "score": 0.915, |
| "headline": false |
| } |
| ] |
| }, |
| "41": { |
| "raw": 0.6169, |
| "skill": 0.4254, |
| "coverage": 1.0, |
| "random": 0.3333, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "42": { |
| "raw": 0.7618, |
| "skill": 0.5366, |
| "coverage": 1.0, |
| "random": 0.486, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "43": { |
| "raw": 0.4719, |
| "skill": 0.1622, |
| "coverage": 1.0, |
| "random": 0.3697, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "44": { |
| "raw": 0.6368, |
| "skill": 0.2736, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "45": { |
| "raw": 0.1297, |
| "skill": 0.0, |
| "coverage": 1.0, |
| "random": 0.1641, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "48": { |
| "raw": 0.1312, |
| "skill": 0.1312, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "vs baseline", |
| "in_index": true |
| }, |
| "50": { |
| "raw": 0.4427, |
| "skill": 0.1912, |
| "coverage": 1.0, |
| "random": 0.311, |
| "rule": "track", |
| "in_index": true, |
| "tracks": [] |
| }, |
| "56": { |
| "raw": 0.6365, |
| "skill": 0.273, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "57": { |
| "raw": 0.5206, |
| "skill": 0.4608, |
| "coverage": 1.0, |
| "random": 0.1109, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "58": { |
| "raw": 0.6379, |
| "skill": 0.4751, |
| "coverage": 1.0, |
| "random": 0.3101, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "59": { |
| "raw": 0.6648, |
| "skill": 0.4306, |
| "coverage": 1.0, |
| "random": 0.4113, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "61": { |
| "raw": 0.6308, |
| "skill": 0.2616, |
| "coverage": 1.0, |
| "random": 0.5, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "62": { |
| "raw": 0.6276, |
| "skill": 0.5035, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "64": { |
| "raw": 0.589, |
| "skill": 0.4862, |
| "coverage": 1.0, |
| "random": 0.2, |
| "rule": "chance", |
| "in_index": true |
| }, |
| "24": { |
| "raw": 0.7375, |
| "skill": 0.65, |
| "coverage": 1.0, |
| "random": 0.25, |
| "rule": "shown, not counted", |
| "in_index": false |
| }, |
| "26": { |
| "raw": 0.9769, |
| "skill": 0.9692, |
| "coverage": 1.0, |
| "random": 0.2502, |
| "rule": "shown, not counted", |
| "in_index": false |
| }, |
| "27": { |
| "raw": 0.9275, |
| "skill": 0.9033, |
| "coverage": 1.0, |
| "random": 0.2502, |
| "rule": "shown, not counted", |
| "in_index": false |
| }, |
| "34": { |
| "raw": 0.1, |
| "skill": 0.0, |
| "coverage": 1.0, |
| "random": 0.1667, |
| "rule": "shown, not counted", |
| "in_index": false |
| } |
| }, |
| "benchmarks": { |
| "1": { |
| "catalog_id": 1, |
| "dataset": "BFCL", |
| "requests": 1694, |
| "answered": 1694, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1694, |
| "metric": "case exact accuracy", |
| "score": 0.9032, |
| "reference_same_cases": null, |
| "median_ms": 94.5, |
| "index_raw": 0.9032, |
| "index_skill": 0.8693, |
| "coverage": 1.0, |
| "chance": 0.2592, |
| "in_index": true |
| }, |
| "2": { |
| "catalog_id": 2, |
| "dataset": "ToolRet", |
| "requests": 1000, |
| "answered": 1000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1000, |
| "metric": "nDCG@10", |
| "score": 0.4126, |
| "reference_same_cases": null, |
| "median_ms": 802.3, |
| "index_raw": 0.4126, |
| "index_skill": 0.3532, |
| "coverage": 1.0, |
| "chance": 0.0918, |
| "in_index": true |
| }, |
| "3": { |
| "catalog_id": 3, |
| "dataset": "API-Bank", |
| "requests": 508, |
| "answered": 508, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 508, |
| "metric": "accuracy", |
| "score": 0.748, |
| "reference_same_cases": null, |
| "median_ms": 425.6, |
| "index_raw": 0.748, |
| "index_skill": 0.7431, |
| "coverage": 1.0, |
| "chance": 0.0189, |
| "in_index": true |
| }, |
| "4": { |
| "catalog_id": 4, |
| "dataset": "BANKING77", |
| "requests": 3080, |
| "answered": 3080, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 3080, |
| "metric": "macro-F1", |
| "score": 0.7149, |
| "reference_same_cases": null, |
| "median_ms": 68.5, |
| "index_raw": 0.7149, |
| "index_skill": 0.7112, |
| "coverage": 1.0, |
| "chance": 0.0127, |
| "in_index": true |
| }, |
| "5": { |
| "catalog_id": 5, |
| "dataset": "CLINC150+OOS", |
| "requests": 5500, |
| "answered": 5500, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5500, |
| "metric": "macro-F1", |
| "score": 0.7391, |
| "reference_same_cases": null, |
| "median_ms": 115.0, |
| "index_raw": 0.7391, |
| "index_skill": 0.7375, |
| "coverage": 1.0, |
| "chance": 0.006, |
| "in_index": true |
| }, |
| "6": { |
| "catalog_id": 6, |
| "dataset": "RouterBench", |
| "requests": 10000, |
| "answered": 10000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 10000, |
| "metric": "selected quality (quality objective)", |
| "score": 0.7905, |
| "reference_same_cases": null, |
| "median_ms": 262.7, |
| "tracks": [ |
| { |
| "track": "RouterBench-0shot", |
| "score": 0.7797, |
| "headline": false |
| }, |
| { |
| "track": "RouterBench-5shot", |
| "score": 0.8014, |
| "headline": false |
| } |
| ], |
| "index_raw": 0.7905, |
| "index_skill": 0.5052, |
| "coverage": 1.0, |
| "chance": 0.5245, |
| "in_index": true |
| }, |
| "9": { |
| "catalog_id": 9, |
| "dataset": "Home appliance simulator", |
| "requests": 160, |
| "answered": 160, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 160, |
| "metric": "case exact accuracy", |
| "score": 0.1187, |
| "reference_same_cases": null, |
| "median_ms": 1148.5, |
| "index_raw": 0.1187, |
| "index_skill": 0.1187, |
| "coverage": 1.0, |
| "chance": 0.0, |
| "in_index": true |
| }, |
| "10": { |
| "catalog_id": 10, |
| "dataset": "SGD/SGD-X", |
| "requests": 2500, |
| "answered": 2500, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2500, |
| "metric": "macro-F1", |
| "score": 0.5004, |
| "reference_same_cases": null, |
| "median_ms": 71.5, |
| "index_raw": 0.5004, |
| "index_skill": 0.1687, |
| "coverage": 1.0, |
| "chance": 0.399, |
| "in_index": true |
| }, |
| "11": { |
| "catalog_id": 11, |
| "dataset": "ContractNLI", |
| "requests": 123, |
| "answered": 123, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 123, |
| "metric": "macro-F1", |
| "score": 0.7612, |
| "reference_same_cases": null, |
| "median_ms": 1993.0, |
| "index_raw": 0.7612, |
| "index_skill": 0.6547, |
| "coverage": 1.0, |
| "chance": 0.3085, |
| "in_index": true |
| }, |
| "12": { |
| "catalog_id": 12, |
| "dataset": "ANLI", |
| "requests": 3200, |
| "answered": 3200, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 3200, |
| "metric": "macro-F1", |
| "score": 0.6012, |
| "reference_same_cases": null, |
| "median_ms": 13.0, |
| "index_raw": 0.6012, |
| "index_skill": 0.4026, |
| "coverage": 1.0, |
| "chance": 0.3324, |
| "in_index": true |
| }, |
| "20": { |
| "catalog_id": 20, |
| "dataset": "BPoMP", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "accuracy", |
| "score": 0.8456, |
| "reference_same_cases": null, |
| "median_ms": 12.5, |
| "index_raw": 0.8467, |
| "index_skill": 0.6933, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "21": { |
| "catalog_id": 21, |
| "dataset": "Humicroedit", |
| "requests": 2628, |
| "answered": 2628, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2628, |
| "metric": "accuracy", |
| "score": 0.6054, |
| "reference_same_cases": null, |
| "median_ms": 8.0, |
| "index_raw": 0.6054, |
| "index_skill": 0.2108, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "22": { |
| "catalog_id": 22, |
| "dataset": "POP909-CL", |
| "requests": 2000, |
| "answered": 2000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2000, |
| "metric": "accuracy", |
| "score": 0.036, |
| "reference_same_cases": null, |
| "median_ms": 571.6, |
| "index_raw": 0.0338, |
| "index_skill": 0.0262, |
| "coverage": 1.0, |
| "chance": 0.0078, |
| "in_index": true |
| }, |
| "23": { |
| "catalog_id": 23, |
| "dataset": "cfcolor", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "accuracy", |
| "score": 0.6016, |
| "reference_same_cases": null, |
| "median_ms": 33.4, |
| "index_raw": 0.5808, |
| "index_skill": 0.1616, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "24": { |
| "catalog_id": 24, |
| "dataset": "MMLU", |
| "requests": 14033, |
| "answered": 14033, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 14033, |
| "metric": "accuracy", |
| "score": 0.7375, |
| "reference_same_cases": null, |
| "median_ms": 13.0, |
| "index_raw": 0.7375, |
| "index_skill": 0.65, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": false |
| }, |
| "25": { |
| "catalog_id": 25, |
| "dataset": "GPQA Diamond", |
| "requests": 196, |
| "answered": 196, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 196, |
| "metric": "accuracy", |
| "score": 0.3724, |
| "reference_same_cases": null, |
| "median_ms": 23.5, |
| "index_raw": 0.3724, |
| "index_skill": 0.1633, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "26": { |
| "catalog_id": 26, |
| "dataset": "ARC-Easy", |
| "requests": 2376, |
| "answered": 2376, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2376, |
| "metric": "accuracy", |
| "score": 0.9769, |
| "reference_same_cases": null, |
| "median_ms": 12.3, |
| "index_raw": 0.9769, |
| "index_skill": 0.9692, |
| "coverage": 1.0, |
| "chance": 0.2502, |
| "in_index": false |
| }, |
| "27": { |
| "catalog_id": 27, |
| "dataset": "ARC-Challenge", |
| "requests": 1172, |
| "answered": 1172, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1172, |
| "metric": "accuracy", |
| "score": 0.9275, |
| "reference_same_cases": null, |
| "median_ms": 12.1, |
| "index_raw": 0.9275, |
| "index_skill": 0.9033, |
| "coverage": 1.0, |
| "chance": 0.2502, |
| "in_index": false |
| }, |
| "28": { |
| "catalog_id": 28, |
| "dataset": "WinoGrande", |
| "requests": 1267, |
| "answered": 1267, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1267, |
| "metric": "accuracy", |
| "score": 0.7032, |
| "reference_same_cases": null, |
| "median_ms": 8.4, |
| "index_raw": 0.7032, |
| "index_skill": 0.4064, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "29": { |
| "catalog_id": 29, |
| "dataset": "HellaSwag", |
| "requests": 10042, |
| "answered": 10042, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 10042, |
| "metric": "accuracy", |
| "score": 0.8194, |
| "reference_same_cases": null, |
| "median_ms": 18.6, |
| "index_raw": 0.8194, |
| "index_skill": 0.7592, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "30": { |
| "catalog_id": 30, |
| "dataset": "GSM8K", |
| "requests": 2638, |
| "answered": 2638, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 2638, |
| "metric": "accuracy", |
| "score": 0.5788, |
| "reference_same_cases": null, |
| "median_ms": 16.4, |
| "tracks": [ |
| { |
| "track": "GSM8K-4choice", |
| "score": 0.5967, |
| "headline": false |
| }, |
| { |
| "track": "GSM8K-10choice", |
| "score": 0.561, |
| "headline": false |
| } |
| ], |
| "index_raw": 0.5788, |
| "index_skill": 0.4872, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "31": { |
| "catalog_id": 31, |
| "dataset": "ChessBench", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "accuracy", |
| "score": 0.1348, |
| "reference_same_cases": null, |
| "median_ms": 88.3, |
| "index_raw": 0.1348, |
| "index_skill": 0.0577, |
| "coverage": 1.0, |
| "chance": 0.0819, |
| "in_index": true |
| }, |
| "32": { |
| "catalog_id": 32, |
| "dataset": "MuSR", |
| "requests": 752, |
| "answered": 752, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 752, |
| "metric": "accuracy", |
| "score": 0.5678, |
| "reference_same_cases": null, |
| "median_ms": 61.7, |
| "index_raw": 0.5678, |
| "index_skill": 0.3129, |
| "coverage": 1.0, |
| "chance": 0.371, |
| "in_index": true |
| }, |
| "33": { |
| "catalog_id": 33, |
| "dataset": "SATA-Bench", |
| "requests": 1650, |
| "answered": 1650, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1650, |
| "metric": "case exact accuracy", |
| "score": 0.2594, |
| "reference_same_cases": null, |
| "median_ms": 213.7, |
| "index_raw": 0.2594, |
| "index_skill": 0.2496, |
| "coverage": 1.0, |
| "chance": 0.0131, |
| "in_index": true |
| }, |
| "34": { |
| "catalog_id": 34, |
| "dataset": "SimpleBench", |
| "requests": 10, |
| "answered": 10, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 10, |
| "metric": "accuracy", |
| "score": 0.1, |
| "reference_same_cases": null, |
| "median_ms": 14.3, |
| "index_raw": 0.1, |
| "index_skill": 0.0, |
| "coverage": 1.0, |
| "chance": 0.1667, |
| "in_index": false |
| }, |
| "36": { |
| "catalog_id": 36, |
| "dataset": "BRIGHT", |
| "requests": 550, |
| "answered": 550, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 550, |
| "metric": "nDCG@10", |
| "score": 0.1751, |
| "reference_same_cases": null, |
| "median_ms": 1002.0, |
| "index_raw": 0.1751, |
| "index_skill": 0.135, |
| "coverage": 1.0, |
| "chance": 0.0464, |
| "in_index": true |
| }, |
| "37": { |
| "catalog_id": 37, |
| "dataset": "Amazon ESCI", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "macro-F1", |
| "score": 0.4309, |
| "reference_same_cases": null, |
| "median_ms": 26.6, |
| "index_raw": 0.4309, |
| "index_skill": 0.2862, |
| "coverage": 1.0, |
| "chance": 0.2027, |
| "in_index": true |
| }, |
| "38": { |
| "catalog_id": 38, |
| "dataset": "ACOS", |
| "requests": 1565, |
| "answered": 1565, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1565, |
| "metric": "case exact accuracy", |
| "score": 0.03, |
| "reference_same_cases": null, |
| "median_ms": 672.9, |
| "index_raw": 0.03, |
| "index_skill": 0.03, |
| "coverage": 1.0, |
| "chance": 0.0, |
| "in_index": true |
| }, |
| "39": { |
| "catalog_id": 39, |
| "dataset": "FinEntity", |
| "requests": 979, |
| "answered": 979, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 979, |
| "metric": "macro-F1", |
| "score": 0.8603, |
| "reference_same_cases": null, |
| "median_ms": 24.9, |
| "index_raw": 0.8603, |
| "index_skill": 0.7945, |
| "coverage": 1.0, |
| "chance": 0.3201, |
| "in_index": true |
| }, |
| "40": { |
| "catalog_id": 40, |
| "dataset": "iSarcasmEval", |
| "requests": 4600, |
| "answered": 4600, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 4600, |
| "metric": "Sarcasm F1 \u00b7 track A, English", |
| "score": 0.4523, |
| "reference_same_cases": null, |
| "median_ms": 8.6, |
| "tracks": [ |
| { |
| "track": "A \u00b7 Arabic", |
| "score": 0.313, |
| "headline": false |
| }, |
| { |
| "track": "A \u00b7 English", |
| "score": 0.4523, |
| "headline": true |
| }, |
| { |
| "track": "C \u00b7 Arabic pairs", |
| "score": 0.645, |
| "headline": false |
| }, |
| { |
| "track": "C \u00b7 English pairs", |
| "score": 0.915, |
| "headline": false |
| } |
| ], |
| "index_raw": 0.4523, |
| "index_skill": 0.2953, |
| "coverage": 1.0, |
| "chance": 0.2227, |
| "in_index": true |
| }, |
| "41": { |
| "catalog_id": 41, |
| "dataset": "VAST", |
| "requests": 3006, |
| "answered": 3006, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 3006, |
| "metric": "macro-F1", |
| "score": 0.6169, |
| "reference_same_cases": null, |
| "median_ms": 14.9, |
| "index_raw": 0.6169, |
| "index_skill": 0.4254, |
| "coverage": 1.0, |
| "chance": 0.3333, |
| "in_index": true |
| }, |
| "42": { |
| "catalog_id": 42, |
| "dataset": "NLI4CT", |
| "requests": 5500, |
| "answered": 5500, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5500, |
| "metric": "macro-F1", |
| "score": 0.7618, |
| "reference_same_cases": null, |
| "median_ms": 37.0, |
| "index_raw": 0.7618, |
| "index_skill": 0.5366, |
| "coverage": 1.0, |
| "chance": 0.486, |
| "in_index": true |
| }, |
| "43": { |
| "catalog_id": 43, |
| "dataset": "CRUXEval", |
| "requests": 570, |
| "answered": 570, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 570, |
| "metric": "accuracy", |
| "score": 0.4719, |
| "reference_same_cases": null, |
| "median_ms": 13.7, |
| "index_raw": 0.4719, |
| "index_skill": 0.1622, |
| "coverage": 1.0, |
| "chance": 0.3697, |
| "in_index": true |
| }, |
| "44": { |
| "catalog_id": 44, |
| "dataset": "CLadder", |
| "requests": 5000, |
| "answered": 5000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 5000, |
| "metric": "accuracy", |
| "score": 0.6368, |
| "reference_same_cases": null, |
| "median_ms": 13.7, |
| "index_raw": 0.6368, |
| "index_skill": 0.2736, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "45": { |
| "catalog_id": 45, |
| "dataset": "HLE", |
| "requests": 501, |
| "answered": 501, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 501, |
| "metric": "accuracy", |
| "score": 0.1297, |
| "reference_same_cases": null, |
| "median_ms": 22.6, |
| "index_raw": 0.1297, |
| "index_skill": 0.0, |
| "coverage": 1.0, |
| "chance": 0.1641, |
| "in_index": true |
| }, |
| "48": { |
| "catalog_id": 48, |
| "dataset": "ForecastBench", |
| "requests": 10139, |
| "answered": 10139, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 10139, |
| "metric": "Brier (lower is better)", |
| "score": 0.2172, |
| "reference_same_cases": null, |
| "median_ms": 49.3, |
| "index_raw": 0.1312, |
| "index_skill": 0.1312, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "50": { |
| "catalog_id": 50, |
| "dataset": "Habermas Machine", |
| "requests": 1676, |
| "answered": 1676, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "scored_requests": 1676, |
| "metric": "accuracy", |
| "score": 0.4427, |
| "reference_same_cases": null, |
| "median_ms": 55.1, |
| "index_raw": 0.4427, |
| "index_skill": 0.1912, |
| "coverage": 1.0, |
| "chance": 0.311, |
| "in_index": true |
| }, |
| "56": { |
| "catalog_id": 56, |
| "dataset": "PhishNChips phishing decisions", |
| "requests": 2000, |
| "answered": 2000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.6365, |
| "median_ms": 140.2, |
| "scored_requests": 2000, |
| "index_raw": 0.6365, |
| "index_skill": 0.273, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "57": { |
| "catalog_id": 57, |
| "dataset": "MMLU-Pro", |
| "requests": 12032, |
| "answered": 12032, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.5206, |
| "median_ms": 20.3, |
| "scored_requests": 12032, |
| "index_raw": 0.5206, |
| "index_skill": 0.4608, |
| "coverage": 1.0, |
| "chance": 0.1109, |
| "in_index": true |
| }, |
| "58": { |
| "catalog_id": 58, |
| "dataset": "BBH fixed-option tasks", |
| "requests": 5507, |
| "answered": 5507, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.6379, |
| "median_ms": 15.0, |
| "scored_requests": 5507, |
| "index_raw": 0.6379, |
| "index_skill": 0.4751, |
| "coverage": 1.0, |
| "chance": 0.3101, |
| "in_index": true |
| }, |
| "59": { |
| "catalog_id": 59, |
| "dataset": "RAGTruth response-level hallucination", |
| "requests": 2700, |
| "answered": 2700, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "F1 on hallucinated class", |
| "score": 0.6648, |
| "median_ms": 50.6, |
| "scored_requests": 2700, |
| "index_raw": 0.6648, |
| "index_skill": 0.4306, |
| "coverage": 1.0, |
| "chance": 0.4113, |
| "in_index": true |
| }, |
| "61": { |
| "catalog_id": 61, |
| "dataset": "HoVer claim verification", |
| "requests": 4000, |
| "answered": 4000, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.6308, |
| "median_ms": 29.9, |
| "scored_requests": 4000, |
| "index_raw": 0.6308, |
| "index_skill": 0.2616, |
| "coverage": 1.0, |
| "chance": 0.5, |
| "in_index": true |
| }, |
| "62": { |
| "catalog_id": 62, |
| "dataset": "When2Call MCQ", |
| "requests": 3652, |
| "answered": 3652, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.6276, |
| "median_ms": 52.3, |
| "scored_requests": 3652, |
| "index_raw": 0.6276, |
| "index_skill": 0.5035, |
| "coverage": 1.0, |
| "chance": 0.25, |
| "in_index": true |
| }, |
| "64": { |
| "catalog_id": 64, |
| "dataset": "New Yorker caption matching", |
| "requests": 528, |
| "answered": 528, |
| "unsupported": 0, |
| "errors": 0, |
| "abstained": 0, |
| "pending": 0, |
| "metric": "accuracy", |
| "score": 0.589, |
| "median_ms": 16.6, |
| "scored_requests": 528, |
| "index_raw": 0.589, |
| "index_skill": 0.4862, |
| "coverage": 1.0, |
| "chance": 0.2, |
| "in_index": true |
| } |
| }, |
| "panel_id": "decision-index-0.2", |
| "note": "Decision Index 0.2 averages 40 benchmarks in five equal-weight areas: each area is the plain mean of its benchmarks and the index is 100 x the mean of the five areas. Each benchmark is chance-corrected first, (score - chance) / (1 - chance) clipped to 0-1, so 0 means random guessing and 100 means perfect. Every score is coverage-adjusted, so an unanswered or unsupported request counts as wrong. ForecastBench enters against its baseline: clip((0.25 - Brier) / 0.25) x coverage, so always predicting 0.5 scores zero. MMLU, ARC-Easy, ARC-Challenge, SimpleBench stay on the board as non-index benchmarks. The six interactive environments are still unrun and stay out. Every entrant on the board has results on all 40 index benchmarks. Point estimates only, no uncertainty intervals yet.", |
| "local_run": { |
| "kit_commit": "19ad28ec9485493cc4f7fc07d91c178f948e6434", |
| "evaluated": "2026-09-25", |
| "note": "Scored locally with the official kit; not a leaderboard submission or result. Requests shared with 0.1 reuse this model's 0.1 predictions; the added requests ran with the same frozen evaluation setup at temperature 1.0. No leaderboard-style exposure penalty is applied: known training exposure (see the model card) stays in these scores.", |
| "without_mmlu_pro": 37.41, |
| "screening": [ |
| { |
| "stage": "T3", |
| "rows": 23156, |
| "matched": 138, |
| "by_source": { |
| "MMLU-Pro": 131, |
| "SuperGPQA": 6, |
| "BoolQ": 1, |
| "MedMCQA": 0 |
| } |
| }, |
| { |
| "stage": "T4", |
| "rows": 42360, |
| "matched": 156, |
| "by_source": { |
| "MMLU-Pro": 148, |
| "SuperGPQA": 7, |
| "BoolQ": 1, |
| "MedMCQA": 0 |
| } |
| } |
| ], |
| "timing_note": "latency_ms and every benchmark's median_ms are each request's share of batched inference time, allocated by prompt tokens. They are not serial-request or HTTP-serving latency and shouldn't be used for serving-latency comparisons." |
| } |
| } |
