{ "easy": { "accuracy": 1.0, "brier_mean": 0.000741052543754835, "calibration_n": 48, "complete": true, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "dataset_hash": "3cbd0e951fa8059741955ef2bd3273b0bb60aecae8f4a68b89872f55f7376b3d", "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8844688534736633, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9957148141049325, "n": 47 } ], "ece": 0.006602810074885651, "n": 48 }, "latency": { "n": 48, "p50_s": 0.5626124106347561, "p95_s": 1.107840776070952 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "ledger_charged_usd": null, "macro_accuracy": 1.0, "model_identities": [ "pcdm:native:runs/nc_v3_tap20_wf" ], "n_attempted": 48, "n_correct": 48, "n_planned": 48, "n_renormalized": 0, "n_scorable": 48, "n_valid": 48, "operational_success": 1.0, "ordinal_mae": null, "paraphrase_consistency": { "agree": 0, "agreement": null, "both_correct_rate_all_pairs": null, "both_valid": 0, "pairs": 0 }, "per_family": { "extraction": { "accuracy": 1.0, "brier_mean": 0.00023703511438985972, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9940488288799921, "n": 12 } ], "ece": 0.00595117112000787, "n": 12 }, "latency": { "n": 12, "p50_s": 0.7644834816455841, "p95_s": 0.9596918206661937 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "fact": { "accuracy": 1.0, "brier_mean": 0.002727175039344051, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8844688534736633, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9881845821033824, "n": 11 } ], "ece": 0.020458395282427478, "n": 12 }, "latency": { "n": 12, "p50_s": 0.6582908667623997, "p95_s": 1.0728746362030503 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 1.0, "brier_mean": 2.1283921091852373e-11, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9999983360370001, "n": 12 } ], "ece": 1.6639629999426475e-06, "n": 12 }, "latency": { "n": 12, "p50_s": 0.5524514839053154, "p95_s": 1.0589902859181164 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tool_selection": { "accuracy": 1.0, "brier_mean": 1.5080045307355258e-15, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9999999900658926, "n": 12 } ], "ece": 9.93410742555767e-09, "n": 12 }, "latency": { "n": 12, "p50_s": 0.544886339455843, "p95_s": 0.7671892851591111 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "price_per_1000_decisions_usd": null, "probability_sources": [ "native" ], "schema_validity": 1.0, "schema_validity_strict": 1.0, "splits": { "public": { "accuracy": 1.0, "brier_mean": 0.000741052543754835, "calibration_n": 48, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8844688534736633, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9957148141049325, "n": 47 } ], "ece": 0.006602810074885651, "n": 48 }, "latency": { "n": 48, "p50_s": 0.5626124106347561, "p95_s": 1.107840776070952 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 48, "n_correct": 48, "n_planned": 48, "n_renormalized": 0, "n_scorable": 48, "n_valid": 48, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "version": "jevbench-v1" }, "hard": { "accuracy": 0.38738738738738737, "brier_mean": 0.8756417814236204, "calibration_n": 111, "complete": true, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "dataset_hash": "e816e106f562dec75a8da12e69b75e957a4bbd9f6300f1145fc71837327a3c61", "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.29647180438041687, "n": 1 }, { "accuracy": 0.2, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.36647668182849885, "n": 10 }, { "accuracy": 0.42857142857142855, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.44530160512243, "n": 7 }, { "accuracy": 0.18181818181818182, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5331228592179038, "n": 11 }, { "accuracy": 0.21428571428571427, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6532700913293021, "n": 14 }, { "accuracy": 0.3, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7405811965465545, "n": 10 }, { "accuracy": 0.38461538461538464, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8388918088032649, "n": 13 }, { "accuracy": 0.5555555555555556, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9753862314754063, "n": 45 } ], "ece": 0.37200241878226, "n": 111 }, "latency": { "n": 111, "p50_s": 1.181565023958683, "p95_s": 1.2096525765955448 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "ledger_charged_usd": null, "macro_accuracy": 0.47499115435647943, "model_identities": [ "pcdm:native:runs/nc_v3_tap20_wf" ], "n_attempted": 111, "n_correct": 43, "n_planned": 111, "n_renormalized": 0, "n_scorable": 111, "n_valid": 111, "operational_success": 1.0, "ordinal_mae": 0.7721535418895655, "paraphrase_consistency": { "agree": 0, "agreement": null, "both_correct_rate_all_pairs": null, "both_valid": 0, "pairs": 0 }, "per_family": { "adversarial": { "accuracy": 0.6666666666666666, "brier_mean": 0.5302267019068412, "calibration_n": 6, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5023947954177856, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7638146877288818, "n": 2 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9972086747487386, "n": 3 } ], "ece": 0.33893475929896033, "n": 6 }, "latency": { "n": 6, "p50_s": 1.18465993180871, "p95_s": 1.1982474736869335 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 4, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": 0.0046126680535962805, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ambiguous": { "accuracy": 0.5714285714285714, "brier_mean": 0.7659289989739032, "calibration_n": 7, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.532402753829956, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7085151076316833, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8497002720832825, "n": 1 }, { "accuracy": 0.5, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9649203270673752, "n": 4 } ], "ece": 0.4048383831977844, "n": 7 }, "latency": { "n": 7, "p50_s": 0.9744239524006844, "p95_s": 1.1390509583055972 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 7, "n_correct": 4, "n_planned": 7, "n_renormalized": 0, "n_scorable": 7, "n_valid": 7, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "judge_hard": { "accuracy": 0.47058823529411764, "brier_mean": 1.0467729097334066, "calibration_n": 17, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.47058823529411764, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9908343132804421, "n": 17 } ], "ece": 0.5202460779863245, "n": 17 }, "latency": { "n": 17, "p50_s": 1.1866157427430153, "p95_s": 1.1948738172650337 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 17, "n_correct": 8, "n_planned": 17, "n_renormalized": 0, "n_scorable": 17, "n_valid": 17, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "long_policy": { "accuracy": 0.15789473684210525, "brier_mean": 1.0325485336532152, "calibration_n": 19, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.29647180438041687, "n": 1 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.36068005859851837, "n": 2 }, { "accuracy": 0.5, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.45221932232379913, "n": 2 }, { "accuracy": 0.25, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5355411916971207, "n": 4 }, { "accuracy": 0.16666666666666666, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6340792576471964, "n": 6 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8239782750606537, "n": 2 }, { "accuracy": 0.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9462971687316895, "n": 2 } ], "ece": 0.45266234090453694, "n": 19 }, "latency": { "n": 19, "p50_s": 1.2041072100400925, "p95_s": 1.2212083987891673 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 19, "n_correct": 3, "n_planned": 19, "n_renormalized": 0, "n_scorable": 19, "n_valid": 19, "operational_success": 1.0, "ordinal_mae": 0.45920406887307763, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "multi_hop": { "accuracy": 0.16666666666666666, "brier_mean": 1.0309613121996857, "calibration_n": 18, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.37918325265248615, "n": 3 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.42796066403388977, "n": 1 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5379397571086884, "n": 2 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.681269109249115, "n": 3 }, { "accuracy": 0.5, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7330727577209473, "n": 2 }, { "accuracy": 0.3333333333333333, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8386213779449463, "n": 6 }, { "accuracy": 0.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9935780167579651, "n": 1 } ], "ece": 0.5098138368792005, "n": 18 }, "latency": { "n": 18, "p50_s": 1.1912375092506409, "p95_s": 1.203717217594385 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 18, "n_correct": 3, "n_planned": 18, "n_renormalized": 0, "n_scorable": 18, "n_valid": 18, "operational_success": 1.0, "ordinal_mae": 0.15101483464241028, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "probability": { "accuracy": 0.3, "brier_mean": 0.9166441995158155, "calibration_n": 10, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4623758941888809, "n": 2 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5498971939086914, "n": 2 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7421726584434509, "n": 2 }, { "accuracy": 0.6666666666666666, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8560118277867635, "n": 3 }, { "accuracy": 0.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9891908168792725, "n": 1 } ], "ece": 0.42166142165660864, "n": 10 }, "latency": { "n": 10, "p50_s": 1.187890186905861, "p95_s": 1.2060833223164082 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 10, "n_correct": 3, "n_planned": 10, "n_renormalized": 0, "n_scorable": 10, "n_valid": 10, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing_hard": { "accuracy": 1.0, "brier_mean": 0.0029613228822248884, "calibration_n": 5, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9767965316772461, "n": 5 } ], "ece": 0.023203468322753884, "n": 5 }, "latency": { "n": 5, "p50_s": 0.9726530313491821, "p95_s": 1.1540774002671241 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 5, "n_correct": 5, "n_planned": 5, "n_renormalized": 0, "n_scorable": 5, "n_valid": 5, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "temporal_numeric": { "accuracy": 0.3333333333333333, "brier_mean": 0.8615734632850287, "calibration_n": 15, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.4, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.36117138862609866, "n": 5 }, { "accuracy": 0.5, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.42998006939888, "n": 2 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5117152333259583, "n": 1 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6441678404808044, "n": 1 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7346175312995911, "n": 2 }, { "accuracy": 0.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8081729412078857, "n": 1 }, { "accuracy": 0.6666666666666666, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9404582381248474, "n": 3 } ], "ece": 0.30592324733734133, "n": 15 }, "latency": { "n": 15, "p50_s": 0.9862628430128098, "p95_s": 1.2137017905712126 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 15, "n_correct": 5, "n_planned": 15, "n_renormalized": 0, "n_scorable": 15, "n_valid": 15, "operational_success": 1.0, "ordinal_mae": 1.6668823510408401, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tradeoff": { "accuracy": 0.3333333333333333, "brier_mean": 1.0548409941648835, "calibration_n": 6, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6642349362373352, "n": 2 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7499415874481201, "n": 1 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.3333333333333333, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9430160721143087, "n": 3 } ], "ece": 0.5679294168949127, "n": 6 }, "latency": { "n": 6, "p50_s": 0.972567968070507, "p95_s": 1.1406548656523228 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 2, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "trap": { "accuracy": 0.75, "brier_mean": 0.5310765981820174, "calibration_n": 8, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6624303460121155, "n": 2 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.6666666666666666, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9645201663176218, "n": 6 } ], "ece": 0.30778253823518753, "n": 8 }, "latency": { "n": 8, "p50_s": 0.9730363674461842, "p95_s": 1.1941582702100275 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 8, "n_correct": 6, "n_planned": 8, "n_renormalized": 0, "n_scorable": 8, "n_valid": 8, "operational_success": 1.0, "ordinal_mae": 1.8920032598543912, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "price_per_1000_decisions_usd": null, "probability_sources": [ "native" ], "schema_validity": 1.0, "schema_validity_strict": 1.0, "splits": { "public": { "accuracy": 0.38738738738738737, "brier_mean": 0.8756417814236204, "calibration_n": 111, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.29647180438041687, "n": 1 }, { "accuracy": 0.2, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.36647668182849885, "n": 10 }, { "accuracy": 0.42857142857142855, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.44530160512243, "n": 7 }, { "accuracy": 0.18181818181818182, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5331228592179038, "n": 11 }, { "accuracy": 0.21428571428571427, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6532700913293021, "n": 14 }, { "accuracy": 0.3, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7405811965465545, "n": 10 }, { "accuracy": 0.38461538461538464, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8388918088032649, "n": 13 }, { "accuracy": 0.5555555555555556, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9753862314754063, "n": 45 } ], "ece": 0.37200241878226, "n": 111 }, "latency": { "n": 111, "p50_s": 1.181565023958683, "p95_s": 1.2096525765955448 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 111, "n_correct": 43, "n_planned": 111, "n_renormalized": 0, "n_scorable": 111, "n_valid": 111, "operational_success": 1.0, "ordinal_mae": 0.7721535418895655, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "version": "jevbench-v1" }, "original": { "accuracy": 0.75, "brier_mean": 0.40276924379203316, "calibration_n": 72, "complete": true, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "dataset_hash": "914276022fa5f0ea70e4231f2af98e68c67b274b61bea8c440b33890401cd2f9", "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.43212921917438507, "n": 2 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5517171770334244, "n": 4 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6635033190250397, "n": 4 }, { "accuracy": 0.6666666666666666, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7310981651147207, "n": 6 }, { "accuracy": 0.6, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8727044343948365, "n": 5 }, { "accuracy": 0.8235294117647058, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9825702835531795, "n": 51 } ], "ece": 0.150803044024441, "n": 72 }, "latency": { "n": 72, "p50_s": 0.5542781464755535, "p95_s": 1.1863042686134575 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "ledger_charged_usd": null, "macro_accuracy": 0.75, "model_identities": [ "pcdm:native:runs/nc_v3_tap20_wf" ], "n_attempted": 72, "n_correct": 54, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_valid": 72, "operational_success": 1.0, "ordinal_mae": 0.34141736199647293, "paraphrase_consistency": { "agree": 30, "agreement": 0.8333333333333334, "both_correct_rate_all_pairs": 0.6666666666666666, "both_valid": 36, "pairs": 36 }, "per_family": { "adequacy": { "accuracy": 0.4166666666666667, "brier_mean": 0.8503524575307818, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5679670572280884, "n": 1 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6368800401687622, "n": 2 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.723824143409729, "n": 1 }, { "accuracy": 0.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8935546875, "n": 1 }, { "accuracy": 0.5714285714285714, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9796636274882725, "n": 7 } ], "ece": 0.4430626134077708, "n": 12 }, "latency": { "n": 12, "p50_s": 0.5596147291362286, "p95_s": 0.9536922551691529 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 5, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 1.0, "brier_mean": 0.013560885788644146, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7313405871391296, "n": 1 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9782732129096985, "n": 11 } ], "ece": 0.04230450590451558, "n": 12 }, "latency": { "n": 12, "p50_s": 0.29115450009703636, "p95_s": 0.7657811347395181 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 0.6666666666666666, "brier_mean": 0.41181819447002926, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.42259737849235535, "n": 1 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6991903185844421, "n": 1 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.734041690826416, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8523933291435242, "n": 1 }, { "accuracy": 0.875, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9949639439582825, "n": 8 } ], "ece": 0.24692896753549576, "n": 12 }, "latency": { "n": 12, "p50_s": 0.9667536541819572, "p95_s": 1.184501038864255 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 8, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 0.8333333333333334, "brier_mean": 0.3210295459915781, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.8333333333333334, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9857434084018072, "n": 12 } ], "ece": 0.15241007506847382, "n": 12 }, "latency": { "n": 12, "p50_s": 0.5506149679422379, "p95_s": 1.129030930250883 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 10, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": 0.34141736199647293, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 0.75, "brier_mean": 0.497069164345908, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5980823040008545, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7690881490707397, "n": 1 }, { "accuracy": 0.5, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8820080161094666, "n": 2 }, { "accuracy": 0.75, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9978939592838287, "n": 8 } ], "ece": 0.2816664377848307, "n": 12 }, "latency": { "n": 12, "p50_s": 0.9681226126849651, "p95_s": 1.1986095048487186 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 9, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 0.8333333333333334, "brier_mean": 0.32278521462525744, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4416610598564148, "n": 1 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5204096734523773, "n": 2 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6810628771781921, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7141472101211548, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8535581231117249, "n": 1 }, { "accuracy": 0.8, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9441299200057983, "n": 5 } ], "ece": 0.19640770554542536, "n": 12 }, "latency": { "n": 12, "p50_s": 0.29117556661367416, "p95_s": 0.9626881990581748 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 10, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "price_per_1000_decisions_usd": null, "probability_sources": [ "native" ], "schema_validity": 1.0, "schema_validity_strict": 1.0, "splits": { "public": { "accuracy": 0.75, "brier_mean": 0.40276924379203316, "calibration_n": 72, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.43212921917438507, "n": 2 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5517171770334244, "n": 4 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6635033190250397, "n": 4 }, { "accuracy": 0.6666666666666666, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7310981651147207, "n": 6 }, { "accuracy": 0.6, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8727044343948365, "n": 5 }, { "accuracy": 0.8235294117647058, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9825702835531795, "n": 51 } ], "ece": 0.150803044024441, "n": 72 }, "latency": { "n": 72, "p50_s": 0.5542781464755535, "p95_s": 1.1863042686134575 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 72, "n_correct": 54, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_valid": 72, "operational_success": 1.0, "ordinal_mae": 0.34141736199647293, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "version": "jevbench-v1" } }