| { |
| "easy": { |
| "accuracy": 1.0, |
| "brier_mean": 0.000741052543754835, |
| "calibration_n": 48, |
| "complete": true, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "dataset_hash": "3cbd0e951fa8059741955ef2bd3273b0bb60aecae8f4a68b89872f55f7376b3d", |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8844688534736633, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9957148141049325, |
| "n": 47 |
| } |
| ], |
| "ece": 0.006602810074885651, |
| "n": 48 |
| }, |
| "latency": { |
| "n": 48, |
| "p50_s": 0.5626124106347561, |
| "p95_s": 1.107840776070952 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "ledger_charged_usd": null, |
| "macro_accuracy": 1.0, |
| "model_identities": [ |
| "pcdm:native:runs/nc_v3_tap20_wf" |
| ], |
| "n_attempted": 48, |
| "n_correct": 48, |
| "n_planned": 48, |
| "n_renormalized": 0, |
| "n_scorable": 48, |
| "n_valid": 48, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "paraphrase_consistency": { |
| "agree": 0, |
| "agreement": null, |
| "both_correct_rate_all_pairs": null, |
| "both_valid": 0, |
| "pairs": 0 |
| }, |
| "per_family": { |
| "extraction": { |
| "accuracy": 1.0, |
| "brier_mean": 0.00023703511438985972, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9940488288799921, |
| "n": 12 |
| } |
| ], |
| "ece": 0.00595117112000787, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.7644834816455841, |
| "p95_s": 0.9596918206661937 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "fact": { |
| "accuracy": 1.0, |
| "brier_mean": 0.002727175039344051, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8844688534736633, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9881845821033824, |
| "n": 11 |
| } |
| ], |
| "ece": 0.020458395282427478, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.6582908667623997, |
| "p95_s": 1.0728746362030503 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "intent": { |
| "accuracy": 1.0, |
| "brier_mean": 2.1283921091852373e-11, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9999983360370001, |
| "n": 12 |
| } |
| ], |
| "ece": 1.6639629999426475e-06, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.5524514839053154, |
| "p95_s": 1.0589902859181164 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "tool_selection": { |
| "accuracy": 1.0, |
| "brier_mean": 1.5080045307355258e-15, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9999999900658926, |
| "n": 12 |
| } |
| ], |
| "ece": 9.93410742555767e-09, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.544886339455843, |
| "p95_s": 0.7671892851591111 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "price_per_1000_decisions_usd": null, |
| "probability_sources": [ |
| "native" |
| ], |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0, |
| "splits": { |
| "public": { |
| "accuracy": 1.0, |
| "brier_mean": 0.000741052543754835, |
| "calibration_n": 48, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8844688534736633, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9957148141049325, |
| "n": 47 |
| } |
| ], |
| "ece": 0.006602810074885651, |
| "n": 48 |
| }, |
| "latency": { |
| "n": 48, |
| "p50_s": 0.5626124106347561, |
| "p95_s": 1.107840776070952 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 48, |
| "n_correct": 48, |
| "n_planned": 48, |
| "n_renormalized": 0, |
| "n_scorable": 48, |
| "n_valid": 48, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "version": "jevbench-v1" |
| }, |
| "hard": { |
| "accuracy": 0.38738738738738737, |
| "brier_mean": 0.8756417814236204, |
| "calibration_n": 111, |
| "complete": true, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "dataset_hash": "e816e106f562dec75a8da12e69b75e957a4bbd9f6300f1145fc71837327a3c61", |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": 0.29647180438041687, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.2, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.36647668182849885, |
| "n": 10 |
| }, |
| { |
| "accuracy": 0.42857142857142855, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.44530160512243, |
| "n": 7 |
| }, |
| { |
| "accuracy": 0.18181818181818182, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5331228592179038, |
| "n": 11 |
| }, |
| { |
| "accuracy": 0.21428571428571427, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6532700913293021, |
| "n": 14 |
| }, |
| { |
| "accuracy": 0.3, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7405811965465545, |
| "n": 10 |
| }, |
| { |
| "accuracy": 0.38461538461538464, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8388918088032649, |
| "n": 13 |
| }, |
| { |
| "accuracy": 0.5555555555555556, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9753862314754063, |
| "n": 45 |
| } |
| ], |
| "ece": 0.37200241878226, |
| "n": 111 |
| }, |
| "latency": { |
| "n": 111, |
| "p50_s": 1.181565023958683, |
| "p95_s": 1.2096525765955448 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "ledger_charged_usd": null, |
| "macro_accuracy": 0.47499115435647943, |
| "model_identities": [ |
| "pcdm:native:runs/nc_v3_tap20_wf" |
| ], |
| "n_attempted": 111, |
| "n_correct": 43, |
| "n_planned": 111, |
| "n_renormalized": 0, |
| "n_scorable": 111, |
| "n_valid": 111, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.7721535418895655, |
| "paraphrase_consistency": { |
| "agree": 0, |
| "agreement": null, |
| "both_correct_rate_all_pairs": null, |
| "both_valid": 0, |
| "pairs": 0 |
| }, |
| "per_family": { |
| "adversarial": { |
| "accuracy": 0.6666666666666666, |
| "brier_mean": 0.5302267019068412, |
| "calibration_n": 6, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5023947954177856, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7638146877288818, |
| "n": 2 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9972086747487386, |
| "n": 3 |
| } |
| ], |
| "ece": 0.33893475929896033, |
| "n": 6 |
| }, |
| "latency": { |
| "n": 6, |
| "p50_s": 1.18465993180871, |
| "p95_s": 1.1982474736869335 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 6, |
| "n_correct": 4, |
| "n_planned": 6, |
| "n_renormalized": 0, |
| "n_scorable": 6, |
| "n_valid": 6, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.0046126680535962805, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "ambiguous": { |
| "accuracy": 0.5714285714285714, |
| "brier_mean": 0.7659289989739032, |
| "calibration_n": 7, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.532402753829956, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7085151076316833, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8497002720832825, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9649203270673752, |
| "n": 4 |
| } |
| ], |
| "ece": 0.4048383831977844, |
| "n": 7 |
| }, |
| "latency": { |
| "n": 7, |
| "p50_s": 0.9744239524006844, |
| "p95_s": 1.1390509583055972 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 7, |
| "n_correct": 4, |
| "n_planned": 7, |
| "n_renormalized": 0, |
| "n_scorable": 7, |
| "n_valid": 7, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "judge_hard": { |
| "accuracy": 0.47058823529411764, |
| "brier_mean": 1.0467729097334066, |
| "calibration_n": 17, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.47058823529411764, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9908343132804421, |
| "n": 17 |
| } |
| ], |
| "ece": 0.5202460779863245, |
| "n": 17 |
| }, |
| "latency": { |
| "n": 17, |
| "p50_s": 1.1866157427430153, |
| "p95_s": 1.1948738172650337 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 17, |
| "n_correct": 8, |
| "n_planned": 17, |
| "n_renormalized": 0, |
| "n_scorable": 17, |
| "n_valid": 17, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "long_policy": { |
| "accuracy": 0.15789473684210525, |
| "brier_mean": 1.0325485336532152, |
| "calibration_n": 19, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": 0.29647180438041687, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.36068005859851837, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.45221932232379913, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.25, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5355411916971207, |
| "n": 4 |
| }, |
| { |
| "accuracy": 0.16666666666666666, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6340792576471964, |
| "n": 6 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8239782750606537, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9462971687316895, |
| "n": 2 |
| } |
| ], |
| "ece": 0.45266234090453694, |
| "n": 19 |
| }, |
| "latency": { |
| "n": 19, |
| "p50_s": 1.2041072100400925, |
| "p95_s": 1.2212083987891673 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 19, |
| "n_correct": 3, |
| "n_planned": 19, |
| "n_renormalized": 0, |
| "n_scorable": 19, |
| "n_valid": 19, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.45920406887307763, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "multi_hop": { |
| "accuracy": 0.16666666666666666, |
| "brier_mean": 1.0309613121996857, |
| "calibration_n": 18, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.37918325265248615, |
| "n": 3 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.42796066403388977, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5379397571086884, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.681269109249115, |
| "n": 3 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7330727577209473, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.3333333333333333, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8386213779449463, |
| "n": 6 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9935780167579651, |
| "n": 1 |
| } |
| ], |
| "ece": 0.5098138368792005, |
| "n": 18 |
| }, |
| "latency": { |
| "n": 18, |
| "p50_s": 1.1912375092506409, |
| "p95_s": 1.203717217594385 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 18, |
| "n_correct": 3, |
| "n_planned": 18, |
| "n_renormalized": 0, |
| "n_scorable": 18, |
| "n_valid": 18, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.15101483464241028, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "probability": { |
| "accuracy": 0.3, |
| "brier_mean": 0.9166441995158155, |
| "calibration_n": 10, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.4623758941888809, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5498971939086914, |
| "n": 2 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7421726584434509, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.6666666666666666, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8560118277867635, |
| "n": 3 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9891908168792725, |
| "n": 1 |
| } |
| ], |
| "ece": 0.42166142165660864, |
| "n": 10 |
| }, |
| "latency": { |
| "n": 10, |
| "p50_s": 1.187890186905861, |
| "p95_s": 1.2060833223164082 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 10, |
| "n_correct": 3, |
| "n_planned": 10, |
| "n_renormalized": 0, |
| "n_scorable": 10, |
| "n_valid": 10, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "routing_hard": { |
| "accuracy": 1.0, |
| "brier_mean": 0.0029613228822248884, |
| "calibration_n": 5, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9767965316772461, |
| "n": 5 |
| } |
| ], |
| "ece": 0.023203468322753884, |
| "n": 5 |
| }, |
| "latency": { |
| "n": 5, |
| "p50_s": 0.9726530313491821, |
| "p95_s": 1.1540774002671241 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 5, |
| "n_correct": 5, |
| "n_planned": 5, |
| "n_renormalized": 0, |
| "n_scorable": 5, |
| "n_valid": 5, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "temporal_numeric": { |
| "accuracy": 0.3333333333333333, |
| "brier_mean": 0.8615734632850287, |
| "calibration_n": 15, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.4, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.36117138862609866, |
| "n": 5 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.42998006939888, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5117152333259583, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6441678404808044, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7346175312995911, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8081729412078857, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.6666666666666666, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9404582381248474, |
| "n": 3 |
| } |
| ], |
| "ece": 0.30592324733734133, |
| "n": 15 |
| }, |
| "latency": { |
| "n": 15, |
| "p50_s": 0.9862628430128098, |
| "p95_s": 1.2137017905712126 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 15, |
| "n_correct": 5, |
| "n_planned": 15, |
| "n_renormalized": 0, |
| "n_scorable": 15, |
| "n_valid": 15, |
| "operational_success": 1.0, |
| "ordinal_mae": 1.6668823510408401, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "tradeoff": { |
| "accuracy": 0.3333333333333333, |
| "brier_mean": 1.0548409941648835, |
| "calibration_n": 6, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6642349362373352, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7499415874481201, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.3333333333333333, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9430160721143087, |
| "n": 3 |
| } |
| ], |
| "ece": 0.5679294168949127, |
| "n": 6 |
| }, |
| "latency": { |
| "n": 6, |
| "p50_s": 0.972567968070507, |
| "p95_s": 1.1406548656523228 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 6, |
| "n_correct": 2, |
| "n_planned": 6, |
| "n_renormalized": 0, |
| "n_scorable": 6, |
| "n_valid": 6, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "trap": { |
| "accuracy": 0.75, |
| "brier_mean": 0.5310765981820174, |
| "calibration_n": 8, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6624303460121155, |
| "n": 2 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.6666666666666666, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9645201663176218, |
| "n": 6 |
| } |
| ], |
| "ece": 0.30778253823518753, |
| "n": 8 |
| }, |
| "latency": { |
| "n": 8, |
| "p50_s": 0.9730363674461842, |
| "p95_s": 1.1941582702100275 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 8, |
| "n_correct": 6, |
| "n_planned": 8, |
| "n_renormalized": 0, |
| "n_scorable": 8, |
| "n_valid": 8, |
| "operational_success": 1.0, |
| "ordinal_mae": 1.8920032598543912, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "price_per_1000_decisions_usd": null, |
| "probability_sources": [ |
| "native" |
| ], |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0, |
| "splits": { |
| "public": { |
| "accuracy": 0.38738738738738737, |
| "brier_mean": 0.8756417814236204, |
| "calibration_n": 111, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": 0.29647180438041687, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.2, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.36647668182849885, |
| "n": 10 |
| }, |
| { |
| "accuracy": 0.42857142857142855, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.44530160512243, |
| "n": 7 |
| }, |
| { |
| "accuracy": 0.18181818181818182, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5331228592179038, |
| "n": 11 |
| }, |
| { |
| "accuracy": 0.21428571428571427, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6532700913293021, |
| "n": 14 |
| }, |
| { |
| "accuracy": 0.3, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7405811965465545, |
| "n": 10 |
| }, |
| { |
| "accuracy": 0.38461538461538464, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8388918088032649, |
| "n": 13 |
| }, |
| { |
| "accuracy": 0.5555555555555556, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9753862314754063, |
| "n": 45 |
| } |
| ], |
| "ece": 0.37200241878226, |
| "n": 111 |
| }, |
| "latency": { |
| "n": 111, |
| "p50_s": 1.181565023958683, |
| "p95_s": 1.2096525765955448 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 111, |
| "n_correct": 43, |
| "n_planned": 111, |
| "n_renormalized": 0, |
| "n_scorable": 111, |
| "n_valid": 111, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.7721535418895655, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "version": "jevbench-v1" |
| }, |
| "original": { |
| "accuracy": 0.75, |
| "brier_mean": 0.40276924379203316, |
| "calibration_n": 72, |
| "complete": true, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "dataset_hash": "914276022fa5f0ea70e4231f2af98e68c67b274b61bea8c440b33890401cd2f9", |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.43212921917438507, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5517171770334244, |
| "n": 4 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6635033190250397, |
| "n": 4 |
| }, |
| { |
| "accuracy": 0.6666666666666666, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7310981651147207, |
| "n": 6 |
| }, |
| { |
| "accuracy": 0.6, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8727044343948365, |
| "n": 5 |
| }, |
| { |
| "accuracy": 0.8235294117647058, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9825702835531795, |
| "n": 51 |
| } |
| ], |
| "ece": 0.150803044024441, |
| "n": 72 |
| }, |
| "latency": { |
| "n": 72, |
| "p50_s": 0.5542781464755535, |
| "p95_s": 1.1863042686134575 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "ledger_charged_usd": null, |
| "macro_accuracy": 0.75, |
| "model_identities": [ |
| "pcdm:native:runs/nc_v3_tap20_wf" |
| ], |
| "n_attempted": 72, |
| "n_correct": 54, |
| "n_planned": 72, |
| "n_renormalized": 0, |
| "n_scorable": 72, |
| "n_valid": 72, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.34141736199647293, |
| "paraphrase_consistency": { |
| "agree": 30, |
| "agreement": 0.8333333333333334, |
| "both_correct_rate_all_pairs": 0.6666666666666666, |
| "both_valid": 36, |
| "pairs": 36 |
| }, |
| "per_family": { |
| "adequacy": { |
| "accuracy": 0.4166666666666667, |
| "brier_mean": 0.8503524575307818, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5679670572280884, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6368800401687622, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.723824143409729, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8935546875, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.5714285714285714, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9796636274882725, |
| "n": 7 |
| } |
| ], |
| "ece": 0.4430626134077708, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.5596147291362286, |
| "p95_s": 0.9536922551691529 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 5, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "extraction": { |
| "accuracy": 1.0, |
| "brier_mean": 0.013560885788644146, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7313405871391296, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9782732129096985, |
| "n": 11 |
| } |
| ], |
| "ece": 0.04230450590451558, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.29115450009703636, |
| "p95_s": 0.7657811347395181 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "intent": { |
| "accuracy": 0.6666666666666666, |
| "brier_mean": 0.41181819447002926, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.42259737849235535, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6991903185844421, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.734041690826416, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8523933291435242, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.875, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9949639439582825, |
| "n": 8 |
| } |
| ], |
| "ece": 0.24692896753549576, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.9667536541819572, |
| "p95_s": 1.184501038864255 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 8, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "ordinal": { |
| "accuracy": 0.8333333333333334, |
| "brier_mean": 0.3210295459915781, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.8333333333333334, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9857434084018072, |
| "n": 12 |
| } |
| ], |
| "ece": 0.15241007506847382, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.5506149679422379, |
| "p95_s": 1.129030930250883 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 10, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.34141736199647293, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "policy": { |
| "accuracy": 0.75, |
| "brier_mean": 0.497069164345908, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5980823040008545, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7690881490707397, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8820080161094666, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.75, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9978939592838287, |
| "n": 8 |
| } |
| ], |
| "ece": 0.2816664377848307, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.9681226126849651, |
| "p95_s": 1.1986095048487186 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 9, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "routing": { |
| "accuracy": 0.8333333333333334, |
| "brier_mean": 0.32278521462525744, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.4416610598564148, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5204096734523773, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6810628771781921, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7141472101211548, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8535581231117249, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.8, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9441299200057983, |
| "n": 5 |
| } |
| ], |
| "ece": 0.19640770554542536, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.29117556661367416, |
| "p95_s": 0.9626881990581748 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 10, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "price_per_1000_decisions_usd": null, |
| "probability_sources": [ |
| "native" |
| ], |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0, |
| "splits": { |
| "public": { |
| "accuracy": 0.75, |
| "brier_mean": 0.40276924379203316, |
| "calibration_n": 72, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.43212921917438507, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5517171770334244, |
| "n": 4 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6635033190250397, |
| "n": 4 |
| }, |
| { |
| "accuracy": 0.6666666666666666, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7310981651147207, |
| "n": 6 |
| }, |
| { |
| "accuracy": 0.6, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8727044343948365, |
| "n": 5 |
| }, |
| { |
| "accuracy": 0.8235294117647058, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9825702835531795, |
| "n": 51 |
| } |
| ], |
| "ece": 0.150803044024441, |
| "n": 72 |
| }, |
| "latency": { |
| "n": 72, |
| "p50_s": 0.5542781464755535, |
| "p95_s": 1.1863042686134575 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 72, |
| "n_correct": 54, |
| "n_planned": 72, |
| "n_renormalized": 0, |
| "n_scorable": 72, |
| "n_valid": 72, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.34141736199647293, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "version": "jevbench-v1" |
| } |
| } |
