| { |
| "objective": -0.3604665485096075, |
| "paired_flip": { |
| "pairs": 80, |
| "flip_rate": 0.95, |
| "both_correct_rate": 0.9375, |
| "invariant_pairs": 32, |
| "invariance_rate": 1.0, |
| "invariant_both_correct_rate": 0.9375 |
| }, |
| "unknowable": null, |
| "clean": { |
| "n": 1264, |
| "nll": 0.5162345825525841, |
| "acc": 0.872626582278481, |
| "ece": 0.07882861339961864, |
| "brier": 0.20370382180198893, |
| "mean_conf": 0.9497319909102234, |
| "confident_error_rate": 0.05300632911392405, |
| "coverage_at_0_9": 0.8393987341772152, |
| "accuracy_at_0_9": 0.9368520263901979, |
| "coverage_at_5pct_error": 0.7982594936708861, |
| "coverage_at_1pct_error": 0.3963607594936709, |
| "aurc": 0.03175002316177704, |
| "error_rate_at_0_9": 0.06314797360980208, |
| "confidence_bias": 0.07710540863174242, |
| "top_bins": { |
| "0.9": { |
| "n": 1061, |
| "errors": 67, |
| "error_rate": 0.06314797360980208 |
| }, |
| "0.95": { |
| "n": 1002, |
| "errors": 49, |
| "error_rate": 0.04890219560878244 |
| }, |
| "0.99": { |
| "n": 898, |
| "errors": 26, |
| "error_rate": 0.028953229398663696 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9873417721518988, |
| "confidence_cutoff": 0.9998771286150151 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.9486166007905138, |
| "confidence_cutoff": 0.9444509701158526 |
| } |
| }, |
| "score_mae": 0.43424875354013565, |
| "ranked_probability_score": 0.07939020551336654 |
| }, |
| "tasks": { |
| "agnews": { |
| "n": 80, |
| "nll": 0.7137886874046098, |
| "acc": 0.8625, |
| "ece": 0.13295572202556327, |
| "brier": 0.2581362811933839, |
| "mean_conf": 0.9762714142518369, |
| "confident_error_rate": 0.1, |
| "coverage_at_0_9": 0.925, |
| "accuracy_at_0_9": 0.8918918918918919, |
| "coverage_at_5pct_error": 0.625, |
| "coverage_at_1pct_error": 0.4625, |
| "aurc": 0.03683036894200468, |
| "error_rate_at_0_9": 0.10810810810810811, |
| "confidence_bias": 0.1137714142518369, |
| "top_bins": { |
| "0.9": { |
| "n": 74, |
| "errors": 8, |
| "error_rate": 0.10810810810810811 |
| }, |
| "0.95": { |
| "n": 72, |
| "errors": 7, |
| "error_rate": 0.09722222222222222 |
| }, |
| "0.99": { |
| "n": 64, |
| "errors": 4, |
| "error_rate": 0.0625 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9992535901004342 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9375, |
| "confidence_cutoff": 0.9903264036490744 |
| } |
| } |
| }, |
| "agnews_yn": { |
| "n": 160, |
| "nll": 0.2780043149657532, |
| "acc": 0.91875, |
| "ece": 0.06317870295443502, |
| "brier": 0.13486952786131337, |
| "mean_conf": 0.9819287029544348, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.93125, |
| "accuracy_at_0_9": 0.9463087248322147, |
| "coverage_at_5pct_error": 0.91875, |
| "coverage_at_1pct_error": 0.66875, |
| "aurc": 0.012381461367063711, |
| "error_rate_at_0_9": 0.053691275167785234, |
| "confidence_bias": 0.06317870295443484, |
| "top_bins": { |
| "0.9": { |
| "n": 149, |
| "errors": 8, |
| "error_rate": 0.053691275167785234 |
| }, |
| "0.95": { |
| "n": 141, |
| "errors": 5, |
| "error_rate": 0.03546099290780142 |
| }, |
| "0.99": { |
| "n": 132, |
| "errors": 4, |
| "error_rate": 0.030303030303030304 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998747367903457 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9765625, |
| "confidence_cutoff": 0.9924665865217894 |
| } |
| } |
| }, |
| "amazon": { |
| "n": 80, |
| "nll": 1.120486990099725, |
| "acc": 0.65, |
| "ece": 0.1452036373571517, |
| "brier": 0.48121742206360574, |
| "mean_conf": 0.7952036373571517, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.375, |
| "accuracy_at_0_9": 0.8666666666666667, |
| "coverage_at_5pct_error": 0.2, |
| "coverage_at_1pct_error": 0.2, |
| "aurc": 0.16172576097763003, |
| "error_rate_at_0_9": 0.13333333333333333, |
| "confidence_bias": 0.1452036373571517, |
| "top_bins": { |
| "0.9": { |
| "n": 30, |
| "errors": 4, |
| "error_rate": 0.13333333333333333 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 2, |
| "error_rate": 0.08333333333333333 |
| }, |
| "0.99": { |
| "n": 10, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.825, |
| "confidence_cutoff": 0.851073766710331 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.703125, |
| "confidence_cutoff": 0.5931310762664513 |
| } |
| }, |
| "score_mae": 0.433674810924642, |
| "ranked_probability_score": 0.07592422931940565 |
| }, |
| "banking77": { |
| "n": 80, |
| "nll": 0.48145296526936593, |
| "acc": 0.9, |
| "ece": 0.0933006640782841, |
| "brier": 0.17748313667579607, |
| "mean_conf": 0.9377179511784941, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.8125, |
| "accuracy_at_0_9": 0.9538461538461539, |
| "coverage_at_5pct_error": 0.8625, |
| "coverage_at_1pct_error": 0.5625, |
| "aurc": 0.02195538568082464, |
| "error_rate_at_0_9": 0.046153846153846156, |
| "confidence_bias": 0.03771795117849408, |
| "top_bins": { |
| "0.9": { |
| "n": 65, |
| "errors": 3, |
| "error_rate": 0.046153846153846156 |
| }, |
| "0.95": { |
| "n": 61, |
| "errors": 2, |
| "error_rate": 0.03278688524590164 |
| }, |
| "0.99": { |
| "n": 55, |
| "errors": 2, |
| "error_rate": 0.03636363636363636 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998607695405539 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9198544674310967 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 80, |
| "nll": 0.4525848363810395, |
| "acc": 0.9125, |
| "ece": 0.07923242291255235, |
| "brier": 0.1663059796228775, |
| "mean_conf": 0.9786756729074163, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.925, |
| "accuracy_at_0_9": 0.918918918918919, |
| "coverage_at_5pct_error": 0.8, |
| "coverage_at_1pct_error": 0.2, |
| "aurc": 0.04134529389423419, |
| "error_rate_at_0_9": 0.08108108108108109, |
| "confidence_bias": 0.06617567290741633, |
| "top_bins": { |
| "0.9": { |
| "n": 74, |
| "errors": 6, |
| "error_rate": 0.08108108108108109 |
| }, |
| "0.95": { |
| "n": 68, |
| "errors": 4, |
| "error_rate": 0.058823529411764705 |
| }, |
| "0.99": { |
| "n": 61, |
| "errors": 3, |
| "error_rate": 0.04918032786885246 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9995304520334682 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.953125, |
| "confidence_cutoff": 0.9829054391409313 |
| } |
| } |
| }, |
| "composition_atom": { |
| "n": 16, |
| "nll": 2.9820085732339667e-06, |
| "acc": 1.0, |
| "ece": 2.982002403895123e-06, |
| "brier": 2.4477619089086167e-11, |
| "mean_conf": 0.9999970179975961, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -2.982002403895123e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999970946309001 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999959761347514 |
| } |
| } |
| }, |
| "composition_conditional": { |
| "n": 16, |
| "nll": 0.28012138871404874, |
| "acc": 0.8125, |
| "ece": 0.1273461735845418, |
| "brier": 0.22036781591057397, |
| "mean_conf": 0.9397310185244409, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.8125, |
| "coverage_at_1pct_error": 0.8125, |
| "aurc": 0.027901785714285712, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.12723101852444085, |
| "top_bins": { |
| "0.9": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5625, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998819108821948 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.7712863202584732 |
| } |
| } |
| }, |
| "composition_conjunction": { |
| "n": 16, |
| "nll": 0.01028125414656798, |
| "acc": 1.0, |
| "ece": 0.010051790951426165, |
| "brier": 0.0008877575747743738, |
| "mean_conf": 0.9899482090485738, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.010051790951426165, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5625, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999902898003522 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9677539667289942 |
| } |
| } |
| }, |
| "composition_disjunction": { |
| "n": 16, |
| "nll": 0.046962216400633236, |
| "acc": 0.9375, |
| "ece": 0.033044212686366634, |
| "brier": 0.03483759547551137, |
| "mean_conf": 0.9704459149367146, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9375, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.9375, |
| "coverage_at_1pct_error": 0.9375, |
| "aurc": 0.00390625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.032945914936714615, |
| "top_bins": { |
| "0.9": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999917491324875 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998163631089358 |
| } |
| } |
| }, |
| "composition_exception": { |
| "n": 16, |
| "nll": 8.24560311982654e-06, |
| "acc": 1.0, |
| "ece": 8.245531508910275e-06, |
| "brier": 2.893279758423789e-10, |
| "mean_conf": 0.9999917544684911, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -8.245531508910275e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999947892800286 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999904230402749 |
| } |
| } |
| }, |
| "composition_negation": { |
| "n": 16, |
| "nll": 3.385252473750771e-05, |
| "acc": 1.0, |
| "ece": 3.384990750143757e-05, |
| "brier": 1.045794100281084e-08, |
| "mean_conf": 0.9999661500924986, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -3.384990750143757e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999967004313122 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999671472562379 |
| } |
| } |
| }, |
| "composition_nested_and": { |
| "n": 16, |
| "nll": 0.38846078289387603, |
| "acc": 0.8125, |
| "ece": 0.16905784975482457, |
| "brier": 0.28419587426147025, |
| "mean_conf": 0.9691361241556393, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.8125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.8125, |
| "coverage_at_1pct_error": 0.8125, |
| "aurc": 0.028385416666666666, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.15663612415563932, |
| "top_bins": { |
| "0.9": { |
| "n": 13, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998956324208319 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9014043337864404 |
| } |
| } |
| }, |
| "composition_nested_or": { |
| "n": 16, |
| "nll": 8.360373813843138e-05, |
| "acc": 1.0, |
| "ece": 8.358792437013207e-05, |
| "brier": 6.323439994947798e-08, |
| "mean_conf": 0.9999164120756299, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -8.358792437013207e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999900619309097 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999772600451928 |
| } |
| } |
| }, |
| "contrastive_age_eligibility": { |
| "n": 24, |
| "nll": 8.918453191185959e-06, |
| "acc": 1.0, |
| "ece": 8.918329865159613e-06, |
| "brier": 4.949861735926648e-10, |
| "mean_conf": 0.9999910816701348, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -8.918329865159613e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999995529542938 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999866425633518 |
| } |
| } |
| }, |
| "contrastive_quantity_limit": { |
| "n": 24, |
| "nll": 0.11432689039921219, |
| "acc": 0.9583333333333334, |
| "ece": 0.06674854949620072, |
| "brier": 0.08281031178744026, |
| "mean_conf": 0.960141823439535, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.875, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.875, |
| "aurc": 0.005441644707949056, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.0018084901062016545, |
| "top_bins": { |
| "0.9": { |
| "n": 21, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 21, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 20, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999990882456489 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998305896429884 |
| } |
| } |
| }, |
| "contrastive_return_window": { |
| "n": 24, |
| "nll": 0.15985187450425817, |
| "acc": 0.9583333333333334, |
| "ece": 0.017002922340390136, |
| "brier": 0.07968701664069926, |
| "mean_conf": 0.9753362556737235, |
| "confident_error_rate": 0.041666666666666664, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 0.9583333333333334, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.6666666666666666, |
| "aurc": 0.0164678826885214, |
| "error_rate_at_0_9": 0.041666666666666664, |
| "confidence_bias": 0.017002922340390136, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 1, |
| "error_rate": 0.041666666666666664 |
| }, |
| "0.95": { |
| "n": 19, |
| "errors": 1, |
| "error_rate": 0.05263157894736842 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9910077489577522 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9489840905432165 |
| } |
| } |
| }, |
| "contrastive_spend_threshold": { |
| "n": 24, |
| "nll": 3.367029789433667e-07, |
| "acc": 1.0, |
| "ece": 3.367029256517995e-07, |
| "brier": 2.7316869295091615e-13, |
| "mean_conf": 0.9999996632970743, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -3.367029256517995e-07, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999997019580142 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999995929679338 |
| } |
| } |
| }, |
| "dbpedia14": { |
| "n": 80, |
| "nll": 0.3225729708469703, |
| "acc": 0.9625, |
| "ece": 0.0333953949930713, |
| "brier": 0.06178517454588339, |
| "mean_conf": 0.9876309216561203, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.975, |
| "accuracy_at_0_9": 0.9743589743589743, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.25, |
| "aurc": 0.018054540877428797, |
| "error_rate_at_0_9": 0.02564102564102564, |
| "confidence_bias": 0.025130921656120297, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 2, |
| "error_rate": 0.02564102564102564 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 2, |
| "error_rate": 0.02564102564102564 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999986417367368 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9999918766575475 |
| } |
| } |
| }, |
| "imdb": { |
| "n": 80, |
| "nll": 0.3340330648730691, |
| "acc": 0.9625, |
| "ece": 0.043889971122268646, |
| "brier": 0.08104374650851562, |
| "mean_conf": 0.988705664765976, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.975, |
| "accuracy_at_0_9": 0.9615384615384616, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.325, |
| "aurc": 0.017244609864498633, |
| "error_rate_at_0_9": 0.038461538461538464, |
| "confidence_bias": 0.026205664765975967, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 3, |
| "error_rate": 0.038461538461538464 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 3, |
| "error_rate": 0.03896103896103896 |
| }, |
| "0.99": { |
| "n": 73, |
| "errors": 2, |
| "error_rate": 0.0273972602739726 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999978336923495 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9998242918029979 |
| } |
| } |
| }, |
| "mnli": { |
| "n": 80, |
| "nll": 0.862303348495556, |
| "acc": 0.925, |
| "ece": 0.059436589706964325, |
| "brier": 0.14150776516303595, |
| "mean_conf": 0.9844365897069643, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.925, |
| "accuracy_at_0_9": 0.9459459459459459, |
| "coverage_at_5pct_error": 0.6875, |
| "coverage_at_1pct_error": 0.1625, |
| "aurc": 0.04135287631159247, |
| "error_rate_at_0_9": 0.05405405405405406, |
| "confidence_bias": 0.059436589706964305, |
| "top_bins": { |
| "0.9": { |
| "n": 74, |
| "errors": 4, |
| "error_rate": 0.05405405405405406 |
| }, |
| "0.95": { |
| "n": 74, |
| "errors": 4, |
| "error_rate": 0.05405405405405406 |
| }, |
| "0.99": { |
| "n": 73, |
| "errors": 4, |
| "error_rate": 0.0547945205479452 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9999991088006394 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9375, |
| "confidence_cutoff": 0.9999003045641519 |
| } |
| } |
| }, |
| "sst5": { |
| "n": 80, |
| "nll": 1.503696914808123, |
| "acc": 0.625, |
| "ece": 0.28428896677547494, |
| "brier": 0.6679423308341643, |
| "mean_conf": 0.8432310728287288, |
| "confident_error_rate": 0.1875, |
| "coverage_at_0_9": 0.5, |
| "accuracy_at_0_9": 0.625, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.3606889455450939, |
| "error_rate_at_0_9": 0.375, |
| "confidence_bias": 0.21823107282872878, |
| "top_bins": { |
| "0.9": { |
| "n": 40, |
| "errors": 15, |
| "error_rate": 0.375 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 8, |
| "error_rate": 0.3076923076923077 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 1, |
| "error_rate": 0.5 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.9018282801599481 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.59375, |
| "confidence_cutoff": 0.6916544106639626 |
| } |
| }, |
| "score_mae": 0.4876080587605968, |
| "ranked_probability_score": 0.09431489975519643 |
| }, |
| "trec": { |
| "n": 80, |
| "nll": 0.34580264586958104, |
| "acc": 0.95, |
| "ece": 0.047956523850530175, |
| "brier": 0.09712037545352312, |
| "mean_conf": 0.9979565238505301, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 0.95, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.7, |
| "aurc": 0.00796046665512359, |
| "error_rate_at_0_9": 0.05, |
| "confidence_bias": 0.047956523850530175, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 4, |
| "error_rate": 0.05 |
| }, |
| "0.95": { |
| "n": 79, |
| "errors": 3, |
| "error_rate": 0.0379746835443038 |
| }, |
| "0.99": { |
| "n": 76, |
| "errors": 3, |
| "error_rate": 0.039473684210526314 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999823983162701 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9998150143864826 |
| } |
| } |
| }, |
| "yelp": { |
| "n": 80, |
| "nll": 0.9747595737320693, |
| "acc": 0.6375, |
| "ece": 0.19595082669672634, |
| "brier": 0.49920645928471624, |
| "mean_conf": 0.8334508266967264, |
| "confident_error_rate": 0.1, |
| "coverage_at_0_9": 0.475, |
| "accuracy_at_0_9": 0.7894736842105263, |
| "coverage_at_5pct_error": 0.2125, |
| "coverage_at_1pct_error": 0.2125, |
| "aurc": 0.19634193730364932, |
| "error_rate_at_0_9": 0.21052631578947367, |
| "confidence_bias": 0.19595082669672648, |
| "top_bins": { |
| "0.9": { |
| "n": 38, |
| "errors": 8, |
| "error_rate": 0.21052631578947367 |
| }, |
| "0.95": { |
| "n": 29, |
| "errors": 7, |
| "error_rate": 0.2413793103448276 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.8, |
| "confidence_cutoff": 0.8925572725643923 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.703125, |
| "confidence_cutoff": 0.677388969661729 |
| } |
| }, |
| "score_mae": 0.3814633909351682, |
| "ranked_probability_score": 0.06793148746549753 |
| }, |
| "yelp_yn": { |
| "n": 80, |
| "nll": 0.2615685053953841, |
| "acc": 0.875, |
| "ece": 0.08916060778865792, |
| "brier": 0.1602256352805817, |
| "mean_conf": 0.9641606077886579, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.85, |
| "accuracy_at_0_9": 0.9852941176470589, |
| "coverage_at_5pct_error": 0.875, |
| "coverage_at_1pct_error": 0.675, |
| "aurc": 0.01390921871402907, |
| "error_rate_at_0_9": 0.014705882352941176, |
| "confidence_bias": 0.08916060778865786, |
| "top_bins": { |
| "0.9": { |
| "n": 68, |
| "errors": 1, |
| "error_rate": 0.014705882352941176 |
| }, |
| "0.95": { |
| "n": 67, |
| "errors": 1, |
| "error_rate": 0.014925373134328358 |
| }, |
| "0.99": { |
| "n": 64, |
| "errors": 1, |
| "error_rate": 0.015625 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997991676694828 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9910321405675893 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 1264, |
| "nll": 0.5162345825525841, |
| "acc": 0.872626582278481, |
| "ece": 0.07882861339961864, |
| "brier": 0.20370382180198893, |
| "mean_conf": 0.9497319909102234, |
| "confident_error_rate": 0.05300632911392405, |
| "coverage_at_0_9": 0.8393987341772152, |
| "accuracy_at_0_9": 0.9368520263901979, |
| "coverage_at_5pct_error": 0.7982594936708861, |
| "coverage_at_1pct_error": 0.3963607594936709, |
| "aurc": 0.03175002316177704, |
| "error_rate_at_0_9": 0.06314797360980208, |
| "confidence_bias": 0.07710540863174242, |
| "top_bins": { |
| "0.9": { |
| "n": 1061, |
| "errors": 67, |
| "error_rate": 0.06314797360980208 |
| }, |
| "0.95": { |
| "n": 1002, |
| "errors": 49, |
| "error_rate": 0.04890219560878244 |
| }, |
| "0.99": { |
| "n": 898, |
| "errors": 26, |
| "error_rate": 0.028953229398663696 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9873417721518988, |
| "confidence_cutoff": 0.9998771286150151 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.9486166007905138, |
| "confidence_cutoff": 0.9444509701158526 |
| } |
| }, |
| "score_mae": 0.43424875354013565, |
| "ranked_probability_score": 0.07939020551336654 |
| }, |
| "none_absent": { |
| "n": 60, |
| "nll": 0.3608079173797808, |
| "acc": 0.9333333333333333, |
| "ece": 0.07941751774577509, |
| "brier": 0.13365874834055136, |
| "mean_conf": 0.965298696299739, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.9166666666666666, |
| "accuracy_at_0_9": 0.9454545454545454, |
| "coverage_at_5pct_error": 0.7666666666666667, |
| "coverage_at_1pct_error": 0.35, |
| "aurc": 0.031487738205623166, |
| "error_rate_at_0_9": 0.05454545454545454, |
| "confidence_bias": 0.03196536296640562, |
| "top_bins": { |
| "0.9": { |
| "n": 55, |
| "errors": 3, |
| "error_rate": 0.05454545454545454 |
| }, |
| "0.95": { |
| "n": 54, |
| "errors": 3, |
| "error_rate": 0.05555555555555555 |
| }, |
| "0.99": { |
| "n": 43, |
| "errors": 2, |
| "error_rate": 0.046511627906976744 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9666666666666667, |
| "confidence_cutoff": 0.9988434531867763 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9375, |
| "confidence_cutoff": 0.9780746480145948 |
| } |
| } |
| }, |
| "none_present": { |
| "n": 60, |
| "nll": 0.7486628676421944, |
| "acc": 0.9, |
| "ece": 0.10408269653644878, |
| "brier": 0.193617997765929, |
| "mean_conf": 0.941888721183197, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.8333333333333334, |
| "accuracy_at_0_9": 0.94, |
| "coverage_at_5pct_error": 0.4, |
| "coverage_at_1pct_error": 0.05, |
| "aurc": 0.0772676202815587, |
| "error_rate_at_0_9": 0.06, |
| "confidence_bias": 0.041888721183196975, |
| "top_bins": { |
| "0.9": { |
| "n": 50, |
| "errors": 3, |
| "error_rate": 0.06 |
| }, |
| "0.95": { |
| "n": 44, |
| "errors": 3, |
| "error_rate": 0.06818181818181818 |
| }, |
| "0.99": { |
| "n": 31, |
| "errors": 2, |
| "error_rate": 0.06451612903225806 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9333333333333333, |
| "confidence_cutoff": 0.9906812627829901 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9375, |
| "confidence_cutoff": 0.9189979211514118 |
| } |
| } |
| }, |
| "permuted": { |
| "n": 84, |
| "nll": 0.37420534360044955, |
| "acc": 0.9523809523809523, |
| "ece": 0.03903323135930105, |
| "brier": 0.07954212216262244, |
| "mean_conf": 0.9791573337511807, |
| "confident_error_rate": 0.023809523809523808, |
| "coverage_at_0_9": 0.9404761904761905, |
| "accuracy_at_0_9": 0.9746835443037974, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.15476190476190477, |
| "aurc": 0.033564435783818146, |
| "error_rate_at_0_9": 0.02531645569620253, |
| "confidence_bias": 0.02677638137022842, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 2, |
| "error_rate": 0.02531645569620253 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| }, |
| "0.99": { |
| "n": 72, |
| "errors": 2, |
| "error_rate": 0.027777777777777776 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9523809523809523, |
| "confidence_cutoff": 0.9999433810714059 |
| }, |
| "0.8": { |
| "coverage": 0.8095238095238095, |
| "accuracy": 0.9705882352941176, |
| "confidence_cutoff": 0.9960369508568402 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": {}, |
| "permutation": { |
| "n": 60, |
| "mean_max_delta": 0.01915095541564352, |
| "flip_rate": 0.016666666666666666 |
| }, |
| "temperature": 2.2973967099940698, |
| "calibrated_clean": { |
| "n": 1264, |
| "nll": 0.3479912777890691, |
| "acc": 0.872626582278481, |
| "ece": 0.01586663164294604, |
| "brier": 0.1824328438059522, |
| "mean_conf": 0.881387307048926, |
| "confident_error_rate": 0.01661392405063291, |
| "coverage_at_0_9": 0.6748417721518988, |
| "accuracy_at_0_9": 0.9753810082063306, |
| "coverage_at_5pct_error": 0.8014240506329114, |
| "coverage_at_1pct_error": 0.3370253164556962, |
| "aurc": 0.030966208087472115, |
| "error_rate_at_0_9": 0.0246189917936694, |
| "confidence_bias": 0.008760724770444983, |
| "top_bins": { |
| "0.9": { |
| "n": 853, |
| "errors": 21, |
| "error_rate": 0.0246189917936694 |
| }, |
| "0.95": { |
| "n": 737, |
| "errors": 12, |
| "error_rate": 0.016282225237449117 |
| }, |
| "0.99": { |
| "n": 461, |
| "errors": 5, |
| "error_rate": 0.010845986984815618 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9889240506329114, |
| "confidence_cutoff": 0.9753801911711253 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.950592885375494, |
| "confidence_cutoff": 0.7410676890660509 |
| } |
| }, |
| "score_mae": 0.4758238299356195, |
| "ranked_probability_score": 0.07265353054441266 |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 1204, |
| "requested_questions": 1468, |
| "evaluated_records": 1204, |
| "evaluated_questions": 1468, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 51.92248449975523, |
| "p95": 135.53371219959445 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 2.2973967099940698, |
| "logits_recorded": true |
| }, |
| "transfer": { |
| "objective": -0.4880013459092438, |
| "paired_flip": { |
| "pairs": 64, |
| "flip_rate": 0.859375, |
| "both_correct_rate": 0.8125, |
| "invariant_pairs": 24, |
| "invariance_rate": 1.0, |
| "invariant_both_correct_rate": 1.0 |
| }, |
| "unknowable": null, |
| "clean": { |
| "n": 656, |
| "nll": 0.5764418952724232, |
| "acc": 0.8170731707317073, |
| "ece": 0.10020924778356732, |
| "brier": 0.2690850170373655, |
| "mean_conf": 0.9128191322541024, |
| "confident_error_rate": 0.05945121951219512, |
| "coverage_at_0_9": 0.7347560975609756, |
| "accuracy_at_0_9": 0.9190871369294605, |
| "coverage_at_5pct_error": 0.5990853658536586, |
| "coverage_at_1pct_error": 0.3978658536585366, |
| "aurc": 0.049144884120614074, |
| "error_rate_at_0_9": 0.08091286307053942, |
| "confidence_bias": 0.09574596152239512, |
| "top_bins": { |
| "0.9": { |
| "n": 482, |
| "errors": 39, |
| "error_rate": 0.08091286307053942 |
| }, |
| "0.95": { |
| "n": 447, |
| "errors": 31, |
| "error_rate": 0.06935123042505593 |
| }, |
| "0.99": { |
| "n": 363, |
| "errors": 17, |
| "error_rate": 0.046831955922865015 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9786585365853658, |
| "confidence_cutoff": 0.9950785493257293 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.900952380952381, |
| "confidence_cutoff": 0.8257571108864877 |
| } |
| }, |
| "score_mae": 0.316824262687028, |
| "ranked_probability_score": 0.12274395686296583 |
| }, |
| "tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.036558273598721235, |
| "acc": 1.0, |
| "ece": 0.030892100510413772, |
| "brier": 0.017881015688346595, |
| "mean_conf": 0.9691078994895861, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.90625, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.03089210051041391, |
| "top_bins": { |
| "0.9": { |
| "n": 29, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 29, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 27, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.53125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999973844026875 |
| }, |
| "0.8": { |
| "coverage": 0.84375, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9908018053072243 |
| } |
| } |
| }, |
| "composition_held_conditional": { |
| "n": 32, |
| "nll": 0.04484538085803764, |
| "acc": 1.0, |
| "ece": 0.04023780814361391, |
| "brier": 0.015210308025896463, |
| "mean_conf": 0.9597621918563862, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.8125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.040237808143613774, |
| "top_bins": { |
| "0.9": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 18, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.53125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9939084631570418 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9661742340828378 |
| } |
| } |
| }, |
| "composition_held_or_not": { |
| "n": 32, |
| "nll": 0.28422348585042867, |
| "acc": 0.9375, |
| "ece": 0.03387826063971708, |
| "brier": 0.10915453853972053, |
| "mean_conf": 0.9713782606397171, |
| "confident_error_rate": 0.03125, |
| "coverage_at_0_9": 0.875, |
| "accuracy_at_0_9": 0.9642857142857143, |
| "coverage_at_5pct_error": 0.96875, |
| "coverage_at_1pct_error": 0.625, |
| "aurc": 0.01524130523374475, |
| "error_rate_at_0_9": 0.03571428571428571, |
| "confidence_bias": 0.033878260639717106, |
| "top_bins": { |
| "0.9": { |
| "n": 28, |
| "errors": 1, |
| "error_rate": 0.03571428571428571 |
| }, |
| "0.95": { |
| "n": 25, |
| "errors": 1, |
| "error_rate": 0.04 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 1, |
| "error_rate": 0.041666666666666664 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9993548255583505 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.9615384615384616, |
| "confidence_cutoff": 0.9372299807347372 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 40, |
| "nll": 5.519474637266503e-06, |
| "acc": 1.0, |
| "ece": 5.519452064506325e-06, |
| "brier": 9.144248319210223e-11, |
| "mean_conf": 0.9999944805479355, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -5.519452064506325e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999956221821245 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999923349753774 |
| } |
| } |
| }, |
| "contrastive_deadline": { |
| "n": 40, |
| "nll": 0.8436244396923623, |
| "acc": 0.65, |
| "ece": 0.27695473137187976, |
| "brier": 0.4842390988197474, |
| "mean_conf": 0.9020071124758093, |
| "confident_error_rate": 0.1, |
| "coverage_at_0_9": 0.65, |
| "accuracy_at_0_9": 0.8461538461538461, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.4, |
| "aurc": 0.11349409024439767, |
| "error_rate_at_0_9": 0.15384615384615385, |
| "confidence_bias": 0.2520071124758093, |
| "top_bins": { |
| "0.9": { |
| "n": 26, |
| "errors": 4, |
| "error_rate": 0.15384615384615385 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 4, |
| "error_rate": 0.15384615384615385 |
| }, |
| "0.99": { |
| "n": 22, |
| "errors": 2, |
| "error_rate": 0.09090909090909091 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9953179512674454 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.78125, |
| "confidence_cutoff": 0.7594178184419988 |
| } |
| }, |
| "score_mae": 0.316824262687028, |
| "ranked_probability_score": 0.12274395686296583 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.7994186231074956, |
| "acc": 0.55, |
| "ece": 0.29613510771975704, |
| "brier": 0.6764238468215377, |
| "mean_conf": 0.846135107719757, |
| "confident_error_rate": 0.15, |
| "coverage_at_0_9": 0.5375, |
| "accuracy_at_0_9": 0.7209302325581395, |
| "coverage_at_5pct_error": 0.15, |
| "coverage_at_1pct_error": 0.15, |
| "aurc": 0.2697507411299849, |
| "error_rate_at_0_9": 0.27906976744186046, |
| "confidence_bias": 0.29613510771975693, |
| "top_bins": { |
| "0.9": { |
| "n": 43, |
| "errors": 12, |
| "error_rate": 0.27906976744186046 |
| }, |
| "0.95": { |
| "n": 35, |
| "errors": 9, |
| "error_rate": 0.2571428571428571 |
| }, |
| "0.99": { |
| "n": 23, |
| "errors": 6, |
| "error_rate": 0.2608695652173913 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.725, |
| "confidence_cutoff": 0.9318937701892213 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.609375, |
| "confidence_cutoff": 0.6904468219734016 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 0.7239653768200205, |
| "acc": 0.725, |
| "ece": 0.1155622225698504, |
| "brier": 0.38191758859093616, |
| "mean_conf": 0.7948846878702913, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.5125, |
| "accuracy_at_0_9": 0.926829268292683, |
| "coverage_at_5pct_error": 0.4, |
| "coverage_at_1pct_error": 0.225, |
| "aurc": 0.10576574303101645, |
| "error_rate_at_0_9": 0.07317073170731707, |
| "confidence_bias": 0.06988468787029134, |
| "top_bins": { |
| "0.9": { |
| "n": 41, |
| "errors": 3, |
| "error_rate": 0.07317073170731707 |
| }, |
| "0.95": { |
| "n": 36, |
| "errors": 2, |
| "error_rate": 0.05555555555555555 |
| }, |
| "0.99": { |
| "n": 27, |
| "errors": 1, |
| "error_rate": 0.037037037037037035 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.925, |
| "confidence_cutoff": 0.9262887306313381 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.5687803486430137 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 0.785768121000188, |
| "acc": 0.7875, |
| "ece": 0.16376117277465274, |
| "brier": 0.35457996865874325, |
| "mean_conf": 0.934871358662847, |
| "confident_error_rate": 0.1375, |
| "coverage_at_0_9": 0.8, |
| "accuracy_at_0_9": 0.828125, |
| "coverage_at_5pct_error": 0.275, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.11157532829806283, |
| "error_rate_at_0_9": 0.171875, |
| "confidence_bias": 0.147371358662847, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 11, |
| "error_rate": 0.171875 |
| }, |
| "0.95": { |
| "n": 60, |
| "errors": 9, |
| "error_rate": 0.15 |
| }, |
| "0.99": { |
| "n": 42, |
| "errors": 6, |
| "error_rate": 0.14285714285714285 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.9926516140967396 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.828125, |
| "confidence_cutoff": 0.9148037688879754 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.30218851707566163, |
| "acc": 0.9125, |
| "ece": 0.06027148972700891, |
| "brier": 0.14840098630475543, |
| "mean_conf": 0.9638953711825529, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.8625, |
| "accuracy_at_0_9": 0.9420289855072463, |
| "coverage_at_5pct_error": 0.8375, |
| "coverage_at_1pct_error": 0.4875, |
| "aurc": 0.021606573578835768, |
| "error_rate_at_0_9": 0.057971014492753624, |
| "confidence_bias": 0.051395371182552885, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 4, |
| "error_rate": 0.057971014492753624 |
| }, |
| "0.95": { |
| "n": 64, |
| "errors": 3, |
| "error_rate": 0.046875 |
| }, |
| "0.99": { |
| "n": 53, |
| "errors": 1, |
| "error_rate": 0.018867924528301886 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9991475900858287 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.953125, |
| "confidence_cutoff": 0.9529522976896565 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.05733198573008065, |
| "acc": 0.975, |
| "ece": 0.028215578518962822, |
| "brier": 0.03727237207148254, |
| "mean_conf": 0.9823463992200709, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.95, |
| "aurc": 0.0009515497687966043, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.007346399220070965, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 73, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 69, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999673481935003 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9985609261980979 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.49008508179404914, |
| "acc": 0.75, |
| "ece": 0.10188382252197292, |
| "brier": 0.3088844829017613, |
| "mean_conf": 0.8518838225219728, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.5, |
| "accuracy_at_0_9": 0.9, |
| "coverage_at_5pct_error": 0.3125, |
| "coverage_at_1pct_error": 0.275, |
| "aurc": 0.09199870714413623, |
| "error_rate_at_0_9": 0.1, |
| "confidence_bias": 0.10188382252197281, |
| "top_bins": { |
| "0.9": { |
| "n": 40, |
| "errors": 4, |
| "error_rate": 0.1 |
| }, |
| "0.95": { |
| "n": 33, |
| "errors": 3, |
| "error_rate": 0.09090909090909091 |
| }, |
| "0.99": { |
| "n": 18, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9152760624369052 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.859375, |
| "confidence_cutoff": 0.6894673314531843 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 656, |
| "nll": 0.5764418952724232, |
| "acc": 0.8170731707317073, |
| "ece": 0.10020924778356732, |
| "brier": 0.2690850170373655, |
| "mean_conf": 0.9128191322541024, |
| "confident_error_rate": 0.05945121951219512, |
| "coverage_at_0_9": 0.7347560975609756, |
| "accuracy_at_0_9": 0.9190871369294605, |
| "coverage_at_5pct_error": 0.5990853658536586, |
| "coverage_at_1pct_error": 0.3978658536585366, |
| "aurc": 0.049144884120614074, |
| "error_rate_at_0_9": 0.08091286307053942, |
| "confidence_bias": 0.09574596152239512, |
| "top_bins": { |
| "0.9": { |
| "n": 482, |
| "errors": 39, |
| "error_rate": 0.08091286307053942 |
| }, |
| "0.95": { |
| "n": 447, |
| "errors": 31, |
| "error_rate": 0.06935123042505593 |
| }, |
| "0.99": { |
| "n": 363, |
| "errors": 17, |
| "error_rate": 0.046831955922865015 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9786585365853658, |
| "confidence_cutoff": 0.9950785493257293 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.900952380952381, |
| "confidence_cutoff": 0.8257571108864877 |
| } |
| }, |
| "score_mae": 0.316824262687028, |
| "ranked_probability_score": 0.12274395686296583 |
| }, |
| "none_absent": { |
| "n": 36, |
| "nll": 0.2382943867360587, |
| "acc": 0.8888888888888888, |
| "ece": 0.06002776857686939, |
| "brier": 0.13660821634616752, |
| "mean_conf": 0.8943886684408269, |
| "confident_error_rate": 0.027777777777777776, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.9629629629629629, |
| "coverage_at_5pct_error": 0.8888888888888888, |
| "coverage_at_1pct_error": 0.6666666666666666, |
| "aurc": 0.016633111236839873, |
| "error_rate_at_0_9": 0.037037037037037035, |
| "confidence_bias": 0.005499779551938033, |
| "top_bins": { |
| "0.9": { |
| "n": 27, |
| "errors": 1, |
| "error_rate": 0.037037037037037035 |
| }, |
| "0.95": { |
| "n": 25, |
| "errors": 1, |
| "error_rate": 0.04 |
| }, |
| "0.99": { |
| "n": 17, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9885625198248165 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.9655172413793104, |
| "confidence_cutoff": 0.8441082925771153 |
| } |
| } |
| }, |
| "none_present": { |
| "n": 36, |
| "nll": 1.4992651509227133, |
| "acc": 0.5555555555555556, |
| "ece": 0.2733507018450212, |
| "brier": 0.6243239275207846, |
| "mean_conf": 0.8289062574005768, |
| "confident_error_rate": 0.1111111111111111, |
| "coverage_at_0_9": 0.5555555555555556, |
| "accuracy_at_0_9": 0.8, |
| "coverage_at_5pct_error": 0.3611111111111111, |
| "coverage_at_1pct_error": 0.3611111111111111, |
| "aurc": 0.1800464786772204, |
| "error_rate_at_0_9": 0.2, |
| "confidence_bias": 0.2733507018450212, |
| "top_bins": { |
| "0.9": { |
| "n": 20, |
| "errors": 4, |
| "error_rate": 0.2 |
| }, |
| "0.95": { |
| "n": 19, |
| "errors": 3, |
| "error_rate": 0.15789473684210525 |
| }, |
| "0.99": { |
| "n": 14, |
| "errors": 1, |
| "error_rate": 0.07142857142857142 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.8333333333333334, |
| "confidence_cutoff": 0.9653464869986069 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.6551724137931034, |
| "confidence_cutoff": 0.5929894291187869 |
| } |
| } |
| }, |
| "permuted": { |
| "n": 36, |
| "nll": 0.6706148135185012, |
| "acc": 0.75, |
| "ece": 0.18165011054121183, |
| "brier": 0.3522686377804207, |
| "mean_conf": 0.896756673361093, |
| "confident_error_rate": 0.027777777777777776, |
| "coverage_at_0_9": 0.6944444444444444, |
| "accuracy_at_0_9": 0.96, |
| "coverage_at_5pct_error": 0.6944444444444444, |
| "coverage_at_1pct_error": 0.5555555555555556, |
| "aurc": 0.056971427946047176, |
| "error_rate_at_0_9": 0.04, |
| "confidence_bias": 0.146756673361093, |
| "top_bins": { |
| "0.9": { |
| "n": 25, |
| "errors": 1, |
| "error_rate": 0.04 |
| }, |
| "0.95": { |
| "n": 25, |
| "errors": 1, |
| "error_rate": 0.04 |
| }, |
| "0.99": { |
| "n": 20, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9961253613933815 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.8620689655172413, |
| "confidence_cutoff": 0.8327445813749598 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.036558273598721235, |
| "acc": 1.0, |
| "ece": 0.030892100510413772, |
| "brier": 0.017881015688346595, |
| "mean_conf": 0.9691078994895861, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.90625, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.03089210051041391, |
| "top_bins": { |
| "0.9": { |
| "n": 29, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 29, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 27, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.53125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999973844026875 |
| }, |
| "0.8": { |
| "coverage": 0.84375, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9908018053072243 |
| } |
| } |
| }, |
| "composition_held_conditional": { |
| "n": 32, |
| "nll": 0.04484538085803764, |
| "acc": 1.0, |
| "ece": 0.04023780814361391, |
| "brier": 0.015210308025896463, |
| "mean_conf": 0.9597621918563862, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.8125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.040237808143613774, |
| "top_bins": { |
| "0.9": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 18, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.53125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9939084631570418 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9661742340828378 |
| } |
| } |
| }, |
| "composition_held_or_not": { |
| "n": 32, |
| "nll": 0.28422348585042867, |
| "acc": 0.9375, |
| "ece": 0.03387826063971708, |
| "brier": 0.10915453853972053, |
| "mean_conf": 0.9713782606397171, |
| "confident_error_rate": 0.03125, |
| "coverage_at_0_9": 0.875, |
| "accuracy_at_0_9": 0.9642857142857143, |
| "coverage_at_5pct_error": 0.96875, |
| "coverage_at_1pct_error": 0.625, |
| "aurc": 0.01524130523374475, |
| "error_rate_at_0_9": 0.03571428571428571, |
| "confidence_bias": 0.033878260639717106, |
| "top_bins": { |
| "0.9": { |
| "n": 28, |
| "errors": 1, |
| "error_rate": 0.03571428571428571 |
| }, |
| "0.95": { |
| "n": 25, |
| "errors": 1, |
| "error_rate": 0.04 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 1, |
| "error_rate": 0.041666666666666664 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9993548255583505 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.9615384615384616, |
| "confidence_cutoff": 0.9372299807347372 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 40, |
| "nll": 5.519474637266503e-06, |
| "acc": 1.0, |
| "ece": 5.519452064506325e-06, |
| "brier": 9.144248319210223e-11, |
| "mean_conf": 0.9999944805479355, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -5.519452064506325e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999956221821245 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999923349753774 |
| } |
| } |
| }, |
| "contrastive_deadline": { |
| "n": 40, |
| "nll": 0.8436244396923623, |
| "acc": 0.65, |
| "ece": 0.27695473137187976, |
| "brier": 0.4842390988197474, |
| "mean_conf": 0.9020071124758093, |
| "confident_error_rate": 0.1, |
| "coverage_at_0_9": 0.65, |
| "accuracy_at_0_9": 0.8461538461538461, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.4, |
| "aurc": 0.11349409024439767, |
| "error_rate_at_0_9": 0.15384615384615385, |
| "confidence_bias": 0.2520071124758093, |
| "top_bins": { |
| "0.9": { |
| "n": 26, |
| "errors": 4, |
| "error_rate": 0.15384615384615385 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 4, |
| "error_rate": 0.15384615384615385 |
| }, |
| "0.99": { |
| "n": 22, |
| "errors": 2, |
| "error_rate": 0.09090909090909091 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9953179512674454 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.78125, |
| "confidence_cutoff": 0.7594178184419988 |
| } |
| }, |
| "score_mae": 0.316824262687028, |
| "ranked_probability_score": 0.12274395686296583 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.7994186231074956, |
| "acc": 0.55, |
| "ece": 0.29613510771975704, |
| "brier": 0.6764238468215377, |
| "mean_conf": 0.846135107719757, |
| "confident_error_rate": 0.15, |
| "coverage_at_0_9": 0.5375, |
| "accuracy_at_0_9": 0.7209302325581395, |
| "coverage_at_5pct_error": 0.15, |
| "coverage_at_1pct_error": 0.15, |
| "aurc": 0.2697507411299849, |
| "error_rate_at_0_9": 0.27906976744186046, |
| "confidence_bias": 0.29613510771975693, |
| "top_bins": { |
| "0.9": { |
| "n": 43, |
| "errors": 12, |
| "error_rate": 0.27906976744186046 |
| }, |
| "0.95": { |
| "n": 35, |
| "errors": 9, |
| "error_rate": 0.2571428571428571 |
| }, |
| "0.99": { |
| "n": 23, |
| "errors": 6, |
| "error_rate": 0.2608695652173913 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.725, |
| "confidence_cutoff": 0.9318937701892213 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.609375, |
| "confidence_cutoff": 0.6904468219734016 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 0.7239653768200205, |
| "acc": 0.725, |
| "ece": 0.1155622225698504, |
| "brier": 0.38191758859093616, |
| "mean_conf": 0.7948846878702913, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.5125, |
| "accuracy_at_0_9": 0.926829268292683, |
| "coverage_at_5pct_error": 0.4, |
| "coverage_at_1pct_error": 0.225, |
| "aurc": 0.10576574303101645, |
| "error_rate_at_0_9": 0.07317073170731707, |
| "confidence_bias": 0.06988468787029134, |
| "top_bins": { |
| "0.9": { |
| "n": 41, |
| "errors": 3, |
| "error_rate": 0.07317073170731707 |
| }, |
| "0.95": { |
| "n": 36, |
| "errors": 2, |
| "error_rate": 0.05555555555555555 |
| }, |
| "0.99": { |
| "n": 27, |
| "errors": 1, |
| "error_rate": 0.037037037037037035 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.925, |
| "confidence_cutoff": 0.9262887306313381 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.5687803486430137 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 0.785768121000188, |
| "acc": 0.7875, |
| "ece": 0.16376117277465274, |
| "brier": 0.35457996865874325, |
| "mean_conf": 0.934871358662847, |
| "confident_error_rate": 0.1375, |
| "coverage_at_0_9": 0.8, |
| "accuracy_at_0_9": 0.828125, |
| "coverage_at_5pct_error": 0.275, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.11157532829806283, |
| "error_rate_at_0_9": 0.171875, |
| "confidence_bias": 0.147371358662847, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 11, |
| "error_rate": 0.171875 |
| }, |
| "0.95": { |
| "n": 60, |
| "errors": 9, |
| "error_rate": 0.15 |
| }, |
| "0.99": { |
| "n": 42, |
| "errors": 6, |
| "error_rate": 0.14285714285714285 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.9926516140967396 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.828125, |
| "confidence_cutoff": 0.9148037688879754 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.30218851707566163, |
| "acc": 0.9125, |
| "ece": 0.06027148972700891, |
| "brier": 0.14840098630475543, |
| "mean_conf": 0.9638953711825529, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.8625, |
| "accuracy_at_0_9": 0.9420289855072463, |
| "coverage_at_5pct_error": 0.8375, |
| "coverage_at_1pct_error": 0.4875, |
| "aurc": 0.021606573578835768, |
| "error_rate_at_0_9": 0.057971014492753624, |
| "confidence_bias": 0.051395371182552885, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 4, |
| "error_rate": 0.057971014492753624 |
| }, |
| "0.95": { |
| "n": 64, |
| "errors": 3, |
| "error_rate": 0.046875 |
| }, |
| "0.99": { |
| "n": 53, |
| "errors": 1, |
| "error_rate": 0.018867924528301886 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9991475900858287 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.953125, |
| "confidence_cutoff": 0.9529522976896565 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.05733198573008065, |
| "acc": 0.975, |
| "ece": 0.028215578518962822, |
| "brier": 0.03727237207148254, |
| "mean_conf": 0.9823463992200709, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.95, |
| "aurc": 0.0009515497687966043, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.007346399220070965, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 73, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 69, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999673481935003 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9985609261980979 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.49008508179404914, |
| "acc": 0.75, |
| "ece": 0.10188382252197292, |
| "brier": 0.3088844829017613, |
| "mean_conf": 0.8518838225219728, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.5, |
| "accuracy_at_0_9": 0.9, |
| "coverage_at_5pct_error": 0.3125, |
| "coverage_at_1pct_error": 0.275, |
| "aurc": 0.09199870714413623, |
| "error_rate_at_0_9": 0.1, |
| "confidence_bias": 0.10188382252197281, |
| "top_bins": { |
| "0.9": { |
| "n": 40, |
| "errors": 4, |
| "error_rate": 0.1 |
| }, |
| "0.95": { |
| "n": 33, |
| "errors": 3, |
| "error_rate": 0.09090909090909091 |
| }, |
| "0.99": { |
| "n": 18, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9152760624369052 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.859375, |
| "confidence_cutoff": 0.6894673314531843 |
| } |
| } |
| } |
| }, |
| "permutation": { |
| "n": 36, |
| "mean_max_delta": 0.026388596882159963, |
| "flip_rate": 0.05555555555555555 |
| }, |
| "temperature": 2.2973967099940698, |
| "calibrated_clean": { |
| "n": 656, |
| "nll": 0.4344257519237035, |
| "acc": 0.8170731707317073, |
| "ece": 0.0405193820148158, |
| "brier": 0.24296359539279005, |
| "mean_conf": 0.8207248136739, |
| "confident_error_rate": 0.012195121951219513, |
| "coverage_at_0_9": 0.49847560975609756, |
| "accuracy_at_0_9": 0.9755351681957186, |
| "coverage_at_5pct_error": 0.6158536585365854, |
| "coverage_at_1pct_error": 0.3780487804878049, |
| "aurc": 0.047235862428547555, |
| "error_rate_at_0_9": 0.024464831804281346, |
| "confidence_bias": 0.0036516429421927787, |
| "top_bins": { |
| "0.9": { |
| "n": 327, |
| "errors": 8, |
| "error_rate": 0.024464831804281346 |
| }, |
| "0.95": { |
| "n": 261, |
| "errors": 3, |
| "error_rate": 0.011494252873563218 |
| }, |
| "0.99": { |
| "n": 98, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975609756097561, |
| "confidence_cutoff": 0.8981225434286827 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.9142857142857143, |
| "confidence_cutoff": 0.6343266379321651 |
| } |
| }, |
| "score_mae": 0.3365660317122896, |
| "ranked_probability_score": 0.10047697969029137 |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 764, |
| "requested_questions": 764, |
| "evaluated_records": 764, |
| "evaluated_questions": 764, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 51.46696049996535, |
| "p95": 54.76280345023952 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 2.2973967099940698, |
| "logits_recorded": true |
| }, |
| "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1" |
| }, |
| "provenance": { |
| "config": { |
| "epochs": 1, |
| "seed": 1, |
| "lr": 2e-05, |
| "lora": 16, |
| "accum": 4, |
| "batch": 2, |
| "perm_kl": 0.0, |
| "perm_frac": 0.3, |
| "ord_w": 0.0, |
| "p_none": 0.1, |
| "p_none_distract": 0.12, |
| "p_distract": 0.15, |
| "p_none_pair": 0.25, |
| "synthetic_repeat": 1, |
| "public_frac": 1.0, |
| "head_lr": 0.0, |
| "weight_decay": 0.01, |
| "anchor_w": 0.0, |
| "label_smoothing": 0.0, |
| "brier_w": 0.0, |
| "focal_gamma": 0.0, |
| "dtype": "bf16", |
| "checkpointing": 1, |
| "max_state": 7552, |
| "base": "Qwen/Qwen3.5-4B-Base", |
| "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", |
| "init_from": "jaredpalmer/kev-4b@957b91e762e883935830246eeb02381f9d2694b6", |
| "data": "evals/round10/skills/train.jsonl", |
| "replay": 4000 |
| }, |
| "config_sha256": "08aa7634eef0409340edd9df72b4b347bdda02e87b25c920e205dc727911525d", |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", |
| "source_hashes": { |
| "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
| "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", |
| "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", |
| "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", |
| "kev/benchmark.py": "10a892d76cb4c200e94d0a68ee74054aa1870d7f484c0530a531cacfa75032ed", |
| "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", |
| "kev/checkpoint.py": "f3edb4d159c0aa12d7006fc599fe6ebc3f2c288b3b69890c289f7dac451dd03d", |
| "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", |
| "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", |
| "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", |
| "kev/cuda_graphs.py": "12f1953fea8f77c3c9dff29468202fbb37545145f93d3618194565a89949fd55", |
| "kev/data.py": "e77a2fffeb8ee7e05b118893be8e38b8358aef97c3ca2e143cdce208201b92f9", |
| "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", |
| "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", |
| "kev/experiment.py": "644adedf43fbd10566830fa6094e16dd001b8cad29d9b9da82f0e0f245ca06b1", |
| "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", |
| "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9", |
| "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", |
| "kev/model.py": "2634ffe7747d69473fb6596942c248e5df2586df3610bd1adb47e7e9acd99f96", |
| "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", |
| "kev/predictors.py": "4d02906c285d78bcbd91908f4fc86577ccc95c320d704817f6444aabbc4fcd4f", |
| "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", |
| "kev/serve.py": "57d379bbcdeb3e4d4c3fdde5471e4dddeaf7710b8ed61ecad3466f3b67e6c610", |
| "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", |
| "kev/suite.py": "32b2882d1ea0fe5d03ba4d676a6180d580545f157e36fa715e7e89169ec99130", |
| "kev/train.py": "68428ed43b3e362e52d4650dc5296ab88767061f7998a525715e1b306eeef474", |
| "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", |
| "modal_app.py": "7c6bc859f7d3e7011539a8f5e9e90390288cc72b5d80f911c8841c068f4d1136", |
| "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d", |
| "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" |
| }, |
| "git_commit": "6d02f5d066cd34958dfd15ffa5d2f6f0f4c21a63", |
| "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", |
| "torch": "2.8.0+cu128", |
| "device": "cuda", |
| "gpu": "NVIDIA H200", |
| "legacy_checkpoint": false, |
| "measured_checkpoint": { |
| "requested": "/runs/r10-skills/00-trial-0/checkpoint", |
| "resolved": "/runs/r10-skills/00-trial-0/checkpoint", |
| "head_sha256": "979aaa60135274973a6dc57fe24dfc4f1ce88e65df46c73d86b2047c94d897df", |
| "adapter_sha256": "90e817356246e7f18bfa7ca3d31794cd4fbeb3332a66a84cb51d9ceae925f2b2", |
| "inference_temperature": 1.0 |
| } |
| }, |
| "mechanism_checks": { |
| "n": 8, |
| "packed_max_delta": 0.0, |
| "sibling_max_delta": 0.00015926361083984375, |
| "tolerance": 0.001, |
| "passed": true |
| }, |
| "gates": { |
| "passed": true, |
| "checks": { |
| "complete_coverage": true, |
| "isolation_and_packing": true, |
| "transfer_complete": true, |
| "heldout_pairs_at_least_70pct": true, |
| "transfer_confident_errors_below_10pct": true |
| }, |
| "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." |
| }, |
| "calibration_fit": { |
| "temperature": 2.2973967099940698, |
| "aggregation": "micro", |
| "objective": "raw-logit NLL", |
| "split": "calibration", |
| "rows_sha256": "54bea908f07d7d7b7972573e5920273c604c967c17bcf36edea176d070930a91", |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", |
| "n": 1148 |
| }, |
| "wall_seconds": 5271.548163347, |
| "promotable": false, |
| "test_evaluated": false, |
| "training_resources": { |
| "wall_seconds": 5031.164765119553, |
| "records_seen": 19994, |
| "requested_records": 15320, |
| "truncated_records": 0, |
| "rejected_records": 0, |
| "optimizer_steps": 1915, |
| "forward_tokens": 8738926, |
| "peak_device_bytes": 47737451520, |
| "device": "cuda", |
| "dtype": "bf16", |
| "batch": 2, |
| "peak_rss_bytes": 31552528384 |
| } |
| } |
|
|
