{ "objective": -0.3604665485096075, "paired_flip": { "pairs": 80, "flip_rate": 0.95, "both_correct_rate": 0.9375, "invariant_pairs": 32, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.9375 }, "unknowable": null, "clean": { "n": 1264, "nll": 0.5162345825525841, "acc": 0.872626582278481, "ece": 0.07882861339961864, "brier": 0.20370382180198893, "mean_conf": 0.9497319909102234, "confident_error_rate": 0.05300632911392405, "coverage_at_0_9": 0.8393987341772152, "accuracy_at_0_9": 0.9368520263901979, "coverage_at_5pct_error": 0.7982594936708861, "coverage_at_1pct_error": 0.3963607594936709, "aurc": 0.03175002316177704, "error_rate_at_0_9": 0.06314797360980208, "confidence_bias": 0.07710540863174242, "top_bins": { "0.9": { "n": 1061, "errors": 67, "error_rate": 0.06314797360980208 }, "0.95": { "n": 1002, "errors": 49, "error_rate": 0.04890219560878244 }, "0.99": { "n": 898, "errors": 26, "error_rate": 0.028953229398663696 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9873417721518988, "confidence_cutoff": 0.9998771286150151 }, "0.8": { "coverage": 0.8006329113924051, "accuracy": 0.9486166007905138, "confidence_cutoff": 0.9444509701158526 } }, "score_mae": 0.43424875354013565, "ranked_probability_score": 0.07939020551336654 }, "tasks": { "agnews": { "n": 80, "nll": 0.7137886874046098, "acc": 0.8625, "ece": 0.13295572202556327, "brier": 0.2581362811933839, "mean_conf": 0.9762714142518369, "confident_error_rate": 0.1, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 0.8918918918918919, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.4625, "aurc": 0.03683036894200468, "error_rate_at_0_9": 0.10810810810810811, "confidence_bias": 0.1137714142518369, "top_bins": { "0.9": { "n": 74, "errors": 8, "error_rate": 0.10810810810810811 }, "0.95": { "n": 72, "errors": 7, "error_rate": 0.09722222222222222 }, "0.99": { "n": 64, "errors": 4, "error_rate": 0.0625 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9992535901004342 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9903264036490744 } } }, "agnews_yn": { "n": 160, "nll": 0.2780043149657532, "acc": 0.91875, "ece": 0.06317870295443502, "brier": 0.13486952786131337, "mean_conf": 0.9819287029544348, "confident_error_rate": 0.05, "coverage_at_0_9": 0.93125, "accuracy_at_0_9": 0.9463087248322147, "coverage_at_5pct_error": 0.91875, "coverage_at_1pct_error": 0.66875, "aurc": 0.012381461367063711, "error_rate_at_0_9": 0.053691275167785234, "confidence_bias": 0.06317870295443484, "top_bins": { "0.9": { "n": 149, "errors": 8, "error_rate": 0.053691275167785234 }, "0.95": { "n": 141, "errors": 5, "error_rate": 0.03546099290780142 }, "0.99": { "n": 132, "errors": 4, "error_rate": 0.030303030303030304 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9998747367903457 }, "0.8": { "coverage": 0.8, "accuracy": 0.9765625, "confidence_cutoff": 0.9924665865217894 } } }, "amazon": { "n": 80, "nll": 1.120486990099725, "acc": 0.65, "ece": 0.1452036373571517, "brier": 0.48121742206360574, "mean_conf": 0.7952036373571517, "confident_error_rate": 0.05, "coverage_at_0_9": 0.375, "accuracy_at_0_9": 0.8666666666666667, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.16172576097763003, "error_rate_at_0_9": 0.13333333333333333, "confidence_bias": 0.1452036373571517, "top_bins": { "0.9": { "n": 30, "errors": 4, "error_rate": 0.13333333333333333 }, "0.95": { "n": 24, "errors": 2, "error_rate": 0.08333333333333333 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.851073766710331 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.5931310762664513 } }, "score_mae": 0.433674810924642, "ranked_probability_score": 0.07592422931940565 }, "banking77": { "n": 80, "nll": 0.48145296526936593, "acc": 0.9, "ece": 0.0933006640782841, "brier": 0.17748313667579607, "mean_conf": 0.9377179511784941, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.9538461538461539, "coverage_at_5pct_error": 0.8625, "coverage_at_1pct_error": 0.5625, "aurc": 0.02195538568082464, "error_rate_at_0_9": 0.046153846153846156, "confidence_bias": 0.03771795117849408, "top_bins": { "0.9": { "n": 65, "errors": 3, "error_rate": 0.046153846153846156 }, "0.95": { "n": 61, "errors": 2, "error_rate": 0.03278688524590164 }, "0.99": { "n": 55, "errors": 2, "error_rate": 0.03636363636363636 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9998607695405539 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9198544674310967 } } }, "boolq": { "n": 80, "nll": 0.4525848363810395, "acc": 0.9125, "ece": 0.07923242291255235, "brier": 0.1663059796228775, "mean_conf": 0.9786756729074163, "confident_error_rate": 0.075, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 0.918918918918919, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.2, "aurc": 0.04134529389423419, "error_rate_at_0_9": 0.08108108108108109, "confidence_bias": 0.06617567290741633, "top_bins": { "0.9": { "n": 74, "errors": 6, "error_rate": 0.08108108108108109 }, "0.95": { "n": 68, "errors": 4, "error_rate": 0.058823529411764705 }, "0.99": { "n": 61, "errors": 3, "error_rate": 0.04918032786885246 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9995304520334682 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.9829054391409313 } } }, "composition_atom": { "n": 16, "nll": 2.9820085732339667e-06, "acc": 1.0, "ece": 2.982002403895123e-06, "brier": 2.4477619089086167e-11, "mean_conf": 0.9999970179975961, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -2.982002403895123e-06, "top_bins": { "0.9": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999970946309001 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9999959761347514 } } }, "composition_conditional": { "n": 16, "nll": 0.28012138871404874, "acc": 0.8125, "ece": 0.1273461735845418, "brier": 0.22036781591057397, "mean_conf": 0.9397310185244409, "confident_error_rate": 0.0, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8125, "coverage_at_1pct_error": 0.8125, "aurc": 0.027901785714285712, "error_rate_at_0_9": 0.0, "confidence_bias": 0.12723101852444085, "top_bins": { "0.9": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 12, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5625, "accuracy": 1.0, "confidence_cutoff": 0.9998819108821948 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.7712863202584732 } } }, "composition_conjunction": { "n": 16, "nll": 0.01028125414656798, "acc": 1.0, "ece": 0.010051790951426165, "brier": 0.0008877575747743738, "mean_conf": 0.9899482090485738, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.010051790951426165, "top_bins": { "0.9": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 12, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5625, "accuracy": 1.0, "confidence_cutoff": 0.9999902898003522 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9677539667289942 } } }, "composition_disjunction": { "n": 16, "nll": 0.046962216400633236, "acc": 0.9375, "ece": 0.033044212686366634, "brier": 0.03483759547551137, "mean_conf": 0.9704459149367146, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9375, "coverage_at_1pct_error": 0.9375, "aurc": 0.00390625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.032945914936714615, "top_bins": { "0.9": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 15, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999917491324875 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9998163631089358 } } }, "composition_exception": { "n": 16, "nll": 8.24560311982654e-06, "acc": 1.0, "ece": 8.245531508910275e-06, "brier": 2.893279758423789e-10, "mean_conf": 0.9999917544684911, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -8.245531508910275e-06, "top_bins": { "0.9": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999947892800286 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9999904230402749 } } }, "composition_negation": { "n": 16, "nll": 3.385252473750771e-05, "acc": 1.0, "ece": 3.384990750143757e-05, "brier": 1.045794100281084e-08, "mean_conf": 0.9999661500924986, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -3.384990750143757e-05, "top_bins": { "0.9": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999967004313122 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9999671472562379 } } }, "composition_nested_and": { "n": 16, "nll": 0.38846078289387603, "acc": 0.8125, "ece": 0.16905784975482457, "brier": 0.28419587426147025, "mean_conf": 0.9691361241556393, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8125, "coverage_at_1pct_error": 0.8125, "aurc": 0.028385416666666666, "error_rate_at_0_9": 0.0, "confidence_bias": 0.15663612415563932, "top_bins": { "0.9": { "n": 13, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 12, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9998956324208319 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9014043337864404 } } }, "composition_nested_or": { "n": 16, "nll": 8.360373813843138e-05, "acc": 1.0, "ece": 8.358792437013207e-05, "brier": 6.323439994947798e-08, "mean_conf": 0.9999164120756299, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -8.358792437013207e-05, "top_bins": { "0.9": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999900619309097 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9999772600451928 } } }, "contrastive_age_eligibility": { "n": 24, "nll": 8.918453191185959e-06, "acc": 1.0, "ece": 8.918329865159613e-06, "brier": 4.949861735926648e-10, "mean_conf": 0.9999910816701348, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -8.918329865159613e-06, "top_bins": { "0.9": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 24, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999995529542938 }, "0.8": { "coverage": 0.8333333333333334, "accuracy": 1.0, "confidence_cutoff": 0.9999866425633518 } } }, "contrastive_quantity_limit": { "n": 24, "nll": 0.11432689039921219, "acc": 0.9583333333333334, "ece": 0.06674854949620072, "brier": 0.08281031178744026, "mean_conf": 0.960141823439535, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.005441644707949056, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0018084901062016545, "top_bins": { "0.9": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999990882456489 }, "0.8": { "coverage": 0.8333333333333334, "accuracy": 1.0, "confidence_cutoff": 0.9998305896429884 } } }, "contrastive_return_window": { "n": 24, "nll": 0.15985187450425817, "acc": 0.9583333333333334, "ece": 0.017002922340390136, "brier": 0.07968701664069926, "mean_conf": 0.9753362556737235, "confident_error_rate": 0.041666666666666664, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.9583333333333334, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.6666666666666666, "aurc": 0.0164678826885214, "error_rate_at_0_9": 0.041666666666666664, "confidence_bias": 0.017002922340390136, "top_bins": { "0.9": { "n": 24, "errors": 1, "error_rate": 0.041666666666666664 }, "0.95": { "n": 19, "errors": 1, "error_rate": 0.05263157894736842 }, "0.99": { "n": 12, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9910077489577522 }, "0.8": { "coverage": 0.8333333333333334, "accuracy": 0.95, "confidence_cutoff": 0.9489840905432165 } } }, "contrastive_spend_threshold": { "n": 24, "nll": 3.367029789433667e-07, "acc": 1.0, "ece": 3.367029256517995e-07, "brier": 2.7316869295091615e-13, "mean_conf": 0.9999996632970743, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -3.367029256517995e-07, "top_bins": { "0.9": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 24, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999997019580142 }, "0.8": { "coverage": 0.8333333333333334, "accuracy": 1.0, "confidence_cutoff": 0.9999995929679338 } } }, "dbpedia14": { "n": 80, "nll": 0.3225729708469703, "acc": 0.9625, "ece": 0.0333953949930713, "brier": 0.06178517454588339, "mean_conf": 0.9876309216561203, "confident_error_rate": 0.025, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 0.9743589743589743, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.25, "aurc": 0.018054540877428797, "error_rate_at_0_9": 0.02564102564102564, "confidence_bias": 0.025130921656120297, "top_bins": { "0.9": { "n": 78, "errors": 2, "error_rate": 0.02564102564102564 }, "0.95": { "n": 78, "errors": 2, "error_rate": 0.02564102564102564 }, "0.99": { "n": 77, "errors": 2, "error_rate": 0.025974025974025976 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9999986417367368 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9999918766575475 } } }, "imdb": { "n": 80, "nll": 0.3340330648730691, "acc": 0.9625, "ece": 0.043889971122268646, "brier": 0.08104374650851562, "mean_conf": 0.988705664765976, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 0.9615384615384616, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.325, "aurc": 0.017244609864498633, "error_rate_at_0_9": 0.038461538461538464, "confidence_bias": 0.026205664765975967, "top_bins": { "0.9": { "n": 78, "errors": 3, "error_rate": 0.038461538461538464 }, "0.95": { "n": 77, "errors": 3, "error_rate": 0.03896103896103896 }, "0.99": { "n": 73, "errors": 2, "error_rate": 0.0273972602739726 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9999978336923495 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9998242918029979 } } }, "mnli": { "n": 80, "nll": 0.862303348495556, "acc": 0.925, "ece": 0.059436589706964325, "brier": 0.14150776516303595, "mean_conf": 0.9844365897069643, "confident_error_rate": 0.05, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 0.9459459459459459, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.1625, "aurc": 0.04135287631159247, "error_rate_at_0_9": 0.05405405405405406, "confidence_bias": 0.059436589706964305, "top_bins": { "0.9": { "n": 74, "errors": 4, "error_rate": 0.05405405405405406 }, "0.95": { "n": 74, "errors": 4, "error_rate": 0.05405405405405406 }, "0.99": { "n": 73, "errors": 4, "error_rate": 0.0547945205479452 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9999991088006394 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9999003045641519 } } }, "sst5": { "n": 80, "nll": 1.503696914808123, "acc": 0.625, "ece": 0.28428896677547494, "brier": 0.6679423308341643, "mean_conf": 0.8432310728287288, "confident_error_rate": 0.1875, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.625, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3606889455450939, "error_rate_at_0_9": 0.375, "confidence_bias": 0.21823107282872878, "top_bins": { "0.9": { "n": 40, "errors": 15, "error_rate": 0.375 }, "0.95": { "n": 26, "errors": 8, "error_rate": 0.3076923076923077 }, "0.99": { "n": 2, "errors": 1, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.9018282801599481 }, "0.8": { "coverage": 0.8, "accuracy": 0.59375, "confidence_cutoff": 0.6916544106639626 } }, "score_mae": 0.4876080587605968, "ranked_probability_score": 0.09431489975519643 }, "trec": { "n": 80, "nll": 0.34580264586958104, "acc": 0.95, "ece": 0.047956523850530175, "brier": 0.09712037545352312, "mean_conf": 0.9979565238505301, "confident_error_rate": 0.05, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.95, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.7, "aurc": 0.00796046665512359, "error_rate_at_0_9": 0.05, "confidence_bias": 0.047956523850530175, "top_bins": { "0.9": { "n": 80, "errors": 4, "error_rate": 0.05 }, "0.95": { "n": 79, "errors": 3, "error_rate": 0.0379746835443038 }, "0.99": { "n": 76, "errors": 3, "error_rate": 0.039473684210526314 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999823983162701 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9998150143864826 } } }, "yelp": { "n": 80, "nll": 0.9747595737320693, "acc": 0.6375, "ece": 0.19595082669672634, "brier": 0.49920645928471624, "mean_conf": 0.8334508266967264, "confident_error_rate": 0.1, "coverage_at_0_9": 0.475, "accuracy_at_0_9": 0.7894736842105263, "coverage_at_5pct_error": 0.2125, "coverage_at_1pct_error": 0.2125, "aurc": 0.19634193730364932, "error_rate_at_0_9": 0.21052631578947367, "confidence_bias": 0.19595082669672648, "top_bins": { "0.9": { "n": 38, "errors": 8, "error_rate": 0.21052631578947367 }, "0.95": { "n": 29, "errors": 7, "error_rate": 0.2413793103448276 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8925572725643923 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.677388969661729 } }, "score_mae": 0.3814633909351682, "ranked_probability_score": 0.06793148746549753 }, "yelp_yn": { "n": 80, "nll": 0.2615685053953841, "acc": 0.875, "ece": 0.08916060778865792, "brier": 0.1602256352805817, "mean_conf": 0.9641606077886579, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.9852941176470589, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.675, "aurc": 0.01390921871402907, "error_rate_at_0_9": 0.014705882352941176, "confidence_bias": 0.08916060778865786, "top_bins": { "0.9": { "n": 68, "errors": 1, "error_rate": 0.014705882352941176 }, "0.95": { "n": 67, "errors": 1, "error_rate": 0.014925373134328358 }, "0.99": { "n": 64, "errors": 1, "error_rate": 0.015625 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997991676694828 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9910321405675893 } } } }, "variants": { "clean": { "n": 1264, "nll": 0.5162345825525841, "acc": 0.872626582278481, "ece": 0.07882861339961864, "brier": 0.20370382180198893, "mean_conf": 0.9497319909102234, "confident_error_rate": 0.05300632911392405, "coverage_at_0_9": 0.8393987341772152, "accuracy_at_0_9": 0.9368520263901979, "coverage_at_5pct_error": 0.7982594936708861, "coverage_at_1pct_error": 0.3963607594936709, "aurc": 0.03175002316177704, "error_rate_at_0_9": 0.06314797360980208, "confidence_bias": 0.07710540863174242, "top_bins": { "0.9": { "n": 1061, "errors": 67, "error_rate": 0.06314797360980208 }, "0.95": { "n": 1002, "errors": 49, "error_rate": 0.04890219560878244 }, "0.99": { "n": 898, "errors": 26, "error_rate": 0.028953229398663696 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9873417721518988, "confidence_cutoff": 0.9998771286150151 }, "0.8": { "coverage": 0.8006329113924051, "accuracy": 0.9486166007905138, "confidence_cutoff": 0.9444509701158526 } }, "score_mae": 0.43424875354013565, "ranked_probability_score": 0.07939020551336654 }, "none_absent": { "n": 60, "nll": 0.3608079173797808, "acc": 0.9333333333333333, "ece": 0.07941751774577509, "brier": 0.13365874834055136, "mean_conf": 0.965298696299739, "confident_error_rate": 0.05, "coverage_at_0_9": 0.9166666666666666, "accuracy_at_0_9": 0.9454545454545454, "coverage_at_5pct_error": 0.7666666666666667, "coverage_at_1pct_error": 0.35, "aurc": 0.031487738205623166, "error_rate_at_0_9": 0.05454545454545454, "confidence_bias": 0.03196536296640562, "top_bins": { "0.9": { "n": 55, "errors": 3, "error_rate": 0.05454545454545454 }, "0.95": { "n": 54, "errors": 3, "error_rate": 0.05555555555555555 }, "0.99": { "n": 43, "errors": 2, "error_rate": 0.046511627906976744 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9666666666666667, "confidence_cutoff": 0.9988434531867763 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9780746480145948 } } }, "none_present": { "n": 60, "nll": 0.7486628676421944, "acc": 0.9, "ece": 0.10408269653644878, "brier": 0.193617997765929, "mean_conf": 0.941888721183197, "confident_error_rate": 0.05, "coverage_at_0_9": 0.8333333333333334, "accuracy_at_0_9": 0.94, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.05, "aurc": 0.0772676202815587, "error_rate_at_0_9": 0.06, "confidence_bias": 0.041888721183196975, "top_bins": { "0.9": { "n": 50, "errors": 3, "error_rate": 0.06 }, "0.95": { "n": 44, "errors": 3, "error_rate": 0.06818181818181818 }, "0.99": { "n": 31, "errors": 2, "error_rate": 0.06451612903225806 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9333333333333333, "confidence_cutoff": 0.9906812627829901 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9189979211514118 } } }, "permuted": { "n": 84, "nll": 0.37420534360044955, "acc": 0.9523809523809523, "ece": 0.03903323135930105, "brier": 0.07954212216262244, "mean_conf": 0.9791573337511807, "confident_error_rate": 0.023809523809523808, "coverage_at_0_9": 0.9404761904761905, "accuracy_at_0_9": 0.9746835443037974, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.15476190476190477, "aurc": 0.033564435783818146, "error_rate_at_0_9": 0.02531645569620253, "confidence_bias": 0.02677638137022842, "top_bins": { "0.9": { "n": 79, "errors": 2, "error_rate": 0.02531645569620253 }, "0.95": { "n": 77, "errors": 2, "error_rate": 0.025974025974025976 }, "0.99": { "n": 72, "errors": 2, "error_rate": 0.027777777777777776 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9523809523809523, "confidence_cutoff": 0.9999433810714059 }, "0.8": { "coverage": 0.8095238095238095, "accuracy": 0.9705882352941176, "confidence_cutoff": 0.9960369508568402 } } } }, "heldout_tasks": {}, "permutation": { "n": 60, "mean_max_delta": 0.01915095541564352, "flip_rate": 0.016666666666666666 }, "temperature": 2.2973967099940698, "calibrated_clean": { "n": 1264, "nll": 0.3479912777890691, "acc": 0.872626582278481, "ece": 0.01586663164294604, "brier": 0.1824328438059522, "mean_conf": 0.881387307048926, "confident_error_rate": 0.01661392405063291, "coverage_at_0_9": 0.6748417721518988, "accuracy_at_0_9": 0.9753810082063306, "coverage_at_5pct_error": 0.8014240506329114, "coverage_at_1pct_error": 0.3370253164556962, "aurc": 0.030966208087472115, "error_rate_at_0_9": 0.0246189917936694, "confidence_bias": 0.008760724770444983, "top_bins": { "0.9": { "n": 853, "errors": 21, "error_rate": 0.0246189917936694 }, "0.95": { "n": 737, "errors": 12, "error_rate": 0.016282225237449117 }, "0.99": { "n": 461, "errors": 5, "error_rate": 0.010845986984815618 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9889240506329114, "confidence_cutoff": 0.9753801911711253 }, "0.8": { "coverage": 0.8006329113924051, "accuracy": 0.950592885375494, "confidence_cutoff": 0.7410676890660509 } }, "score_mae": 0.4758238299356195, "ranked_probability_score": 0.07265353054441266 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 }, "coverage": { "requested_records": 1204, "requested_questions": 1468, "evaluated_records": 1204, "evaluated_questions": 1468, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 51.92248449975523, "p95": 135.53371219959445 }, "calibration": { "inference_temperature": 1.0, "additional_temperature": 2.2973967099940698, "logits_recorded": true }, "transfer": { "objective": -0.4880013459092438, "paired_flip": { "pairs": 64, "flip_rate": 0.859375, "both_correct_rate": 0.8125, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 1.0 }, "unknowable": null, "clean": { "n": 656, "nll": 0.5764418952724232, "acc": 0.8170731707317073, "ece": 0.10020924778356732, "brier": 0.2690850170373655, "mean_conf": 0.9128191322541024, "confident_error_rate": 0.05945121951219512, "coverage_at_0_9": 0.7347560975609756, "accuracy_at_0_9": 0.9190871369294605, "coverage_at_5pct_error": 0.5990853658536586, "coverage_at_1pct_error": 0.3978658536585366, "aurc": 0.049144884120614074, "error_rate_at_0_9": 0.08091286307053942, "confidence_bias": 0.09574596152239512, "top_bins": { "0.9": { "n": 482, "errors": 39, "error_rate": 0.08091286307053942 }, "0.95": { "n": 447, "errors": 31, "error_rate": 0.06935123042505593 }, "0.99": { "n": 363, "errors": 17, "error_rate": 0.046831955922865015 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9786585365853658, "confidence_cutoff": 0.9950785493257293 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.900952380952381, "confidence_cutoff": 0.8257571108864877 } }, "score_mae": 0.316824262687028, "ranked_probability_score": 0.12274395686296583 }, "tasks": { "composition_held_and_or": { "n": 32, "nll": 0.036558273598721235, "acc": 1.0, "ece": 0.030892100510413772, "brier": 0.017881015688346595, "mean_conf": 0.9691078994895861, "confident_error_rate": 0.0, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03089210051041391, "top_bins": { "0.9": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 27, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.999973844026875 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.9908018053072243 } } }, "composition_held_conditional": { "n": 32, "nll": 0.04484538085803764, "acc": 1.0, "ece": 0.04023780814361391, "brier": 0.015210308025896463, "mean_conf": 0.9597621918563862, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.040237808143613774, "top_bins": { "0.9": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 18, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.9939084631570418 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9661742340828378 } } }, "composition_held_or_not": { "n": 32, "nll": 0.28422348585042867, "acc": 0.9375, "ece": 0.03387826063971708, "brier": 0.10915453853972053, "mean_conf": 0.9713782606397171, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.9642857142857143, "coverage_at_5pct_error": 0.96875, "coverage_at_1pct_error": 0.625, "aurc": 0.01524130523374475, "error_rate_at_0_9": 0.03571428571428571, "confidence_bias": 0.033878260639717106, "top_bins": { "0.9": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 }, "0.95": { "n": 25, "errors": 1, "error_rate": 0.04 }, "0.99": { "n": 24, "errors": 1, "error_rate": 0.041666666666666664 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9993548255583505 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.9372299807347372 } } }, "contrastive_authorization": { "n": 40, "nll": 5.519474637266503e-06, "acc": 1.0, "ece": 5.519452064506325e-06, "brier": 9.144248319210223e-11, "mean_conf": 0.9999944805479355, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -5.519452064506325e-06, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 40, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999956221821245 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9999923349753774 } } }, "contrastive_deadline": { "n": 40, "nll": 0.8436244396923623, "acc": 0.65, "ece": 0.27695473137187976, "brier": 0.4842390988197474, "mean_conf": 0.9020071124758093, "confident_error_rate": 0.1, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.8461538461538461, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.4, "aurc": 0.11349409024439767, "error_rate_at_0_9": 0.15384615384615385, "confidence_bias": 0.2520071124758093, "top_bins": { "0.9": { "n": 26, "errors": 4, "error_rate": 0.15384615384615385 }, "0.95": { "n": 26, "errors": 4, "error_rate": 0.15384615384615385 }, "0.99": { "n": 22, "errors": 2, "error_rate": 0.09090909090909091 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9953179512674454 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7594178184419988 } }, "score_mae": 0.316824262687028, "ranked_probability_score": 0.12274395686296583 }, "emotion": { "n": 80, "nll": 1.7994186231074956, "acc": 0.55, "ece": 0.29613510771975704, "brier": 0.6764238468215377, "mean_conf": 0.846135107719757, "confident_error_rate": 0.15, "coverage_at_0_9": 0.5375, "accuracy_at_0_9": 0.7209302325581395, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.2697507411299849, "error_rate_at_0_9": 0.27906976744186046, "confidence_bias": 0.29613510771975693, "top_bins": { "0.9": { "n": 43, "errors": 12, "error_rate": 0.27906976744186046 }, "0.95": { "n": 35, "errors": 9, "error_rate": 0.2571428571428571 }, "0.99": { "n": 23, "errors": 6, "error_rate": 0.2608695652173913 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.725, "confidence_cutoff": 0.9318937701892213 }, "0.8": { "coverage": 0.8, "accuracy": 0.609375, "confidence_cutoff": 0.6904468219734016 } } }, "mmlu": { "n": 80, "nll": 0.7239653768200205, "acc": 0.725, "ece": 0.1155622225698504, "brier": 0.38191758859093616, "mean_conf": 0.7948846878702913, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.5125, "accuracy_at_0_9": 0.926829268292683, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.225, "aurc": 0.10576574303101645, "error_rate_at_0_9": 0.07317073170731707, "confidence_bias": 0.06988468787029134, "top_bins": { "0.9": { "n": 41, "errors": 3, "error_rate": 0.07317073170731707 }, "0.95": { "n": 36, "errors": 2, "error_rate": 0.05555555555555555 }, "0.99": { "n": 27, "errors": 1, "error_rate": 0.037037037037037035 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9262887306313381 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.5687803486430137 } } }, "paws": { "n": 80, "nll": 0.785768121000188, "acc": 0.7875, "ece": 0.16376117277465274, "brier": 0.35457996865874325, "mean_conf": 0.934871358662847, "confident_error_rate": 0.1375, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.828125, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.0625, "aurc": 0.11157532829806283, "error_rate_at_0_9": 0.171875, "confidence_bias": 0.147371358662847, "top_bins": { "0.9": { "n": 64, "errors": 11, "error_rate": 0.171875 }, "0.95": { "n": 60, "errors": 9, "error_rate": 0.15 }, "0.99": { "n": 42, "errors": 6, "error_rate": 0.14285714285714285 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.9926516140967396 }, "0.8": { "coverage": 0.8, "accuracy": 0.828125, "confidence_cutoff": 0.9148037688879754 } } }, "qnli": { "n": 80, "nll": 0.30218851707566163, "acc": 0.9125, "ece": 0.06027148972700891, "brier": 0.14840098630475543, "mean_conf": 0.9638953711825529, "confident_error_rate": 0.05, "coverage_at_0_9": 0.8625, "accuracy_at_0_9": 0.9420289855072463, "coverage_at_5pct_error": 0.8375, "coverage_at_1pct_error": 0.4875, "aurc": 0.021606573578835768, "error_rate_at_0_9": 0.057971014492753624, "confidence_bias": 0.051395371182552885, "top_bins": { "0.9": { "n": 69, "errors": 4, "error_rate": 0.057971014492753624 }, "0.95": { "n": 64, "errors": 3, "error_rate": 0.046875 }, "0.99": { "n": 53, "errors": 1, "error_rate": 0.018867924528301886 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9991475900858287 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.9529522976896565 } } }, "sciq": { "n": 80, "nll": 0.05733198573008065, "acc": 0.975, "ece": 0.028215578518962822, "brier": 0.03727237207148254, "mean_conf": 0.9823463992200709, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "aurc": 0.0009515497687966043, "error_rate_at_0_9": 0.0, "confidence_bias": 0.007346399220070965, "top_bins": { "0.9": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 73, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 69, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999673481935003 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9985609261980979 } } }, "tweet_offensive": { "n": 80, "nll": 0.49008508179404914, "acc": 0.75, "ece": 0.10188382252197292, "brier": 0.3088844829017613, "mean_conf": 0.8518838225219728, "confident_error_rate": 0.05, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.3125, "coverage_at_1pct_error": 0.275, "aurc": 0.09199870714413623, "error_rate_at_0_9": 0.1, "confidence_bias": 0.10188382252197281, "top_bins": { "0.9": { "n": 40, "errors": 4, "error_rate": 0.1 }, "0.95": { "n": 33, "errors": 3, "error_rate": 0.09090909090909091 }, "0.99": { "n": 18, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9152760624369052 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.6894673314531843 } } } }, "variants": { "clean": { "n": 656, "nll": 0.5764418952724232, "acc": 0.8170731707317073, "ece": 0.10020924778356732, "brier": 0.2690850170373655, "mean_conf": 0.9128191322541024, "confident_error_rate": 0.05945121951219512, "coverage_at_0_9": 0.7347560975609756, "accuracy_at_0_9": 0.9190871369294605, "coverage_at_5pct_error": 0.5990853658536586, "coverage_at_1pct_error": 0.3978658536585366, "aurc": 0.049144884120614074, "error_rate_at_0_9": 0.08091286307053942, "confidence_bias": 0.09574596152239512, "top_bins": { "0.9": { "n": 482, "errors": 39, "error_rate": 0.08091286307053942 }, "0.95": { "n": 447, "errors": 31, "error_rate": 0.06935123042505593 }, "0.99": { "n": 363, "errors": 17, "error_rate": 0.046831955922865015 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9786585365853658, "confidence_cutoff": 0.9950785493257293 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.900952380952381, "confidence_cutoff": 0.8257571108864877 } }, "score_mae": 0.316824262687028, "ranked_probability_score": 0.12274395686296583 }, "none_absent": { "n": 36, "nll": 0.2382943867360587, "acc": 0.8888888888888888, "ece": 0.06002776857686939, "brier": 0.13660821634616752, "mean_conf": 0.8943886684408269, "confident_error_rate": 0.027777777777777776, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.9629629629629629, "coverage_at_5pct_error": 0.8888888888888888, "coverage_at_1pct_error": 0.6666666666666666, "aurc": 0.016633111236839873, "error_rate_at_0_9": 0.037037037037037035, "confidence_bias": 0.005499779551938033, "top_bins": { "0.9": { "n": 27, "errors": 1, "error_rate": 0.037037037037037035 }, "0.95": { "n": 25, "errors": 1, "error_rate": 0.04 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9885625198248165 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.9655172413793104, "confidence_cutoff": 0.8441082925771153 } } }, "none_present": { "n": 36, "nll": 1.4992651509227133, "acc": 0.5555555555555556, "ece": 0.2733507018450212, "brier": 0.6243239275207846, "mean_conf": 0.8289062574005768, "confident_error_rate": 0.1111111111111111, "coverage_at_0_9": 0.5555555555555556, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.3611111111111111, "coverage_at_1pct_error": 0.3611111111111111, "aurc": 0.1800464786772204, "error_rate_at_0_9": 0.2, "confidence_bias": 0.2733507018450212, "top_bins": { "0.9": { "n": 20, "errors": 4, "error_rate": 0.2 }, "0.95": { "n": 19, "errors": 3, "error_rate": 0.15789473684210525 }, "0.99": { "n": 14, "errors": 1, "error_rate": 0.07142857142857142 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.9653464869986069 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.6551724137931034, "confidence_cutoff": 0.5929894291187869 } } }, "permuted": { "n": 36, "nll": 0.6706148135185012, "acc": 0.75, "ece": 0.18165011054121183, "brier": 0.3522686377804207, "mean_conf": 0.896756673361093, "confident_error_rate": 0.027777777777777776, "coverage_at_0_9": 0.6944444444444444, "accuracy_at_0_9": 0.96, "coverage_at_5pct_error": 0.6944444444444444, "coverage_at_1pct_error": 0.5555555555555556, "aurc": 0.056971427946047176, "error_rate_at_0_9": 0.04, "confidence_bias": 0.146756673361093, "top_bins": { "0.9": { "n": 25, "errors": 1, "error_rate": 0.04 }, "0.95": { "n": 25, "errors": 1, "error_rate": 0.04 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9961253613933815 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.8620689655172413, "confidence_cutoff": 0.8327445813749598 } } } }, "heldout_tasks": { "composition_held_and_or": { "n": 32, "nll": 0.036558273598721235, "acc": 1.0, "ece": 0.030892100510413772, "brier": 0.017881015688346595, "mean_conf": 0.9691078994895861, "confident_error_rate": 0.0, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03089210051041391, "top_bins": { "0.9": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 27, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.999973844026875 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.9908018053072243 } } }, "composition_held_conditional": { "n": 32, "nll": 0.04484538085803764, "acc": 1.0, "ece": 0.04023780814361391, "brier": 0.015210308025896463, "mean_conf": 0.9597621918563862, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.040237808143613774, "top_bins": { "0.9": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 18, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.9939084631570418 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9661742340828378 } } }, "composition_held_or_not": { "n": 32, "nll": 0.28422348585042867, "acc": 0.9375, "ece": 0.03387826063971708, "brier": 0.10915453853972053, "mean_conf": 0.9713782606397171, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.9642857142857143, "coverage_at_5pct_error": 0.96875, "coverage_at_1pct_error": 0.625, "aurc": 0.01524130523374475, "error_rate_at_0_9": 0.03571428571428571, "confidence_bias": 0.033878260639717106, "top_bins": { "0.9": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 }, "0.95": { "n": 25, "errors": 1, "error_rate": 0.04 }, "0.99": { "n": 24, "errors": 1, "error_rate": 0.041666666666666664 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9993548255583505 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.9372299807347372 } } }, "contrastive_authorization": { "n": 40, "nll": 5.519474637266503e-06, "acc": 1.0, "ece": 5.519452064506325e-06, "brier": 9.144248319210223e-11, "mean_conf": 0.9999944805479355, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -5.519452064506325e-06, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 40, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999956221821245 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9999923349753774 } } }, "contrastive_deadline": { "n": 40, "nll": 0.8436244396923623, "acc": 0.65, "ece": 0.27695473137187976, "brier": 0.4842390988197474, "mean_conf": 0.9020071124758093, "confident_error_rate": 0.1, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.8461538461538461, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.4, "aurc": 0.11349409024439767, "error_rate_at_0_9": 0.15384615384615385, "confidence_bias": 0.2520071124758093, "top_bins": { "0.9": { "n": 26, "errors": 4, "error_rate": 0.15384615384615385 }, "0.95": { "n": 26, "errors": 4, "error_rate": 0.15384615384615385 }, "0.99": { "n": 22, "errors": 2, "error_rate": 0.09090909090909091 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9953179512674454 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7594178184419988 } }, "score_mae": 0.316824262687028, "ranked_probability_score": 0.12274395686296583 }, "emotion": { "n": 80, "nll": 1.7994186231074956, "acc": 0.55, "ece": 0.29613510771975704, "brier": 0.6764238468215377, "mean_conf": 0.846135107719757, "confident_error_rate": 0.15, "coverage_at_0_9": 0.5375, "accuracy_at_0_9": 0.7209302325581395, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.2697507411299849, "error_rate_at_0_9": 0.27906976744186046, "confidence_bias": 0.29613510771975693, "top_bins": { "0.9": { "n": 43, "errors": 12, "error_rate": 0.27906976744186046 }, "0.95": { "n": 35, "errors": 9, "error_rate": 0.2571428571428571 }, "0.99": { "n": 23, "errors": 6, "error_rate": 0.2608695652173913 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.725, "confidence_cutoff": 0.9318937701892213 }, "0.8": { "coverage": 0.8, "accuracy": 0.609375, "confidence_cutoff": 0.6904468219734016 } } }, "mmlu": { "n": 80, "nll": 0.7239653768200205, "acc": 0.725, "ece": 0.1155622225698504, "brier": 0.38191758859093616, "mean_conf": 0.7948846878702913, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.5125, "accuracy_at_0_9": 0.926829268292683, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.225, "aurc": 0.10576574303101645, "error_rate_at_0_9": 0.07317073170731707, "confidence_bias": 0.06988468787029134, "top_bins": { "0.9": { "n": 41, "errors": 3, "error_rate": 0.07317073170731707 }, "0.95": { "n": 36, "errors": 2, "error_rate": 0.05555555555555555 }, "0.99": { "n": 27, "errors": 1, "error_rate": 0.037037037037037035 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9262887306313381 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.5687803486430137 } } }, "paws": { "n": 80, "nll": 0.785768121000188, "acc": 0.7875, "ece": 0.16376117277465274, "brier": 0.35457996865874325, "mean_conf": 0.934871358662847, "confident_error_rate": 0.1375, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.828125, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.0625, "aurc": 0.11157532829806283, "error_rate_at_0_9": 0.171875, "confidence_bias": 0.147371358662847, "top_bins": { "0.9": { "n": 64, "errors": 11, "error_rate": 0.171875 }, "0.95": { "n": 60, "errors": 9, "error_rate": 0.15 }, "0.99": { "n": 42, "errors": 6, "error_rate": 0.14285714285714285 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.9926516140967396 }, "0.8": { "coverage": 0.8, "accuracy": 0.828125, "confidence_cutoff": 0.9148037688879754 } } }, "qnli": { "n": 80, "nll": 0.30218851707566163, "acc": 0.9125, "ece": 0.06027148972700891, "brier": 0.14840098630475543, "mean_conf": 0.9638953711825529, "confident_error_rate": 0.05, "coverage_at_0_9": 0.8625, "accuracy_at_0_9": 0.9420289855072463, "coverage_at_5pct_error": 0.8375, "coverage_at_1pct_error": 0.4875, "aurc": 0.021606573578835768, "error_rate_at_0_9": 0.057971014492753624, "confidence_bias": 0.051395371182552885, "top_bins": { "0.9": { "n": 69, "errors": 4, "error_rate": 0.057971014492753624 }, "0.95": { "n": 64, "errors": 3, "error_rate": 0.046875 }, "0.99": { "n": 53, "errors": 1, "error_rate": 0.018867924528301886 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9991475900858287 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.9529522976896565 } } }, "sciq": { "n": 80, "nll": 0.05733198573008065, "acc": 0.975, "ece": 0.028215578518962822, "brier": 0.03727237207148254, "mean_conf": 0.9823463992200709, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "aurc": 0.0009515497687966043, "error_rate_at_0_9": 0.0, "confidence_bias": 0.007346399220070965, "top_bins": { "0.9": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 73, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 69, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999673481935003 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9985609261980979 } } }, "tweet_offensive": { "n": 80, "nll": 0.49008508179404914, "acc": 0.75, "ece": 0.10188382252197292, "brier": 0.3088844829017613, "mean_conf": 0.8518838225219728, "confident_error_rate": 0.05, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.3125, "coverage_at_1pct_error": 0.275, "aurc": 0.09199870714413623, "error_rate_at_0_9": 0.1, "confidence_bias": 0.10188382252197281, "top_bins": { "0.9": { "n": 40, "errors": 4, "error_rate": 0.1 }, "0.95": { "n": 33, "errors": 3, "error_rate": 0.09090909090909091 }, "0.99": { "n": 18, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9152760624369052 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.6894673314531843 } } } }, "permutation": { "n": 36, "mean_max_delta": 0.026388596882159963, "flip_rate": 0.05555555555555555 }, "temperature": 2.2973967099940698, "calibrated_clean": { "n": 656, "nll": 0.4344257519237035, "acc": 0.8170731707317073, "ece": 0.0405193820148158, "brier": 0.24296359539279005, "mean_conf": 0.8207248136739, "confident_error_rate": 0.012195121951219513, "coverage_at_0_9": 0.49847560975609756, "accuracy_at_0_9": 0.9755351681957186, "coverage_at_5pct_error": 0.6158536585365854, "coverage_at_1pct_error": 0.3780487804878049, "aurc": 0.047235862428547555, "error_rate_at_0_9": 0.024464831804281346, "confidence_bias": 0.0036516429421927787, "top_bins": { "0.9": { "n": 327, "errors": 8, "error_rate": 0.024464831804281346 }, "0.95": { "n": 261, "errors": 3, "error_rate": 0.011494252873563218 }, "0.99": { "n": 98, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975609756097561, "confidence_cutoff": 0.8981225434286827 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.9142857142857143, "confidence_cutoff": 0.6343266379321651 } }, "score_mae": 0.3365660317122896, "ranked_probability_score": 0.10047697969029137 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 }, "coverage": { "requested_records": 764, "requested_questions": 764, "evaluated_records": 764, "evaluated_questions": 764, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 51.46696049996535, "p95": 54.76280345023952 }, "calibration": { "inference_temperature": 1.0, "additional_temperature": 2.2973967099940698, "logits_recorded": true }, "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1" }, "provenance": { "config": { "epochs": 1, "seed": 1, "lr": 2e-05, "lora": 16, "accum": 4, "batch": 2, "perm_kl": 0.0, "perm_frac": 0.3, "ord_w": 0.0, "p_none": 0.1, "p_none_distract": 0.12, "p_distract": 0.15, "p_none_pair": 0.25, "synthetic_repeat": 1, "public_frac": 1.0, "head_lr": 0.0, "weight_decay": 0.01, "anchor_w": 0.0, "label_smoothing": 0.0, "brier_w": 0.0, "focal_gamma": 0.0, "dtype": "bf16", "checkpointing": 1, "max_state": 7552, "base": "Qwen/Qwen3.5-4B-Base", "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", "init_from": "jaredpalmer/kev-4b@957b91e762e883935830246eeb02381f9d2694b6", "data": "evals/round10/skills/train.jsonl", "replay": 4000 }, "config_sha256": "08aa7634eef0409340edd9df72b4b347bdda02e87b25c920e205dc727911525d", "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", "source_hashes": { "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", "kev/benchmark.py": "10a892d76cb4c200e94d0a68ee74054aa1870d7f484c0530a531cacfa75032ed", "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", "kev/checkpoint.py": "f3edb4d159c0aa12d7006fc599fe6ebc3f2c288b3b69890c289f7dac451dd03d", "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", "kev/cuda_graphs.py": "12f1953fea8f77c3c9dff29468202fbb37545145f93d3618194565a89949fd55", "kev/data.py": "e77a2fffeb8ee7e05b118893be8e38b8358aef97c3ca2e143cdce208201b92f9", "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", "kev/experiment.py": "644adedf43fbd10566830fa6094e16dd001b8cad29d9b9da82f0e0f245ca06b1", "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9", "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", "kev/model.py": "2634ffe7747d69473fb6596942c248e5df2586df3610bd1adb47e7e9acd99f96", "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", "kev/predictors.py": "4d02906c285d78bcbd91908f4fc86577ccc95c320d704817f6444aabbc4fcd4f", "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", "kev/serve.py": "57d379bbcdeb3e4d4c3fdde5471e4dddeaf7710b8ed61ecad3466f3b67e6c610", "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", "kev/suite.py": "32b2882d1ea0fe5d03ba4d676a6180d580545f157e36fa715e7e89169ec99130", "kev/train.py": "68428ed43b3e362e52d4650dc5296ab88767061f7998a525715e1b306eeef474", "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", "modal_app.py": "7c6bc859f7d3e7011539a8f5e9e90390288cc72b5d80f911c8841c068f4d1136", "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d", "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" }, "git_commit": "6d02f5d066cd34958dfd15ffa5d2f6f0f4c21a63", "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", "torch": "2.8.0+cu128", "device": "cuda", "gpu": "NVIDIA H200", "legacy_checkpoint": false, "measured_checkpoint": { "requested": "/runs/r10-skills/00-trial-0/checkpoint", "resolved": "/runs/r10-skills/00-trial-0/checkpoint", "head_sha256": "979aaa60135274973a6dc57fe24dfc4f1ce88e65df46c73d86b2047c94d897df", "adapter_sha256": "90e817356246e7f18bfa7ca3d31794cd4fbeb3332a66a84cb51d9ceae925f2b2", "inference_temperature": 1.0 } }, "mechanism_checks": { "n": 8, "packed_max_delta": 0.0, "sibling_max_delta": 0.00015926361083984375, "tolerance": 0.001, "passed": true }, "gates": { "passed": true, "checks": { "complete_coverage": true, "isolation_and_packing": true, "transfer_complete": true, "heldout_pairs_at_least_70pct": true, "transfer_confident_errors_below_10pct": true }, "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." }, "calibration_fit": { "temperature": 2.2973967099940698, "aggregation": "micro", "objective": "raw-logit NLL", "split": "calibration", "rows_sha256": "54bea908f07d7d7b7972573e5920273c604c967c17bcf36edea176d070930a91", "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", "n": 1148 }, "wall_seconds": 5271.548163347, "promotable": false, "test_evaluated": false, "training_resources": { "wall_seconds": 5031.164765119553, "records_seen": 19994, "requested_records": 15320, "truncated_records": 0, "rejected_records": 0, "optimizer_steps": 1915, "forward_tokens": 8738926, "peak_device_bytes": 47737451520, "device": "cuda", "dtype": "bf16", "batch": 2, "peak_rss_bytes": 31552528384 } }