| { |
| "objective": -0.44946179013506016, |
| "paired_flip": { |
| "pairs": 80, |
| "flip_rate": 0.95, |
| "both_correct_rate": 0.95, |
| "invariant_pairs": 32, |
| "invariance_rate": 1.0, |
| "invariant_both_correct_rate": 0.9375 |
| }, |
| "unknowable": null, |
| "clean": { |
| "n": 1264, |
| "nll": 0.6168926722690661, |
| "acc": 0.872626582278481, |
| "ece": 0.08624432523402295, |
| "brier": 0.20582430898522214, |
| "mean_conf": 0.9579030816260836, |
| "confident_error_rate": 0.06724683544303797, |
| "coverage_at_0_9": 0.8734177215189873, |
| "accuracy_at_0_9": 0.9230072463768116, |
| "coverage_at_5pct_error": 0.807753164556962, |
| "coverage_at_1pct_error": 0.012658227848101266, |
| "aurc": 0.03488689793265612, |
| "error_rate_at_0_9": 0.0769927536231884, |
| "confidence_bias": 0.08527649934760262, |
| "top_bins": { |
| "0.9": { |
| "n": 1104, |
| "errors": 85, |
| "error_rate": 0.0769927536231884 |
| }, |
| "0.95": { |
| "n": 1060, |
| "errors": 68, |
| "error_rate": 0.06415094339622641 |
| }, |
| "0.99": { |
| "n": 970, |
| "errors": 35, |
| "error_rate": 0.03608247422680412 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9857594936708861, |
| "confidence_cutoff": 0.9999853520719274 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.9515810276679841, |
| "confidence_cutoff": 0.9756593913753835 |
| } |
| }, |
| "score_mae": 0.42582366131690075, |
| "ranked_probability_score": 0.07949507614242349 |
| }, |
| "tasks": { |
| "agnews": { |
| "n": 80, |
| "nll": 1.088087841683712, |
| "acc": 0.8625, |
| "ece": 0.14149794590514259, |
| "brier": 0.268037735349122, |
| "mean_conf": 0.9830186623522476, |
| "confident_error_rate": 0.1125, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 0.881578947368421, |
| "coverage_at_5pct_error": 0.5625, |
| "coverage_at_1pct_error": 0.475, |
| "aurc": 0.040719994570916664, |
| "error_rate_at_0_9": 0.11842105263157894, |
| "confidence_bias": 0.12051866235224751, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 9, |
| "error_rate": 0.11842105263157894 |
| }, |
| "0.95": { |
| "n": 75, |
| "errors": 9, |
| "error_rate": 0.12 |
| }, |
| "0.99": { |
| "n": 73, |
| "errors": 7, |
| "error_rate": 0.0958904109589041 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999781047633173 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.921875, |
| "confidence_cutoff": 0.9993162733876088 |
| } |
| } |
| }, |
| "agnews_yn": { |
| "n": 160, |
| "nll": 0.3684986515629499, |
| "acc": 0.925, |
| "ece": 0.07080036453142347, |
| "brier": 0.14191875012263128, |
| "mean_conf": 0.9904318711658459, |
| "confident_error_rate": 0.06875, |
| "coverage_at_0_9": 0.98125, |
| "accuracy_at_0_9": 0.9299363057324841, |
| "coverage_at_5pct_error": 0.9125, |
| "coverage_at_1pct_error": 0.5125, |
| "aurc": 0.015426883965016677, |
| "error_rate_at_0_9": 0.07006369426751592, |
| "confidence_bias": 0.06543187116584581, |
| "top_bins": { |
| "0.9": { |
| "n": 157, |
| "errors": 11, |
| "error_rate": 0.07006369426751592 |
| }, |
| "0.95": { |
| "n": 149, |
| "errors": 9, |
| "error_rate": 0.06040268456375839 |
| }, |
| "0.99": { |
| "n": 138, |
| "errors": 5, |
| "error_rate": 0.036231884057971016 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999283159732779 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9765625, |
| "confidence_cutoff": 0.9980463005644982 |
| } |
| } |
| }, |
| "amazon": { |
| "n": 80, |
| "nll": 1.176018556239162, |
| "acc": 0.6125, |
| "ece": 0.18137689608782254, |
| "brier": 0.4977355567070353, |
| "mean_conf": 0.7938768960878225, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.425, |
| "accuracy_at_0_9": 0.8529411764705882, |
| "coverage_at_5pct_error": 0.25, |
| "coverage_at_1pct_error": 0.2125, |
| "aurc": 0.1690225243329798, |
| "error_rate_at_0_9": 0.14705882352941177, |
| "confidence_bias": 0.18137689608782248, |
| "top_bins": { |
| "0.9": { |
| "n": 34, |
| "errors": 5, |
| "error_rate": 0.14705882352941177 |
| }, |
| "0.95": { |
| "n": 27, |
| "errors": 3, |
| "error_rate": 0.1111111111111111 |
| }, |
| "0.99": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.825, |
| "confidence_cutoff": 0.8506096822990515 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.71875, |
| "confidence_cutoff": 0.5751792299676172 |
| } |
| }, |
| "score_mae": 0.44938028819980486, |
| "ranked_probability_score": 0.07977985685011632 |
| }, |
| "banking77": { |
| "n": 80, |
| "nll": 0.4635284533347269, |
| "acc": 0.9125, |
| "ece": 0.06510758088814228, |
| "brier": 0.15688024282767815, |
| "mean_conf": 0.9528486541244575, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.85, |
| "accuracy_at_0_9": 0.9411764705882353, |
| "coverage_at_5pct_error": 0.8375, |
| "coverage_at_1pct_error": 0.5375, |
| "aurc": 0.019995786215425037, |
| "error_rate_at_0_9": 0.058823529411764705, |
| "confidence_bias": 0.04034865412445754, |
| "top_bins": { |
| "0.9": { |
| "n": 68, |
| "errors": 4, |
| "error_rate": 0.058823529411764705 |
| }, |
| "0.95": { |
| "n": 67, |
| "errors": 3, |
| "error_rate": 0.04477611940298507 |
| }, |
| "0.99": { |
| "n": 57, |
| "errors": 2, |
| "error_rate": 0.03508771929824561 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999568082541851 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.953125, |
| "confidence_cutoff": 0.9657829895649422 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 80, |
| "nll": 0.5519322868596024, |
| "acc": 0.9125, |
| "ece": 0.0865069251642396, |
| "brier": 0.1559152293261062, |
| "mean_conf": 0.9810412667154444, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 0.9210526315789473, |
| "coverage_at_5pct_error": 0.75, |
| "coverage_at_1pct_error": 0.35, |
| "aurc": 0.0338245365853573, |
| "error_rate_at_0_9": 0.07894736842105263, |
| "confidence_bias": 0.06854126671544447, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 6, |
| "error_rate": 0.07894736842105263 |
| }, |
| "0.95": { |
| "n": 74, |
| "errors": 5, |
| "error_rate": 0.06756756756756757 |
| }, |
| "0.99": { |
| "n": 71, |
| "errors": 4, |
| "error_rate": 0.056338028169014086 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999555161966023 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9375, |
| "confidence_cutoff": 0.9969850660718333 |
| } |
| } |
| }, |
| "composition_atom": { |
| "n": 16, |
| "nll": 5.587525980704396e-07, |
| "acc": 1.0, |
| "ece": 5.58752369261839e-07, |
| "brier": 1.0589627199968719e-12, |
| "mean_conf": 0.9999994412476307, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -5.58752369261839e-07, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999997385055852 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999989170810404 |
| } |
| } |
| }, |
| "composition_conditional": { |
| "n": 16, |
| "nll": 0.5862268029006099, |
| "acc": 0.8125, |
| "ece": 0.17657065265426275, |
| "brier": 0.34248733847315155, |
| "mean_conf": 0.9890706526542627, |
| "confident_error_rate": 0.1875, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 0.8125, |
| "coverage_at_5pct_error": 0.8125, |
| "coverage_at_1pct_error": 0.8125, |
| "aurc": 0.028385416666666666, |
| "error_rate_at_0_9": 0.1875, |
| "confidence_bias": 0.17657065265426275, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 3, |
| "error_rate": 0.1875 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 3, |
| "error_rate": 0.1875 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999546055940167 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9595710587789154 |
| } |
| } |
| }, |
| "composition_conjunction": { |
| "n": 16, |
| "nll": 0.0014957885933607818, |
| "acc": 1.0, |
| "ece": 0.0014898847851706476, |
| "brier": 2.3490712686390572e-05, |
| "mean_conf": 0.9985101152148294, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0014898847851706476, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999950008268189 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999490590400107 |
| } |
| } |
| }, |
| "composition_disjunction": { |
| "n": 16, |
| "nll": 0.12174322044047148, |
| "acc": 0.9375, |
| "ece": 0.05359792559000376, |
| "brier": 0.09189231131503667, |
| "mean_conf": 0.9910773458368907, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9375, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.9375, |
| "coverage_at_1pct_error": 0.9375, |
| "aurc": 0.00390625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.053577345836890666, |
| "top_bins": { |
| "0.9": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999988091175268 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999961085481334 |
| } |
| } |
| }, |
| "composition_exception": { |
| "n": 16, |
| "nll": 2.455481704878314e-06, |
| "acc": 1.0, |
| "ece": 2.4554759838757434e-06, |
| "brier": 2.3014830474470046e-11, |
| "mean_conf": 0.9999975445240161, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -2.4554759838757434e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999989408444062 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999950638491678 |
| } |
| } |
| }, |
| "composition_negation": { |
| "n": 16, |
| "nll": 2.3543106879567546e-06, |
| "acc": 1.0, |
| "ece": 2.3542971585577277e-06, |
| "brier": 5.334216154583253e-11, |
| "mean_conf": 0.9999976457028414, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -2.3542971585577277e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999999645056845 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999988154752283 |
| } |
| } |
| }, |
| "composition_nested_and": { |
| "n": 16, |
| "nll": 0.77960504564072, |
| "acc": 0.8125, |
| "ece": 0.1837522072127682, |
| "brier": 0.36330684634954047, |
| "mean_conf": 0.9962522072127682, |
| "confident_error_rate": 0.1875, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 0.8125, |
| "coverage_at_5pct_error": 0.8125, |
| "coverage_at_1pct_error": 0.8125, |
| "aurc": 0.028385416666666666, |
| "error_rate_at_0_9": 0.1875, |
| "confidence_bias": 0.1837522072127682, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 3, |
| "error_rate": 0.1875 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 3, |
| "error_rate": 0.1875 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5625, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.99996807236974 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9875134522207604 |
| } |
| } |
| }, |
| "composition_nested_or": { |
| "n": 16, |
| "nll": 1.093226828449568e-05, |
| "acc": 1.0, |
| "ece": 1.0932081188075315e-05, |
| "brier": 7.487674731318452e-10, |
| "mean_conf": 0.9999890679188119, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -1.0932081188075315e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999969424662678 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999925980083295 |
| } |
| } |
| }, |
| "contrastive_age_eligibility": { |
| "n": 24, |
| "nll": 5.006770668290524e-07, |
| "acc": 1.0, |
| "ece": 5.006768900583936e-07, |
| "brier": 9.244256634768012e-13, |
| "mean_conf": 0.9999994993231099, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -5.006768900583936e-07, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999996360830347 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999994255795048 |
| } |
| } |
| }, |
| "contrastive_quantity_limit": { |
| "n": 24, |
| "nll": 1.3929967331016708e-05, |
| "acc": 1.0, |
| "ece": 1.3928931700823988e-05, |
| "brier": 4.132248138653341e-09, |
| "mean_conf": 0.9999860710682992, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -1.3928931700823988e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999999877529739 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999994545316442 |
| } |
| } |
| }, |
| "contrastive_return_window": { |
| "n": 24, |
| "nll": 0.31056791234726155, |
| "acc": 0.9583333333333334, |
| "ece": 0.0640438652373011, |
| "brier": 0.09770288155198424, |
| "mean_conf": 0.9732028167534837, |
| "confident_error_rate": 0.041666666666666664, |
| "coverage_at_0_9": 0.9166666666666666, |
| "accuracy_at_0_9": 0.9545454545454546, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.5833333333333334, |
| "aurc": 0.021849827132965845, |
| "error_rate_at_0_9": 0.045454545454545456, |
| "confidence_bias": 0.014869483420150376, |
| "top_bins": { |
| "0.9": { |
| "n": 22, |
| "errors": 1, |
| "error_rate": 0.045454545454545456 |
| }, |
| "0.95": { |
| "n": 22, |
| "errors": 1, |
| "error_rate": 0.045454545454545456 |
| }, |
| "0.99": { |
| "n": 20, |
| "errors": 1, |
| "error_rate": 0.05 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999100215405365 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.990736211417129 |
| } |
| } |
| }, |
| "contrastive_spend_threshold": { |
| "n": 24, |
| "nll": 3.735462479764618e-08, |
| "acc": 1.0, |
| "ece": 3.7354620174845365e-08, |
| "brier": 2.3881922218447637e-15, |
| "mean_conf": 0.9999999626453798, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -3.7354620174845365e-08, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999999653479973 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999999556227255 |
| } |
| } |
| }, |
| "dbpedia14": { |
| "n": 80, |
| "nll": 0.42304812389495156, |
| "acc": 0.975, |
| "ece": 0.02932681717155103, |
| "brier": 0.05385781169677199, |
| "mean_conf": 0.9944886376890493, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.9875, |
| "accuracy_at_0_9": 0.9746835443037974, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.1125, |
| "aurc": 0.027674600083788314, |
| "error_rate_at_0_9": 0.02531645569620253, |
| "confidence_bias": 0.019488637689049293, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 2, |
| "error_rate": 0.02531645569620253 |
| }, |
| "0.95": { |
| "n": 79, |
| "errors": 2, |
| "error_rate": 0.02531645569620253 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999999934143395 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9999999347799529 |
| } |
| } |
| }, |
| "imdb": { |
| "n": 80, |
| "nll": 0.3393769807278961, |
| "acc": 0.9625, |
| "ece": 0.0393599985606192, |
| "brier": 0.0752463437000028, |
| "mean_conf": 0.9934010979791104, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.975, |
| "accuracy_at_0_9": 0.9615384615384616, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.3375, |
| "aurc": 0.016063430120988282, |
| "error_rate_at_0_9": 0.038461538461538464, |
| "confidence_bias": 0.030901097979110337, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 3, |
| "error_rate": 0.038461538461538464 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 3, |
| "error_rate": 0.03896103896103896 |
| }, |
| "0.99": { |
| "n": 74, |
| "errors": 2, |
| "error_rate": 0.02702702702702703 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.999999701806285 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9999559709412794 |
| } |
| } |
| }, |
| "mnli": { |
| "n": 80, |
| "nll": 1.1504605172469486, |
| "acc": 0.9125, |
| "ece": 0.08443622134266553, |
| "brier": 0.16791837255349212, |
| "mean_conf": 0.9969362213426658, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.9875, |
| "accuracy_at_0_9": 0.9240506329113924, |
| "coverage_at_5pct_error": 0.8375, |
| "coverage_at_1pct_error": 0.1375, |
| "aurc": 0.04950172008707943, |
| "error_rate_at_0_9": 0.0759493670886076, |
| "confidence_bias": 0.08443622134266582, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 6, |
| "error_rate": 0.0759493670886076 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 6, |
| "error_rate": 0.07692307692307693 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 5, |
| "error_rate": 0.06493506493506493 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9999999752595827 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.953125, |
| "confidence_cutoff": 0.9999923938776696 |
| } |
| } |
| }, |
| "sst5": { |
| "n": 80, |
| "nll": 1.6075716823764101, |
| "acc": 0.6125, |
| "ece": 0.28824579721934074, |
| "brier": 0.640125725434039, |
| "mean_conf": 0.878584840967417, |
| "confident_error_rate": 0.1875, |
| "coverage_at_0_9": 0.5875, |
| "accuracy_at_0_9": 0.6808510638297872, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.3136211989261356, |
| "error_rate_at_0_9": 0.3191489361702128, |
| "confidence_bias": 0.2660848409674169, |
| "top_bins": { |
| "0.9": { |
| "n": 47, |
| "errors": 15, |
| "error_rate": 0.3191489361702128 |
| }, |
| "0.95": { |
| "n": 38, |
| "errors": 10, |
| "error_rate": 0.2631578947368421 |
| }, |
| "0.99": { |
| "n": 17, |
| "errors": 3, |
| "error_rate": 0.17647058823529413 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.9438309726342528 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.640625, |
| "confidence_cutoff": 0.7701640331718456 |
| } |
| }, |
| "score_mae": 0.4506116888085069, |
| "ranked_probability_score": 0.09071892097481096 |
| }, |
| "trec": { |
| "n": 80, |
| "nll": 0.5498752511376328, |
| "acc": 0.95, |
| "ece": 0.049752106930267104, |
| "brier": 0.09947315940143245, |
| "mean_conf": 0.9997521069302671, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 0.95, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.5875, |
| "aurc": 0.010228250702194944, |
| "error_rate_at_0_9": 0.05, |
| "confidence_bias": 0.049752106930267104, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 4, |
| "error_rate": 0.05 |
| }, |
| "0.95": { |
| "n": 80, |
| "errors": 4, |
| "error_rate": 0.05 |
| }, |
| "0.99": { |
| "n": 79, |
| "errors": 3, |
| "error_rate": 0.0379746835443038 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999999352563271 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9999986380425966 |
| } |
| } |
| }, |
| "yelp": { |
| "n": 80, |
| "nll": 0.964132890901699, |
| "acc": 0.6375, |
| "ece": 0.20879169675234072, |
| "brier": 0.4929403852546871, |
| "mean_conf": 0.8310000477468991, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.4875, |
| "accuracy_at_0_9": 0.7435897435897436, |
| "coverage_at_5pct_error": 0.2625, |
| "coverage_at_1pct_error": 0.25, |
| "aurc": 0.19143002309963925, |
| "error_rate_at_0_9": 0.2564102564102564, |
| "confidence_bias": 0.1935000477468991, |
| "top_bins": { |
| "0.9": { |
| "n": 39, |
| "errors": 10, |
| "error_rate": 0.2564102564102564 |
| }, |
| "0.95": { |
| "n": 28, |
| "errors": 5, |
| "error_rate": 0.17857142857142858 |
| }, |
| "0.99": { |
| "n": 19, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.8965825455533782 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.71875, |
| "confidence_cutoff": 0.6472665439576823 |
| } |
| }, |
| "score_mae": 0.3774790069423904, |
| "ranked_probability_score": 0.06798645060234317 |
| }, |
| "yelp_yn": { |
| "n": 80, |
| "nll": 0.30488218854102955, |
| "acc": 0.8875, |
| "ece": 0.07508091469896844, |
| "brier": 0.17120315623001298, |
| "mean_conf": 0.9621212064255577, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.875, |
| "accuracy_at_0_9": 0.9571428571428572, |
| "coverage_at_5pct_error": 0.875, |
| "coverage_at_1pct_error": 0.6625, |
| "aurc": 0.015744398791700327, |
| "error_rate_at_0_9": 0.04285714285714286, |
| "confidence_bias": 0.0746212064255577, |
| "top_bins": { |
| "0.9": { |
| "n": 70, |
| "errors": 3, |
| "error_rate": 0.04285714285714286 |
| }, |
| "0.95": { |
| "n": 67, |
| "errors": 2, |
| "error_rate": 0.029850746268656716 |
| }, |
| "0.99": { |
| "n": 62, |
| "errors": 1, |
| "error_rate": 0.016129032258064516 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998813596426201 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9837482122145327 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 1264, |
| "nll": 0.6168926722690661, |
| "acc": 0.872626582278481, |
| "ece": 0.08624432523402295, |
| "brier": 0.20582430898522214, |
| "mean_conf": 0.9579030816260836, |
| "confident_error_rate": 0.06724683544303797, |
| "coverage_at_0_9": 0.8734177215189873, |
| "accuracy_at_0_9": 0.9230072463768116, |
| "coverage_at_5pct_error": 0.807753164556962, |
| "coverage_at_1pct_error": 0.012658227848101266, |
| "aurc": 0.03488689793265612, |
| "error_rate_at_0_9": 0.0769927536231884, |
| "confidence_bias": 0.08527649934760262, |
| "top_bins": { |
| "0.9": { |
| "n": 1104, |
| "errors": 85, |
| "error_rate": 0.0769927536231884 |
| }, |
| "0.95": { |
| "n": 1060, |
| "errors": 68, |
| "error_rate": 0.06415094339622641 |
| }, |
| "0.99": { |
| "n": 970, |
| "errors": 35, |
| "error_rate": 0.03608247422680412 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9857594936708861, |
| "confidence_cutoff": 0.9999853520719274 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.9515810276679841, |
| "confidence_cutoff": 0.9756593913753835 |
| } |
| }, |
| "score_mae": 0.42582366131690075, |
| "ranked_probability_score": 0.07949507614242349 |
| }, |
| "none_absent": { |
| "n": 60, |
| "nll": 0.5750708488188147, |
| "acc": 0.8833333333333333, |
| "ece": 0.09672233307148694, |
| "brier": 0.1946567603170117, |
| "mean_conf": 0.9636866873975226, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 0.9444444444444444, |
| "coverage_at_5pct_error": 0.7333333333333333, |
| "coverage_at_1pct_error": 0.2, |
| "aurc": 0.04686992361578679, |
| "error_rate_at_0_9": 0.05555555555555555, |
| "confidence_bias": 0.08035335406418931, |
| "top_bins": { |
| "0.9": { |
| "n": 54, |
| "errors": 3, |
| "error_rate": 0.05555555555555555 |
| }, |
| "0.95": { |
| "n": 54, |
| "errors": 3, |
| "error_rate": 0.05555555555555555 |
| }, |
| "0.99": { |
| "n": 47, |
| "errors": 3, |
| "error_rate": 0.06382978723404255 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9666666666666667, |
| "confidence_cutoff": 0.9998527746924852 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9375, |
| "confidence_cutoff": 0.9877892246847566 |
| } |
| } |
| }, |
| "none_present": { |
| "n": 60, |
| "nll": 0.7256526534658846, |
| "acc": 0.9166666666666666, |
| "ece": 0.07485845385044373, |
| "brier": 0.13614155592450627, |
| "mean_conf": 0.9703975852828999, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.9166666666666666, |
| "accuracy_at_0_9": 0.9454545454545454, |
| "coverage_at_5pct_error": 0.75, |
| "coverage_at_1pct_error": 0.05, |
| "aurc": 0.05872786035794387, |
| "error_rate_at_0_9": 0.05454545454545454, |
| "confidence_bias": 0.05373091861623325, |
| "top_bins": { |
| "0.9": { |
| "n": 55, |
| "errors": 3, |
| "error_rate": 0.05454545454545454 |
| }, |
| "0.95": { |
| "n": 53, |
| "errors": 3, |
| "error_rate": 0.05660377358490566 |
| }, |
| "0.99": { |
| "n": 49, |
| "errors": 3, |
| "error_rate": 0.061224489795918366 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9666666666666667, |
| "confidence_cutoff": 0.999700409676498 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9375, |
| "confidence_cutoff": 0.9939915175771098 |
| } |
| } |
| }, |
| "permuted": { |
| "n": 84, |
| "nll": 0.44617124681813825, |
| "acc": 0.9523809523809523, |
| "ece": 0.04257200563999509, |
| "brier": 0.08180703209551511, |
| "mean_conf": 0.9869312935684673, |
| "confident_error_rate": 0.023809523809523808, |
| "coverage_at_0_9": 0.9523809523809523, |
| "accuracy_at_0_9": 0.975, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.05952380952380952, |
| "aurc": 0.03935129081916679, |
| "error_rate_at_0_9": 0.025, |
| "confidence_bias": 0.03455034118751499, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 2, |
| "error_rate": 0.025 |
| }, |
| "0.95": { |
| "n": 79, |
| "errors": 2, |
| "error_rate": 0.02531645569620253 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9761904761904762, |
| "confidence_cutoff": 0.99999704787547 |
| }, |
| "0.8": { |
| "coverage": 0.8095238095238095, |
| "accuracy": 0.9705882352941176, |
| "confidence_cutoff": 0.9993851721771361 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": {}, |
| "permutation": { |
| "n": 60, |
| "mean_max_delta": 0.024041959134575408, |
| "flip_rate": 0.016666666666666666 |
| }, |
| "temperature": 2.732080513508791, |
| "calibrated_clean": { |
| "n": 1264, |
| "nll": 0.3515788273092323, |
| "acc": 0.872626582278481, |
| "ece": 0.02305460667741268, |
| "brier": 0.1813435470997088, |
| "mean_conf": 0.8871547347082455, |
| "confident_error_rate": 0.021360759493670885, |
| "coverage_at_0_9": 0.7001582278481012, |
| "accuracy_at_0_9": 0.9694915254237289, |
| "coverage_at_5pct_error": 0.7990506329113924, |
| "coverage_at_1pct_error": 0.011867088607594937, |
| "aurc": 0.03507879287066272, |
| "error_rate_at_0_9": 0.030508474576271188, |
| "confidence_bias": 0.014528152429764463, |
| "top_bins": { |
| "0.9": { |
| "n": 885, |
| "errors": 27, |
| "error_rate": 0.030508474576271188 |
| }, |
| "0.95": { |
| "n": 786, |
| "errors": 20, |
| "error_rate": 0.02544529262086514 |
| }, |
| "0.99": { |
| "n": 476, |
| "errors": 6, |
| "error_rate": 0.012605042016806723 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9873417721518988, |
| "confidence_cutoff": 0.978302275330191 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.9496047430830039, |
| "confidence_cutoff": 0.7611815169166877 |
| } |
| }, |
| "score_mae": 0.4720223859631723, |
| "ranked_probability_score": 0.07130551407018675 |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 1204, |
| "requested_questions": 1468, |
| "evaluated_records": 1204, |
| "evaluated_questions": 1468, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 49.6572095000829, |
| "p95": 136.3470733003851 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 2.732080513508791, |
| "logits_recorded": true |
| }, |
| "transfer": { |
| "objective": -0.7354206605171923, |
| "paired_flip": { |
| "pairs": 64, |
| "flip_rate": 0.8125, |
| "both_correct_rate": 0.78125, |
| "invariant_pairs": 24, |
| "invariance_rate": 1.0, |
| "invariant_both_correct_rate": 0.9166666666666666 |
| }, |
| "unknowable": null, |
| "clean": { |
| "n": 656, |
| "nll": 0.8420064323719835, |
| "acc": 0.801829268292683, |
| "ece": 0.14524044074533254, |
| "brier": 0.3265035782727766, |
| "mean_conf": 0.9442720446627839, |
| "confident_error_rate": 0.10060975609756098, |
| "coverage_at_0_9": 0.8277439024390244, |
| "accuracy_at_0_9": 0.8784530386740331, |
| "coverage_at_5pct_error": 0.5259146341463414, |
| "coverage_at_1pct_error": 0.27439024390243905, |
| "aurc": 0.059246832389346436, |
| "error_rate_at_0_9": 0.12154696132596685, |
| "confidence_bias": 0.14244277637010094, |
| "top_bins": { |
| "0.9": { |
| "n": 543, |
| "errors": 66, |
| "error_rate": 0.12154696132596685 |
| }, |
| "0.95": { |
| "n": 509, |
| "errors": 51, |
| "error_rate": 0.10019646365422397 |
| }, |
| "0.99": { |
| "n": 444, |
| "errors": 34, |
| "error_rate": 0.07657657657657657 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9573170731707317, |
| "confidence_cutoff": 0.9997165252969916 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.8914285714285715, |
| "confidence_cutoff": 0.926982780530237 |
| } |
| }, |
| "score_mae": 0.3868049169616327, |
| "ranked_probability_score": 0.1808253510701079 |
| }, |
| "tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.21275731832987918, |
| "acc": 0.90625, |
| "ece": 0.08960640747049482, |
| "brier": 0.14806787785708828, |
| "mean_conf": 0.968119000247354, |
| "confident_error_rate": 0.03125, |
| "coverage_at_0_9": 0.84375, |
| "accuracy_at_0_9": 0.9629629629629629, |
| "coverage_at_5pct_error": 0.84375, |
| "coverage_at_1pct_error": 0.8125, |
| "aurc": 0.016607299648786718, |
| "error_rate_at_0_9": 0.037037037037037035, |
| "confidence_bias": 0.06186900024735398, |
| "top_bins": { |
| "0.9": { |
| "n": 27, |
| "errors": 1, |
| "error_rate": 0.037037037037037035 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999867426553156 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997005752724791 |
| } |
| } |
| }, |
| "composition_held_conditional": { |
| "n": 32, |
| "nll": 0.014373469714188625, |
| "acc": 1.0, |
| "ece": 0.01341902983845485, |
| "brier": 0.003340072727106961, |
| "mean_conf": 0.9865809701615452, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.96875, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.013419029838454777, |
| "top_bins": { |
| "0.9": { |
| "n": 31, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 30, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 25, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.997536508908986 |
| }, |
| "0.8": { |
| "coverage": 0.84375, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9860660764373641 |
| } |
| } |
| }, |
| "composition_held_or_not": { |
| "n": 32, |
| "nll": 0.39586312721931666, |
| "acc": 0.875, |
| "ece": 0.09422189894535778, |
| "brier": 0.1800316303783951, |
| "mean_conf": 0.969221898945358, |
| "confident_error_rate": 0.03125, |
| "coverage_at_0_9": 0.875, |
| "accuracy_at_0_9": 0.9642857142857143, |
| "coverage_at_5pct_error": 0.90625, |
| "coverage_at_1pct_error": 0.53125, |
| "aurc": 0.026035965425144192, |
| "error_rate_at_0_9": 0.03571428571428571, |
| "confidence_bias": 0.09422189894535804, |
| "top_bins": { |
| "0.9": { |
| "n": 28, |
| "errors": 1, |
| "error_rate": 0.03571428571428571 |
| }, |
| "0.95": { |
| "n": 27, |
| "errors": 1, |
| "error_rate": 0.037037037037037035 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 1, |
| "error_rate": 0.041666666666666664 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998200130139856 |
| }, |
| "0.8": { |
| "coverage": 0.84375, |
| "accuracy": 0.9629629629629629, |
| "confidence_cutoff": 0.9864744883957841 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 40, |
| "nll": 2.626769435101001e-06, |
| "acc": 1.0, |
| "ece": 2.6267639215982186e-06, |
| "brier": 2.200791979309494e-11, |
| "mean_conf": 0.9999973732360784, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -2.6267639215982186e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999982121058788 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999951777711299 |
| } |
| } |
| }, |
| "contrastive_deadline": { |
| "n": 40, |
| "nll": 1.6227537153922078, |
| "acc": 0.6, |
| "ece": 0.3704952209007848, |
| "brier": 0.721418824936752, |
| "mean_conf": 0.9704952209007848, |
| "confident_error_rate": 0.325, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 0.6388888888888888, |
| "coverage_at_5pct_error": 0.575, |
| "coverage_at_1pct_error": 0.45, |
| "aurc": 0.12570899891487713, |
| "error_rate_at_0_9": 0.3611111111111111, |
| "confidence_bias": 0.3704952209007848, |
| "top_bins": { |
| "0.9": { |
| "n": 36, |
| "errors": 13, |
| "error_rate": 0.3611111111111111 |
| }, |
| "0.95": { |
| "n": 33, |
| "errors": 11, |
| "error_rate": 0.3333333333333333 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 4, |
| "error_rate": 0.15384615384615385 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9998875216084744 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.6875, |
| "confidence_cutoff": 0.9653250745881129 |
| } |
| }, |
| "score_mae": 0.3868049169616327, |
| "ranked_probability_score": 0.1808253510701079 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 2.01845007225599, |
| "acc": 0.5875, |
| "ece": 0.3034439572619928, |
| "brier": 0.659342484681167, |
| "mean_conf": 0.8816687443113388, |
| "confident_error_rate": 0.1625, |
| "coverage_at_0_9": 0.6125, |
| "accuracy_at_0_9": 0.7346938775510204, |
| "coverage_at_5pct_error": 0.025, |
| "coverage_at_1pct_error": 0.025, |
| "aurc": 0.2725565889995515, |
| "error_rate_at_0_9": 0.2653061224489796, |
| "confidence_bias": 0.29416874431133877, |
| "top_bins": { |
| "0.9": { |
| "n": 49, |
| "errors": 13, |
| "error_rate": 0.2653061224489796 |
| }, |
| "0.95": { |
| "n": 43, |
| "errors": 10, |
| "error_rate": 0.23255813953488372 |
| }, |
| "0.99": { |
| "n": 30, |
| "errors": 8, |
| "error_rate": 0.26666666666666666 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.775, |
| "confidence_cutoff": 0.9654279207142605 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.7552159935350502 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 1.0623469179109404, |
| "acc": 0.7375, |
| "ece": 0.17148582929965572, |
| "brier": 0.4389275967782478, |
| "mean_conf": 0.8598028670228626, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.625, |
| "accuracy_at_0_9": 0.86, |
| "coverage_at_5pct_error": 0.1125, |
| "coverage_at_1pct_error": 0.1125, |
| "aurc": 0.1331753286671519, |
| "error_rate_at_0_9": 0.14, |
| "confidence_bias": 0.12230286702286253, |
| "top_bins": { |
| "0.9": { |
| "n": 50, |
| "errors": 7, |
| "error_rate": 0.14 |
| }, |
| "0.95": { |
| "n": 43, |
| "errors": 4, |
| "error_rate": 0.09302325581395349 |
| }, |
| "0.99": { |
| "n": 33, |
| "errors": 3, |
| "error_rate": 0.09090909090909091 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.925, |
| "confidence_cutoff": 0.974014602758476 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.78125, |
| "confidence_cutoff": 0.6995644294224197 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 1.62106974555167, |
| "acc": 0.725, |
| "ece": 0.24858545944074148, |
| "brier": 0.47996204925896696, |
| "mean_conf": 0.9735854594407414, |
| "confident_error_rate": 0.225, |
| "coverage_at_0_9": 0.9375, |
| "accuracy_at_0_9": 0.76, |
| "coverage_at_5pct_error": 0.4, |
| "coverage_at_1pct_error": 0.1375, |
| "aurc": 0.1160986678934329, |
| "error_rate_at_0_9": 0.24, |
| "confidence_bias": 0.2485854594407414, |
| "top_bins": { |
| "0.9": { |
| "n": 75, |
| "errors": 18, |
| "error_rate": 0.24 |
| }, |
| "0.95": { |
| "n": 70, |
| "errors": 15, |
| "error_rate": 0.21428571428571427 |
| }, |
| "0.99": { |
| "n": 65, |
| "errors": 13, |
| "error_rate": 0.2 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9999292199405727 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.9935345534056593 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.2861285544171043, |
| "acc": 0.925, |
| "ece": 0.06519801763745295, |
| "brier": 0.13858164311006516, |
| "mean_conf": 0.9633829066567101, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.8875, |
| "accuracy_at_0_9": 0.9577464788732394, |
| "coverage_at_5pct_error": 0.8875, |
| "coverage_at_1pct_error": 0.575, |
| "aurc": 0.013965998776904765, |
| "error_rate_at_0_9": 0.04225352112676056, |
| "confidence_bias": 0.03838290665671007, |
| "top_bins": { |
| "0.9": { |
| "n": 71, |
| "errors": 3, |
| "error_rate": 0.04225352112676056 |
| }, |
| "0.95": { |
| "n": 70, |
| "errors": 3, |
| "error_rate": 0.04285714285714286 |
| }, |
| "0.99": { |
| "n": 64, |
| "errors": 1, |
| "error_rate": 0.015625 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999454840710632 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9905160244738724 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.05712256491977856, |
| "acc": 0.9625, |
| "ece": 0.024095647099121015, |
| "brier": 0.04132271841446281, |
| "mean_conf": 0.9815817586969505, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.95, |
| "aurc": 0.0012680561790530144, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.0190817586969505, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 74, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 72, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999996614790427 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998869569876149 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.7987591532086051, |
| "acc": 0.725, |
| "ece": 0.2208875858819574, |
| "brier": 0.42590760472944245, |
| "mean_conf": 0.9281939852960915, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.8333333333333334, |
| "coverage_at_5pct_error": 0.3625, |
| "coverage_at_1pct_error": 0.15, |
| "aurc": 0.10808477691679863, |
| "error_rate_at_0_9": 0.16666666666666666, |
| "confidence_bias": 0.20319398529609156, |
| "top_bins": { |
| "0.9": { |
| "n": 60, |
| "errors": 10, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.95": { |
| "n": 53, |
| "errors": 7, |
| "error_rate": 0.1320754716981132 |
| }, |
| "0.99": { |
| "n": 39, |
| "errors": 4, |
| "error_rate": 0.10256410256410256 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9898656520526313 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.828125, |
| "confidence_cutoff": 0.8874677287419479 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 656, |
| "nll": 0.8420064323719835, |
| "acc": 0.801829268292683, |
| "ece": 0.14524044074533254, |
| "brier": 0.3265035782727766, |
| "mean_conf": 0.9442720446627839, |
| "confident_error_rate": 0.10060975609756098, |
| "coverage_at_0_9": 0.8277439024390244, |
| "accuracy_at_0_9": 0.8784530386740331, |
| "coverage_at_5pct_error": 0.5259146341463414, |
| "coverage_at_1pct_error": 0.27439024390243905, |
| "aurc": 0.059246832389346436, |
| "error_rate_at_0_9": 0.12154696132596685, |
| "confidence_bias": 0.14244277637010094, |
| "top_bins": { |
| "0.9": { |
| "n": 543, |
| "errors": 66, |
| "error_rate": 0.12154696132596685 |
| }, |
| "0.95": { |
| "n": 509, |
| "errors": 51, |
| "error_rate": 0.10019646365422397 |
| }, |
| "0.99": { |
| "n": 444, |
| "errors": 34, |
| "error_rate": 0.07657657657657657 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9573170731707317, |
| "confidence_cutoff": 0.9997165252969916 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.8914285714285715, |
| "confidence_cutoff": 0.926982780530237 |
| } |
| }, |
| "score_mae": 0.3868049169616327, |
| "ranked_probability_score": 0.1808253510701079 |
| }, |
| "none_absent": { |
| "n": 36, |
| "nll": 0.6045433705207882, |
| "acc": 0.8333333333333334, |
| "ece": 0.13280966055516136, |
| "brier": 0.2692087471452192, |
| "mean_conf": 0.9302829455612941, |
| "confident_error_rate": 0.08333333333333333, |
| "coverage_at_0_9": 0.8055555555555556, |
| "accuracy_at_0_9": 0.896551724137931, |
| "coverage_at_5pct_error": 0.25, |
| "coverage_at_1pct_error": 0.25, |
| "aurc": 0.0709278531457756, |
| "error_rate_at_0_9": 0.10344827586206896, |
| "confidence_bias": 0.09694961222796072, |
| "top_bins": { |
| "0.9": { |
| "n": 29, |
| "errors": 3, |
| "error_rate": 0.10344827586206896 |
| }, |
| "0.95": { |
| "n": 25, |
| "errors": 2, |
| "error_rate": 0.08 |
| }, |
| "0.99": { |
| "n": 17, |
| "errors": 1, |
| "error_rate": 0.058823529411764705 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9444444444444444, |
| "confidence_cutoff": 0.9867735272600054 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.896551724137931, |
| "confidence_cutoff": 0.9021773367725505 |
| } |
| } |
| }, |
| "none_present": { |
| "n": 36, |
| "nll": 1.8636645679022996, |
| "acc": 0.6111111111111112, |
| "ece": 0.2968259055464007, |
| "brier": 0.6471666151139533, |
| "mean_conf": 0.9079370166575118, |
| "confident_error_rate": 0.2222222222222222, |
| "coverage_at_0_9": 0.6944444444444444, |
| "accuracy_at_0_9": 0.68, |
| "coverage_at_5pct_error": 0.3055555555555556, |
| "coverage_at_1pct_error": 0.3055555555555556, |
| "aurc": 0.1675614581367646, |
| "error_rate_at_0_9": 0.32, |
| "confidence_bias": 0.29682590554640065, |
| "top_bins": { |
| "0.9": { |
| "n": 25, |
| "errors": 8, |
| "error_rate": 0.32 |
| }, |
| "0.95": { |
| "n": 23, |
| "errors": 6, |
| "error_rate": 0.2608695652173913 |
| }, |
| "0.99": { |
| "n": 19, |
| "errors": 3, |
| "error_rate": 0.15789473684210525 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.8888888888888888, |
| "confidence_cutoff": 0.996923503041837 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.6551724137931034, |
| "confidence_cutoff": 0.7998003295333731 |
| } |
| } |
| }, |
| "permuted": { |
| "n": 36, |
| "nll": 0.9001924115234267, |
| "acc": 0.7777777777777778, |
| "ece": 0.16577047477357917, |
| "brier": 0.362691945174966, |
| "mean_conf": 0.9435482525513572, |
| "confident_error_rate": 0.05555555555555555, |
| "coverage_at_0_9": 0.7222222222222222, |
| "accuracy_at_0_9": 0.9230769230769231, |
| "coverage_at_5pct_error": 0.6944444444444444, |
| "coverage_at_1pct_error": 0.4444444444444444, |
| "aurc": 0.06003225907915171, |
| "error_rate_at_0_9": 0.07692307692307693, |
| "confidence_bias": 0.16577047477357942, |
| "top_bins": { |
| "0.9": { |
| "n": 26, |
| "errors": 2, |
| "error_rate": 0.07692307692307693 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 2, |
| "error_rate": 0.07692307692307693 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 1, |
| "error_rate": 0.041666666666666664 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9444444444444444, |
| "confidence_cutoff": 0.9994163361463438 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.8620689655172413, |
| "confidence_cutoff": 0.8723240949648822 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.21275731832987918, |
| "acc": 0.90625, |
| "ece": 0.08960640747049482, |
| "brier": 0.14806787785708828, |
| "mean_conf": 0.968119000247354, |
| "confident_error_rate": 0.03125, |
| "coverage_at_0_9": 0.84375, |
| "accuracy_at_0_9": 0.9629629629629629, |
| "coverage_at_5pct_error": 0.84375, |
| "coverage_at_1pct_error": 0.8125, |
| "aurc": 0.016607299648786718, |
| "error_rate_at_0_9": 0.037037037037037035, |
| "confidence_bias": 0.06186900024735398, |
| "top_bins": { |
| "0.9": { |
| "n": 27, |
| "errors": 1, |
| "error_rate": 0.037037037037037035 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999867426553156 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997005752724791 |
| } |
| } |
| }, |
| "composition_held_conditional": { |
| "n": 32, |
| "nll": 0.014373469714188625, |
| "acc": 1.0, |
| "ece": 0.01341902983845485, |
| "brier": 0.003340072727106961, |
| "mean_conf": 0.9865809701615452, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.96875, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.013419029838454777, |
| "top_bins": { |
| "0.9": { |
| "n": 31, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 30, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 25, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.997536508908986 |
| }, |
| "0.8": { |
| "coverage": 0.84375, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9860660764373641 |
| } |
| } |
| }, |
| "composition_held_or_not": { |
| "n": 32, |
| "nll": 0.39586312721931666, |
| "acc": 0.875, |
| "ece": 0.09422189894535778, |
| "brier": 0.1800316303783951, |
| "mean_conf": 0.969221898945358, |
| "confident_error_rate": 0.03125, |
| "coverage_at_0_9": 0.875, |
| "accuracy_at_0_9": 0.9642857142857143, |
| "coverage_at_5pct_error": 0.90625, |
| "coverage_at_1pct_error": 0.53125, |
| "aurc": 0.026035965425144192, |
| "error_rate_at_0_9": 0.03571428571428571, |
| "confidence_bias": 0.09422189894535804, |
| "top_bins": { |
| "0.9": { |
| "n": 28, |
| "errors": 1, |
| "error_rate": 0.03571428571428571 |
| }, |
| "0.95": { |
| "n": 27, |
| "errors": 1, |
| "error_rate": 0.037037037037037035 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 1, |
| "error_rate": 0.041666666666666664 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998200130139856 |
| }, |
| "0.8": { |
| "coverage": 0.84375, |
| "accuracy": 0.9629629629629629, |
| "confidence_cutoff": 0.9864744883957841 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 40, |
| "nll": 2.626769435101001e-06, |
| "acc": 1.0, |
| "ece": 2.6267639215982186e-06, |
| "brier": 2.200791979309494e-11, |
| "mean_conf": 0.9999973732360784, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -2.6267639215982186e-06, |
| "top_bins": { |
| "0.9": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999982121058788 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999951777711299 |
| } |
| } |
| }, |
| "contrastive_deadline": { |
| "n": 40, |
| "nll": 1.6227537153922078, |
| "acc": 0.6, |
| "ece": 0.3704952209007848, |
| "brier": 0.721418824936752, |
| "mean_conf": 0.9704952209007848, |
| "confident_error_rate": 0.325, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 0.6388888888888888, |
| "coverage_at_5pct_error": 0.575, |
| "coverage_at_1pct_error": 0.45, |
| "aurc": 0.12570899891487713, |
| "error_rate_at_0_9": 0.3611111111111111, |
| "confidence_bias": 0.3704952209007848, |
| "top_bins": { |
| "0.9": { |
| "n": 36, |
| "errors": 13, |
| "error_rate": 0.3611111111111111 |
| }, |
| "0.95": { |
| "n": 33, |
| "errors": 11, |
| "error_rate": 0.3333333333333333 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 4, |
| "error_rate": 0.15384615384615385 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9998875216084744 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.6875, |
| "confidence_cutoff": 0.9653250745881129 |
| } |
| }, |
| "score_mae": 0.3868049169616327, |
| "ranked_probability_score": 0.1808253510701079 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 2.01845007225599, |
| "acc": 0.5875, |
| "ece": 0.3034439572619928, |
| "brier": 0.659342484681167, |
| "mean_conf": 0.8816687443113388, |
| "confident_error_rate": 0.1625, |
| "coverage_at_0_9": 0.6125, |
| "accuracy_at_0_9": 0.7346938775510204, |
| "coverage_at_5pct_error": 0.025, |
| "coverage_at_1pct_error": 0.025, |
| "aurc": 0.2725565889995515, |
| "error_rate_at_0_9": 0.2653061224489796, |
| "confidence_bias": 0.29416874431133877, |
| "top_bins": { |
| "0.9": { |
| "n": 49, |
| "errors": 13, |
| "error_rate": 0.2653061224489796 |
| }, |
| "0.95": { |
| "n": 43, |
| "errors": 10, |
| "error_rate": 0.23255813953488372 |
| }, |
| "0.99": { |
| "n": 30, |
| "errors": 8, |
| "error_rate": 0.26666666666666666 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.775, |
| "confidence_cutoff": 0.9654279207142605 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.7552159935350502 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 1.0623469179109404, |
| "acc": 0.7375, |
| "ece": 0.17148582929965572, |
| "brier": 0.4389275967782478, |
| "mean_conf": 0.8598028670228626, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.625, |
| "accuracy_at_0_9": 0.86, |
| "coverage_at_5pct_error": 0.1125, |
| "coverage_at_1pct_error": 0.1125, |
| "aurc": 0.1331753286671519, |
| "error_rate_at_0_9": 0.14, |
| "confidence_bias": 0.12230286702286253, |
| "top_bins": { |
| "0.9": { |
| "n": 50, |
| "errors": 7, |
| "error_rate": 0.14 |
| }, |
| "0.95": { |
| "n": 43, |
| "errors": 4, |
| "error_rate": 0.09302325581395349 |
| }, |
| "0.99": { |
| "n": 33, |
| "errors": 3, |
| "error_rate": 0.09090909090909091 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.925, |
| "confidence_cutoff": 0.974014602758476 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.78125, |
| "confidence_cutoff": 0.6995644294224197 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 1.62106974555167, |
| "acc": 0.725, |
| "ece": 0.24858545944074148, |
| "brier": 0.47996204925896696, |
| "mean_conf": 0.9735854594407414, |
| "confident_error_rate": 0.225, |
| "coverage_at_0_9": 0.9375, |
| "accuracy_at_0_9": 0.76, |
| "coverage_at_5pct_error": 0.4, |
| "coverage_at_1pct_error": 0.1375, |
| "aurc": 0.1160986678934329, |
| "error_rate_at_0_9": 0.24, |
| "confidence_bias": 0.2485854594407414, |
| "top_bins": { |
| "0.9": { |
| "n": 75, |
| "errors": 18, |
| "error_rate": 0.24 |
| }, |
| "0.95": { |
| "n": 70, |
| "errors": 15, |
| "error_rate": 0.21428571428571427 |
| }, |
| "0.99": { |
| "n": 65, |
| "errors": 13, |
| "error_rate": 0.2 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9999292199405727 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.9935345534056593 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.2861285544171043, |
| "acc": 0.925, |
| "ece": 0.06519801763745295, |
| "brier": 0.13858164311006516, |
| "mean_conf": 0.9633829066567101, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.8875, |
| "accuracy_at_0_9": 0.9577464788732394, |
| "coverage_at_5pct_error": 0.8875, |
| "coverage_at_1pct_error": 0.575, |
| "aurc": 0.013965998776904765, |
| "error_rate_at_0_9": 0.04225352112676056, |
| "confidence_bias": 0.03838290665671007, |
| "top_bins": { |
| "0.9": { |
| "n": 71, |
| "errors": 3, |
| "error_rate": 0.04225352112676056 |
| }, |
| "0.95": { |
| "n": 70, |
| "errors": 3, |
| "error_rate": 0.04285714285714286 |
| }, |
| "0.99": { |
| "n": 64, |
| "errors": 1, |
| "error_rate": 0.015625 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999454840710632 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9905160244738724 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.05712256491977856, |
| "acc": 0.9625, |
| "ece": 0.024095647099121015, |
| "brier": 0.04132271841446281, |
| "mean_conf": 0.9815817586969505, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.95, |
| "aurc": 0.0012680561790530144, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.0190817586969505, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 74, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 72, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999996614790427 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998869569876149 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.7987591532086051, |
| "acc": 0.725, |
| "ece": 0.2208875858819574, |
| "brier": 0.42590760472944245, |
| "mean_conf": 0.9281939852960915, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.8333333333333334, |
| "coverage_at_5pct_error": 0.3625, |
| "coverage_at_1pct_error": 0.15, |
| "aurc": 0.10808477691679863, |
| "error_rate_at_0_9": 0.16666666666666666, |
| "confidence_bias": 0.20319398529609156, |
| "top_bins": { |
| "0.9": { |
| "n": 60, |
| "errors": 10, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.95": { |
| "n": 53, |
| "errors": 7, |
| "error_rate": 0.1320754716981132 |
| }, |
| "0.99": { |
| "n": 39, |
| "errors": 4, |
| "error_rate": 0.10256410256410256 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9898656520526313 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.828125, |
| "confidence_cutoff": 0.8874677287419479 |
| } |
| } |
| } |
| }, |
| "permutation": { |
| "n": 36, |
| "mean_max_delta": 0.04771277529822956, |
| "flip_rate": 0.05555555555555555 |
| }, |
| "temperature": 2.732080513508791, |
| "calibrated_clean": { |
| "n": 656, |
| "nll": 0.4734830253813679, |
| "acc": 0.801829268292683, |
| "ece": 0.05261345517511798, |
| "brier": 0.26777059962010535, |
| "mean_conf": 0.850709804046946, |
| "confident_error_rate": 0.03048780487804878, |
| "coverage_at_0_9": 0.5838414634146342, |
| "accuracy_at_0_9": 0.9477806788511749, |
| "coverage_at_5pct_error": 0.5411585365853658, |
| "coverage_at_1pct_error": 0.3490853658536585, |
| "aurc": 0.05769454271433421, |
| "error_rate_at_0_9": 0.05221932114882506, |
| "confidence_bias": 0.04888053575426299, |
| "top_bins": { |
| "0.9": { |
| "n": 383, |
| "errors": 20, |
| "error_rate": 0.05221932114882506 |
| }, |
| "0.95": { |
| "n": 325, |
| "errors": 14, |
| "error_rate": 0.043076923076923075 |
| }, |
| "0.99": { |
| "n": 115, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9542682926829268, |
| "confidence_cutoff": 0.9487693566866638 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.8914285714285715, |
| "confidence_cutoff": 0.6847608977037521 |
| } |
| }, |
| "score_mae": 0.36649026598278567, |
| "ranked_probability_score": 0.1335905179824414 |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 764, |
| "requested_questions": 764, |
| "evaluated_records": 764, |
| "evaluated_questions": 764, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 48.01533849990847, |
| "p95": 53.9990647502691 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 2.732080513508791, |
| "logits_recorded": true |
| }, |
| "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1" |
| }, |
| "provenance": { |
| "config": { |
| "epochs": 1, |
| "seed": 2, |
| "lr": 2e-05, |
| "lora": 16, |
| "accum": 4, |
| "batch": 2, |
| "perm_kl": 0.0, |
| "perm_frac": 0.3, |
| "ord_w": 0.0, |
| "p_none": 0.1, |
| "p_none_distract": 0.12, |
| "p_distract": 0.15, |
| "p_none_pair": 0.25, |
| "synthetic_repeat": 1, |
| "public_frac": 1.0, |
| "head_lr": 0.0, |
| "weight_decay": 0.01, |
| "anchor_w": 0.0, |
| "label_smoothing": 0.0, |
| "brier_w": 0.0, |
| "focal_gamma": 0.0, |
| "dtype": "bf16", |
| "checkpointing": 1, |
| "replay": 2000, |
| "max_state": 7552, |
| "data": "evals/documents-v1/train.jsonl", |
| "base": "Qwen/Qwen3.5-4B-Base", |
| "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", |
| "init_from": "jaredpalmer/kev-4b" |
| }, |
| "config_sha256": "6fb0132214a793f590cce025ce669b85433bab47a63522c369c226f0fb34ff3c", |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", |
| "source_hashes": { |
| "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
| "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", |
| "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", |
| "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", |
| "kev/benchmark.py": "9c442efd8b6b852c574a646192c61ee91afa1306e3b2978bb6391ae1b91a3fa0", |
| "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", |
| "kev/checkpoint.py": "7cd4fbedfcfd11c2997490df2bd88eaa12af60e5ad60da2e5390e6df1ac1f626", |
| "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", |
| "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", |
| "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", |
| "kev/data.py": "ef1be396e7218ba5c94f0f67de2ceff23a39656b57199cb3aa3192cde53e8320", |
| "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", |
| "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", |
| "kev/experiment.py": "644adedf43fbd10566830fa6094e16dd001b8cad29d9b9da82f0e0f245ca06b1", |
| "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", |
| "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9", |
| "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", |
| "kev/model.py": "387f281bc0b72da1620a8d2fcf508996fc30b9d8e342211ecd67c6d930f9cbaa", |
| "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", |
| "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3", |
| "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", |
| "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735", |
| "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", |
| "kev/suite.py": "d524663fe2656d873c548a6216a2ddfebaf7299650f07ba31ae24e4c453398f1", |
| "kev/train.py": "68428ed43b3e362e52d4650dc5296ab88767061f7998a525715e1b306eeef474", |
| "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", |
| "modal_app.py": "28318ef72dbeba7855f3c6b05339ee8c0f2e57562d21ede72352fba98d0e6ae3", |
| "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d", |
| "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" |
| }, |
| "git_commit": "ea271bf9dd0f2a280b1ef096a7d237388484620c", |
| "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", |
| "torch": "2.8.0+cu128", |
| "device": "cuda", |
| "gpu": "NVIDIA H200", |
| "legacy_checkpoint": false, |
| "measured_checkpoint": { |
| "requested": "/runs/r8-small/00-trial-0/checkpoint", |
| "resolved": "/runs/r8-small/00-trial-0/checkpoint", |
| "head_sha256": "19fdcdfb65380b51e90458edeb7e8dd7554532f357379c4cda44152cbbcd4195", |
| "adapter_sha256": "c2c99077b58cd4a604304b743b7db5391187ef1d58e575f90c554c744aa23d65", |
| "inference_temperature": 1.0 |
| } |
| }, |
| "mechanism_checks": { |
| "n": 8, |
| "packed_max_delta": 0.0, |
| "sibling_max_delta": 0.0002613663673400879, |
| "tolerance": 0.001, |
| "passed": true |
| }, |
| "gates": { |
| "passed": false, |
| "checks": { |
| "complete_coverage": true, |
| "isolation_and_packing": true, |
| "transfer_complete": true, |
| "heldout_pairs_at_least_70pct": true, |
| "transfer_confident_errors_below_10pct": false |
| }, |
| "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." |
| }, |
| "calibration_fit": { |
| "temperature": 2.732080513508791, |
| "aggregation": "micro", |
| "objective": "raw-logit NLL", |
| "split": "calibration", |
| "rows_sha256": "7c08bad532793a9c6ab113e0ad7b9b862bdf999641cca068858d73669a2b675f", |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", |
| "n": 1148 |
| }, |
| "wall_seconds": 2855.117294672, |
| "promotable": false, |
| "test_evaluated": false, |
| "training_resources": { |
| "wall_seconds": 2574.7955305576324, |
| "records_seen": 10179, |
| "requested_records": 7219, |
| "truncated_records": 0, |
| "rejected_records": 0, |
| "optimizer_steps": 903, |
| "forward_tokens": 6749150, |
| "peak_device_bytes": 39839947776, |
| "device": "cuda", |
| "dtype": "bf16", |
| "batch": 2, |
| "peak_rss_bytes": 31563161600 |
| } |
| } |
|
|
