kev-4b preview, decision-v7 recipe (transfer 0.790 dev / 0.806 locked; held-out pairs 0.73)
2bd3bb9 verified | { | |
| "objective": -0.3781113270188752, | |
| "paired_flip": { | |
| "pairs": 80, | |
| "flip_rate": 0.8875, | |
| "both_correct_rate": 0.8875, | |
| "invariant_pairs": 32, | |
| "invariance_rate": 1.0, | |
| "invariant_both_correct_rate": 0.90625 | |
| }, | |
| "clean": { | |
| "n": 1264, | |
| "nll": 0.47096940162233064, | |
| "acc": 0.8536392405063291, | |
| "ece": 0.0654366260188363, | |
| "brier": 0.21322991316558398, | |
| "mean_conf": 0.9178467644873951, | |
| "confident_error_rate": 0.041930379746835444, | |
| "coverage_at_0_9": 0.757120253164557, | |
| "accuracy_at_0_9": 0.9446185997910136, | |
| "confidence_bias": 0.064207523981066, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 957, | |
| "errors": 53, | |
| "error_rate": 0.055381400208986416 | |
| }, | |
| "0.95": { | |
| "n": 911, | |
| "errors": 41, | |
| "error_rate": 0.04500548847420417 | |
| }, | |
| "0.99": { | |
| "n": 795, | |
| "errors": 27, | |
| "error_rate": 0.033962264150943396 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5007911392405063, | |
| "accuracy": 0.9778830963665087, | |
| "confidence_cutoff": 0.9976429106296094 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8006329113924051, | |
| "accuracy": 0.9268774703557312, | |
| "confidence_cutoff": 0.829332684596399 | |
| } | |
| }, | |
| "score_mae": 0.45394055553275015, | |
| "ranked_probability_score": 0.0717298259542393 | |
| }, | |
| "tasks": { | |
| "agnews": { | |
| "n": 80, | |
| "nll": 0.7764970089350782, | |
| "acc": 0.8375, | |
| "ece": 0.14810546152452678, | |
| "brier": 0.2815909368447759, | |
| "mean_conf": 0.9768289098497732, | |
| "confident_error_rate": 0.1125, | |
| "coverage_at_0_9": 0.9375, | |
| "accuracy_at_0_9": 0.88, | |
| "confidence_bias": 0.1393289098497732, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 75, | |
| "errors": 9, | |
| "error_rate": 0.12 | |
| }, | |
| "0.95": { | |
| "n": 72, | |
| "errors": 8, | |
| "error_rate": 0.1111111111111111 | |
| }, | |
| "0.99": { | |
| "n": 63, | |
| "errors": 6, | |
| "error_rate": 0.09523809523809523 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.975, | |
| "confidence_cutoff": 0.9990865056765899 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.90625, | |
| "confidence_cutoff": 0.9894646850622684 | |
| } | |
| } | |
| }, | |
| "agnews_yn": { | |
| "n": 160, | |
| "nll": 0.24504913251170982, | |
| "acc": 0.925, | |
| "ece": 0.06460526620007352, | |
| "brier": 0.13290747891560833, | |
| "mean_conf": 0.9730792763846988, | |
| "confident_error_rate": 0.05, | |
| "coverage_at_0_9": 0.93125, | |
| "accuracy_at_0_9": 0.9463087248322147, | |
| "confidence_bias": 0.04807927638469878, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 149, | |
| "errors": 8, | |
| "error_rate": 0.053691275167785234 | |
| }, | |
| "0.95": { | |
| "n": 140, | |
| "errors": 4, | |
| "error_rate": 0.02857142857142857 | |
| }, | |
| "0.99": { | |
| "n": 119, | |
| "errors": 2, | |
| "error_rate": 0.01680672268907563 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9978675951482209 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9765625, | |
| "confidence_cutoff": 0.9823942798793209 | |
| } | |
| } | |
| }, | |
| "amazon": { | |
| "n": 80, | |
| "nll": 0.9351267400695216, | |
| "acc": 0.575, | |
| "ece": 0.11115798555393083, | |
| "brier": 0.49294492359563036, | |
| "mean_conf": 0.6772779379261196, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.125, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": 0.10227793792611961, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 10, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 8, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 2, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8, | |
| "confidence_cutoff": 0.6379188511283084 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.671875, | |
| "confidence_cutoff": 0.502037328376669 | |
| } | |
| }, | |
| "score_mae": 0.5075561225779872, | |
| "ranked_probability_score": 0.080820187811701 | |
| }, | |
| "banking77": { | |
| "n": 80, | |
| "nll": 0.43079029036341554, | |
| "acc": 0.8875, | |
| "ece": 0.06732893165588627, | |
| "brier": 0.18477308183131425, | |
| "mean_conf": 0.9039776123393157, | |
| "confident_error_rate": 0.025, | |
| "coverage_at_0_9": 0.725, | |
| "accuracy_at_0_9": 0.9655172413793104, | |
| "confidence_bias": 0.016477612339315773, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 58, | |
| "errors": 2, | |
| "error_rate": 0.034482758620689655 | |
| }, | |
| "0.95": { | |
| "n": 52, | |
| "errors": 2, | |
| "error_rate": 0.038461538461538464 | |
| }, | |
| "0.99": { | |
| "n": 45, | |
| "errors": 1, | |
| "error_rate": 0.022222222222222223 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.975, | |
| "confidence_cutoff": 0.9935029742060698 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.921875, | |
| "confidence_cutoff": 0.8365431118140658 | |
| } | |
| } | |
| }, | |
| "boolq": { | |
| "n": 80, | |
| "nll": 0.5692292766048166, | |
| "acc": 0.85, | |
| "ece": 0.12247463111478857, | |
| "brier": 0.2618067497766957, | |
| "mean_conf": 0.9492440640971932, | |
| "confident_error_rate": 0.075, | |
| "coverage_at_0_9": 0.825, | |
| "accuracy_at_0_9": 0.9090909090909091, | |
| "confidence_bias": 0.09924406409719322, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 66, | |
| "errors": 6, | |
| "error_rate": 0.09090909090909091 | |
| }, | |
| "0.95": { | |
| "n": 64, | |
| "errors": 6, | |
| "error_rate": 0.09375 | |
| }, | |
| "0.99": { | |
| "n": 51, | |
| "errors": 3, | |
| "error_rate": 0.058823529411764705 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.95, | |
| "confidence_cutoff": 0.997317726504134 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.90625, | |
| "confidence_cutoff": 0.95449530011607 | |
| } | |
| } | |
| }, | |
| "composition_atom": { | |
| "n": 16, | |
| "nll": 0.0003952823957936077, | |
| "acc": 1.0, | |
| "ece": 0.0003950750452166307, | |
| "brier": 8.287164835155329e-07, | |
| "mean_conf": 0.9996049249547834, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.0003950750452166307, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5625, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9998752945554933 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9991505367027831 | |
| } | |
| } | |
| }, | |
| "composition_conditional": { | |
| "n": 16, | |
| "nll": 0.5019992150539019, | |
| "acc": 0.75, | |
| "ece": 0.1902398997443278, | |
| "brier": 0.3599381608463673, | |
| "mean_conf": 0.7634217561511185, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.3125, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": 0.013421756151118514, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 5, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 4, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.744914077070775 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.6923076923076923, | |
| "confidence_cutoff": 0.598910492691085 | |
| } | |
| } | |
| }, | |
| "composition_conjunction": { | |
| "n": 16, | |
| "nll": 0.0016869379224466628, | |
| "acc": 1.0, | |
| "ece": 0.0016845554564111076, | |
| "brier": 9.51034121939477e-06, | |
| "mean_conf": 0.9983154445435889, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.0016845554564111076, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9993657412742123 | |
| }, | |
| "0.8": { | |
| "coverage": 0.875, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9967811070502508 | |
| } | |
| } | |
| }, | |
| "composition_disjunction": { | |
| "n": 16, | |
| "nll": 0.5432369947671923, | |
| "acc": 0.75, | |
| "ece": 0.20894149081016355, | |
| "brier": 0.38181577671679756, | |
| "mean_conf": 0.9589414908101636, | |
| "confident_error_rate": 0.125, | |
| "coverage_at_0_9": 0.875, | |
| "accuracy_at_0_9": 0.8571428571428571, | |
| "confidence_bias": 0.20894149081016355, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 14, | |
| "errors": 2, | |
| "error_rate": 0.14285714285714285 | |
| }, | |
| "0.95": { | |
| "n": 12, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 9, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9934810838022192 | |
| }, | |
| "0.8": { | |
| "coverage": 0.875, | |
| "accuracy": 0.8571428571428571, | |
| "confidence_cutoff": 0.9103974050520304 | |
| } | |
| } | |
| }, | |
| "composition_exception": { | |
| "n": 16, | |
| "nll": 0.1551373177250381, | |
| "acc": 0.9375, | |
| "ece": 0.05574866742756779, | |
| "brier": 0.10455386244590524, | |
| "mean_conf": 0.9932486674275678, | |
| "confident_error_rate": 0.0625, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 0.9375, | |
| "confidence_bias": 0.05574866742756779, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 1, | |
| "error_rate": 0.0625 | |
| }, | |
| "0.95": { | |
| "n": 15, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 15, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9990134402017437 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9962711415415374 | |
| } | |
| } | |
| }, | |
| "composition_negation": { | |
| "n": 16, | |
| "nll": 0.003782036848642094, | |
| "acc": 1.0, | |
| "ece": 0.0037719108100304766, | |
| "brier": 4.034726206637131e-05, | |
| "mean_conf": 0.9962280891899695, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.0037719108100304766, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5625, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9958158766414407 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.994085511181873 | |
| } | |
| } | |
| }, | |
| "composition_nested_and": { | |
| "n": 16, | |
| "nll": 0.6928116765567718, | |
| "acc": 0.8125, | |
| "ece": 0.17331043627000298, | |
| "brier": 0.35599699730189693, | |
| "mean_conf": 0.985810436270003, | |
| "confident_error_rate": 0.1875, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 0.8125, | |
| "confidence_bias": 0.17331043627000298, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 3, | |
| "error_rate": 0.1875 | |
| }, | |
| "0.95": { | |
| "n": 16, | |
| "errors": 3, | |
| "error_rate": 0.1875 | |
| }, | |
| "0.99": { | |
| "n": 9, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5625, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9903228993107223 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.9230769230769231, | |
| "confidence_cutoff": 0.9728845808970246 | |
| } | |
| } | |
| }, | |
| "composition_nested_or": { | |
| "n": 16, | |
| "nll": 0.016058082714040366, | |
| "acc": 1.0, | |
| "ece": 0.01574006900953684, | |
| "brier": 0.0012317899851369283, | |
| "mean_conf": 0.9842599309904632, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.01574006900953684, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 15, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 11, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9949054004552627 | |
| }, | |
| "0.8": { | |
| "coverage": 0.875, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.95337582700002 | |
| } | |
| } | |
| }, | |
| "contrastive_age_eligibility": { | |
| "n": 24, | |
| "nll": 5.0117464260243464e-05, | |
| "acc": 1.0, | |
| "ece": 5.011085399608817e-05, | |
| "brier": 2.6433003191483944e-08, | |
| "mean_conf": 0.9999498891460039, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -5.011085399608817e-05, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9999858161181404 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8333333333333334, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9999397590096755 | |
| } | |
| } | |
| }, | |
| "contrastive_quantity_limit": { | |
| "n": 24, | |
| "nll": 0.009079523471059399, | |
| "acc": 1.0, | |
| "ece": 0.008472354885487591, | |
| "brier": 0.0021768652024033363, | |
| "mean_conf": 0.9915276451145125, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.9583333333333334, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.008472354885487476, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 23, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 23, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 22, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.999989081677853 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8333333333333334, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.99856468186255 | |
| } | |
| } | |
| }, | |
| "contrastive_return_window": { | |
| "n": 24, | |
| "nll": 0.48560927718172114, | |
| "acc": 0.875, | |
| "ece": 0.14552832909469873, | |
| "brier": 0.24864852616575747, | |
| "mean_conf": 0.948072710063192, | |
| "confident_error_rate": 0.125, | |
| "coverage_at_0_9": 0.8333333333333334, | |
| "accuracy_at_0_9": 0.85, | |
| "confidence_bias": 0.07307271006319205, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 20, | |
| "errors": 3, | |
| "error_rate": 0.15 | |
| }, | |
| "0.95": { | |
| "n": 17, | |
| "errors": 2, | |
| "error_rate": 0.11764705882352941 | |
| }, | |
| "0.99": { | |
| "n": 10, | |
| "errors": 1, | |
| "error_rate": 0.1 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9166666666666666, | |
| "confidence_cutoff": 0.9877289487620399 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8333333333333334, | |
| "accuracy": 0.85, | |
| "confidence_cutoff": 0.9337131559678992 | |
| } | |
| } | |
| }, | |
| "contrastive_spend_threshold": { | |
| "n": 24, | |
| "nll": 1.653518426304661e-06, | |
| "acc": 1.0, | |
| "ece": 1.6535165444642175e-06, | |
| "brier": 5.729719218057532e-12, | |
| "mean_conf": 0.9999983464834555, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -1.6535165444642175e-06, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.99999888450003 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8333333333333334, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9999975114889905 | |
| } | |
| } | |
| }, | |
| "dbpedia14": { | |
| "n": 80, | |
| "nll": 0.352393166043517, | |
| "acc": 0.9625, | |
| "ece": 0.032725262113352305, | |
| "brier": 0.06411155116331552, | |
| "mean_conf": 0.9952252621133522, | |
| "confident_error_rate": 0.025, | |
| "coverage_at_0_9": 0.9875, | |
| "accuracy_at_0_9": 0.9746835443037974, | |
| "confidence_bias": 0.03272526211335214, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 79, | |
| "errors": 2, | |
| "error_rate": 0.02531645569620253 | |
| }, | |
| "0.95": { | |
| "n": 78, | |
| "errors": 2, | |
| "error_rate": 0.02564102564102564 | |
| }, | |
| "0.99": { | |
| "n": 77, | |
| "errors": 2, | |
| "error_rate": 0.025974025974025976 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.975, | |
| "confidence_cutoff": 0.9999907673769224 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.984375, | |
| "confidence_cutoff": 0.9999730956432238 | |
| } | |
| } | |
| }, | |
| "imdb": { | |
| "n": 80, | |
| "nll": 0.28706193547875214, | |
| "acc": 0.925, | |
| "ece": 0.048198774610847796, | |
| "brier": 0.11596031960348152, | |
| "mean_conf": 0.9730559394793742, | |
| "confident_error_rate": 0.025, | |
| "coverage_at_0_9": 0.9125, | |
| "accuracy_at_0_9": 0.9726027397260274, | |
| "confidence_bias": 0.04805593947937414, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 73, | |
| "errors": 2, | |
| "error_rate": 0.0273972602739726 | |
| }, | |
| "0.95": { | |
| "n": 72, | |
| "errors": 2, | |
| "error_rate": 0.027777777777777776 | |
| }, | |
| "0.99": { | |
| "n": 65, | |
| "errors": 2, | |
| "error_rate": 0.03076923076923077 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.975, | |
| "confidence_cutoff": 0.9995681686832004 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.96875, | |
| "confidence_cutoff": 0.9938746881478918 | |
| } | |
| } | |
| }, | |
| "mnli": { | |
| "n": 80, | |
| "nll": 0.888157101486706, | |
| "acc": 0.875, | |
| "ece": 0.11179936394023159, | |
| "brier": 0.23549742863127898, | |
| "mean_conf": 0.9842962604308741, | |
| "confident_error_rate": 0.1125, | |
| "coverage_at_0_9": 0.9375, | |
| "accuracy_at_0_9": 0.88, | |
| "confidence_bias": 0.10929626043087415, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 75, | |
| "errors": 9, | |
| "error_rate": 0.12 | |
| }, | |
| "0.95": { | |
| "n": 74, | |
| "errors": 8, | |
| "error_rate": 0.10810810810810811 | |
| }, | |
| "0.99": { | |
| "n": 72, | |
| "errors": 7, | |
| "error_rate": 0.09722222222222222 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.925, | |
| "confidence_cutoff": 0.9999273470630932 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9375, | |
| "confidence_cutoff": 0.9993270111828301 | |
| } | |
| } | |
| }, | |
| "sst5": { | |
| "n": 80, | |
| "nll": 0.9756452930629113, | |
| "acc": 0.575, | |
| "ece": 0.09928021239575943, | |
| "brier": 0.5462656054729043, | |
| "mean_conf": 0.6724271814545316, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.0125, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": 0.09742718145453166, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 1, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.65, | |
| "confidence_cutoff": 0.6593349139603858 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.640625, | |
| "confidence_cutoff": 0.5640514758708144 | |
| } | |
| }, | |
| "score_mae": 0.49195305628447744, | |
| "ranked_probability_score": 0.07798845711611709 | |
| }, | |
| "trec": { | |
| "n": 80, | |
| "nll": 0.29521071463972587, | |
| "acc": 0.9625, | |
| "ece": 0.04335316463941281, | |
| "brier": 0.0751019611146547, | |
| "mean_conf": 0.9896871670671918, | |
| "confident_error_rate": 0.0375, | |
| "coverage_at_0_9": 0.9625, | |
| "accuracy_at_0_9": 0.961038961038961, | |
| "confidence_bias": 0.02718716706719182, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 77, | |
| "errors": 3, | |
| "error_rate": 0.03896103896103896 | |
| }, | |
| "0.95": { | |
| "n": 76, | |
| "errors": 2, | |
| "error_rate": 0.02631578947368421 | |
| }, | |
| "0.99": { | |
| "n": 74, | |
| "errors": 2, | |
| "error_rate": 0.02702702702702703 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9999414239640388 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.96875, | |
| "confidence_cutoff": 0.9994822242519307 | |
| } | |
| } | |
| }, | |
| "yelp": { | |
| "n": 80, | |
| "nll": 0.7097296189599043, | |
| "acc": 0.675, | |
| "ece": 0.11383632871560348, | |
| "brier": 0.4167826081797442, | |
| "mean_conf": 0.7656471258802438, | |
| "confident_error_rate": 0.0125, | |
| "coverage_at_0_9": 0.225, | |
| "accuracy_at_0_9": 0.9444444444444444, | |
| "confidence_bias": 0.0906471258802437, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 18, | |
| "errors": 1, | |
| "error_rate": 0.05555555555555555 | |
| }, | |
| "0.95": { | |
| "n": 14, | |
| "errors": 1, | |
| "error_rate": 0.07142857142857142 | |
| }, | |
| "0.99": { | |
| "n": 2, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.825, | |
| "confidence_cutoff": 0.7783977437192959 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.75, | |
| "confidence_cutoff": 0.6039501956802334 | |
| } | |
| }, | |
| "score_mae": 0.36231248773578584, | |
| "ranked_probability_score": 0.056380832934899795 | |
| }, | |
| "yelp_yn": { | |
| "n": 80, | |
| "nll": 0.19993345467765128, | |
| "acc": 0.9, | |
| "ece": 0.05032214018379682, | |
| "brier": 0.11241742390597173, | |
| "mean_conf": 0.9503221401837969, | |
| "confident_error_rate": 0.025, | |
| "coverage_at_0_9": 0.875, | |
| "accuracy_at_0_9": 0.9714285714285714, | |
| "confidence_bias": 0.05032214018379688, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 70, | |
| "errors": 2, | |
| "error_rate": 0.02857142857142857 | |
| }, | |
| "0.95": { | |
| "n": 63, | |
| "errors": 1, | |
| "error_rate": 0.015873015873015872 | |
| }, | |
| "0.99": { | |
| "n": 53, | |
| "errors": 1, | |
| "error_rate": 0.018867924528301886 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9976448653637842 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.984375, | |
| "confidence_cutoff": 0.9449425130872713 | |
| } | |
| } | |
| } | |
| }, | |
| "variants": { | |
| "clean": { | |
| "n": 1264, | |
| "nll": 0.47096940162233064, | |
| "acc": 0.8536392405063291, | |
| "ece": 0.0654366260188363, | |
| "brier": 0.21322991316558398, | |
| "mean_conf": 0.9178467644873951, | |
| "confident_error_rate": 0.041930379746835444, | |
| "coverage_at_0_9": 0.757120253164557, | |
| "accuracy_at_0_9": 0.9446185997910136, | |
| "confidence_bias": 0.064207523981066, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 957, | |
| "errors": 53, | |
| "error_rate": 0.055381400208986416 | |
| }, | |
| "0.95": { | |
| "n": 911, | |
| "errors": 41, | |
| "error_rate": 0.04500548847420417 | |
| }, | |
| "0.99": { | |
| "n": 795, | |
| "errors": 27, | |
| "error_rate": 0.033962264150943396 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5007911392405063, | |
| "accuracy": 0.9778830963665087, | |
| "confidence_cutoff": 0.9976429106296094 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8006329113924051, | |
| "accuracy": 0.9268774703557312, | |
| "confidence_cutoff": 0.829332684596399 | |
| } | |
| }, | |
| "score_mae": 0.45394055553275015, | |
| "ranked_probability_score": 0.0717298259542393 | |
| }, | |
| "none_absent": { | |
| "n": 60, | |
| "nll": 0.2677466069278561, | |
| "acc": 0.9, | |
| "ece": 0.04481354248148454, | |
| "brier": 0.12271424327012309, | |
| "mean_conf": 0.9448135424814846, | |
| "confident_error_rate": 0.016666666666666666, | |
| "coverage_at_0_9": 0.8333333333333334, | |
| "accuracy_at_0_9": 0.98, | |
| "confidence_bias": 0.044813542481484614, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 50, | |
| "errors": 1, | |
| "error_rate": 0.02 | |
| }, | |
| "0.95": { | |
| "n": 43, | |
| "errors": 1, | |
| "error_rate": 0.023255813953488372 | |
| }, | |
| "0.99": { | |
| "n": 37, | |
| "errors": 1, | |
| "error_rate": 0.02702702702702703 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9666666666666667, | |
| "confidence_cutoff": 0.9961424168910517 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9791666666666666, | |
| "confidence_cutoff": 0.9062051924985772 | |
| } | |
| } | |
| }, | |
| "none_present": { | |
| "n": 60, | |
| "nll": 0.559854866685432, | |
| "acc": 0.8666666666666667, | |
| "ece": 0.05224881763254032, | |
| "brier": 0.19237936181427168, | |
| "mean_conf": 0.9124065831677801, | |
| "confident_error_rate": 0.03333333333333333, | |
| "coverage_at_0_9": 0.7, | |
| "accuracy_at_0_9": 0.9523809523809523, | |
| "confidence_bias": 0.04573991650111342, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 42, | |
| "errors": 2, | |
| "error_rate": 0.047619047619047616 | |
| }, | |
| "0.95": { | |
| "n": 40, | |
| "errors": 1, | |
| "error_rate": 0.025 | |
| }, | |
| "0.99": { | |
| "n": 26, | |
| "errors": 1, | |
| "error_rate": 0.038461538461538464 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9666666666666667, | |
| "confidence_cutoff": 0.984968995229399 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9375, | |
| "confidence_cutoff": 0.846107297275265 | |
| } | |
| } | |
| }, | |
| "permuted": { | |
| "n": 84, | |
| "nll": 0.26296669876591966, | |
| "acc": 0.9642857142857143, | |
| "ece": 0.038646583486446284, | |
| "brier": 0.06057403759598415, | |
| "mean_conf": 0.9717462012238827, | |
| "confident_error_rate": 0.023809523809523808, | |
| "coverage_at_0_9": 0.9166666666666666, | |
| "accuracy_at_0_9": 0.974025974025974, | |
| "confidence_bias": 0.007460486938168365, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 77, | |
| "errors": 2, | |
| "error_rate": 0.025974025974025976 | |
| }, | |
| "0.95": { | |
| "n": 74, | |
| "errors": 2, | |
| "error_rate": 0.02702702702702703 | |
| }, | |
| "0.99": { | |
| "n": 63, | |
| "errors": 1, | |
| "error_rate": 0.015873015873015872 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9761904761904762, | |
| "confidence_cutoff": 0.9997112982180948 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8095238095238095, | |
| "accuracy": 0.9852941176470589, | |
| "confidence_cutoff": 0.9815243603295957 | |
| } | |
| } | |
| } | |
| }, | |
| "heldout_tasks": {}, | |
| "permutation": { | |
| "n": 60, | |
| "mean_max_delta": 0.02218218472923495, | |
| "flip_rate": 0.016666666666666666 | |
| }, | |
| "temperature": 1.6817928305074288, | |
| "calibrated_clean": { | |
| "n": 1264, | |
| "nll": 0.3928855341201546, | |
| "acc": 0.8536392405063291, | |
| "ece": 0.02783520772554953, | |
| "brier": 0.2051613865533046, | |
| "mean_conf": 0.8663882981180834, | |
| "confident_error_rate": 0.0245253164556962, | |
| "coverage_at_0_9": 0.6677215189873418, | |
| "accuracy_at_0_9": 0.9632701421800948, | |
| "confidence_bias": 0.012749057611754266, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 844, | |
| "errors": 31, | |
| "error_rate": 0.03672985781990521 | |
| }, | |
| "0.95": { | |
| "n": 736, | |
| "errors": 24, | |
| "error_rate": 0.03260869565217391 | |
| }, | |
| "0.99": { | |
| "n": 419, | |
| "errors": 11, | |
| "error_rate": 0.026252983293556086 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9746835443037974, | |
| "confidence_cutoff": 0.970184381160819 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8006329113924051, | |
| "accuracy": 0.9288537549407114, | |
| "confidence_cutoff": 0.6712393014030038 | |
| } | |
| }, | |
| "score_mae": 0.47852238769255256, | |
| "ranked_probability_score": 0.07118696124768613 | |
| }, | |
| "metric_policy": { | |
| "nll_floor": 1e-09, | |
| "renormalize_returned_probabilities": true, | |
| "raw_sums_outside_1e_5": 0, | |
| "returned_zeros": 0 | |
| }, | |
| "coverage": { | |
| "requested_records": 1204, | |
| "requested_questions": 1468, | |
| "evaluated_records": 1204, | |
| "evaluated_questions": 1468, | |
| "rejected_records": 0, | |
| "truncated_records": 0 | |
| }, | |
| "latency_ms": { | |
| "median": 51.816085000155, | |
| "p95": 173.47100315005264 | |
| }, | |
| "transfer": { | |
| "objective": -0.6958444571045241, | |
| "paired_flip": { | |
| "pairs": 64, | |
| "flip_rate": 0.734375, | |
| "both_correct_rate": 0.734375, | |
| "invariant_pairs": 24, | |
| "invariance_rate": 0.9583333333333334, | |
| "invariant_both_correct_rate": 0.8333333333333334 | |
| }, | |
| "clean": { | |
| "n": 656, | |
| "nll": 0.6976569692441479, | |
| "acc": 0.7896341463414634, | |
| "ece": 0.10175734176675508, | |
| "brier": 0.32825872907075376, | |
| "mean_conf": 0.8873174290671823, | |
| "confident_error_rate": 0.08231707317073171, | |
| "coverage_at_0_9": 0.6783536585365854, | |
| "accuracy_at_0_9": 0.8786516853932584, | |
| "confidence_bias": 0.0976832827257188, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 445, | |
| "errors": 54, | |
| "error_rate": 0.12134831460674157 | |
| }, | |
| "0.95": { | |
| "n": 400, | |
| "errors": 39, | |
| "error_rate": 0.0975 | |
| }, | |
| "0.99": { | |
| "n": 262, | |
| "errors": 19, | |
| "error_rate": 0.07251908396946564 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9146341463414634, | |
| "confidence_cutoff": 0.9790200062252951 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8003048780487805, | |
| "accuracy": 0.8476190476190476, | |
| "confidence_cutoff": 0.75841224193573 | |
| } | |
| }, | |
| "score_mae": 0.45259144588683287, | |
| "ranked_probability_score": 0.20820131499001437 | |
| }, | |
| "tasks": { | |
| "composition_held_and_or": { | |
| "n": 32, | |
| "nll": 0.7456535624236502, | |
| "acc": 0.78125, | |
| "ece": 0.17958555475745153, | |
| "brier": 0.3359556523619844, | |
| "mean_conf": 0.9225585020071866, | |
| "confident_error_rate": 0.09375, | |
| "coverage_at_0_9": 0.78125, | |
| "accuracy_at_0_9": 0.88, | |
| "confidence_bias": 0.1413085020071866, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 25, | |
| "errors": 3, | |
| "error_rate": 0.12 | |
| }, | |
| "0.95": { | |
| "n": 25, | |
| "errors": 3, | |
| "error_rate": 0.12 | |
| }, | |
| "0.99": { | |
| "n": 24, | |
| "errors": 3, | |
| "error_rate": 0.125 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9375, | |
| "confidence_cutoff": 0.997011398040403 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.8846153846153846, | |
| "confidence_cutoff": 0.8473277470765 | |
| } | |
| } | |
| }, | |
| "composition_held_conditional": { | |
| "n": 32, | |
| "nll": 0.3332870284344554, | |
| "acc": 0.875, | |
| "ece": 0.13686614272503672, | |
| "brier": 0.21284482647640357, | |
| "mean_conf": 0.7938492418508554, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.40625, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.08115075814914463, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 13, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 11, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 2, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.7695624232292175 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.8461538461538461, | |
| "confidence_cutoff": 0.6414507818654412 | |
| } | |
| } | |
| }, | |
| "composition_held_or_not": { | |
| "n": 32, | |
| "nll": 0.15833464456584684, | |
| "acc": 0.96875, | |
| "ece": 0.07149179201435074, | |
| "brier": 0.08199247173625367, | |
| "mean_conf": 0.8972582079856491, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.625, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.07149179201435085, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 20, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 17, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9687480655265324 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.791817462688164 | |
| } | |
| } | |
| }, | |
| "contrastive_authorization": { | |
| "n": 40, | |
| "nll": 0.0003820984206954688, | |
| "acc": 1.0, | |
| "ece": 0.0003819875787638427, | |
| "brier": 4.431386725588458e-07, | |
| "mean_conf": 0.9996180124212362, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.0003819875787638427, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 40, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 40, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 40, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9997253036940529 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9993739016215527 | |
| } | |
| } | |
| }, | |
| "contrastive_deadline": { | |
| "n": 40, | |
| "nll": 2.381891379766067, | |
| "acc": 0.525, | |
| "ece": 0.4445411347971735, | |
| "brier": 0.8313373993065791, | |
| "mean_conf": 0.9505143411607072, | |
| "confident_error_rate": 0.35, | |
| "coverage_at_0_9": 0.85, | |
| "accuracy_at_0_9": 0.5882352941176471, | |
| "confidence_bias": 0.4255143411607072, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 34, | |
| "errors": 14, | |
| "error_rate": 0.4117647058823529 | |
| }, | |
| "0.95": { | |
| "n": 32, | |
| "errors": 12, | |
| "error_rate": 0.375 | |
| }, | |
| "0.99": { | |
| "n": 25, | |
| "errors": 8, | |
| "error_rate": 0.32 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.75, | |
| "confidence_cutoff": 0.9963926273512684 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.625, | |
| "confidence_cutoff": 0.9538424791841433 | |
| } | |
| }, | |
| "score_mae": 0.45259144588683287, | |
| "ranked_probability_score": 0.20820131499001437 | |
| }, | |
| "emotion": { | |
| "n": 80, | |
| "nll": 1.2263539675718254, | |
| "acc": 0.6625, | |
| "ece": 0.14963081389980043, | |
| "brier": 0.5245823410352607, | |
| "mean_conf": 0.7403260524211139, | |
| "confident_error_rate": 0.075, | |
| "coverage_at_0_9": 0.3, | |
| "accuracy_at_0_9": 0.75, | |
| "confidence_bias": 0.07782605242111396, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 24, | |
| "errors": 6, | |
| "error_rate": 0.25 | |
| }, | |
| "0.95": { | |
| "n": 15, | |
| "errors": 3, | |
| "error_rate": 0.2 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.7, | |
| "confidence_cutoff": 0.797799825668335 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.71875, | |
| "confidence_cutoff": 0.5535729084804598 | |
| } | |
| } | |
| }, | |
| "mmlu": { | |
| "n": 80, | |
| "nll": 0.8274234004446729, | |
| "acc": 0.65, | |
| "ece": 0.18294625228578548, | |
| "brier": 0.42173974468001135, | |
| "mean_conf": 0.7970014825127686, | |
| "confident_error_rate": 0.0125, | |
| "coverage_at_0_9": 0.45, | |
| "accuracy_at_0_9": 0.9722222222222222, | |
| "confidence_bias": 0.14700148251276857, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 36, | |
| "errors": 1, | |
| "error_rate": 0.027777777777777776 | |
| }, | |
| "0.95": { | |
| "n": 30, | |
| "errors": 1, | |
| "error_rate": 0.03333333333333333 | |
| }, | |
| "0.99": { | |
| "n": 23, | |
| "errors": 1, | |
| "error_rate": 0.043478260869565216 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.925, | |
| "confidence_cutoff": 0.8614584193836526 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.765625, | |
| "confidence_cutoff": 0.5891912744154578 | |
| } | |
| } | |
| }, | |
| "paws": { | |
| "n": 80, | |
| "nll": 0.9617935415055063, | |
| "acc": 0.725, | |
| "ece": 0.23949173491457365, | |
| "brier": 0.4834975530791322, | |
| "mean_conf": 0.9521085317061271, | |
| "confident_error_rate": 0.2, | |
| "coverage_at_0_9": 0.875, | |
| "accuracy_at_0_9": 0.7714285714285715, | |
| "confidence_bias": 0.2271085317061271, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 70, | |
| "errors": 16, | |
| "error_rate": 0.22857142857142856 | |
| }, | |
| "0.95": { | |
| "n": 63, | |
| "errors": 13, | |
| "error_rate": 0.20634920634920634 | |
| }, | |
| "0.99": { | |
| "n": 33, | |
| "errors": 5, | |
| "error_rate": 0.15151515151515152 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.825, | |
| "confidence_cutoff": 0.9852659738611202 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.796875, | |
| "confidence_cutoff": 0.9460779604702808 | |
| } | |
| } | |
| }, | |
| "qnli": { | |
| "n": 80, | |
| "nll": 0.28925677259060756, | |
| "acc": 0.8875, | |
| "ece": 0.07341603957619632, | |
| "brier": 0.18009551904084561, | |
| "mean_conf": 0.9343277556724386, | |
| "confident_error_rate": 0.0625, | |
| "coverage_at_0_9": 0.8125, | |
| "accuracy_at_0_9": 0.9230769230769231, | |
| "confidence_bias": 0.04682775567243869, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 65, | |
| "errors": 5, | |
| "error_rate": 0.07692307692307693 | |
| }, | |
| "0.95": { | |
| "n": 59, | |
| "errors": 2, | |
| "error_rate": 0.03389830508474576 | |
| }, | |
| "0.99": { | |
| "n": 32, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9858154563418124 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9375, | |
| "confidence_cutoff": 0.9094368151617287 | |
| } | |
| } | |
| }, | |
| "sciq": { | |
| "n": 80, | |
| "nll": 0.05181181132785821, | |
| "acc": 0.9875, | |
| "ece": 0.03175391680550414, | |
| "brier": 0.02513979546365842, | |
| "mean_conf": 0.9682344692294615, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.9, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.019265530770538497, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 72, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 69, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 64, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9995528209501393 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9920047199640097 | |
| } | |
| } | |
| }, | |
| "tweet_offensive": { | |
| "n": 80, | |
| "nll": 0.6781008210985803, | |
| "acc": 0.75, | |
| "ece": 0.15591583779095422, | |
| "brier": 0.3886805236287897, | |
| "mean_conf": 0.8634720692805369, | |
| "confident_error_rate": 0.1125, | |
| "coverage_at_0_9": 0.575, | |
| "accuracy_at_0_9": 0.8043478260869565, | |
| "confidence_bias": 0.11347206928053688, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 46, | |
| "errors": 9, | |
| "error_rate": 0.1956521739130435 | |
| }, | |
| "0.95": { | |
| "n": 39, | |
| "errors": 5, | |
| "error_rate": 0.1282051282051282 | |
| }, | |
| "0.99": { | |
| "n": 19, | |
| "errors": 2, | |
| "error_rate": 0.10526315789473684 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.875, | |
| "confidence_cutoff": 0.9428173514340226 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.796875, | |
| "confidence_cutoff": 0.7029978842436637 | |
| } | |
| } | |
| } | |
| }, | |
| "variants": { | |
| "clean": { | |
| "n": 656, | |
| "nll": 0.6976569692441479, | |
| "acc": 0.7896341463414634, | |
| "ece": 0.10175734176675508, | |
| "brier": 0.32825872907075376, | |
| "mean_conf": 0.8873174290671823, | |
| "confident_error_rate": 0.08231707317073171, | |
| "coverage_at_0_9": 0.6783536585365854, | |
| "accuracy_at_0_9": 0.8786516853932584, | |
| "confidence_bias": 0.0976832827257188, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 445, | |
| "errors": 54, | |
| "error_rate": 0.12134831460674157 | |
| }, | |
| "0.95": { | |
| "n": 400, | |
| "errors": 39, | |
| "error_rate": 0.0975 | |
| }, | |
| "0.99": { | |
| "n": 262, | |
| "errors": 19, | |
| "error_rate": 0.07251908396946564 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9146341463414634, | |
| "confidence_cutoff": 0.9790200062252951 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8003048780487805, | |
| "accuracy": 0.8476190476190476, | |
| "confidence_cutoff": 0.75841224193573 | |
| } | |
| }, | |
| "score_mae": 0.45259144588683287, | |
| "ranked_probability_score": 0.20820131499001437 | |
| }, | |
| "none_absent": { | |
| "n": 36, | |
| "nll": 0.33905683993850544, | |
| "acc": 0.8611111111111112, | |
| "ece": 0.1400074421282228, | |
| "brier": 0.21437423073204523, | |
| "mean_conf": 0.8859076835917248, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.6388888888888888, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": 0.02479657248061362, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 23, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 8, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9417026261862247 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8055555555555556, | |
| "accuracy": 0.9310344827586207, | |
| "confidence_cutoff": 0.8415947782198402 | |
| } | |
| } | |
| }, | |
| "none_present": { | |
| "n": 36, | |
| "nll": 1.1768707979298563, | |
| "acc": 0.5833333333333334, | |
| "ece": 0.2854879866906561, | |
| "brier": 0.5694624721478241, | |
| "mean_conf": 0.8057177842126594, | |
| "confident_error_rate": 0.08333333333333333, | |
| "coverage_at_0_9": 0.5277777777777778, | |
| "accuracy_at_0_9": 0.8421052631578947, | |
| "confidence_bias": 0.222384450879326, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 19, | |
| "errors": 3, | |
| "error_rate": 0.15789473684210525 | |
| }, | |
| "0.95": { | |
| "n": 17, | |
| "errors": 2, | |
| "error_rate": 0.11764705882352941 | |
| }, | |
| "0.99": { | |
| "n": 9, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8333333333333334, | |
| "confidence_cutoff": 0.9343934025851296 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8055555555555556, | |
| "accuracy": 0.6206896551724138, | |
| "confidence_cutoff": 0.6258565324429688 | |
| } | |
| } | |
| }, | |
| "permuted": { | |
| "n": 36, | |
| "nll": 0.5841450773580558, | |
| "acc": 0.8055555555555556, | |
| "ece": 0.09385834965988679, | |
| "brier": 0.2762947760202843, | |
| "mean_conf": 0.8743721324628857, | |
| "confident_error_rate": 0.05555555555555555, | |
| "coverage_at_0_9": 0.6666666666666666, | |
| "accuracy_at_0_9": 0.9166666666666666, | |
| "confidence_bias": 0.06881657690733012, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 24, | |
| "errors": 2, | |
| "error_rate": 0.08333333333333333 | |
| }, | |
| "0.95": { | |
| "n": 22, | |
| "errors": 2, | |
| "error_rate": 0.09090909090909091 | |
| }, | |
| "0.99": { | |
| "n": 14, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8888888888888888, | |
| "confidence_cutoff": 0.9793524354721298 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8055555555555556, | |
| "accuracy": 0.896551724137931, | |
| "confidence_cutoff": 0.7735453959337681 | |
| } | |
| } | |
| } | |
| }, | |
| "heldout_tasks": { | |
| "composition_held_and_or": { | |
| "n": 32, | |
| "nll": 0.7456535624236502, | |
| "acc": 0.78125, | |
| "ece": 0.17958555475745153, | |
| "brier": 0.3359556523619844, | |
| "mean_conf": 0.9225585020071866, | |
| "confident_error_rate": 0.09375, | |
| "coverage_at_0_9": 0.78125, | |
| "accuracy_at_0_9": 0.88, | |
| "confidence_bias": 0.1413085020071866, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 25, | |
| "errors": 3, | |
| "error_rate": 0.12 | |
| }, | |
| "0.95": { | |
| "n": 25, | |
| "errors": 3, | |
| "error_rate": 0.12 | |
| }, | |
| "0.99": { | |
| "n": 24, | |
| "errors": 3, | |
| "error_rate": 0.125 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9375, | |
| "confidence_cutoff": 0.997011398040403 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.8846153846153846, | |
| "confidence_cutoff": 0.8473277470765 | |
| } | |
| } | |
| }, | |
| "composition_held_conditional": { | |
| "n": 32, | |
| "nll": 0.3332870284344554, | |
| "acc": 0.875, | |
| "ece": 0.13686614272503672, | |
| "brier": 0.21284482647640357, | |
| "mean_conf": 0.7938492418508554, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.40625, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.08115075814914463, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 13, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 11, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 2, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.7695624232292175 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.8461538461538461, | |
| "confidence_cutoff": 0.6414507818654412 | |
| } | |
| } | |
| }, | |
| "composition_held_or_not": { | |
| "n": 32, | |
| "nll": 0.15833464456584684, | |
| "acc": 0.96875, | |
| "ece": 0.07149179201435074, | |
| "brier": 0.08199247173625367, | |
| "mean_conf": 0.8972582079856491, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.625, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.07149179201435085, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 20, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 17, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9687480655265324 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.791817462688164 | |
| } | |
| } | |
| }, | |
| "contrastive_authorization": { | |
| "n": 40, | |
| "nll": 0.0003820984206954688, | |
| "acc": 1.0, | |
| "ece": 0.0003819875787638427, | |
| "brier": 4.431386725588458e-07, | |
| "mean_conf": 0.9996180124212362, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.0003819875787638427, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 40, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 40, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 40, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9997253036940529 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9993739016215527 | |
| } | |
| } | |
| }, | |
| "contrastive_deadline": { | |
| "n": 40, | |
| "nll": 2.381891379766067, | |
| "acc": 0.525, | |
| "ece": 0.4445411347971735, | |
| "brier": 0.8313373993065791, | |
| "mean_conf": 0.9505143411607072, | |
| "confident_error_rate": 0.35, | |
| "coverage_at_0_9": 0.85, | |
| "accuracy_at_0_9": 0.5882352941176471, | |
| "confidence_bias": 0.4255143411607072, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 34, | |
| "errors": 14, | |
| "error_rate": 0.4117647058823529 | |
| }, | |
| "0.95": { | |
| "n": 32, | |
| "errors": 12, | |
| "error_rate": 0.375 | |
| }, | |
| "0.99": { | |
| "n": 25, | |
| "errors": 8, | |
| "error_rate": 0.32 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.75, | |
| "confidence_cutoff": 0.9963926273512684 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.625, | |
| "confidence_cutoff": 0.9538424791841433 | |
| } | |
| }, | |
| "score_mae": 0.45259144588683287, | |
| "ranked_probability_score": 0.20820131499001437 | |
| }, | |
| "emotion": { | |
| "n": 80, | |
| "nll": 1.2263539675718254, | |
| "acc": 0.6625, | |
| "ece": 0.14963081389980043, | |
| "brier": 0.5245823410352607, | |
| "mean_conf": 0.7403260524211139, | |
| "confident_error_rate": 0.075, | |
| "coverage_at_0_9": 0.3, | |
| "accuracy_at_0_9": 0.75, | |
| "confidence_bias": 0.07782605242111396, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 24, | |
| "errors": 6, | |
| "error_rate": 0.25 | |
| }, | |
| "0.95": { | |
| "n": 15, | |
| "errors": 3, | |
| "error_rate": 0.2 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.7, | |
| "confidence_cutoff": 0.797799825668335 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.71875, | |
| "confidence_cutoff": 0.5535729084804598 | |
| } | |
| } | |
| }, | |
| "mmlu": { | |
| "n": 80, | |
| "nll": 0.8274234004446729, | |
| "acc": 0.65, | |
| "ece": 0.18294625228578548, | |
| "brier": 0.42173974468001135, | |
| "mean_conf": 0.7970014825127686, | |
| "confident_error_rate": 0.0125, | |
| "coverage_at_0_9": 0.45, | |
| "accuracy_at_0_9": 0.9722222222222222, | |
| "confidence_bias": 0.14700148251276857, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 36, | |
| "errors": 1, | |
| "error_rate": 0.027777777777777776 | |
| }, | |
| "0.95": { | |
| "n": 30, | |
| "errors": 1, | |
| "error_rate": 0.03333333333333333 | |
| }, | |
| "0.99": { | |
| "n": 23, | |
| "errors": 1, | |
| "error_rate": 0.043478260869565216 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.925, | |
| "confidence_cutoff": 0.8614584193836526 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.765625, | |
| "confidence_cutoff": 0.5891912744154578 | |
| } | |
| } | |
| }, | |
| "paws": { | |
| "n": 80, | |
| "nll": 0.9617935415055063, | |
| "acc": 0.725, | |
| "ece": 0.23949173491457365, | |
| "brier": 0.4834975530791322, | |
| "mean_conf": 0.9521085317061271, | |
| "confident_error_rate": 0.2, | |
| "coverage_at_0_9": 0.875, | |
| "accuracy_at_0_9": 0.7714285714285715, | |
| "confidence_bias": 0.2271085317061271, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 70, | |
| "errors": 16, | |
| "error_rate": 0.22857142857142856 | |
| }, | |
| "0.95": { | |
| "n": 63, | |
| "errors": 13, | |
| "error_rate": 0.20634920634920634 | |
| }, | |
| "0.99": { | |
| "n": 33, | |
| "errors": 5, | |
| "error_rate": 0.15151515151515152 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.825, | |
| "confidence_cutoff": 0.9852659738611202 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.796875, | |
| "confidence_cutoff": 0.9460779604702808 | |
| } | |
| } | |
| }, | |
| "qnli": { | |
| "n": 80, | |
| "nll": 0.28925677259060756, | |
| "acc": 0.8875, | |
| "ece": 0.07341603957619632, | |
| "brier": 0.18009551904084561, | |
| "mean_conf": 0.9343277556724386, | |
| "confident_error_rate": 0.0625, | |
| "coverage_at_0_9": 0.8125, | |
| "accuracy_at_0_9": 0.9230769230769231, | |
| "confidence_bias": 0.04682775567243869, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 65, | |
| "errors": 5, | |
| "error_rate": 0.07692307692307693 | |
| }, | |
| "0.95": { | |
| "n": 59, | |
| "errors": 2, | |
| "error_rate": 0.03389830508474576 | |
| }, | |
| "0.99": { | |
| "n": 32, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9858154563418124 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9375, | |
| "confidence_cutoff": 0.9094368151617287 | |
| } | |
| } | |
| }, | |
| "sciq": { | |
| "n": 80, | |
| "nll": 0.05181181132785821, | |
| "acc": 0.9875, | |
| "ece": 0.03175391680550414, | |
| "brier": 0.02513979546365842, | |
| "mean_conf": 0.9682344692294615, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.9, | |
| "accuracy_at_0_9": 1.0, | |
| "confidence_bias": -0.019265530770538497, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 72, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 69, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 64, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9995528209501393 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9920047199640097 | |
| } | |
| } | |
| }, | |
| "tweet_offensive": { | |
| "n": 80, | |
| "nll": 0.6781008210985803, | |
| "acc": 0.75, | |
| "ece": 0.15591583779095422, | |
| "brier": 0.3886805236287897, | |
| "mean_conf": 0.8634720692805369, | |
| "confident_error_rate": 0.1125, | |
| "coverage_at_0_9": 0.575, | |
| "accuracy_at_0_9": 0.8043478260869565, | |
| "confidence_bias": 0.11347206928053688, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 46, | |
| "errors": 9, | |
| "error_rate": 0.1956521739130435 | |
| }, | |
| "0.95": { | |
| "n": 39, | |
| "errors": 5, | |
| "error_rate": 0.1282051282051282 | |
| }, | |
| "0.99": { | |
| "n": 19, | |
| "errors": 2, | |
| "error_rate": 0.10526315789473684 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.875, | |
| "confidence_cutoff": 0.9428173514340226 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.796875, | |
| "confidence_cutoff": 0.7029978842436637 | |
| } | |
| } | |
| } | |
| }, | |
| "permutation": { | |
| "n": 36, | |
| "mean_max_delta": 0.04950095278980541, | |
| "flip_rate": 0.05555555555555555 | |
| }, | |
| "temperature": 1.6817928305074288, | |
| "calibrated_clean": { | |
| "n": 656, | |
| "nll": 0.5645439847672864, | |
| "acc": 0.7896341463414634, | |
| "ece": 0.0621233006894711, | |
| "brier": 0.3098930396153164, | |
| "mean_conf": 0.8183328357646178, | |
| "confident_error_rate": 0.042682926829268296, | |
| "coverage_at_0_9": 0.5076219512195121, | |
| "accuracy_at_0_9": 0.9159159159159159, | |
| "confidence_bias": 0.028698689423154344, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 333, | |
| "errors": 28, | |
| "error_rate": 0.08408408408408409 | |
| }, | |
| "0.95": { | |
| "n": 224, | |
| "errors": 14, | |
| "error_rate": 0.0625 | |
| }, | |
| "0.99": { | |
| "n": 85, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9146341463414634, | |
| "confidence_cutoff": 0.9045152638026881 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8003048780487805, | |
| "accuracy": 0.8514285714285714, | |
| "confidence_cutoff": 0.6315002649515302 | |
| } | |
| }, | |
| "score_mae": 0.4483170247504605, | |
| "ranked_probability_score": 0.18781810424878748 | |
| }, | |
| "metric_policy": { | |
| "nll_floor": 1e-09, | |
| "renormalize_returned_probabilities": true, | |
| "raw_sums_outside_1e_5": 0, | |
| "returned_zeros": 0 | |
| }, | |
| "coverage": { | |
| "requested_records": 764, | |
| "requested_questions": 764, | |
| "evaluated_records": 764, | |
| "evaluated_questions": 764, | |
| "rejected_records": 0, | |
| "truncated_records": 0 | |
| }, | |
| "latency_ms": { | |
| "median": 44.0643375000036, | |
| "p95": 52.84930554976199 | |
| }, | |
| "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1" | |
| }, | |
| "provenance": { | |
| "config": { | |
| "epochs": 2, | |
| "seed": 1, | |
| "lr": 5e-05, | |
| "lora": 16, | |
| "accum": 2, | |
| "batch": 4, | |
| "perm_kl": 0.0, | |
| "perm_frac": 0.3, | |
| "ord_w": 0.0, | |
| "p_none": 0.1, | |
| "p_none_distract": 0.12, | |
| "p_distract": 0.15, | |
| "p_none_pair": 0.25, | |
| "synthetic_repeat": 1, | |
| "public_frac": 1.0, | |
| "head_lr": 0.0, | |
| "weight_decay": 0.01, | |
| "anchor_w": 0.0, | |
| "base": "Qwen/Qwen3-4B-Base", | |
| "dtype": "bf16", | |
| "checkpointing": 1 | |
| }, | |
| "config_sha256": "7401c9d55a0961c57c94d559125675d98461bff8e78cd47575e279dc6a40d832", | |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", | |
| "source_hashes": { | |
| "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", | |
| "kev/anchors.py": "089d8a5493502bb26f540eb1c5e681780ca0bb01276073d4e1733211f15d0e10", | |
| "kev/api.py": "cdb0602684d798ccd3fc2c86be622f2a8e7bcdeee64d07f95604c7d3fd4701ec", | |
| "kev/autoresearch.py": "0a8aa6b57c1c9ba93d25b2cf631b03aed2e686e374c1148002eec48765b7b1fc", | |
| "kev/benchmark.py": "4dccec28be6becf6c3dcc8c186e8e1908a6482f15f602a60b440cc9cb4ac8a59", | |
| "kev/compare.py": "bd0445f021de59e35c7bff9304e39dd2e1211e3877a453575594ae7b81b0ada4", | |
| "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", | |
| "kev/contrastive.py": "b1ce87a389b34b4d54a9eaca3ec2ec0809f64ae6640f4fa99785cea9293e3831", | |
| "kev/data.py": "9729f8f497b02aca542915d6f1bea3ff55956629d7d9610370aea91ee66efd23", | |
| "kev/evaluate.py": "0567c95bf8e6e6600bd7d356da064aaffcdad475be75c8bdf3a4d2b669a12e35", | |
| "kev/experiment.py": "c635394fe56c9aa11765f8bbf8f133dcf3e57cbe8f2bce4ebdc7b4e85fa536ff", | |
| "kev/jev.py": "e0213782359ba2f95adbf045ddaf0a008b08b4162bf6a9b66d4ce55fc91a51cb", | |
| "kev/model.py": "839dd7632ec291ac4740035c86f780f8f13da2b74b54a146ce90ff0ce8de527b", | |
| "kev/plot.py": "d689c7dd18cde7f9ea77cb4f55c50ff7da1880b21cb2ecf244348e45842a1382", | |
| "kev/publish.py": "b88c710140e7717ba42d539fdbe785507ac8b9d56b673641ade61c4136bb7dc9", | |
| "kev/serve.py": "7c8807379e915ceafc743e663278262c5c9e9e4a13d214c988cff21e11e4ebb5", | |
| "kev/study_v3.py": "7aa85ef92dd519dcf03a74b07dba400d7c533e9e457e7849b94b19cabafc4fba", | |
| "kev/suite.py": "5678fdcaf9a3e13efc3a77f514e2ddc5687e7407dddb86487461076f717c96fc", | |
| "kev/train.py": "e0df6503506d6a0cb5ccd35ae38bb072a6103383adf7beab4269bb1a1f5fcdce", | |
| "modal_app.py": "64381e069ddf3f560e8a8d8844268a888d7689c72377be899eec2dfc97d4d6d1", | |
| "pyproject.toml": "ce566ee11b68c0cb587bb591943641f403d761d820817693dae7590faacf9888", | |
| "uv.lock": "846fdc64a8bd6279a17a4365f18cd4e92ba9622d50d378aafcb5be6604296138" | |
| }, | |
| "git_commit": "29d71c78368657b3a522729a01c748ea15272abc", | |
| "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", | |
| "torch": "2.8.0+cu128", | |
| "device": "cuda", | |
| "gpu": "NVIDIA H200", | |
| "legacy_checkpoint": false | |
| }, | |
| "mechanism_checks": { | |
| "n": 8, | |
| "packed_max_delta": 0.0, | |
| "sibling_max_delta": 1.7285346984863281e-06, | |
| "tolerance": 0.001, | |
| "passed": true | |
| }, | |
| "gates": { | |
| "passed": true, | |
| "checks": { | |
| "complete_coverage": true, | |
| "isolation_and_packing": true, | |
| "transfer_complete": true, | |
| "heldout_pairs_at_least_70pct": true, | |
| "transfer_confident_errors_below_10pct": true | |
| }, | |
| "policy": "Research screening only: 1e-3 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." | |
| }, | |
| "wall_seconds": 2587.115574389, | |
| "promotable": false, | |
| "test_evaluated": false, | |
| "training_resources": { | |
| "wall_seconds": 2378.645946741104, | |
| "records_seen": 30530, | |
| "requested_records": 25152, | |
| "truncated_records": 0, | |
| "rejected_records": 0, | |
| "optimizer_steps": 3144, | |
| "forward_tokens": 5860024, | |
| "peak_device_bytes": 23007304704, | |
| "device": "cuda", | |
| "dtype": "bf16", | |
| "batch": 4, | |
| "peak_rss_bytes": 25029832704 | |
| } | |
| } | |