Kev-0.8B: dates+unknowable delta (locked test 0.834 / 0.684); previous weights at tag v7-base
2256796 verified | { | |
| "objective": -0.5837225842185488, | |
| "paired_flip": { | |
| "pairs": 80, | |
| "flip_rate": 0.775, | |
| "both_correct_rate": 0.7625, | |
| "invariant_pairs": 32, | |
| "invariance_rate": 1.0, | |
| "invariant_both_correct_rate": 0.8125 | |
| }, | |
| "unknowable": null, | |
| "clean": { | |
| "n": 1264, | |
| "nll": 0.677277750842542, | |
| "acc": 0.8251582278481012, | |
| "ece": 0.10968515087361608, | |
| "brier": 0.2759729702219201, | |
| "mean_conf": 0.9329791785729107, | |
| "confident_error_rate": 0.07911392405063292, | |
| "coverage_at_0_9": 0.8061708860759493, | |
| "accuracy_at_0_9": 0.9018645731108931, | |
| "coverage_at_5pct_error": 0.5632911392405063, | |
| "coverage_at_1pct_error": 0.17325949367088608, | |
| "confidence_bias": 0.10782095072480946, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 1019, | |
| "errors": 100, | |
| "error_rate": 0.09813542688910697 | |
| }, | |
| "0.95": { | |
| "n": 934, | |
| "errors": 75, | |
| "error_rate": 0.08029978586723768 | |
| }, | |
| "0.99": { | |
| "n": 786, | |
| "errors": 43, | |
| "error_rate": 0.054707379134860054 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9620253164556962, | |
| "confidence_cutoff": 0.998565945579216 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8006329113924051, | |
| "accuracy": 0.9021739130434783, | |
| "confidence_cutoff": 0.9062608044404319 | |
| } | |
| }, | |
| "score_mae": 0.49005627978925503, | |
| "ranked_probability_score": 0.08996848010227619 | |
| }, | |
| "tasks": { | |
| "agnews": { | |
| "n": 80, | |
| "nll": 0.9832496835003219, | |
| "acc": 0.8375, | |
| "ece": 0.15513109503024222, | |
| "brier": 0.3025716783162474, | |
| "mean_conf": 0.9695536627637408, | |
| "confident_error_rate": 0.1125, | |
| "coverage_at_0_9": 0.9125, | |
| "accuracy_at_0_9": 0.8767123287671232, | |
| "coverage_at_5pct_error": 0.6375, | |
| "coverage_at_1pct_error": 0.45, | |
| "confidence_bias": 0.13205366276374075, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 73, | |
| "errors": 9, | |
| "error_rate": 0.1232876712328767 | |
| }, | |
| "0.95": { | |
| "n": 72, | |
| "errors": 9, | |
| "error_rate": 0.125 | |
| }, | |
| "0.99": { | |
| "n": 64, | |
| "errors": 7, | |
| "error_rate": 0.109375 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.975, | |
| "confidence_cutoff": 0.9999153535825172 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.890625, | |
| "confidence_cutoff": 0.9912545823258385 | |
| } | |
| } | |
| }, | |
| "agnews_yn": { | |
| "n": 160, | |
| "nll": 0.3115965548177428, | |
| "acc": 0.925, | |
| "ece": 0.0603462185065268, | |
| "brier": 0.1318512819711422, | |
| "mean_conf": 0.980261737940278, | |
| "confident_error_rate": 0.05, | |
| "coverage_at_0_9": 0.9375, | |
| "accuracy_at_0_9": 0.9466666666666667, | |
| "coverage_at_5pct_error": 0.9125, | |
| "coverage_at_1pct_error": 0.5, | |
| "confidence_bias": 0.055261737940278, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 150, | |
| "errors": 8, | |
| "error_rate": 0.05333333333333334 | |
| }, | |
| "0.95": { | |
| "n": 147, | |
| "errors": 8, | |
| "error_rate": 0.05442176870748299 | |
| }, | |
| "0.99": { | |
| "n": 136, | |
| "errors": 4, | |
| "error_rate": 0.029411764705882353 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9994875802345914 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.96875, | |
| "confidence_cutoff": 0.9969696708954863 | |
| } | |
| } | |
| }, | |
| "amazon": { | |
| "n": 80, | |
| "nll": 1.2366158624032093, | |
| "acc": 0.575, | |
| "ece": 0.20630787161160322, | |
| "brier": 0.5776284368946147, | |
| "mean_conf": 0.7813078716116031, | |
| "confident_error_rate": 0.05, | |
| "coverage_at_0_9": 0.3625, | |
| "accuracy_at_0_9": 0.8620689655172413, | |
| "coverage_at_5pct_error": 0.15, | |
| "coverage_at_1pct_error": 0.15, | |
| "confidence_bias": 0.20630787161160313, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 29, | |
| "errors": 4, | |
| "error_rate": 0.13793103448275862 | |
| }, | |
| "0.95": { | |
| "n": 20, | |
| "errors": 3, | |
| "error_rate": 0.15 | |
| }, | |
| "0.99": { | |
| "n": 5, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8, | |
| "confidence_cutoff": 0.804787614863595 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.65625, | |
| "confidence_cutoff": 0.6037764923189102 | |
| } | |
| }, | |
| "score_mae": 0.5011906275246643, | |
| "ranked_probability_score": 0.08998982899356231 | |
| }, | |
| "banking77": { | |
| "n": 80, | |
| "nll": 0.4023205818047388, | |
| "acc": 0.85, | |
| "ece": 0.08929501496409659, | |
| "brier": 0.2009334459047487, | |
| "mean_conf": 0.8971821022144549, | |
| "confident_error_rate": 0.0125, | |
| "coverage_at_0_9": 0.75, | |
| "accuracy_at_0_9": 0.9833333333333333, | |
| "coverage_at_5pct_error": 0.8, | |
| "coverage_at_1pct_error": 0.725, | |
| "confidence_bias": 0.04718210221445496, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 60, | |
| "errors": 1, | |
| "error_rate": 0.016666666666666666 | |
| }, | |
| "0.95": { | |
| "n": 58, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 48, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9989341306330574 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.953125, | |
| "confidence_cutoff": 0.8131230570950913 | |
| } | |
| } | |
| }, | |
| "boolq": { | |
| "n": 80, | |
| "nll": 1.1931741060605592, | |
| "acc": 0.7625, | |
| "ece": 0.18683346049861238, | |
| "brier": 0.4130432350526035, | |
| "mean_conf": 0.9465267519654077, | |
| "confident_error_rate": 0.1625, | |
| "coverage_at_0_9": 0.8125, | |
| "accuracy_at_0_9": 0.8, | |
| "coverage_at_5pct_error": 0.0125, | |
| "coverage_at_1pct_error": 0.0125, | |
| "confidence_bias": 0.18402675196540774, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 65, | |
| "errors": 13, | |
| "error_rate": 0.2 | |
| }, | |
| "0.95": { | |
| "n": 61, | |
| "errors": 12, | |
| "error_rate": 0.19672131147540983 | |
| }, | |
| "0.99": { | |
| "n": 51, | |
| "errors": 10, | |
| "error_rate": 0.19607843137254902 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.825, | |
| "confidence_cutoff": 0.9969401287601459 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.8125, | |
| "confidence_cutoff": 0.9391503613934029 | |
| } | |
| } | |
| }, | |
| "composition_atom": { | |
| "n": 16, | |
| "nll": 0.0018204253681370606, | |
| "acc": 1.0, | |
| "ece": 0.0018176200570680923, | |
| "brier": 1.1192189656453611e-05, | |
| "mean_conf": 0.9981823799429319, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "coverage_at_5pct_error": 1.0, | |
| "coverage_at_1pct_error": 1.0, | |
| "confidence_bias": -0.0018176200570680923, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9979979161178252 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9979044254752123 | |
| } | |
| } | |
| }, | |
| "composition_conditional": { | |
| "n": 16, | |
| "nll": 1.407375578542251, | |
| "acc": 0.75, | |
| "ece": 0.3408496346386551, | |
| "brier": 0.5686769779830284, | |
| "mean_conf": 0.8849633266106525, | |
| "confident_error_rate": 0.25, | |
| "coverage_at_0_9": 0.6875, | |
| "accuracy_at_0_9": 0.6363636363636364, | |
| "coverage_at_5pct_error": 0.0625, | |
| "coverage_at_1pct_error": 0.0625, | |
| "confidence_bias": 0.13496332661065247, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 11, | |
| "errors": 4, | |
| "error_rate": 0.36363636363636365 | |
| }, | |
| "0.95": { | |
| "n": 11, | |
| "errors": 4, | |
| "error_rate": 0.36363636363636365 | |
| }, | |
| "0.99": { | |
| "n": 7, | |
| "errors": 3, | |
| "error_rate": 0.42857142857142855 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.5, | |
| "confidence_cutoff": 0.9663209643099036 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.6923076923076923, | |
| "confidence_cutoff": 0.786160076283213 | |
| } | |
| } | |
| }, | |
| "composition_conjunction": { | |
| "n": 16, | |
| "nll": 0.33671509514484377, | |
| "acc": 0.9375, | |
| "ece": 0.037238846975122586, | |
| "brier": 0.1268404957604162, | |
| "mean_conf": 0.9747388469751226, | |
| "confident_error_rate": 0.0625, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 0.9375, | |
| "coverage_at_5pct_error": 0.5625, | |
| "coverage_at_1pct_error": 0.5625, | |
| "confidence_bias": 0.037238846975122586, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 1, | |
| "error_rate": 0.0625 | |
| }, | |
| "0.95": { | |
| "n": 12, | |
| "errors": 1, | |
| "error_rate": 0.08333333333333333 | |
| }, | |
| "0.99": { | |
| "n": 10, | |
| "errors": 1, | |
| "error_rate": 0.1 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9947336048940597 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.9230769230769231, | |
| "confidence_cutoff": 0.9366926690443258 | |
| } | |
| } | |
| }, | |
| "composition_disjunction": { | |
| "n": 16, | |
| "nll": 0.7980323464815665, | |
| "acc": 0.625, | |
| "ece": 0.34042637355738437, | |
| "brier": 0.5754585999291262, | |
| "mean_conf": 0.9236151581808647, | |
| "confident_error_rate": 0.125, | |
| "coverage_at_0_9": 0.6875, | |
| "accuracy_at_0_9": 0.8181818181818182, | |
| "coverage_at_5pct_error": 0.5625, | |
| "coverage_at_1pct_error": 0.5625, | |
| "confidence_bias": 0.2986151581808647, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 11, | |
| "errors": 2, | |
| "error_rate": 0.18181818181818182 | |
| }, | |
| "0.95": { | |
| "n": 9, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 5, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9873694740706329 | |
| }, | |
| "0.8": { | |
| "coverage": 0.875, | |
| "accuracy": 0.6428571428571429, | |
| "confidence_cutoff": 0.8276452175484533 | |
| } | |
| } | |
| }, | |
| "composition_exception": { | |
| "n": 16, | |
| "nll": 0.5900095558121554, | |
| "acc": 0.5625, | |
| "ece": 0.2816931411850216, | |
| "brier": 0.39583397870207226, | |
| "mean_conf": 0.8441931411850218, | |
| "confident_error_rate": 0.0625, | |
| "coverage_at_0_9": 0.625, | |
| "accuracy_at_0_9": 0.9, | |
| "coverage_at_5pct_error": 0.5, | |
| "coverage_at_1pct_error": 0.5, | |
| "confidence_bias": 0.28169314118502176, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 10, | |
| "errors": 1, | |
| "error_rate": 0.1 | |
| }, | |
| "0.95": { | |
| "n": 10, | |
| "errors": 1, | |
| "error_rate": 0.1 | |
| }, | |
| "0.99": { | |
| "n": 5, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9760010333359338 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.6923076923076923, | |
| "confidence_cutoff": 0.631696857911161 | |
| } | |
| } | |
| }, | |
| "composition_negation": { | |
| "n": 16, | |
| "nll": 0.06080032875137842, | |
| "acc": 1.0, | |
| "ece": 0.05737638597491783, | |
| "brier": 0.01223893910076251, | |
| "mean_conf": 0.9426236140250821, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.875, | |
| "accuracy_at_0_9": 1.0, | |
| "coverage_at_5pct_error": 1.0, | |
| "coverage_at_1pct_error": 1.0, | |
| "confidence_bias": -0.0573763859749179, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 14, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 11, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 1, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9676642634197532 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9302116355236877 | |
| } | |
| } | |
| }, | |
| "composition_nested_and": { | |
| "n": 16, | |
| "nll": 0.6256150963330884, | |
| "acc": 0.8125, | |
| "ece": 0.14464532099308625, | |
| "brier": 0.348125633054877, | |
| "mean_conf": 0.9571453209930862, | |
| "confident_error_rate": 0.1875, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 0.8125, | |
| "coverage_at_5pct_error": 0.3125, | |
| "coverage_at_1pct_error": 0.3125, | |
| "confidence_bias": 0.14464532099308625, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 3, | |
| "error_rate": 0.1875 | |
| }, | |
| "0.95": { | |
| "n": 10, | |
| "errors": 3, | |
| "error_rate": 0.3 | |
| }, | |
| "0.99": { | |
| "n": 4, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.875, | |
| "confidence_cutoff": 0.9574078703506022 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.7692307692307693, | |
| "confidence_cutoff": 0.9318581472390091 | |
| } | |
| } | |
| }, | |
| "composition_nested_or": { | |
| "n": 16, | |
| "nll": 0.07625810559742767, | |
| "acc": 0.9375, | |
| "ece": 0.05914285391270866, | |
| "brier": 0.04200748661529065, | |
| "mean_conf": 0.9485557788640286, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.9375, | |
| "accuracy_at_0_9": 1.0, | |
| "coverage_at_5pct_error": 0.9375, | |
| "coverage_at_1pct_error": 0.9375, | |
| "confidence_bias": 0.011055778864028643, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 15, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 12, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 6, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9898232160226875 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9273196502137393 | |
| } | |
| } | |
| }, | |
| "contrastive_age_eligibility": { | |
| "n": 24, | |
| "nll": 1.8499446775205282e-05, | |
| "acc": 1.0, | |
| "ece": 1.8499237824265435e-05, | |
| "brier": 8.357891696338058e-10, | |
| "mean_conf": 0.9999815007621757, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "coverage_at_5pct_error": 1.0, | |
| "coverage_at_1pct_error": 1.0, | |
| "confidence_bias": -1.8499237824265435e-05, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9999830436286485 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8333333333333334, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.999974184122501 | |
| } | |
| } | |
| }, | |
| "contrastive_quantity_limit": { | |
| "n": 24, | |
| "nll": 0.1125956044001396, | |
| "acc": 0.9583333333333334, | |
| "ece": 0.03624299722303426, | |
| "brier": 0.07197842498551321, | |
| "mean_conf": 0.9945763305563676, | |
| "confident_error_rate": 0.041666666666666664, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 0.9583333333333334, | |
| "coverage_at_5pct_error": 1.0, | |
| "coverage_at_1pct_error": 0.9583333333333334, | |
| "confidence_bias": 0.03624299722303426, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 24, | |
| "errors": 1, | |
| "error_rate": 0.041666666666666664 | |
| }, | |
| "0.95": { | |
| "n": 23, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 21, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9999334638396701 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8333333333333334, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9981896653224097 | |
| } | |
| } | |
| }, | |
| "contrastive_return_window": { | |
| "n": 24, | |
| "nll": 0.6051529398630384, | |
| "acc": 0.6666666666666666, | |
| "ece": 0.21995455952093168, | |
| "brier": 0.371917448872205, | |
| "mean_conf": 0.8675996478133303, | |
| "confident_error_rate": 0.08333333333333333, | |
| "coverage_at_0_9": 0.625, | |
| "accuracy_at_0_9": 0.8666666666666667, | |
| "coverage_at_5pct_error": 0.041666666666666664, | |
| "coverage_at_1pct_error": 0.041666666666666664, | |
| "confidence_bias": 0.2009329811466637, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 15, | |
| "errors": 2, | |
| "error_rate": 0.13333333333333333 | |
| }, | |
| "0.95": { | |
| "n": 12, | |
| "errors": 2, | |
| "error_rate": 0.16666666666666666 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8333333333333334, | |
| "confidence_cutoff": 0.9521516320165855 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8333333333333334, | |
| "accuracy": 0.8, | |
| "confidence_cutoff": 0.7327820281023392 | |
| } | |
| } | |
| }, | |
| "contrastive_spend_threshold": { | |
| "n": 24, | |
| "nll": 0.00030279504157641773, | |
| "acc": 1.0, | |
| "ece": 0.0003026639771004369, | |
| "brier": 5.21775523659121e-07, | |
| "mean_conf": 0.9996973360228996, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 1.0, | |
| "coverage_at_5pct_error": 1.0, | |
| "coverage_at_1pct_error": 1.0, | |
| "confidence_bias": -0.0003026639771004369, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 24, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9999916121909319 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8333333333333334, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9993623150563549 | |
| } | |
| } | |
| }, | |
| "dbpedia14": { | |
| "n": 80, | |
| "nll": 0.4604112284196398, | |
| "acc": 0.9625, | |
| "ece": 0.03734234636353262, | |
| "brier": 0.0749167806142053, | |
| "mean_conf": 0.9998423463635326, | |
| "confident_error_rate": 0.0375, | |
| "coverage_at_0_9": 1.0, | |
| "accuracy_at_0_9": 0.9625, | |
| "coverage_at_5pct_error": 1.0, | |
| "coverage_at_1pct_error": 0.35, | |
| "confidence_bias": 0.03734234636353262, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 80, | |
| "errors": 3, | |
| "error_rate": 0.0375 | |
| }, | |
| "0.95": { | |
| "n": 80, | |
| "errors": 3, | |
| "error_rate": 0.0375 | |
| }, | |
| "0.99": { | |
| "n": 80, | |
| "errors": 3, | |
| "error_rate": 0.0375 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.975, | |
| "confidence_cutoff": 0.9999999475161038 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.984375, | |
| "confidence_cutoff": 0.9999997520431655 | |
| } | |
| } | |
| }, | |
| "imdb": { | |
| "n": 80, | |
| "nll": 0.5716301725699824, | |
| "acc": 0.8875, | |
| "ece": 0.09875640553643103, | |
| "brier": 0.19712326669594268, | |
| "mean_conf": 0.9785553763225823, | |
| "confident_error_rate": 0.0875, | |
| "coverage_at_0_9": 0.9375, | |
| "accuracy_at_0_9": 0.9066666666666666, | |
| "coverage_at_5pct_error": 0.8375, | |
| "coverage_at_1pct_error": 0.3625, | |
| "confidence_bias": 0.09105537632258232, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 75, | |
| "errors": 7, | |
| "error_rate": 0.09333333333333334 | |
| }, | |
| "0.95": { | |
| "n": 72, | |
| "errors": 5, | |
| "error_rate": 0.06944444444444445 | |
| }, | |
| "0.99": { | |
| "n": 68, | |
| "errors": 4, | |
| "error_rate": 0.058823529411764705 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.975, | |
| "confidence_cutoff": 0.9999773407237053 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.953125, | |
| "confidence_cutoff": 0.9974316348485517 | |
| } | |
| } | |
| }, | |
| "mnli": { | |
| "n": 80, | |
| "nll": 1.1387845715116893, | |
| "acc": 0.8125, | |
| "ece": 0.16543833890354487, | |
| "brier": 0.3362546680191699, | |
| "mean_conf": 0.9719655603456216, | |
| "confident_error_rate": 0.1625, | |
| "coverage_at_0_9": 0.95, | |
| "accuracy_at_0_9": 0.8289473684210527, | |
| "coverage_at_5pct_error": 0.25, | |
| "coverage_at_1pct_error": 0.1875, | |
| "confidence_bias": 0.15946556034562165, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 76, | |
| "errors": 13, | |
| "error_rate": 0.17105263157894737 | |
| }, | |
| "0.95": { | |
| "n": 71, | |
| "errors": 12, | |
| "error_rate": 0.16901408450704225 | |
| }, | |
| "0.99": { | |
| "n": 64, | |
| "errors": 6, | |
| "error_rate": 0.09375 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9, | |
| "confidence_cutoff": 0.9992974611653868 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.90625, | |
| "confidence_cutoff": 0.9905674125869428 | |
| } | |
| } | |
| }, | |
| "sst5": { | |
| "n": 80, | |
| "nll": 1.5534444317420577, | |
| "acc": 0.5875, | |
| "ece": 0.23933171099649792, | |
| "brier": 0.7022323612305323, | |
| "mean_conf": 0.8124146265105597, | |
| "confident_error_rate": 0.2, | |
| "coverage_at_0_9": 0.4125, | |
| "accuracy_at_0_9": 0.5151515151515151, | |
| "coverage_at_5pct_error": 0.0375, | |
| "coverage_at_1pct_error": 0.0375, | |
| "confidence_bias": 0.2249146265105597, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 33, | |
| "errors": 16, | |
| "error_rate": 0.48484848484848486 | |
| }, | |
| "0.95": { | |
| "n": 13, | |
| "errors": 5, | |
| "error_rate": 0.38461538461538464 | |
| }, | |
| "0.99": { | |
| "n": 2, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.575, | |
| "confidence_cutoff": 0.8458941242777314 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.578125, | |
| "confidence_cutoff": 0.6599644726386572 | |
| } | |
| }, | |
| "score_mae": 0.5693146229296997, | |
| "ranked_probability_score": 0.10965566832128679 | |
| }, | |
| "trec": { | |
| "n": 80, | |
| "nll": 0.2886405792332978, | |
| "acc": 0.9625, | |
| "ece": 0.039050754551288716, | |
| "brier": 0.07288648289502976, | |
| "mean_conf": 0.9895815825818847, | |
| "confident_error_rate": 0.025, | |
| "coverage_at_0_9": 0.975, | |
| "accuracy_at_0_9": 0.9743589743589743, | |
| "coverage_at_5pct_error": 1.0, | |
| "coverage_at_1pct_error": 0.6875, | |
| "confidence_bias": 0.027081582581884645, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 78, | |
| "errors": 2, | |
| "error_rate": 0.02564102564102564 | |
| }, | |
| "0.95": { | |
| "n": 76, | |
| "errors": 2, | |
| "error_rate": 0.02631578947368421 | |
| }, | |
| "0.99": { | |
| "n": 76, | |
| "errors": 2, | |
| "error_rate": 0.02631578947368421 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9999995069938787 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.984375, | |
| "confidence_cutoff": 0.9999976468822103 | |
| } | |
| } | |
| }, | |
| "yelp": { | |
| "n": 80, | |
| "nll": 0.9293500772388761, | |
| "acc": 0.65, | |
| "ece": 0.20917757536782644, | |
| "brier": 0.5126908269440805, | |
| "mean_conf": 0.8206727370156578, | |
| "confident_error_rate": 0.0875, | |
| "coverage_at_0_9": 0.425, | |
| "accuracy_at_0_9": 0.7941176470588235, | |
| "coverage_at_5pct_error": 0.2625, | |
| "coverage_at_1pct_error": 0.175, | |
| "confidence_bias": 0.17067273701565777, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 34, | |
| "errors": 7, | |
| "error_rate": 0.20588235294117646 | |
| }, | |
| "0.95": { | |
| "n": 22, | |
| "errors": 2, | |
| "error_rate": 0.09090909090909091 | |
| }, | |
| "0.99": { | |
| "n": 8, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.775, | |
| "confidence_cutoff": 0.8554426543815328 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.671875, | |
| "confidence_cutoff": 0.6485221260511311 | |
| } | |
| }, | |
| "score_mae": 0.3996635889134011, | |
| "ranked_probability_score": 0.07025994299197942 | |
| }, | |
| "yelp_yn": { | |
| "n": 80, | |
| "nll": 0.32542780116067804, | |
| "acc": 0.8875, | |
| "ece": 0.08498293375955343, | |
| "brier": 0.15938160338912247, | |
| "mean_conf": 0.9595849699746, | |
| "confident_error_rate": 0.0375, | |
| "coverage_at_0_9": 0.875, | |
| "accuracy_at_0_9": 0.9571428571428572, | |
| "coverage_at_5pct_error": 0.875, | |
| "coverage_at_1pct_error": 0.625, | |
| "confidence_bias": 0.07208496997460001, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 70, | |
| "errors": 3, | |
| "error_rate": 0.04285714285714286 | |
| }, | |
| "0.95": { | |
| "n": 68, | |
| "errors": 3, | |
| "error_rate": 0.04411764705882353 | |
| }, | |
| "0.99": { | |
| "n": 61, | |
| "errors": 3, | |
| "error_rate": 0.04918032786885246 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9996372033356943 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.953125, | |
| "confidence_cutoff": 0.9804108300621437 | |
| } | |
| } | |
| } | |
| }, | |
| "variants": { | |
| "clean": { | |
| "n": 1264, | |
| "nll": 0.677277750842542, | |
| "acc": 0.8251582278481012, | |
| "ece": 0.10968515087361608, | |
| "brier": 0.2759729702219201, | |
| "mean_conf": 0.9329791785729107, | |
| "confident_error_rate": 0.07911392405063292, | |
| "coverage_at_0_9": 0.8061708860759493, | |
| "accuracy_at_0_9": 0.9018645731108931, | |
| "coverage_at_5pct_error": 0.5632911392405063, | |
| "coverage_at_1pct_error": 0.17325949367088608, | |
| "confidence_bias": 0.10782095072480946, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 1019, | |
| "errors": 100, | |
| "error_rate": 0.09813542688910697 | |
| }, | |
| "0.95": { | |
| "n": 934, | |
| "errors": 75, | |
| "error_rate": 0.08029978586723768 | |
| }, | |
| "0.99": { | |
| "n": 786, | |
| "errors": 43, | |
| "error_rate": 0.054707379134860054 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9620253164556962, | |
| "confidence_cutoff": 0.998565945579216 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8006329113924051, | |
| "accuracy": 0.9021739130434783, | |
| "confidence_cutoff": 0.9062608044404319 | |
| } | |
| }, | |
| "score_mae": 0.49005627978925503, | |
| "ranked_probability_score": 0.08996848010227619 | |
| }, | |
| "none_absent": { | |
| "n": 60, | |
| "nll": 0.5141479457352874, | |
| "acc": 0.85, | |
| "ece": 0.11106580603119935, | |
| "brier": 0.22718497312140562, | |
| "mean_conf": 0.9610658060311994, | |
| "confident_error_rate": 0.08333333333333333, | |
| "coverage_at_0_9": 0.9, | |
| "accuracy_at_0_9": 0.9074074074074074, | |
| "coverage_at_5pct_error": 0.6666666666666666, | |
| "coverage_at_1pct_error": 0.23333333333333334, | |
| "confidence_bias": 0.11106580603119942, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 54, | |
| "errors": 5, | |
| "error_rate": 0.09259259259259259 | |
| }, | |
| "0.95": { | |
| "n": 51, | |
| "errors": 5, | |
| "error_rate": 0.09803921568627451 | |
| }, | |
| "0.99": { | |
| "n": 36, | |
| "errors": 2, | |
| "error_rate": 0.05555555555555555 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9666666666666667, | |
| "confidence_cutoff": 0.998077132042754 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.8958333333333334, | |
| "confidence_cutoff": 0.957759179102288 | |
| } | |
| } | |
| }, | |
| "none_present": { | |
| "n": 60, | |
| "nll": 0.8304065371572367, | |
| "acc": 0.8333333333333334, | |
| "ece": 0.11153206529511675, | |
| "brier": 0.27340139863288243, | |
| "mean_conf": 0.9448653986284501, | |
| "confident_error_rate": 0.06666666666666667, | |
| "coverage_at_0_9": 0.8166666666666667, | |
| "accuracy_at_0_9": 0.9183673469387755, | |
| "coverage_at_5pct_error": 0.6333333333333333, | |
| "coverage_at_1pct_error": 0.03333333333333333, | |
| "confidence_bias": 0.11153206529511672, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 49, | |
| "errors": 4, | |
| "error_rate": 0.08163265306122448 | |
| }, | |
| "0.95": { | |
| "n": 43, | |
| "errors": 4, | |
| "error_rate": 0.09302325581395349 | |
| }, | |
| "0.99": { | |
| "n": 32, | |
| "errors": 1, | |
| "error_rate": 0.03125 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9666666666666667, | |
| "confidence_cutoff": 0.992714404701043 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9166666666666666, | |
| "confidence_cutoff": 0.9168633320077479 | |
| } | |
| } | |
| }, | |
| "permuted": { | |
| "n": 84, | |
| "nll": 0.48436865419846886, | |
| "acc": 0.9047619047619048, | |
| "ece": 0.07211805581863377, | |
| "brier": 0.15448812106112633, | |
| "mean_conf": 0.9768799605805386, | |
| "confident_error_rate": 0.05952380952380952, | |
| "coverage_at_0_9": 0.9404761904761905, | |
| "accuracy_at_0_9": 0.9367088607594937, | |
| "coverage_at_5pct_error": 0.9047619047619048, | |
| "coverage_at_1pct_error": 0.09523809523809523, | |
| "confidence_bias": 0.07211805581863384, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 79, | |
| "errors": 5, | |
| "error_rate": 0.06329113924050633 | |
| }, | |
| "0.95": { | |
| "n": 77, | |
| "errors": 4, | |
| "error_rate": 0.05194805194805195 | |
| }, | |
| "0.99": { | |
| "n": 73, | |
| "errors": 3, | |
| "error_rate": 0.0410958904109589 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9761904761904762, | |
| "confidence_cutoff": 0.9998084861230152 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8095238095238095, | |
| "accuracy": 0.9705882352941176, | |
| "confidence_cutoff": 0.9936823144468198 | |
| } | |
| } | |
| } | |
| }, | |
| "heldout_tasks": {}, | |
| "permutation": { | |
| "n": 60, | |
| "mean_max_delta": 0.02025688803068432, | |
| "flip_rate": 0.016666666666666666 | |
| }, | |
| "temperature": 2.6390158215457884, | |
| "calibrated_clean": { | |
| "n": 1264, | |
| "nll": 0.46507794327280627, | |
| "acc": 0.8251582278481012, | |
| "ece": 0.024966358925870097, | |
| "brier": 0.2496238544020784, | |
| "mean_conf": 0.8187350664838222, | |
| "confident_error_rate": 0.022151898734177215, | |
| "coverage_at_0_9": 0.5158227848101266, | |
| "accuracy_at_0_9": 0.9570552147239264, | |
| "coverage_at_5pct_error": 0.5435126582278481, | |
| "coverage_at_1pct_error": 0.15822784810126583, | |
| "confidence_bias": -0.006423161364279006, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 652, | |
| "errors": 28, | |
| "error_rate": 0.04294478527607362 | |
| }, | |
| "0.95": { | |
| "n": 483, | |
| "errors": 13, | |
| "error_rate": 0.026915113871635612 | |
| }, | |
| "0.99": { | |
| "n": 142, | |
| "errors": 1, | |
| "error_rate": 0.007042253521126761 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9572784810126582, | |
| "confidence_cutoff": 0.9093676413006413 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8006329113924051, | |
| "accuracy": 0.900197628458498, | |
| "confidence_cutoff": 0.628961610919671 | |
| } | |
| }, | |
| "score_mae": 0.5430814474849706, | |
| "ranked_probability_score": 0.08275479929730008 | |
| }, | |
| "metric_policy": { | |
| "nll_floor": 1e-09, | |
| "renormalize_returned_probabilities": true, | |
| "raw_sums_outside_1e_5": 0, | |
| "returned_zeros": 0 | |
| }, | |
| "coverage": { | |
| "requested_records": 1204, | |
| "requested_questions": 1468, | |
| "evaluated_records": 1204, | |
| "evaluated_questions": 1468, | |
| "rejected_records": 0, | |
| "truncated_records": 0 | |
| }, | |
| "latency_ms": { | |
| "median": 41.915772000010065, | |
| "p95": 44.87425409996604 | |
| }, | |
| "transfer": { | |
| "objective": -0.9002227022737674, | |
| "paired_flip": { | |
| "pairs": 64, | |
| "flip_rate": 0.484375, | |
| "both_correct_rate": 0.421875, | |
| "invariant_pairs": 24, | |
| "invariance_rate": 0.9583333333333334, | |
| "invariant_both_correct_rate": 0.5416666666666666 | |
| }, | |
| "unknowable": null, | |
| "clean": { | |
| "n": 656, | |
| "nll": 0.9222275811844616, | |
| "acc": 0.6524390243902439, | |
| "ece": 0.17939440499990641, | |
| "brier": 0.49870669415283536, | |
| "mean_conf": 0.8314353498515511, | |
| "confident_error_rate": 0.09908536585365854, | |
| "coverage_at_0_9": 0.5060975609756098, | |
| "accuracy_at_0_9": 0.8042168674698795, | |
| "coverage_at_5pct_error": 0.22865853658536586, | |
| "coverage_at_1pct_error": 0.08841463414634146, | |
| "confidence_bias": 0.17899632546130717, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 332, | |
| "errors": 65, | |
| "error_rate": 0.19578313253012047 | |
| }, | |
| "0.95": { | |
| "n": 249, | |
| "errors": 35, | |
| "error_rate": 0.14056224899598393 | |
| }, | |
| "0.99": { | |
| "n": 157, | |
| "errors": 8, | |
| "error_rate": 0.050955414012738856 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8079268292682927, | |
| "confidence_cutoff": 0.9024031758308411 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8003048780487805, | |
| "accuracy": 0.7219047619047619, | |
| "confidence_cutoff": 0.6487010024681538 | |
| } | |
| }, | |
| "score_mae": 0.6515860158402881, | |
| "ranked_probability_score": 0.23189794760605645 | |
| }, | |
| "tasks": { | |
| "composition_held_and_or": { | |
| "n": 32, | |
| "nll": 1.1899194065298853, | |
| "acc": 0.65625, | |
| "ece": 0.3062826763573561, | |
| "brier": 0.6297875816783778, | |
| "mean_conf": 0.9625326763573561, | |
| "confident_error_rate": 0.3125, | |
| "coverage_at_0_9": 0.9375, | |
| "accuracy_at_0_9": 0.6666666666666666, | |
| "coverage_at_5pct_error": 0.125, | |
| "coverage_at_1pct_error": 0.125, | |
| "confidence_bias": 0.30628267635735607, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 30, | |
| "errors": 10, | |
| "error_rate": 0.3333333333333333 | |
| }, | |
| "0.95": { | |
| "n": 24, | |
| "errors": 8, | |
| "error_rate": 0.3333333333333333 | |
| }, | |
| "0.99": { | |
| "n": 4, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8125, | |
| "confidence_cutoff": 0.976970766787314 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.6153846153846154, | |
| "confidence_cutoff": 0.9484939888568524 | |
| } | |
| } | |
| }, | |
| "composition_held_conditional": { | |
| "n": 32, | |
| "nll": 0.675725781843872, | |
| "acc": 0.59375, | |
| "ece": 0.18786636617174748, | |
| "brier": 0.4705037394069092, | |
| "mean_conf": 0.7816163661717475, | |
| "confident_error_rate": 0.0625, | |
| "coverage_at_0_9": 0.25, | |
| "accuracy_at_0_9": 0.75, | |
| "coverage_at_5pct_error": 0.1875, | |
| "coverage_at_1pct_error": 0.1875, | |
| "confidence_bias": 0.18786636617174746, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 8, | |
| "errors": 2, | |
| "error_rate": 0.25 | |
| }, | |
| "0.95": { | |
| "n": 3, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8125, | |
| "confidence_cutoff": 0.8704544098878388 | |
| }, | |
| "0.8": { | |
| "coverage": 0.84375, | |
| "accuracy": 0.6666666666666666, | |
| "confidence_cutoff": 0.5778257422991319 | |
| } | |
| } | |
| }, | |
| "composition_held_or_not": { | |
| "n": 32, | |
| "nll": 0.8207520004606448, | |
| "acc": 0.5625, | |
| "ece": 0.2816247209338629, | |
| "brier": 0.5616030854083928, | |
| "mean_conf": 0.8278648881415858, | |
| "confident_error_rate": 0.03125, | |
| "coverage_at_0_9": 0.3125, | |
| "accuracy_at_0_9": 0.9, | |
| "coverage_at_5pct_error": 0.09375, | |
| "coverage_at_1pct_error": 0.09375, | |
| "confidence_bias": 0.26536488814158576, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 10, | |
| "errors": 1, | |
| "error_rate": 0.1 | |
| }, | |
| "0.95": { | |
| "n": 5, | |
| "errors": 1, | |
| "error_rate": 0.2 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.53125, | |
| "accuracy": 0.7058823529411765, | |
| "confidence_cutoff": 0.867844091834961 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.6538461538461539, | |
| "confidence_cutoff": 0.6750653184930885 | |
| } | |
| } | |
| }, | |
| "contrastive_authorization": { | |
| "n": 40, | |
| "nll": 0.06778782153149417, | |
| "acc": 0.975, | |
| "ece": 0.031234135570157376, | |
| "brier": 0.036192439631432066, | |
| "mean_conf": 0.9497480518201697, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.85, | |
| "accuracy_at_0_9": 1.0, | |
| "coverage_at_5pct_error": 1.0, | |
| "coverage_at_1pct_error": 0.95, | |
| "confidence_bias": -0.02525194817983023, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 34, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 31, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 27, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.99714271867781 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9445846571260915 | |
| } | |
| } | |
| }, | |
| "contrastive_deadline": { | |
| "n": 40, | |
| "nll": 1.3889026704649357, | |
| "acc": 0.375, | |
| "ece": 0.3829928525155385, | |
| "brier": 0.7922274946031982, | |
| "mean_conf": 0.7042415556863528, | |
| "confident_error_rate": 0.1, | |
| "coverage_at_0_9": 0.225, | |
| "accuracy_at_0_9": 0.5555555555555556, | |
| "coverage_at_5pct_error": 0.0, | |
| "coverage_at_1pct_error": 0.0, | |
| "confidence_bias": 0.3292415556863528, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 9, | |
| "errors": 4, | |
| "error_rate": 0.4444444444444444 | |
| }, | |
| "0.95": { | |
| "n": 5, | |
| "errors": 2, | |
| "error_rate": 0.4 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.55, | |
| "confidence_cutoff": 0.6421262312885925 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.40625, | |
| "confidence_cutoff": 0.5615110481010533 | |
| } | |
| }, | |
| "score_mae": 0.6515860158402881, | |
| "ranked_probability_score": 0.23189794760605645 | |
| }, | |
| "emotion": { | |
| "n": 80, | |
| "nll": 1.6769960423140386, | |
| "acc": 0.5375, | |
| "ece": 0.22586976855913896, | |
| "brier": 0.6678512282134752, | |
| "mean_conf": 0.7472712517372814, | |
| "confident_error_rate": 0.075, | |
| "coverage_at_0_9": 0.3125, | |
| "accuracy_at_0_9": 0.76, | |
| "coverage_at_5pct_error": 0.0, | |
| "coverage_at_1pct_error": 0.0, | |
| "confidence_bias": 0.2097712517372814, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 25, | |
| "errors": 6, | |
| "error_rate": 0.24 | |
| }, | |
| "0.95": { | |
| "n": 19, | |
| "errors": 4, | |
| "error_rate": 0.21052631578947367 | |
| }, | |
| "0.99": { | |
| "n": 9, | |
| "errors": 3, | |
| "error_rate": 0.3333333333333333 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.65, | |
| "confidence_cutoff": 0.7725859213122825 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.578125, | |
| "confidence_cutoff": 0.56208528383785 | |
| } | |
| } | |
| }, | |
| "mmlu": { | |
| "n": 80, | |
| "nll": 1.406507662100838, | |
| "acc": 0.425, | |
| "ece": 0.23086071722009255, | |
| "brier": 0.7337000166926988, | |
| "mean_conf": 0.6558607172200925, | |
| "confident_error_rate": 0.05, | |
| "coverage_at_0_9": 0.225, | |
| "accuracy_at_0_9": 0.7777777777777778, | |
| "coverage_at_5pct_error": 0.1, | |
| "coverage_at_1pct_error": 0.1, | |
| "confidence_bias": 0.23086071722009255, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 18, | |
| "errors": 4, | |
| "error_rate": 0.2222222222222222 | |
| }, | |
| "0.95": { | |
| "n": 16, | |
| "errors": 3, | |
| "error_rate": 0.1875 | |
| }, | |
| "0.99": { | |
| "n": 11, | |
| "errors": 1, | |
| "error_rate": 0.09090909090909091 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.525, | |
| "confidence_cutoff": 0.618546923871794 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.453125, | |
| "confidence_cutoff": 0.4300697519593815 | |
| } | |
| } | |
| }, | |
| "paws": { | |
| "n": 80, | |
| "nll": 1.1248226915361372, | |
| "acc": 0.55, | |
| "ece": 0.3200331593407043, | |
| "brier": 0.7235656563851223, | |
| "mean_conf": 0.8654864054853849, | |
| "confident_error_rate": 0.2625, | |
| "coverage_at_0_9": 0.5, | |
| "accuracy_at_0_9": 0.475, | |
| "coverage_at_5pct_error": 0.0125, | |
| "coverage_at_1pct_error": 0.0125, | |
| "confidence_bias": 0.31548640548538487, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 40, | |
| "errors": 21, | |
| "error_rate": 0.525 | |
| }, | |
| "0.95": { | |
| "n": 12, | |
| "errors": 5, | |
| "error_rate": 0.4166666666666667 | |
| }, | |
| "0.99": { | |
| "n": 2, | |
| "errors": 1, | |
| "error_rate": 0.5 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.475, | |
| "confidence_cutoff": 0.9014056727918538 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.546875, | |
| "confidence_cutoff": 0.8111088751091231 | |
| } | |
| } | |
| }, | |
| "qnli": { | |
| "n": 80, | |
| "nll": 0.4122149692919013, | |
| "acc": 0.85, | |
| "ece": 0.09433166058006281, | |
| "brier": 0.24752108114076724, | |
| "mean_conf": 0.8990905626919042, | |
| "confident_error_rate": 0.0375, | |
| "coverage_at_0_9": 0.6625, | |
| "accuracy_at_0_9": 0.9433962264150944, | |
| "coverage_at_5pct_error": 0.5125, | |
| "coverage_at_1pct_error": 0.3625, | |
| "confidence_bias": 0.04909056269190426, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 53, | |
| "errors": 3, | |
| "error_rate": 0.05660377358490566 | |
| }, | |
| "0.95": { | |
| "n": 43, | |
| "errors": 3, | |
| "error_rate": 0.06976744186046512 | |
| }, | |
| "0.99": { | |
| "n": 29, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.95, | |
| "confidence_cutoff": 0.9583688735354272 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.890625, | |
| "confidence_cutoff": 0.793002498045601 | |
| } | |
| } | |
| }, | |
| "sciq": { | |
| "n": 80, | |
| "nll": 0.24132037549325877, | |
| "acc": 0.9125, | |
| "ece": 0.04025150891820237, | |
| "brier": 0.12820506248919616, | |
| "mean_conf": 0.9387513504714097, | |
| "confident_error_rate": 0.025, | |
| "coverage_at_0_9": 0.8375, | |
| "accuracy_at_0_9": 0.9701492537313433, | |
| "coverage_at_5pct_error": 0.925, | |
| "coverage_at_1pct_error": 0.7, | |
| "confidence_bias": 0.026251350471409696, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 67, | |
| "errors": 2, | |
| "error_rate": 0.029850746268656716 | |
| }, | |
| "0.95": { | |
| "n": 63, | |
| "errors": 1, | |
| "error_rate": 0.015873015873015872 | |
| }, | |
| "0.99": { | |
| "n": 58, | |
| "errors": 1, | |
| "error_rate": 0.017241379310344827 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9998130113129569 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.96875, | |
| "confidence_cutoff": 0.9417403771936594 | |
| } | |
| } | |
| }, | |
| "tweet_offensive": { | |
| "n": 80, | |
| "nll": 0.8975003034444349, | |
| "acc": 0.675, | |
| "ece": 0.18809786512896348, | |
| "brier": 0.5095841174172036, | |
| "mean_conf": 0.8555092051551096, | |
| "confident_error_rate": 0.15, | |
| "coverage_at_0_9": 0.475, | |
| "accuracy_at_0_9": 0.6842105263157895, | |
| "coverage_at_5pct_error": 0.0375, | |
| "coverage_at_1pct_error": 0.0375, | |
| "confidence_bias": 0.1805092051551096, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 38, | |
| "errors": 12, | |
| "error_rate": 0.3157894736842105 | |
| }, | |
| "0.95": { | |
| "n": 28, | |
| "errors": 8, | |
| "error_rate": 0.2857142857142857 | |
| }, | |
| "0.99": { | |
| "n": 17, | |
| "errors": 2, | |
| "error_rate": 0.11764705882352941 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.675, | |
| "confidence_cutoff": 0.8908563720881537 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.703125, | |
| "confidence_cutoff": 0.7286377114869528 | |
| } | |
| } | |
| } | |
| }, | |
| "variants": { | |
| "clean": { | |
| "n": 656, | |
| "nll": 0.9222275811844616, | |
| "acc": 0.6524390243902439, | |
| "ece": 0.17939440499990641, | |
| "brier": 0.49870669415283536, | |
| "mean_conf": 0.8314353498515511, | |
| "confident_error_rate": 0.09908536585365854, | |
| "coverage_at_0_9": 0.5060975609756098, | |
| "accuracy_at_0_9": 0.8042168674698795, | |
| "coverage_at_5pct_error": 0.22865853658536586, | |
| "coverage_at_1pct_error": 0.08841463414634146, | |
| "confidence_bias": 0.17899632546130717, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 332, | |
| "errors": 65, | |
| "error_rate": 0.19578313253012047 | |
| }, | |
| "0.95": { | |
| "n": 249, | |
| "errors": 35, | |
| "error_rate": 0.14056224899598393 | |
| }, | |
| "0.99": { | |
| "n": 157, | |
| "errors": 8, | |
| "error_rate": 0.050955414012738856 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8079268292682927, | |
| "confidence_cutoff": 0.9024031758308411 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8003048780487805, | |
| "accuracy": 0.7219047619047619, | |
| "confidence_cutoff": 0.6487010024681538 | |
| } | |
| }, | |
| "score_mae": 0.6515860158402881, | |
| "ranked_probability_score": 0.23189794760605645 | |
| }, | |
| "none_absent": { | |
| "n": 36, | |
| "nll": 0.38820769746181283, | |
| "acc": 0.8333333333333334, | |
| "ece": 0.08916457948776643, | |
| "brier": 0.22282945335191004, | |
| "mean_conf": 0.8007323164732282, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.4444444444444444, | |
| "accuracy_at_0_9": 1.0, | |
| "coverage_at_5pct_error": 0.75, | |
| "coverage_at_1pct_error": 0.4444444444444444, | |
| "confidence_bias": -0.03260101686010519, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 16, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 13, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 6, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.9444444444444444, | |
| "confidence_cutoff": 0.8828300404102825 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8055555555555556, | |
| "accuracy": 0.9310344827586207, | |
| "confidence_cutoff": 0.6286187226089864 | |
| } | |
| } | |
| }, | |
| "none_present": { | |
| "n": 36, | |
| "nll": 1.264206963123794, | |
| "acc": 0.4444444444444444, | |
| "ece": 0.31744480152703275, | |
| "brier": 0.653531262440586, | |
| "mean_conf": 0.7618892459714771, | |
| "confident_error_rate": 0.05555555555555555, | |
| "coverage_at_0_9": 0.3888888888888889, | |
| "accuracy_at_0_9": 0.8571428571428571, | |
| "coverage_at_5pct_error": 0.3055555555555556, | |
| "coverage_at_1pct_error": 0.3055555555555556, | |
| "confidence_bias": 0.3174448015270327, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 14, | |
| "errors": 2, | |
| "error_rate": 0.14285714285714285 | |
| }, | |
| "0.95": { | |
| "n": 12, | |
| "errors": 1, | |
| "error_rate": 0.08333333333333333 | |
| }, | |
| "0.99": { | |
| "n": 8, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.7777777777777778, | |
| "confidence_cutoff": 0.8565133594916252 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8055555555555556, | |
| "accuracy": 0.5517241379310345, | |
| "confidence_cutoff": 0.5275042540090179 | |
| } | |
| } | |
| }, | |
| "permuted": { | |
| "n": 36, | |
| "nll": 0.6298818160485206, | |
| "acc": 0.6944444444444444, | |
| "ece": 0.16106824698543198, | |
| "brier": 0.36143960629508687, | |
| "mean_conf": 0.7996874698756644, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.5277777777777778, | |
| "accuracy_at_0_9": 1.0, | |
| "coverage_at_5pct_error": 0.6111111111111112, | |
| "coverage_at_1pct_error": 0.5833333333333334, | |
| "confidence_bias": 0.10524302543122, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 19, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 17, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 13, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9428915688779251 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8055555555555556, | |
| "accuracy": 0.7586206896551724, | |
| "confidence_cutoff": 0.5394178434772673 | |
| } | |
| } | |
| } | |
| }, | |
| "heldout_tasks": { | |
| "composition_held_and_or": { | |
| "n": 32, | |
| "nll": 1.1899194065298853, | |
| "acc": 0.65625, | |
| "ece": 0.3062826763573561, | |
| "brier": 0.6297875816783778, | |
| "mean_conf": 0.9625326763573561, | |
| "confident_error_rate": 0.3125, | |
| "coverage_at_0_9": 0.9375, | |
| "accuracy_at_0_9": 0.6666666666666666, | |
| "coverage_at_5pct_error": 0.125, | |
| "coverage_at_1pct_error": 0.125, | |
| "confidence_bias": 0.30628267635735607, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 30, | |
| "errors": 10, | |
| "error_rate": 0.3333333333333333 | |
| }, | |
| "0.95": { | |
| "n": 24, | |
| "errors": 8, | |
| "error_rate": 0.3333333333333333 | |
| }, | |
| "0.99": { | |
| "n": 4, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8125, | |
| "confidence_cutoff": 0.976970766787314 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.6153846153846154, | |
| "confidence_cutoff": 0.9484939888568524 | |
| } | |
| } | |
| }, | |
| "composition_held_conditional": { | |
| "n": 32, | |
| "nll": 0.675725781843872, | |
| "acc": 0.59375, | |
| "ece": 0.18786636617174748, | |
| "brier": 0.4705037394069092, | |
| "mean_conf": 0.7816163661717475, | |
| "confident_error_rate": 0.0625, | |
| "coverage_at_0_9": 0.25, | |
| "accuracy_at_0_9": 0.75, | |
| "coverage_at_5pct_error": 0.1875, | |
| "coverage_at_1pct_error": 0.1875, | |
| "confidence_bias": 0.18786636617174746, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 8, | |
| "errors": 2, | |
| "error_rate": 0.25 | |
| }, | |
| "0.95": { | |
| "n": 3, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8125, | |
| "confidence_cutoff": 0.8704544098878388 | |
| }, | |
| "0.8": { | |
| "coverage": 0.84375, | |
| "accuracy": 0.6666666666666666, | |
| "confidence_cutoff": 0.5778257422991319 | |
| } | |
| } | |
| }, | |
| "composition_held_or_not": { | |
| "n": 32, | |
| "nll": 0.8207520004606448, | |
| "acc": 0.5625, | |
| "ece": 0.2816247209338629, | |
| "brier": 0.5616030854083928, | |
| "mean_conf": 0.8278648881415858, | |
| "confident_error_rate": 0.03125, | |
| "coverage_at_0_9": 0.3125, | |
| "accuracy_at_0_9": 0.9, | |
| "coverage_at_5pct_error": 0.09375, | |
| "coverage_at_1pct_error": 0.09375, | |
| "confidence_bias": 0.26536488814158576, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 10, | |
| "errors": 1, | |
| "error_rate": 0.1 | |
| }, | |
| "0.95": { | |
| "n": 5, | |
| "errors": 1, | |
| "error_rate": 0.2 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.53125, | |
| "accuracy": 0.7058823529411765, | |
| "confidence_cutoff": 0.867844091834961 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8125, | |
| "accuracy": 0.6538461538461539, | |
| "confidence_cutoff": 0.6750653184930885 | |
| } | |
| } | |
| }, | |
| "contrastive_authorization": { | |
| "n": 40, | |
| "nll": 0.06778782153149417, | |
| "acc": 0.975, | |
| "ece": 0.031234135570157376, | |
| "brier": 0.036192439631432066, | |
| "mean_conf": 0.9497480518201697, | |
| "confident_error_rate": 0.0, | |
| "coverage_at_0_9": 0.85, | |
| "accuracy_at_0_9": 1.0, | |
| "coverage_at_5pct_error": 1.0, | |
| "coverage_at_1pct_error": 0.95, | |
| "confidence_bias": -0.02525194817983023, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 34, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.95": { | |
| "n": 31, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 27, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.99714271867781 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9445846571260915 | |
| } | |
| } | |
| }, | |
| "contrastive_deadline": { | |
| "n": 40, | |
| "nll": 1.3889026704649357, | |
| "acc": 0.375, | |
| "ece": 0.3829928525155385, | |
| "brier": 0.7922274946031982, | |
| "mean_conf": 0.7042415556863528, | |
| "confident_error_rate": 0.1, | |
| "coverage_at_0_9": 0.225, | |
| "accuracy_at_0_9": 0.5555555555555556, | |
| "coverage_at_5pct_error": 0.0, | |
| "coverage_at_1pct_error": 0.0, | |
| "confidence_bias": 0.3292415556863528, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 9, | |
| "errors": 4, | |
| "error_rate": 0.4444444444444444 | |
| }, | |
| "0.95": { | |
| "n": 5, | |
| "errors": 2, | |
| "error_rate": 0.4 | |
| }, | |
| "0.99": { | |
| "n": 0, | |
| "errors": 0, | |
| "error_rate": null | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.55, | |
| "confidence_cutoff": 0.6421262312885925 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.40625, | |
| "confidence_cutoff": 0.5615110481010533 | |
| } | |
| }, | |
| "score_mae": 0.6515860158402881, | |
| "ranked_probability_score": 0.23189794760605645 | |
| }, | |
| "emotion": { | |
| "n": 80, | |
| "nll": 1.6769960423140386, | |
| "acc": 0.5375, | |
| "ece": 0.22586976855913896, | |
| "brier": 0.6678512282134752, | |
| "mean_conf": 0.7472712517372814, | |
| "confident_error_rate": 0.075, | |
| "coverage_at_0_9": 0.3125, | |
| "accuracy_at_0_9": 0.76, | |
| "coverage_at_5pct_error": 0.0, | |
| "coverage_at_1pct_error": 0.0, | |
| "confidence_bias": 0.2097712517372814, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 25, | |
| "errors": 6, | |
| "error_rate": 0.24 | |
| }, | |
| "0.95": { | |
| "n": 19, | |
| "errors": 4, | |
| "error_rate": 0.21052631578947367 | |
| }, | |
| "0.99": { | |
| "n": 9, | |
| "errors": 3, | |
| "error_rate": 0.3333333333333333 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.65, | |
| "confidence_cutoff": 0.7725859213122825 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.578125, | |
| "confidence_cutoff": 0.56208528383785 | |
| } | |
| } | |
| }, | |
| "mmlu": { | |
| "n": 80, | |
| "nll": 1.406507662100838, | |
| "acc": 0.425, | |
| "ece": 0.23086071722009255, | |
| "brier": 0.7337000166926988, | |
| "mean_conf": 0.6558607172200925, | |
| "confident_error_rate": 0.05, | |
| "coverage_at_0_9": 0.225, | |
| "accuracy_at_0_9": 0.7777777777777778, | |
| "coverage_at_5pct_error": 0.1, | |
| "coverage_at_1pct_error": 0.1, | |
| "confidence_bias": 0.23086071722009255, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 18, | |
| "errors": 4, | |
| "error_rate": 0.2222222222222222 | |
| }, | |
| "0.95": { | |
| "n": 16, | |
| "errors": 3, | |
| "error_rate": 0.1875 | |
| }, | |
| "0.99": { | |
| "n": 11, | |
| "errors": 1, | |
| "error_rate": 0.09090909090909091 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.525, | |
| "confidence_cutoff": 0.618546923871794 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.453125, | |
| "confidence_cutoff": 0.4300697519593815 | |
| } | |
| } | |
| }, | |
| "paws": { | |
| "n": 80, | |
| "nll": 1.1248226915361372, | |
| "acc": 0.55, | |
| "ece": 0.3200331593407043, | |
| "brier": 0.7235656563851223, | |
| "mean_conf": 0.8654864054853849, | |
| "confident_error_rate": 0.2625, | |
| "coverage_at_0_9": 0.5, | |
| "accuracy_at_0_9": 0.475, | |
| "coverage_at_5pct_error": 0.0125, | |
| "coverage_at_1pct_error": 0.0125, | |
| "confidence_bias": 0.31548640548538487, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 40, | |
| "errors": 21, | |
| "error_rate": 0.525 | |
| }, | |
| "0.95": { | |
| "n": 12, | |
| "errors": 5, | |
| "error_rate": 0.4166666666666667 | |
| }, | |
| "0.99": { | |
| "n": 2, | |
| "errors": 1, | |
| "error_rate": 0.5 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.475, | |
| "confidence_cutoff": 0.9014056727918538 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.546875, | |
| "confidence_cutoff": 0.8111088751091231 | |
| } | |
| } | |
| }, | |
| "qnli": { | |
| "n": 80, | |
| "nll": 0.4122149692919013, | |
| "acc": 0.85, | |
| "ece": 0.09433166058006281, | |
| "brier": 0.24752108114076724, | |
| "mean_conf": 0.8990905626919042, | |
| "confident_error_rate": 0.0375, | |
| "coverage_at_0_9": 0.6625, | |
| "accuracy_at_0_9": 0.9433962264150944, | |
| "coverage_at_5pct_error": 0.5125, | |
| "coverage_at_1pct_error": 0.3625, | |
| "confidence_bias": 0.04909056269190426, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 53, | |
| "errors": 3, | |
| "error_rate": 0.05660377358490566 | |
| }, | |
| "0.95": { | |
| "n": 43, | |
| "errors": 3, | |
| "error_rate": 0.06976744186046512 | |
| }, | |
| "0.99": { | |
| "n": 29, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.95, | |
| "confidence_cutoff": 0.9583688735354272 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.890625, | |
| "confidence_cutoff": 0.793002498045601 | |
| } | |
| } | |
| }, | |
| "sciq": { | |
| "n": 80, | |
| "nll": 0.24132037549325877, | |
| "acc": 0.9125, | |
| "ece": 0.04025150891820237, | |
| "brier": 0.12820506248919616, | |
| "mean_conf": 0.9387513504714097, | |
| "confident_error_rate": 0.025, | |
| "coverage_at_0_9": 0.8375, | |
| "accuracy_at_0_9": 0.9701492537313433, | |
| "coverage_at_5pct_error": 0.925, | |
| "coverage_at_1pct_error": 0.7, | |
| "confidence_bias": 0.026251350471409696, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 67, | |
| "errors": 2, | |
| "error_rate": 0.029850746268656716 | |
| }, | |
| "0.95": { | |
| "n": 63, | |
| "errors": 1, | |
| "error_rate": 0.015873015873015872 | |
| }, | |
| "0.99": { | |
| "n": 58, | |
| "errors": 1, | |
| "error_rate": 0.017241379310344827 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 1.0, | |
| "confidence_cutoff": 0.9998130113129569 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.96875, | |
| "confidence_cutoff": 0.9417403771936594 | |
| } | |
| } | |
| }, | |
| "tweet_offensive": { | |
| "n": 80, | |
| "nll": 0.8975003034444349, | |
| "acc": 0.675, | |
| "ece": 0.18809786512896348, | |
| "brier": 0.5095841174172036, | |
| "mean_conf": 0.8555092051551096, | |
| "confident_error_rate": 0.15, | |
| "coverage_at_0_9": 0.475, | |
| "accuracy_at_0_9": 0.6842105263157895, | |
| "coverage_at_5pct_error": 0.0375, | |
| "coverage_at_1pct_error": 0.0375, | |
| "confidence_bias": 0.1805092051551096, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 38, | |
| "errors": 12, | |
| "error_rate": 0.3157894736842105 | |
| }, | |
| "0.95": { | |
| "n": 28, | |
| "errors": 8, | |
| "error_rate": 0.2857142857142857 | |
| }, | |
| "0.99": { | |
| "n": 17, | |
| "errors": 2, | |
| "error_rate": 0.11764705882352941 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.675, | |
| "confidence_cutoff": 0.8908563720881537 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.703125, | |
| "confidence_cutoff": 0.7286377114869528 | |
| } | |
| } | |
| } | |
| }, | |
| "permutation": { | |
| "n": 36, | |
| "mean_max_delta": 0.0699928005819115, | |
| "flip_rate": 0.08333333333333333 | |
| }, | |
| "temperature": 2.6390158215457884, | |
| "calibrated_clean": { | |
| "n": 656, | |
| "nll": 0.71295343189403, | |
| "acc": 0.6524390243902439, | |
| "ece": 0.04090334952670928, | |
| "brier": 0.42880721405783623, | |
| "mean_conf": 0.672610937208296, | |
| "confident_error_rate": 0.003048780487804878, | |
| "coverage_at_0_9": 0.15548780487804878, | |
| "accuracy_at_0_9": 0.9803921568627451, | |
| "coverage_at_5pct_error": 0.22865853658536586, | |
| "coverage_at_1pct_error": 0.14939024390243902, | |
| "confidence_bias": 0.020171912818052085, | |
| "top_bins": { | |
| "0.9": { | |
| "n": 102, | |
| "errors": 2, | |
| "error_rate": 0.0196078431372549 | |
| }, | |
| "0.95": { | |
| "n": 52, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| }, | |
| "0.99": { | |
| "n": 8, | |
| "errors": 0, | |
| "error_rate": 0.0 | |
| } | |
| }, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.5, | |
| "accuracy": 0.8079268292682927, | |
| "confidence_cutoff": 0.6812517987895854 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8003048780487805, | |
| "accuracy": 0.7257142857142858, | |
| "confidence_cutoff": 0.5086893876402309 | |
| } | |
| }, | |
| "score_mae": 0.69030504615076, | |
| "ranked_probability_score": 0.21277812800381835 | |
| }, | |
| "metric_policy": { | |
| "nll_floor": 1e-09, | |
| "renormalize_returned_probabilities": true, | |
| "raw_sums_outside_1e_5": 0, | |
| "returned_zeros": 0 | |
| }, | |
| "coverage": { | |
| "requested_records": 764, | |
| "requested_questions": 764, | |
| "evaluated_records": 764, | |
| "evaluated_questions": 764, | |
| "rejected_records": 0, | |
| "truncated_records": 0 | |
| }, | |
| "latency_ms": { | |
| "median": 40.60463100006473, | |
| "p95": 43.923992149950664 | |
| }, | |
| "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1" | |
| }, | |
| "provenance": { | |
| "config": { | |
| "epochs": 1, | |
| "seed": 1, | |
| "lr": 4e-05, | |
| "lora": 16, | |
| "accum": 1, | |
| "batch": 8, | |
| "perm_kl": 0.0, | |
| "perm_frac": 0.3, | |
| "ord_w": 0.0, | |
| "p_none": 0.1, | |
| "p_none_distract": 0.12, | |
| "p_distract": 0.15, | |
| "p_none_pair": 0.25, | |
| "synthetic_repeat": 1, | |
| "public_frac": 1.0, | |
| "head_lr": 0.0, | |
| "weight_decay": 0.01, | |
| "anchor_w": 0.0, | |
| "dtype": "bf16", | |
| "replay": 2000, | |
| "base": "Qwen/Qwen3.5-0.8B-Base", | |
| "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", | |
| "init_from": "jaredpalmer/kev-0.8b", | |
| "data": "evals/night2/dates_unknowable.jsonl" | |
| }, | |
| "config_sha256": "0e130e9f338c8a4c911c6dc37493b36e6be7ca972cbafbb42466043b6daa7a4b", | |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", | |
| "source_hashes": { | |
| "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", | |
| "kev/anchors.py": "089d8a5493502bb26f540eb1c5e681780ca0bb01276073d4e1733211f15d0e10", | |
| "kev/api.py": "c9eebbdb6c625563d33f030299ce0fbe8b50493dd1e92592df9514c30f5720af", | |
| "kev/autoresearch.py": "0a8aa6b57c1c9ba93d25b2cf631b03aed2e686e374c1148002eec48765b7b1fc", | |
| "kev/benchmark.py": "4192ec3b26b065452f84bde38a091e6854a28fe185d2e0f39a0c91b2efe69df7", | |
| "kev/compare.py": "bd0445f021de59e35c7bff9304e39dd2e1211e3877a453575594ae7b81b0ada4", | |
| "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", | |
| "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", | |
| "kev/data.py": "997c31d737c2a130ade49edd6534aa47d910786c98af883745b1a97ec858a704", | |
| "kev/evaluate.py": "1b20e3f9edf417aa8dae924b1526e52f74b710cadf7213c5ec68334f6e7f8fe1", | |
| "kev/experiment.py": "ba034856f79bd500d8b2eeeb7382e4db3fccb30b437f9f3cf55cf1a7c565a6c9", | |
| "kev/jev.py": "e0213782359ba2f95adbf045ddaf0a008b08b4162bf6a9b66d4ce55fc91a51cb", | |
| "kev/model.py": "a17d57a52da3fe6ebef7146ce548bd3cedc09e21cc2b7738d9077771f5f16989", | |
| "kev/plot.py": "d689c7dd18cde7f9ea77cb4f55c50ff7da1880b21cb2ecf244348e45842a1382", | |
| "kev/publish.py": "8f23cda767008756e0f169249eb44577db76bd6c1e12e9cf64c302bad04c8f16", | |
| "kev/serve.py": "d095bbee3a210fa8807d1a7b073b56181c93aa69b00f2e243a323ab0a19fa8c9", | |
| "kev/study_v3.py": "9fc44d44dad09a4f1ee29a7bcb2eb3c7aa373d09186d0e9533666b69ec95c401", | |
| "kev/suite.py": "44858f9a99df086a47d1ae36141631fab6fb6a8293a9e1d0c6ad397ddcfaf94a", | |
| "kev/train.py": "c2a3770072b73dd4c0769c8f188ddedbb402f2f8195df6e10b11e5112cab734b", | |
| "kev/transfer_v9.py": "588aa2ff3ec0823c2e31733bef9a3748b263849c966ef6a80ebd686539c605e9", | |
| "modal_app.py": "d3700b2914be5ef6baa6d588a967d3a242903e1ed96d0f5cc525a8908aa58340", | |
| "pyproject.toml": "52da5eea3efc6f2b1c0589acebad62e56a214294bb02c1a4218c93efd4af3182", | |
| "uv.lock": "18b3e5ea0f25d2e8546fab81f16cb965ae05c3289fffaaa1ce27d114adee47f3" | |
| }, | |
| "git_commit": "19dcae9b6e3e1a48200c5825aad9fc200d31e20a", | |
| "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", | |
| "torch": "2.8.0+cu128", | |
| "device": "cuda", | |
| "gpu": "NVIDIA H100 80GB HBM3", | |
| "legacy_checkpoint": false | |
| }, | |
| "mechanism_checks": { | |
| "n": 8, | |
| "packed_max_delta": 0.0, | |
| "sibling_max_delta": 0.0008976459503173828, | |
| "tolerance": 0.001, | |
| "passed": true | |
| }, | |
| "gates": { | |
| "passed": false, | |
| "checks": { | |
| "complete_coverage": true, | |
| "isolation_and_packing": true, | |
| "transfer_complete": true, | |
| "heldout_pairs_at_least_70pct": false, | |
| "transfer_confident_errors_below_10pct": true | |
| }, | |
| "policy": "Research screening only: 1e-3 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." | |
| }, | |
| "wall_seconds": 546.105594389, | |
| "promotable": false, | |
| "test_evaluated": false, | |
| "training_resources": { | |
| "wall_seconds": 313.94338822364807, | |
| "records_seen": 3937, | |
| "requested_records": 3425, | |
| "truncated_records": 0, | |
| "rejected_records": 0, | |
| "optimizer_steps": 429, | |
| "forward_tokens": 640184, | |
| "peak_device_bytes": 43896273920, | |
| "device": "cuda", | |
| "dtype": "bf16", | |
| "batch": 8, | |
| "peak_rss_bytes": 10316816384 | |
| } | |
| } | |