{ "run": "runs/kev", "holdout_sources": [], "baseline_zero_shot_base": { "agnews": { "n": 150, "acc": 0.8133333333333334, "ece": 0.06872746845086417, "mean_conf": 0.7705230579773585 }, "agnews_yn": { "n": 300, "acc": 0.78, "ece": 0.1028619595368703, "mean_conf": 0.7659800553321838 }, "sst5": { "n": 150, "acc": 0.37333333333333335, "ece": 0.08330158472061158, "mean_conf": 0.4375393835703532 }, "boolq": { "n": 150, "acc": 0.4266666666666667, "ece": 0.27390830198923743, "mean_conf": 0.7005749686559042 }, "yelp": { "n": 150, "acc": 0.31333333333333335, "ece": 0.0428644096851349, "mean_conf": 0.2963246401151021 }, "yelp_yn": { "n": 150, "acc": 0.8333333333333334, "ece": 0.12900178631146747, "mean_conf": 0.711561503012975 }, "mnli": { "n": 150, "acc": 0.46, "ece": 0.2252740067243576, "mean_conf": 0.6829265818993251 } }, "baseline_zero_shot_instruct": { "agnews": { "n": 150, "acc": 0.7866666666666666, "ece": 0.1601808597644171, "mean_conf": 0.9435027374823888 }, "agnews_yn": { "n": 300, "acc": 0.8533333333333334, "ece": 0.06154396494229632, "mean_conf": 0.791789368391037 }, "sst5": { "n": 150, "acc": 0.44666666666666666, "ece": 0.3441617304086685, "mean_conf": 0.7542105954885483 }, "boolq": { "n": 150, "acc": 0.6066666666666667, "ece": 0.0836830314000448, "mean_conf": 0.6903496980667114 }, "yelp": { "n": 150, "acc": 0.35333333333333333, "ece": 0.07781609197457631, "mean_conf": 0.3684885170062383 }, "yelp_yn": { "n": 150, "acc": 0.8333333333333334, "ece": 0.06565124352773032, "mean_conf": 0.8569089023272196 }, "mnli": { "n": 150, "acc": 0.43333333333333335, "ece": 0.3897599758704503, "mean_conf": 0.8230933092037837 } }, "accuracy_calibration": { "agnews": { "n": 150, "acc": 0.94, "ece": 0.02787075857321425, "nll": 0.21451582382221224, "mean_conf": 0.9441667677958806 }, "agnews_yn": { "n": 300, "acc": 0.96, "ece": 0.01691638271013903, "nll": 0.10442558608399229, "mean_conf": 0.9658287052313487 }, "sst5": { "n": 150, "acc": 0.5333333333333333, "ece": 0.12093879044055943, "nll": 1.169872780334628, "mean_conf": 0.6458062372605006, "score_mae_levels": 0.5936220041739337 }, "boolq": { "n": 150, "acc": 0.7533333333333333, "ece": 0.13565061767896014, "nll": 0.6336200484640118, "mean_conf": 0.8791843442122141 }, "yelp": { "n": 150, "acc": 0.5533333333333333, "ece": 0.11765717585881552, "nll": 0.9509858549967819, "mean_conf": 0.6709905091921489, "score_mae_levels": 0.5347741969060735 }, "yelp_yn": { "n": 150, "acc": 0.8866666666666667, "ece": 0.083875914812088, "nll": 0.3330675296269493, "mean_conf": 0.9531527554988861 }, "mnli": { "n": 150, "acc": 0.7466666666666667, "ece": 0.09995454927285516, "nll": 0.6312906145540002, "mean_conf": 0.8385461642344793 }, "banking77": { "n": 150, "acc": 0.86, "ece": 0.05680914471546812, "nll": 0.5588821548329137, "mean_conf": 0.9066748427351315 }, "ALL": { "n": 1350, "acc": 0.7992592592592592, "ece": 0.0653030041080934, "mean_conf": 0.8633532257102154 } }, "temperature_scaling": { "T": 1.4741506576538086, "n_fit": 676, "n_held": 674, "before": { "nll": 0.5052552819252014, "ece": 0.05687867120483156 }, "after": { "nll": 0.48128193616867065, "ece": 0.031318123875191346 } }, "permutation": { "n": 81, "argmax_flip_rate": 0.07407407407407407, "mean_prob_spread": 0.06532220676811527, "p90_prob_spread": 0.2485833764076233 }, "iia": { "n": 86, "mean_abs_logodds_shift": 0.13282728950854877, "mean_shift": 0.0404207781784885, "p90_abs_shift": 0.34304980440445876 }, "isolation": { "sibling": { "mean_p_code": 0.030429982068017124, "min": 0.010374528355896473, "max": 0.07068686187267303 }, "state": { "mean_p_code": 0.9903109312057495, "min": 0.9674482941627502, "max": 0.99826979637146 }, "absent": { "mean_p_code": 0.03043001419864595, "min": 0.01037475187331438, "max": 0.07068636268377304 } }, "packed_vs_separate": { "n_questions": 80, "max_abs_prob_diff": 3.6954879760742188e-06, "mean_abs_prob_diff": 2.1806145014124923e-07, "packed_s": 2.3287577629089355, "separate_s": 4.706204175949097, "speedup": 2.020907563211043 } }