eval.json

{
 "run": "runs/kev",
 "holdout_sources": [],
 "baseline_zero_shot_base": {
  "agnews": {
   "n": 150,
   "acc": 0.8133333333333334,
   "ece": 0.06872746845086417,
   "mean_conf": 0.7705230579773585
  },
  "agnews_yn": {
   "n": 300,
   "acc": 0.78,
   "ece": 0.1028619595368703,
   "mean_conf": 0.7659800553321838
  },
  "sst5": {
   "n": 150,
   "acc": 0.37333333333333335,
   "ece": 0.08330158472061158,
   "mean_conf": 0.4375393835703532
  },
  "boolq": {
   "n": 150,
   "acc": 0.4266666666666667,
   "ece": 0.27390830198923743,
   "mean_conf": 0.7005749686559042
  },
  "yelp": {
   "n": 150,
   "acc": 0.31333333333333335,
   "ece": 0.0428644096851349,
   "mean_conf": 0.2963246401151021
  },
  "yelp_yn": {
   "n": 150,
   "acc": 0.8333333333333334,
   "ece": 0.12900178631146747,
   "mean_conf": 0.711561503012975
  },
  "mnli": {
   "n": 150,
   "acc": 0.46,
   "ece": 0.2252740067243576,
   "mean_conf": 0.6829265818993251
  }
 },
 "baseline_zero_shot_instruct": {
  "agnews": {
   "n": 150,
   "acc": 0.7866666666666666,
   "ece": 0.1601808597644171,
   "mean_conf": 0.9435027374823888
  },
  "agnews_yn": {
   "n": 300,
   "acc": 0.8533333333333334,
   "ece": 0.06154396494229632,
   "mean_conf": 0.791789368391037
  },
  "sst5": {
   "n": 150,
   "acc": 0.44666666666666666,
   "ece": 0.3441617304086685,
   "mean_conf": 0.7542105954885483
  },
  "boolq": {
   "n": 150,
   "acc": 0.6066666666666667,
   "ece": 0.0836830314000448,
   "mean_conf": 0.6903496980667114
  },
  "yelp": {
   "n": 150,
   "acc": 0.35333333333333333,
   "ece": 0.07781609197457631,
   "mean_conf": 0.3684885170062383
  },
  "yelp_yn": {
   "n": 150,
   "acc": 0.8333333333333334,
   "ece": 0.06565124352773032,
   "mean_conf": 0.8569089023272196
  },
  "mnli": {
   "n": 150,
   "acc": 0.43333333333333335,
   "ece": 0.3897599758704503,
   "mean_conf": 0.8230933092037837
  }
 },
 "accuracy_calibration": {
  "agnews": {
   "n": 150,
   "acc": 0.94,
   "ece": 0.02787075857321425,
   "nll": 0.21451582382221224,
   "mean_conf": 0.9441667677958806
  },
  "agnews_yn": {
   "n": 300,
   "acc": 0.96,
   "ece": 0.01691638271013903,
   "nll": 0.10442558608399229,
   "mean_conf": 0.9658287052313487
  },
  "sst5": {
   "n": 150,
   "acc": 0.5333333333333333,
   "ece": 0.12093879044055943,
   "nll": 1.169872780334628,
   "mean_conf": 0.6458062372605006,
   "score_mae_levels": 0.5936220041739337
  },
  "boolq": {
   "n": 150,
   "acc": 0.7533333333333333,
   "ece": 0.13565061767896014,
   "nll": 0.6336200484640118,
   "mean_conf": 0.8791843442122141
  },
  "yelp": {
   "n": 150,
   "acc": 0.5533333333333333,
   "ece": 0.11765717585881552,
   "nll": 0.9509858549967819,
   "mean_conf": 0.6709905091921489,
   "score_mae_levels": 0.5347741969060735
  },
  "yelp_yn": {
   "n": 150,
   "acc": 0.8866666666666667,
   "ece": 0.083875914812088,
   "nll": 0.3330675296269493,
   "mean_conf": 0.9531527554988861
  },
  "mnli": {
   "n": 150,
   "acc": 0.7466666666666667,
   "ece": 0.09995454927285516,
   "nll": 0.6312906145540002,
   "mean_conf": 0.8385461642344793
  },
  "banking77": {
   "n": 150,
   "acc": 0.86,
   "ece": 0.05680914471546812,
   "nll": 0.5588821548329137,
   "mean_conf": 0.9066748427351315
  },
  "ALL": {
   "n": 1350,
   "acc": 0.7992592592592592,
   "ece": 0.0653030041080934,
   "mean_conf": 0.8633532257102154
  }
 },
 "temperature_scaling": {
  "T": 1.4741506576538086,
  "n_fit": 676,
  "n_held": 674,
  "before": {
   "nll": 0.5052552819252014,
   "ece": 0.05687867120483156
  },
  "after": {
   "nll": 0.48128193616867065,
   "ece": 0.031318123875191346
  }
 },
 "permutation": {
  "n": 81,
  "argmax_flip_rate": 0.07407407407407407,
  "mean_prob_spread": 0.06532220676811527,
  "p90_prob_spread": 0.2485833764076233
 },
 "iia": {
  "n": 86,
  "mean_abs_logodds_shift": 0.13282728950854877,
  "mean_shift": 0.0404207781784885,
  "p90_abs_shift": 0.34304980440445876
 },
 "isolation": {
  "sibling": {
   "mean_p_code": 0.030429982068017124,
   "min": 0.010374528355896473,
   "max": 0.07068686187267303
  },
  "state": {
   "mean_p_code": 0.9903109312057495,
   "min": 0.9674482941627502,
   "max": 0.99826979637146
  },
  "absent": {
   "mean_p_code": 0.03043001419864595,
   "min": 0.01037475187331438,
   "max": 0.07068636268377304
  }
 },
 "packed_vs_separate": {
  "n_questions": 80,
  "max_abs_prob_diff": 3.6954879760742188e-06,
  "mean_abs_prob_diff": 2.1806145014124923e-07,
  "packed_s": 2.3287577629089355,
  "separate_s": 4.706204175949097,
  "speedup": 2.020907563211043
 }
}