{
  "T": 1.03979841848149,
  "val_nll": 0.3844288446342489,
  "args": {
    "name": "nc_v3_tap20_wf",
    "backbone": "Qwen/Qwen3-1.7B-Base",
    "lora_layers": 8,
    "lora_r": 16,
    "steps": 12000,
    "bs": 64,
    "lr": 0.0003,
    "lora_lr": 0.0001,
    "seed": 0,
    "no_hybrid": false,
    "no_cand_null": false,
    "no_null": false,
    "null": "factored",
    "hard_only": false,
    "mix": "full",
    "eval_every": 6000,
    "val_every": 1000,
    "ckpt_every": 2000,
    "eval_bs": 128,
    "device": "auto",
    "wandb": true,
    "hf_repo": "guychuk/pcdm-runs",
    "data": "data_v5",
    "extra_data": "data_kb,data_wf,data_wf_hf",
    "eval_cap": 1500,
    "family_weights": "E:0.35,K:0.25,W:0.40",
    "bucket_map": null,
    "smoke": false,
    "eval_limit": 0,
    "tap_layer": 20,
    "zscore": true,
    "eval_only": false,
    "dump_logits": null,
    "tower_d": 512,
    "tower_layers": 2,
    "tower_heads": 8,
    "joint": false,
    "listwise": false,
    "extra_tap": 0,
    "init_from": null,
    "readout": "native",
    "nc_head": "n3",
    "nc_render": "letters_nonull",
    "perm_lambda": 0.0,
    "no_shuffle": false,
    "zero_shot": false,
    "shots": 0,
    "cand_encoder": "backbone",
    "head": "mlp",
    "z_dim": 128,
    "z_probes": 8,
    "tiny_layers": 2,
    "distill_alpha": 1.0,
    "distill_beta": 0.0,
    "distill_T": 2.0,
    "delta_gamma": 0.0
  },
  "eval": {
    "anli_test": {
      "raw": {
        "nll": 1.102941199093527,
        "brier": 0.6654224947329161,
        "jsd": 0.2776569035848503,
        "acc": 0.480625,
        "acc_k": 0.480625,
        "ece": 0.19827818479388953,
        "sel_acc80": 0.5078125,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.0888644635622768,
        "brier": 0.6598136085192928,
        "jsd": 0.2771962879481138,
        "acc": 0.480625,
        "acc_k": 0.480625,
        "ece": 0.18998124125413596,
        "sel_acc80": 0.506640625,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "banking77_k": {
      "raw": {
        "nll": 0.8803650682647327,
        "brier": 0.4301744118516424,
        "jsd": 0.18126308231810428,
        "acc": 0.7012353706111834,
        "acc_k": 0.8533685601056803,
        "ece": 0.12095656268230806,
        "sel_acc80": 0.7704185290532304,
        "conf_wrong": 0.08898847631241998,
        "coverage": 0.640442132639792,
        "false_abstain": 0.10501981505944517,
        "null_recall": 0.6062740076824584,
        "auroc_null": 0.869071762144864,
        "auroc_conf": 0.8650941617037398
      },
      "scaled": {
        "nll": 0.8744278074702166,
        "brier": 0.42846344428894817,
        "jsd": 0.18141129837781403,
        "acc": 0.7009102730819246,
        "acc_k": 0.8533685601056803,
        "ece": 0.11870673383127112,
        "sel_acc80": 0.7712312068264933,
        "conf_wrong": 0.08770806658130602,
        "coverage": 0.6397919375812744,
        "false_abstain": 0.10634081902245707,
        "null_recall": 0.6062740076824584,
        "auroc_null": 0.869071762144864,
        "auroc_conf": 0.8651785215919028
      }
    },
    "banking77_test": {
      "raw": {
        "nll": 1.8943291566507845,
        "brier": 0.593270217263453,
        "jsd": 0.2957812292956273,
        "acc": 0.5718465539661899,
        "acc_k": 0.6089076723016905,
        "ece": 0.09358625776184087,
        "sel_acc80": 0.653799268590004,
        "coverage": 0.881664499349805,
        "false_abstain": 0.11833550065019506,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.8612821539463003,
        "brier": 0.5919693859966786,
        "jsd": 0.2973040180361969,
        "acc": 0.5711963589076723,
        "acc_k": 0.6089076723016905,
        "ece": 0.08643925216131311,
        "sel_acc80": 0.6525802519301097,
        "coverage": 0.8780884265279584,
        "false_abstain": 0.12191157347204161,
        "null_recall": "n/a"
      }
    },
    "boolq_val": {
      "raw": {
        "nll": 0.39380937671763266,
        "brier": 0.24073265857614617,
        "jsd": 0.10124721371819573,
        "acc": 0.8314984709480122,
        "acc_k": 0.8318042813455657,
        "ece": 0.0429865891415774,
        "sel_acc80": 0.8937308868501529,
        "coverage": 0.9996941896024465,
        "false_abstain": 0.0003058103975535168,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.39024824040639783,
        "brier": 0.2397877640930743,
        "jsd": 0.10181249185984385,
        "acc": 0.8314984709480122,
        "acc_k": 0.8318042813455657,
        "ece": 0.038267658646318155,
        "sel_acc80": 0.8937308868501529,
        "coverage": 0.9996941896024465,
        "false_abstain": 0.0003058103975535168,
        "null_recall": "n/a"
      }
    },
    "chaos_mnli": {
      "raw": {
        "nll": 1.1710537505967527,
        "brier": 0.22073145417462792,
        "jsd": 0.08313512961435864
      },
      "scaled": {
        "nll": 1.1500571436778484,
        "brier": 0.21457783233844271,
        "jsd": 0.08064832731516772
      }
    },
    "clinc_heldout": {
      "raw": {
        "nll": 0.27074162635707527,
        "brier": 0.11428612065096902,
        "jsd": 0.0543765164648202,
        "acc": 0.9242761692650334,
        "acc_k": 0.9621380846325167,
        "ece": 0.01579013417053861,
        "sel_acc80": 0.9833101529902643,
        "coverage": 0.9443207126948775,
        "false_abstain": 0.0556792873051225,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.2684241498104494,
        "brier": 0.11447972908114325,
        "jsd": 0.054635113203736436,
        "acc": 0.9242761692650334,
        "acc_k": 0.9621380846325167,
        "ece": 0.013791607257252527,
        "sel_acc80": 0.9833101529902643,
        "coverage": 0.9443207126948775,
        "false_abstain": 0.0556792873051225,
        "null_recall": "n/a"
      }
    },
    "clinc_k": {
      "raw": {
        "nll": 0.311515750792269,
        "brier": 0.16297286687242818,
        "jsd": 0.07596730831547199,
        "acc": 0.8914118705035972,
        "acc_k": 0.9633437639696022,
        "ece": 0.016932064215083087,
        "sel_acc80": 0.9530767069401517,
        "conf_wrong": 0.028493894165535955,
        "coverage": 0.5476618705035972,
        "false_abstain": 0.05185516316495306,
        "null_recall": 0.8575305291723202,
        "auroc_null": 0.968572628384877,
        "auroc_conf": 0.9670774829069189
      },
      "scaled": {
        "nll": 0.31090634881130236,
        "brier": 0.16266923363280875,
        "jsd": 0.07609676101977222,
        "acc": 0.8920863309352518,
        "acc_k": 0.9633437639696022,
        "ece": 0.016195901120148588,
        "sel_acc80": 0.9530767069401517,
        "conf_wrong": 0.028493894165535955,
        "coverage": 0.5469874100719425,
        "false_abstain": 0.05185516316495306,
        "null_recall": 0.858887381275441,
        "auroc_null": 0.968572628384877,
        "auroc_conf": 0.9670253196164098
      }
    },
    "clinc_oos": {
      "raw": {
        "nll": 0.3819562982125116,
        "brier": 0.18383285615491454,
        "jsd": 0.11433253299812943,
        "acc": 0.9009009009009009,
        "ece": 0.1304006703623064,
        "sel_acc80": 0.93625,
        "conf_wrong": 0.006006006006006006,
        "coverage": 0.09909909909909909,
        "false_abstain": "n/a",
        "null_recall": 0.9009009009009009
      },
      "scaled": {
        "nll": 0.3819562920724052,
        "brier": 0.18188391253101058,
        "jsd": 0.11433253017955829,
        "acc": 0.9029029029029029,
        "ece": 0.13308967943783395,
        "sel_acc80": 0.9375,
        "conf_wrong": 0.006006006006006006,
        "coverage": 0.09709709709709713,
        "false_abstain": "n/a",
        "null_recall": 0.9029029029029029
      }
    },
    "clinc_test": {
      "raw": {
        "nll": 0.9257443901513311,
        "brier": 0.36654857219284437,
        "jsd": 0.1835993970985268,
        "acc": 0.7360611510791367,
        "acc_k": 0.8223920863309353,
        "ece": 0.04578179257030756,
        "sel_acc80": 0.8328182073616184,
        "coverage": 0.835431654676259,
        "false_abstain": 0.164568345323741,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.920197700698808,
        "brier": 0.36788022046321867,
        "jsd": 0.1850340584565983,
        "acc": 0.7335881294964028,
        "acc_k": 0.8223920863309353,
        "ece": 0.04579438925084237,
        "sel_acc80": 0.8316942961506041,
        "coverage": 0.8322841726618705,
        "false_abstain": 0.1677158273381295,
        "null_recall": "n/a"
      }
    },
    "cse_banking77": {
      "raw": {
        "nll": 0.9572221236913963,
        "brier": 0.3772673010367225,
        "jsd": 0.16592351634826877,
        "acc": 0.7,
        "acc_k": 0.764,
        "ece": 0.1161508621672789,
        "sel_acc80": 0.7754166666666666,
        "conf_wrong": 0.098,
        "coverage": 0.8089999999999999,
        "false_abstain": 0.1016,
        "null_recall": 0.638,
        "auroc_null": 0.8789352,
        "auroc_conf": 0.8513904000000001
      },
      "scaled": {
        "nll": 0.9426415598509017,
        "brier": 0.375676489733989,
        "jsd": 0.16591535574509875,
        "acc": 0.7,
        "acc_k": 0.764,
        "ece": 0.11323217004537582,
        "sel_acc80": 0.7754166666666666,
        "conf_wrong": 0.096,
        "coverage": 0.808,
        "false_abstain": 0.1024,
        "null_recall": 0.64,
        "auroc_null": 0.8789352,
        "auroc_conf": 0.8510016
      },
      "cse": {
        "add_irr/p_irr": 0.0005661446037615407,
        "add_irr/dlo_top2": 0.2631000002575834,
        "remove_gold/dp_null": 0.4630419385624409,
        "dup/mass_err": 0.034684130056172505,
        "dup/slot_gap": 0.1855905814659261,
        "dup/acc_dedup": 0.838,
        "reorder/max_dp": 0.08897487102333913,
        "near_dup/cluster_mass_err": 0.03278506057004932
      }
    },
    "cse_clinc": {
      "raw": {
        "nll": 0.32437873855368415,
        "brier": 0.13432812964214,
        "jsd": 0.06377881461172775,
        "acc": 0.861178091629349,
        "acc_k": 0.8977980889073536,
        "ece": 0.0228896729501822,
        "sel_acc80": 0.9466207490314249,
        "conf_wrong": 0.03024193548387097,
        "coverage": 0.8256975542542198,
        "false_abstain": 0.0402991275446614,
        "null_recall": 0.8245967741935484,
        "auroc_null": 0.9744327700121956,
        "auroc_conf": 0.959220921505823
      },
      "scaled": {
        "nll": 0.322856150605895,
        "brier": 0.1338994864013624,
        "jsd": 0.06387929138342104,
        "acc": 0.8615225628660007,
        "acc_k": 0.8977980889073536,
        "ece": 0.021907006774406955,
        "sel_acc80": 0.9461902712010332,
        "conf_wrong": 0.03024193548387097,
        "coverage": 0.825353083017568,
        "false_abstain": 0.0402991275446614,
        "null_recall": 0.8266129032258065,
        "auroc_null": 0.9744327700121956,
        "auroc_conf": 0.9589512108500743
      },
      "cse": {
        "add_irr/p_irr": 0.0006255071544984961,
        "add_irr/dlo_top2": 0.3369036140496365,
        "remove_gold/dp_null": 0.7144053382439192,
        "dup/mass_err": 0.01891620019684563,
        "dup/slot_gap": 0.1731909674514862,
        "dup/acc_dedup": 0.9758064516129032,
        "reorder/max_dp": 0.035958222781689086,
        "near_dup/cluster_mass_err": 0.01875680701157674
      }
    },
    "cse_hwu64": {
      "raw": {
        "nll": 0.5650941517704147,
        "brier": 0.2537955674753089,
        "jsd": 0.11948498837404782,
        "acc": 0.773,
        "acc_k": 0.822,
        "ece": 0.04737755128741265,
        "sel_acc80": 0.84625,
        "conf_wrong": 0.058,
        "coverage": 0.8406666666666667,
        "false_abstain": 0.0548,
        "null_recall": 0.682,
        "auroc_null": 0.9259464000000002,
        "auroc_conf": 0.8916696
      },
      "scaled": {
        "nll": 0.5615799890226735,
        "brier": 0.2529652757233588,
        "jsd": 0.11981138093082586,
        "acc": 0.773,
        "acc_k": 0.822,
        "ece": 0.044626818845669436,
        "sel_acc80": 0.8466666666666667,
        "conf_wrong": 0.058,
        "coverage": 0.8406666666666667,
        "false_abstain": 0.0548,
        "null_recall": 0.682,
        "auroc_null": 0.9259464000000002,
        "auroc_conf": 0.8913199999999999
      },
      "cse": {
        "add_irr/p_irr": 0.009121834453501867,
        "add_irr/dlo_top2": 0.2421149150189948,
        "remove_gold/dp_null": 0.5026223030890106,
        "dup/mass_err": 0.036702267746052814,
        "dup/slot_gap": 0.1844224565153616,
        "dup/acc_dedup": 0.94,
        "reorder/max_dp": 0.06679525845381432,
        "near_dup/cluster_mass_err": 0.0402173605489788
      }
    },
    "cse_snli": {
      "raw": {
        "nll": 0.42645938467085154,
        "brier": 0.17942825051131092,
        "jsd": 0.09235424546372771,
        "acc": 0.8393124065769806,
        "acc_k": 0.8474264705882353,
        "ece": 0.04809902380088343,
        "sel_acc80": 0.9093881363848669,
        "conf_wrong": 0.03,
        "coverage": 0.8497757847533632,
        "false_abstain": 0.0,
        "null_recall": 0.804,
        "auroc_null": 0.9997564338235294,
        "auroc_conf": 0.9327977941176472
      },
      "scaled": {
        "nll": 0.4289722985328097,
        "brier": 0.1800702618260962,
        "jsd": 0.09380953611654914,
        "acc": 0.8396860986547086,
        "acc_k": 0.8474264705882353,
        "ece": 0.053872445317038144,
        "sel_acc80": 0.9093881363848669,
        "conf_wrong": 0.028,
        "coverage": 0.8494020926756353,
        "false_abstain": 0.0,
        "null_recall": 0.806,
        "auroc_null": 0.9997564338235294,
        "auroc_conf": 0.9316098345588235
      },
      "cse": {
        "add_irr/p_irr": 0.017130310322203912,
        "add_irr/dlo_top2": 0.16061721221456687,
        "remove_gold/dp_null": 0.7290901585564826,
        "dup/mass_err": 0.048134207372087984,
        "dup/slot_gap": 0.07666136570906383,
        "dup/acc_dedup": 0.906,
        "reorder/max_dp": 0.019010702397325076,
        "near_dup/cluster_mass_err": 0.05454888817257597
      }
    },
    "hwu64_test": {
      "raw": {
        "nll": 0.9872592959129582,
        "brier": 0.37264704437162927,
        "jsd": 0.20387588981527172,
        "acc": 0.7360594795539034,
        "acc_k": 0.7685873605947955,
        "ece": 0.050404875778132685,
        "sel_acc80": 0.8362369337979094,
        "coverage": 0.9237918215613383,
        "false_abstain": 0.0762081784386617,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.9838593010731094,
        "brier": 0.374179300802023,
        "jsd": 0.20625670639465926,
        "acc": 0.7351301115241635,
        "acc_k": 0.7685873605947955,
        "ece": 0.05403944545297376,
        "sel_acc80": 0.8385598141695703,
        "coverage": 0.9191449814126395,
        "false_abstain": 0.08085501858736059,
        "null_recall": "n/a"
      }
    },
    "ksweep_banking77": {
      "raw": {
        "nll": 0.9074376456631574,
        "brier": 0.4173011333165641,
        "jsd": 0.18435439268294618,
        "acc": 0.7067857142857142,
        "acc_k": 0.835,
        "ece": 0.10016975336841179,
        "sel_acc80": 0.7852678571428572,
        "conf_wrong": 0.055357142857142855,
        "coverage": 0.6191071428571429,
        "false_abstain": 0.11857142857142858,
        "null_recall": 0.6432142857142857,
        "auroc_null": 0.8660471938775509,
        "auroc_conf": 0.8641072704081633
      },
      "scaled": {
        "nll": 0.9013350930664579,
        "brier": 0.41534687241103474,
        "jsd": 0.18468721347220082,
        "acc": 0.7075,
        "acc_k": 0.835,
        "ece": 0.09552091195381116,
        "sel_acc80": 0.7866071428571428,
        "conf_wrong": 0.055,
        "coverage": 0.6182142857142857,
        "false_abstain": 0.11892857142857143,
        "null_recall": 0.6446428571428572,
        "auroc_null": 0.8660471938775509,
        "auroc_conf": 0.8642051020408164
      },
      "ksweep": {
        "K2/p_null_absent": 0.9124126675825391,
        "K2/p_null_present": 0.21334632581547794,
        "K2/acc": 0.975,
        "K2/auroc": 0.9396500000000001,
        "K3/p_null_absent": 0.8384780986316036,
        "K3/p_null_present": 0.15296061804300165,
        "K3/acc": 0.9475,
        "K3/auroc": 0.94301875,
        "K5/p_null_absent": 0.6920988737579319,
        "K5/p_null_present": 0.10306336762551155,
        "K5/acc": 0.9325,
        "K5/auroc": 0.92894375,
        "K10/p_null_absent": 0.5897006903094006,
        "K10/p_null_present": 0.13136198390784556,
        "K10/acc": 0.8875,
        "K10/auroc": 0.8729625,
        "K20/p_null_absent": 0.5052241023955867,
        "K20/p_null_present": 0.15151537593046668,
        "K20/acc": 0.8175,
        "K20/auroc": 0.8350124999999999,
        "K50/p_null_absent": 0.3282962662074715,
        "K50/p_null_present": 0.1271415389876347,
        "K50/acc": 0.6775,
        "K50/auroc": 0.7611312499999999,
        "K77/p_null_absent": 0.2986328311264515,
        "K77/p_null_present": 0.14399044846883044,
        "K77/acc": 0.6075,
        "K77/auroc": 0.727425,
        "p_null_absent_range": 0.6137798364560876
      }
    },
    "ksweep_clinc": {
      "raw": {
        "nll": 0.45495889803216055,
        "brier": 0.2303747224801235,
        "jsd": 0.10721702841363247,
        "acc": 0.8410087719298246,
        "acc_k": 0.931390977443609,
        "ece": 0.021296209616021692,
        "sel_acc80": 0.9091620986687549,
        "conf_wrong": 0.03477443609022556,
        "coverage": 0.5596804511278195,
        "false_abstain": 0.07957393483709273,
        "null_recall": 0.8010651629072681,
        "auroc_null": 0.9493543146713903,
        "auroc_conf": 0.943741814592873
      },
      "scaled": {
        "nll": 0.4534248016144343,
        "brier": 0.22990815748137056,
        "jsd": 0.10749978171784527,
        "acc": 0.8416353383458647,
        "acc_k": 0.931390977443609,
        "ece": 0.021236134109491114,
        "sel_acc80": 0.9095536413469069,
        "conf_wrong": 0.03320802005012531,
        "coverage": 0.5584273182957393,
        "false_abstain": 0.08051378446115288,
        "null_recall": 0.8026315789473685,
        "auroc_null": 0.9493543146713903,
        "auroc_conf": 0.9435183843851482
      },
      "ksweep": {
        "K2/p_null_absent": 0.9052965747044284,
        "K2/p_null_present": 0.05470474093286318,
        "K2/acc": 0.9974937343358395,
        "K2/auroc": 0.9891332340877256,
        "K3/p_null_absent": 0.8784392188938642,
        "K3/p_null_present": 0.042013873441291844,
        "K3/acc": 0.9924812030075187,
        "K3/auroc": 0.9859108925195194,
        "K5/p_null_absent": 0.8487086221536349,
        "K5/p_null_present": 0.04932880515136326,
        "K5/acc": 0.9849624060150376,
        "K5/auroc": 0.9836370374557948,
        "K10/p_null_absent": 0.7949157231020505,
        "K10/p_null_present": 0.07082589385284982,
        "K10/acc": 0.9774436090225563,
        "K10/auroc": 0.9739700127511762,
        "K20/p_null_absent": 0.7415750534238672,
        "K20/p_null_present": 0.11202508829274964,
        "K20/acc": 0.9598997493734336,
        "K20/auroc": 0.946331995402039,
        "K50/p_null_absent": 0.6288358126008383,
        "K50/p_null_present": 0.1512641449672471,
        "K50/acc": 0.9022556390977443,
        "K50/auroc": 0.9079025885515795,
        "K100/p_null_absent": 0.5329392300918698,
        "K100/p_null_present": 0.16651480242535285,
        "K100/acc": 0.8571428571428571,
        "K100/auroc": 0.8676453037355293,
        "K150/p_null_absent": 0.5002122879224388,
        "K150/p_null_present": 0.21401167929611334,
        "K150/acc": 0.7794486215538847,
        "K150/auroc": 0.8178717470367649,
        "p_null_absent_range": 0.4050842867819896
      }
    },
    "mmlu_cf": {
      "raw": {
        "nll": 1.9737032157151875,
        "brier": 0.7070673220421707,
        "jsd": 0.39682694280253844,
        "acc": 0.3486666666666667,
        "acc_k": 0.358,
        "ece": 0.08559551896651585,
        "sel_acc80": 0.3908333333333333,
        "coverage": 0.978,
        "false_abstain": 0.022,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.9647858546383499,
        "brier": 0.7052778805649923,
        "jsd": 0.3981142799732584,
        "acc": 0.3486666666666667,
        "acc_k": 0.358,
        "ece": 0.07654446144898731,
        "sel_acc80": 0.39166666666666666,
        "coverage": 0.978,
        "false_abstain": 0.022,
        "null_recall": "n/a"
      },
      "cf": {
        "orig/acc": 0.376,
        "orig/among_k": 0.376,
        "remove3/acc": 0.372,
        "remove3/among_k": 0.424,
        "add3_unrel/acc": 0.344,
        "add3_unrel/among_k": 0.348,
        "replace_hard/acc": 0.384,
        "replace_hard/among_k": 0.384,
        "add_neardup/acc": 0.248,
        "add_neardup/among_k": 0.248,
        "reorder/acc": 0.368,
        "reorder/among_k": 0.368,
        "add3_unrel/dlo_top2": 0.2423946739889807,
        "reorder/max_dp": 0.13986547412350775,
        "add_neardup/slot_gap": 0.17452707809314597,
        "add_neardup/cluster_mass_err": 0.10785954518744256
      }
    },
    "mmlu_cf_teacher": {
      "raw": {
        "nll": 2.0121815929819125,
        "brier": 0.7285061878659447,
        "jsd": 0.4095139954925811,
        "acc": 0.3173611111111111,
        "acc_k": 0.3279166666666667,
        "ece": 0.08924061194165714,
        "sel_acc80": 0.3581597222222222,
        "coverage": 0.97875,
        "false_abstain": 0.02125,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 2.003204512134324,
        "brier": 0.7264348569473305,
        "jsd": 0.4105225353643012,
        "acc": 0.3173611111111111,
        "acc_k": 0.3279166666666667,
        "ece": 0.08019677360335156,
        "sel_acc80": 0.3585069444444444,
        "coverage": 0.97875,
        "false_abstain": 0.02125,
        "null_recall": "n/a"
      },
      "cf": {
        "orig/acc": 0.33,
        "orig/among_k": 0.3308333333333333,
        "orig/kl_teacher": 0.3248038523416908,
        "remove3/acc": 0.35,
        "remove3/among_k": 0.4066666666666667,
        "remove3/kl_teacher": 0.9387059597443529,
        "add3_unrel/acc": 0.32083333333333336,
        "add3_unrel/among_k": 0.32166666666666666,
        "add3_unrel/kl_teacher": 0.5062704442603856,
        "replace_hard/acc": 0.33666666666666667,
        "replace_hard/among_k": 0.34,
        "replace_hard/kl_teacher": 0.39902467457106233,
        "add_neardup/acc": 0.2225,
        "add_neardup/among_k": 0.2225,
        "add_neardup/kl_teacher": 0.3746501627036028,
        "reorder/acc": 0.3441666666666667,
        "reorder/among_k": 0.3458333333333333,
        "reorder/kl_teacher": 0.3114207331236338,
        "remove3/delta_mae": 1.1855408093066204,
        "remove3/delta_teacher_abs": 1.135840148062828,
        "add3_unrel/delta_mae": 0.7900708989092671,
        "add3_unrel/delta_teacher_abs": 0.7390631803199154,
        "replace_hard/delta_mae": 0.7568652725640471,
        "replace_hard/delta_teacher_abs": 0.6681882120617108,
        "add_neardup/delta_mae": 0.74848264109364,
        "add_neardup/delta_teacher_abs": 0.7305883395327554,
        "reorder/delta_mae": 1.027615587327876,
        "reorder/delta_teacher_abs": 0.7763704285195197,
        "add3_unrel/dlo_top2": 0.22777400764209627,
        "reorder/max_dp": 0.1413952209048466,
        "add_neardup/slot_gap": 0.1525056871988272,
        "add_neardup/cluster_mass_err": 0.09664878458231972
      }
    },
    "mmlu_choicesonly": {
      "raw": {
        "nll": 2.2140372835626967,
        "brier": 0.8731933106846517,
        "jsd": 0.4800525530271047,
        "acc": 0.22333333333333333,
        "acc_k": 0.22333333333333333,
        "ece": 0.09060597602277995,
        "sel_acc80": 0.24375,
        "coverage": 0.9975,
        "false_abstain": 0.0025,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 2.2027042994257227,
        "brier": 0.8707815808273419,
        "jsd": 0.4804888539877334,
        "acc": 0.22333333333333333,
        "acc_k": 0.22333333333333333,
        "ece": 0.08494451654454074,
        "sel_acc80": 0.24375,
        "coverage": 0.9975,
        "false_abstain": 0.0025,
        "null_recall": "n/a"
      }
    },
    "mmlu_pro": {
      "raw": {
        "nll": 1.900452030855777,
        "brier": 0.7831113370273955,
        "jsd": 0.4215778910382713,
        "acc": 0.33,
        "acc_k": 0.3308333333333333,
        "ece": 0.06917534026006857,
        "sel_acc80": 0.3697916666666667,
        "coverage": 0.9941666666666666,
        "false_abstain": 0.005833333333333334,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.8933466892965118,
        "brier": 0.7815541704992409,
        "jsd": 0.42284726531134265,
        "acc": 0.33,
        "acc_k": 0.3308333333333333,
        "ece": 0.06307571604847909,
        "sel_acc80": 0.3697916666666667,
        "coverage": 0.9941666666666666,
        "false_abstain": 0.005833333333333334,
        "null_recall": "n/a"
      }
    },
    "mmlu_shuffledq": {
      "raw": {
        "nll": 2.40635102023092,
        "brier": 0.92227431248264,
        "jsd": 0.49396211895175013,
        "acc": 0.2075,
        "acc_k": 0.21166666666666667,
        "ece": 0.1442431898104648,
        "sel_acc80": 0.22395833333333334,
        "coverage": 0.9775,
        "false_abstain": 0.0225,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 2.3869170155851944,
        "brier": 0.9183096395248436,
        "jsd": 0.4940199949436655,
        "acc": 0.2075,
        "acc_k": 0.21166666666666667,
        "ece": 0.1356223477423191,
        "sel_acc80": 0.22395833333333334,
        "coverage": 0.9775,
        "false_abstain": 0.0225,
        "null_recall": "n/a"
      }
    },
    "mnli_val": {
      "raw": {
        "nll": 0.38822534996826497,
        "brier": 0.21632864323315787,
        "jsd": 0.10595431629698053,
        "acc": 0.8493197278911565,
        "acc_k": 0.8493197278911565,
        "ece": 0.022966662576409422,
        "sel_acc80": 0.9209183673469388,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.39044109534972526,
        "brier": 0.21707419982724052,
        "jsd": 0.10778225595049809,
        "acc": 0.8493197278911565,
        "acc_k": 0.8493197278911565,
        "ece": 0.027150291964715845,
        "sel_acc80": 0.9204931972789115,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "ng20_test": {
      "raw": {
        "nll": 1.6372119960226885,
        "brier": 0.6259482143658505,
        "jsd": 0.3162359946431149,
        "acc": 0.5235816814764184,
        "acc_k": 0.5696514012303486,
        "ece": 0.08290197773494705,
        "sel_acc80": 0.6006493506493507,
        "coverage": 0.8281613123718387,
        "false_abstain": 0.17183868762816132,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.622403194321037,
        "brier": 0.6257776708518196,
        "jsd": 0.31806354999429126,
        "acc": 0.5216678058783322,
        "acc_k": 0.5696514012303486,
        "ece": 0.07770895541162419,
        "sel_acc80": 0.5979152426520847,
        "coverage": 0.82365003417635,
        "false_abstain": 0.17634996582365003,
        "null_recall": "n/a"
      }
    },
    "null_irrq_intent": {
      "raw": {
        "nll": 0.2980190814605242,
        "brier": 0.14585620980287914,
        "jsd": 0.08084519615677742,
        "acc": 0.8998988877654196,
        "acc_k": 0.9642857142857143,
        "ece": 0.05771066426206286,
        "sel_acc80": 0.9665192672141504,
        "conf_wrong": 0.001002004008016032,
        "coverage": 0.5399393326592518,
        "false_abstain": 0.044897959183673466,
        "null_recall": 0.8677354709418837,
        "auroc_null": 0.9771737352255532,
        "auroc_conf": 0.9805253363870599
      },
      "scaled": {
        "nll": 0.29709066455898037,
        "brier": 0.14508696242491342,
        "jsd": 0.0809374973561244,
        "acc": 0.9009100101112234,
        "acc_k": 0.9642857142857143,
        "ece": 0.05521615729474199,
        "sel_acc80": 0.967150979153506,
        "conf_wrong": 0.0,
        "coverage": 0.5384226491405459,
        "false_abstain": 0.04591836734693878,
        "null_recall": 0.8697394789579158,
        "auroc_null": 0.9771737352255532,
        "auroc_conf": 0.9806715471759847
      }
    },
    "null_irrq_squad": {
      "raw": {
        "nll": 0.15154312186284882,
        "brier": 0.08788833430991315,
        "jsd": 0.03980749293912364,
        "acc": 0.9385,
        "acc_k": 0.995,
        "ece": 0.009013604998588616,
        "sel_acc80": 0.989375,
        "conf_wrong": 0.017,
        "coverage": 0.5445,
        "false_abstain": 0.015,
        "null_recall": 0.896,
        "auroc_null": 0.9931030000000001,
        "auroc_conf": 0.9928480000000001
      },
      "scaled": {
        "nll": 0.1514942799887926,
        "brier": 0.08774260814706673,
        "jsd": 0.039821659736051646,
        "acc": 0.9385,
        "acc_k": 0.995,
        "ece": 0.009389410480856907,
        "sel_acc80": 0.989375,
        "conf_wrong": 0.017,
        "coverage": 0.5445,
        "false_abstain": 0.015,
        "null_recall": 0.896,
        "auroc_null": 0.9931030000000001,
        "auroc_conf": 0.9928390000000001
      }
    },
    "null_nearmiss_banking77": {
      "raw": {
        "nll": 0.9520833679756119,
        "brier": 0.4805285036043141,
        "jsd": 0.20084081249467256,
        "acc": 0.665,
        "acc_k": 0.878,
        "ece": 0.14005471447110177,
        "sel_acc80": 0.72875,
        "conf_wrong": 0.104,
        "coverage": 0.7010000000000001,
        "false_abstain": 0.094,
        "null_recall": 0.504,
        "auroc_null": 0.854664,
        "auroc_conf": 0.8435440000000001
      },
      "scaled": {
        "nll": 0.9474953750672853,
        "brier": 0.4787477860806431,
        "jsd": 0.20112935951834626,
        "acc": 0.665,
        "acc_k": 0.878,
        "ece": 0.13741894581913947,
        "sel_acc80": 0.73125,
        "conf_wrong": 0.102,
        "coverage": 0.7010000000000001,
        "false_abstain": 0.094,
        "null_recall": 0.504,
        "auroc_null": 0.854664,
        "auroc_conf": 0.843312
      }
    },
    "null_nearmiss_clinc": {
      "raw": {
        "nll": 0.6851653943413197,
        "brier": 0.37972759426437286,
        "jsd": 0.14847968896737876,
        "acc": 0.7467071935157041,
        "acc_k": 0.9373737373737374,
        "ece": 0.13139047720511998,
        "sel_acc80": 0.8063291139240506,
        "conf_wrong": 0.1565040650406504,
        "coverage": 0.6757852077001013,
        "false_abstain": 0.052525252525252523,
        "null_recall": 0.5975609756097561,
        "auroc_null": 0.9133653609263365,
        "auroc_conf": 0.882996632996633
      },
      "scaled": {
        "nll": 0.6842461953661407,
        "brier": 0.3792214233262601,
        "jsd": 0.14869438900479817,
        "acc": 0.7467071935157041,
        "acc_k": 0.9373737373737374,
        "ece": 0.13018060612219567,
        "sel_acc80": 0.8063291139240506,
        "conf_wrong": 0.15447154471544716,
        "coverage": 0.6757852077001013,
        "false_abstain": 0.052525252525252523,
        "null_recall": 0.5975609756097561,
        "auroc_null": 0.9133653609263365,
        "auroc_conf": 0.8821630943582163
      }
    },
    "null_nearmiss_hwu64": {
      "raw": {
        "nll": 0.7480553781771985,
        "brier": 0.42575202811506824,
        "jsd": 0.1816428992506284,
        "acc": 0.689,
        "acc_k": 0.906,
        "ece": 0.11218150165677071,
        "sel_acc80": 0.75875,
        "conf_wrong": 0.086,
        "coverage": 0.723,
        "false_abstain": 0.05,
        "null_recall": 0.504,
        "auroc_null": 0.893416,
        "auroc_conf": 0.864184
      },
      "scaled": {
        "nll": 0.747809683498297,
        "brier": 0.42425010959360987,
        "jsd": 0.18204554811066134,
        "acc": 0.69,
        "acc_k": 0.906,
        "ece": 0.1076480211019516,
        "sel_acc80": 0.75875,
        "conf_wrong": 0.086,
        "coverage": 0.722,
        "false_abstain": 0.05,
        "null_recall": 0.506,
        "auroc_null": 0.893416,
        "auroc_conf": 0.8633120000000001
      }
    },
    "snli_null": {
      "raw": {
        "nll": 0.38093771683344796,
        "brier": 0.21158908219324113,
        "jsd": 0.09917117969867743,
        "acc": 0.8548807376227701,
        "acc_k": 0.9408906882591093,
        "ece": 0.009903888708127223,
        "sel_acc80": 0.9210921843687375,
        "conf_wrong": 0.03215561730845574,
        "coverage": 0.5908999799559029,
        "false_abstain": 0.022672064777327937,
        "null_recall": 0.7880111155220325,
        "auroc_null": 0.9758374009350796,
        "auroc_conf": 0.9598274168947578
      },
      "scaled": {
        "nll": 0.38155742627478745,
        "brier": 0.21133330351090482,
        "jsd": 0.09963839430183072,
        "acc": 0.8552816195630387,
        "acc_k": 0.9408906882591093,
        "ece": 0.011238936343062077,
        "sel_acc80": 0.9215931863727455,
        "conf_wrong": 0.030964668519253672,
        "coverage": 0.5904990980156344,
        "false_abstain": 0.022672064777327937,
        "null_recall": 0.7888050813815006,
        "auroc_null": 0.9758374009350796,
        "auroc_conf": 0.9593634129602873
      }
    },
    "snli_test": {
      "raw": {
        "nll": 0.2944368899492289,
        "brier": 0.16127000903644761,
        "jsd": 0.08039721279595598,
        "acc": 0.8935658448586891,
        "acc_k": 0.8935658448586891,
        "ece": 0.025348633677095857,
        "sel_acc80": 0.9521543086172345,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.2968152009716646,
        "brier": 0.16198921686136677,
        "jsd": 0.08209697670114373,
        "acc": 0.8935658448586891,
        "acc_k": 0.8935658448586891,
        "ece": 0.030635995678730602,
        "sel_acc80": 0.9519038076152304,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "snli_test_hyponly": {
      "raw": {
        "nll": 1.5253299458289893,
        "brier": 0.8161567119018235,
        "jsd": 0.32832325568109827,
        "acc": 0.41130487071557426,
        "acc_k": 0.41130487071557426,
        "ece": 0.314992772150575,
        "sel_acc80": 0.43361723446893785,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.4912990926911411,
        "brier": 0.8073056544838173,
        "jsd": 0.3266241508995174,
        "acc": 0.41130487071557426,
        "acc_k": 0.41130487071557426,
        "ece": 0.30664448151251045,
        "sel_acc80": 0.4338677354709419,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "snli_test_paraphrase": {
      "raw": {
        "nll": 0.3621301663516111,
        "brier": 0.19344863996217362,
        "jsd": 0.1055394463756449,
        "acc": 0.8817398276207656,
        "acc_k": 0.8817398276207656,
        "ece": 0.08257404655534728,
        "sel_acc80": 0.9408817635270541,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.36799802624918665,
        "brier": 0.19604129055167072,
        "jsd": 0.1080423642559439,
        "acc": 0.8817398276207656,
        "acc_k": 0.8817398276207656,
        "ece": 0.09054662749891076,
        "sel_acc80": 0.9411322645290581,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "snli_test_qpara": {
      "raw": {
        "nll": 0.3062878039158672,
        "brier": 0.16650863974613417,
        "jsd": 0.08552545369825479,
        "acc": 0.8856569709127382,
        "acc_k": 0.8856569709127382,
        "ece": 0.037319196367574685,
        "sel_acc80": 0.9555137844611529,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.3097593663004104,
        "brier": 0.16754294958887214,
        "jsd": 0.08746037039837277,
        "acc": 0.8856569709127382,
        "acc_k": 0.8856569709127382,
        "ece": 0.043105250301069334,
        "sel_acc80": 0.9555137844611529,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "snli_test_soft": {
      "raw": {
        "nll": 0.5461843753820834,
        "brier": 0.10901544554920943,
        "jsd": 0.05977115001837822,
        "acc": 0.8935658448586891,
        "acc_k": 0.8935658448586891,
        "ece": 0.025348633677095857,
        "sel_acc80": 0.9521543086172345,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.5389270214964142,
        "brier": 0.10806343070784269,
        "jsd": 0.060134176388955525,
        "acc": 0.8935658448586891,
        "acc_k": 0.8935658448586891,
        "ece": 0.030635995678730602,
        "sel_acc80": 0.9519038076152304,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "squad_null": {
      "raw": {
        "nll": 0.08111459801249543,
        "brier": 0.04076323473737811,
        "jsd": 0.01734145744096447,
        "acc": 0.97375,
        "acc_k": 0.996,
        "ece": 0.008793617211282249,
        "sel_acc80": 0.99625,
        "conf_wrong": 0.017,
        "coverage": 0.51075,
        "false_abstain": 0.014,
        "null_recall": 0.9645,
        "auroc_null": 0.9950402500000001,
        "auroc_conf": 0.9949386250000001
      },
      "scaled": {
        "nll": 0.08106301545108649,
        "brier": 0.040767455693852005,
        "jsd": 0.017360626133039504,
        "acc": 0.97375,
        "acc_k": 0.996,
        "ece": 0.008595976613461967,
        "sel_acc80": 0.99625,
        "conf_wrong": 0.017,
        "coverage": 0.51075,
        "false_abstain": 0.014,
        "null_recall": 0.9645,
        "auroc_null": 0.9950402500000001,
        "auroc_conf": 0.994932
      }
    },
    "trec_coarse": {
      "raw": {
        "nll": 1.5918411787616937,
        "brier": 0.6349207986106156,
        "jsd": 0.29208722584601526,
        "acc": 0.532,
        "acc_k": 0.584,
        "ece": 0.13663677060604096,
        "sel_acc80": 0.5825,
        "coverage": 0.86,
        "false_abstain": 0.14,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.5655300627415383,
        "brier": 0.6341792940622437,
        "jsd": 0.2930151878513147,
        "acc": 0.532,
        "acc_k": 0.584,
        "ece": 0.1302748074531555,
        "sel_acc80": 0.58,
        "coverage": 0.858,
        "false_abstain": 0.142,
        "null_recall": "n/a"
      }
    },
    "trec_fine": {
      "raw": {
        "nll": 2.169659186192131,
        "brier": 0.8538986718630976,
        "jsd": 0.3937861872099405,
        "acc": 0.376,
        "acc_k": 0.5,
        "ece": 0.24738949897885326,
        "sel_acc80": 0.3925,
        "coverage": 0.6679999999999999,
        "false_abstain": 0.332,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 2.143684417143925,
        "brier": 0.8532065630013449,
        "jsd": 0.39535413252242657,
        "acc": 0.372,
        "acc_k": 0.5,
        "ece": 0.24531754493713384,
        "sel_acc80": 0.3925,
        "coverage": 0.6579999999999999,
        "false_abstain": 0.342,
        "null_recall": "n/a"
      }
    },
    "unli_test": {
      "raw": {
        "nll": 0.5318298036808603,
        "brier": 0.13167920018417006,
        "jsd": 0.04981357317021074,
        "auroc_null": 0.8890171083529017,
        "auroc_conf": 0.8890171083529017
      },
      "scaled": {
        "nll": 0.5318297999376992,
        "brier": 0.13167919956989954,
        "jsd": 0.04981357268943865,
        "auroc_null": 0.8890171083529017,
        "auroc_conf": 0.8890171083529017
      }
    },
    "data_kb_val": {
      "raw": {
        "nll": 0.8995894797702431,
        "brier": 0.46785703357109987,
        "jsd": 0.23309134218604935,
        "acc": 0.6293333333333333,
        "acc_k": 0.609704641350211,
        "ece": 0.02164344120025636,
        "sel_acc80": 0.7133333333333334,
        "conf_wrong": 0.01282051282051282,
        "coverage": 0.9486666666666667,
        "false_abstain": 0.0,
        "null_recall": 0.9871794871794872,
        "auroc_null": 0.9999639366727975,
        "auroc_conf": 0.9893613184752426
      },
      "scaled": {
        "nll": 0.8988669709475279,
        "brier": 0.4678002285972357,
        "jsd": 0.23442374370611735,
        "acc": 0.6293333333333333,
        "acc_k": 0.609704641350211,
        "ece": 0.0223244069914023,
        "sel_acc80": 0.7125,
        "conf_wrong": 0.01282051282051282,
        "coverage": 0.9486666666666667,
        "false_abstain": 0.0,
        "null_recall": 0.9871794871794872,
        "auroc_null": 0.9999639366727975,
        "auroc_conf": 0.9892982076526381
      }
    },
    "truthfulqa_mc1": {
      "raw": {
        "nll": 2.3583552921727957,
        "brier": 0.9532752488732227,
        "jsd": 0.43745611493995573,
        "acc": 0.2668298653610771,
        "acc_k": 0.3268053855569155,
        "ece": 0.28699413596708306,
        "sel_acc80": 0.28440366972477066,
        "coverage": 0.8837209302325582,
        "false_abstain": 0.11627906976744186,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 2.330985705103918,
        "brier": 0.9483202788516172,
        "jsd": 0.4370409987040793,
        "acc": 0.2668298653610771,
        "acc_k": 0.3268053855569155,
        "ece": 0.2787222843457611,
        "sel_acc80": 0.28593272171253825,
        "coverage": 0.8837209302325582,
        "false_abstain": 0.11627906976744186,
        "null_recall": "n/a"
      }
    },
    "data_wf_val": {
      "raw": {
        "nll": 0.24610411725258516,
        "brier": 0.1394640994048586,
        "jsd": 0.0622050245204808,
        "acc": 0.8986666666666666,
        "acc_k": 0.8991138377641446,
        "ece": 0.014609658221403754,
        "sel_acc80": 0.9733333333333334,
        "conf_wrong": 0.030303030303030304,
        "coverage": 0.9806666666666667,
        "false_abstain": 0.0,
        "null_recall": 0.8787878787878788,
        "auroc_null": 0.9997727789138832,
        "auroc_conf": 0.9778356158724256
      },
      "scaled": {
        "nll": 0.24495817350751345,
        "brier": 0.1391821391803038,
        "jsd": 0.06268002385019018,
        "acc": 0.8986666666666666,
        "acc_k": 0.8991138377641446,
        "ece": 0.013083436906337762,
        "sel_acc80": 0.9733333333333334,
        "conf_wrong": 0.030303030303030304,
        "coverage": 0.9806666666666667,
        "false_abstain": 0.0,
        "null_recall": 0.8787878787878788,
        "auroc_null": 0.9997727789138832,
        "auroc_conf": 0.9776910206358059
      }
    },
    "wf_heldout_choice": {
      "raw": {
        "nll": 0.04069819362514963,
        "brier": 0.018002550660832672,
        "jsd": 0.008412281696821153,
        "acc": 0.9893333333333333,
        "acc_k": 0.9943382873319179,
        "ece": 0.005535807013511662,
        "sel_acc80": 1.0,
        "conf_wrong": 0.06896551724137931,
        "coverage": 0.9473333333333334,
        "false_abstain": 0.0,
        "null_recall": 0.9080459770114943,
        "auroc_null": 0.9994468441646127,
        "auroc_conf": 0.9956154265400916
      },
      "scaled": {
        "nll": 0.04119968250350716,
        "brier": 0.018094206464215678,
        "jsd": 0.008601874984231025,
        "acc": 0.9893333333333333,
        "acc_k": 0.9943382873319179,
        "ece": 0.005908609946568721,
        "sel_acc80": 1.0,
        "conf_wrong": 0.06896551724137931,
        "coverage": 0.9473333333333334,
        "false_abstain": 0.0,
        "null_recall": 0.9080459770114943,
        "auroc_null": 0.9994468441646127,
        "auroc_conf": 0.9953957911348643
      }
    },
    "wf_heldout_noul": {
      "raw": {
        "nll": 0.7866961798391291,
        "brier": 0.4496349124070911,
        "jsd": 0.1659911488117415,
        "acc": 0.6986666666666667,
        "acc_k": 0.6986666666666667,
        "ece": 0.18171938677628838,
        "sel_acc80": 0.7525,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.7674072662661018,
        "brier": 0.44586403670437996,
        "jsd": 0.16535243468099545,
        "acc": 0.6986666666666667,
        "acc_k": 0.6986666666666667,
        "ece": 0.1772376935084661,
        "sel_acc80": 0.7525,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "wf_heldout_score": {
      "raw": {
        "nll": 2.0987200270240858,
        "brier": 0.8091494680533715,
        "jsd": 0.29950320499754934,
        "acc": 0.49733333333333335,
        "acc_k": 0.49733333333333335,
        "ece": 0.37649274045228964,
        "sel_acc80": 0.5458333333333333,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 2.0312639152806957,
        "brier": 0.8016462098572795,
        "jsd": 0.2979000494751449,
        "acc": 0.49733333333333335,
        "acc_k": 0.49733333333333335,
        "ece": 0.37102700203657146,
        "sel_acc80": 0.5458333333333333,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "wf_heldout_style": {
      "raw": {
        "nll": 0.24614928496086558,
        "brier": 0.1391935848872391,
        "jsd": 0.06184622570106476,
        "acc": 0.8993333333333333,
        "acc_k": 0.8977657413676371,
        "ece": 0.01664232387145353,
        "sel_acc80": 0.9683333333333334,
        "conf_wrong": 0.0,
        "coverage": 0.9846666666666667,
        "false_abstain": 0.0,
        "null_recall": 1.0,
        "auroc_null": 1.0,
        "auroc_conf": 1.0
      },
      "scaled": {
        "nll": 0.24471071542524464,
        "brier": 0.13901082740892692,
        "jsd": 0.06236480920470213,
        "acc": 0.8993333333333333,
        "acc_k": 0.8977657413676371,
        "ece": 0.013723565757274658,
        "sel_acc80": 0.9683333333333334,
        "conf_wrong": 0.0,
        "coverage": 0.9846666666666667,
        "false_abstain": 0.0,
        "null_recall": 1.0,
        "auroc_null": 1.0,
        "auroc_conf": 1.0
      }
    },
    "wf_rubric_flip": {
      "raw": {
        "nll": 0.5307086168197724,
        "brier": 0.26619245796638785,
        "jsd": 0.09958348690925925,
        "acc": 0.8273333333333334,
        "acc_k": 0.8273333333333334,
        "ece": 0.10746072290341058,
        "sel_acc80": 0.9,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.5168277089308791,
        "brier": 0.26401031532355146,
        "jsd": 0.09929868841268231,
        "acc": 0.8273333333333334,
        "acc_k": 0.8273333333333334,
        "ece": 0.10272792806227998,
        "sel_acc80": 0.9,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "flip": {
        "n_pairs": 750,
        "flip_rate": 0.7466666666666667,
        "both_correct": 0.6946666666666667,
        "acc": 0.8273333333333334
      }
    },
    "wf_rubric_shuffled": {
      "raw": {
        "nll": 4.5102297803204126,
        "brier": 1.0442621780841999,
        "jsd": 0.3776491722774307,
        "acc": 0.39866666666666667,
        "acc_k": 0.39866666666666667,
        "ece": 0.4885187878410022,
        "sel_acc80": 0.405,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 4.348945097725256,
        "brier": 1.0389491271745737,
        "jsd": 0.3764887276833897,
        "acc": 0.39866666666666667,
        "acc_k": 0.39866666666666667,
        "ece": 0.4842325220704079,
        "sel_acc80": 0.4041666666666667,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "data_wf_hf_val": {
      "raw": {
        "nll": 0.010585208018264583,
        "brier": 0.005168010358705758,
        "jsd": 0.0020250369475977915,
        "acc": 0.9966666666666667,
        "acc_k": 0.9966666666666667,
        "ece": 0.0028440783818563034,
        "sel_acc80": 1.0,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.010354351768417104,
        "brier": 0.005121358381636529,
        "jsd": 0.002035180293716168,
        "acc": 0.9966666666666667,
        "acc_k": 0.9966666666666667,
        "ece": 0.002598859508832256,
        "sel_acc80": 1.0,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "cua_s1_forms_demo": {
      "raw": {
        "nll": 9.68217684130784e-05,
        "brier": 2.42991571051238e-06,
        "jsd": 3.3502051815451166e-05,
        "acc": 1.0,
        "acc_k": 1.0,
        "ece": 9.620737056348361e-05,
        "sel_acc80": 1.0,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.0001166535209764421,
        "brier": 3.334119317415377e-06,
        "jsd": 4.0353015412178415e-05,
        "acc": 1.0,
        "acc_k": 1.0,
        "ece": 0.00011580908785058242,
        "sel_acc80": 1.0,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "cua_s1_forms_test": {
      "raw": {
        "nll": 0.010497363812569337,
        "brier": 0.005111427197274268,
        "jsd": 0.0020249838705119843,
        "acc": 0.996,
        "acc_k": 0.996,
        "ece": 0.0026506466865538827,
        "sel_acc80": 1.0,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.010267529708334153,
        "brier": 0.00506279966705414,
        "jsd": 0.002029791549827035,
        "acc": 0.996,
        "acc_k": 0.996,
        "ece": 0.002581955194473174,
        "sel_acc80": 1.0,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "jev4b_adversarial": {
      "raw": {
        "nll": 0.6563798084706536,
        "brier": 0.2509300255617882,
        "jsd": 0.09856897364784455,
        "acc": 0.8433333333333334,
        "acc_k": 0.8433333333333334,
        "ece": 0.09749235555529595,
        "sel_acc80": 0.9229166666666667,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.6374143261836251,
        "brier": 0.24906241688616718,
        "jsd": 0.09839928565661872,
        "acc": 0.8433333333333334,
        "acc_k": 0.8433333333333334,
        "ece": 0.09604769155383108,
        "sel_acc80": 0.9229166666666667,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "jevlogs_triage": {
      "raw": {
        "nll": 0.9845623682584176,
        "brier": 0.6729167311320402,
        "jsd": 0.2490643395451554,
        "acc": 0.49733333333333335,
        "acc_k": 0.49733333333333335,
        "ece": 0.27361580948034925,
        "sel_acc80": 0.5175,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.9654136745276661,
        "brier": 0.6654463383788486,
        "jsd": 0.2473301509447621,
        "acc": 0.49733333333333335,
        "acc_k": 0.49733333333333335,
        "ece": 0.2673972271680832,
        "sel_acc80": 0.5175,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "mind2web_choice": {
      "raw": {
        "nll": 1.6608707584399627,
        "brier": 0.7200349551249486,
        "jsd": 0.32339959282045455,
        "acc": 0.422,
        "acc_k": 0.422,
        "ece": 0.2274525083303452,
        "sel_acc80": 0.4841666666666667,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.6333868224230361,
        "brier": 0.7136883533617026,
        "jsd": 0.3232182688768001,
        "acc": 0.422,
        "acc_k": 0.422,
        "ece": 0.2181771738330523,
        "sel_acc80": 0.48583333333333334,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "pagerduty_trigger": {
      "raw": {
        "nll": 1.3089463058665416,
        "brier": 0.5937149639931447,
        "jsd": 0.225649696977532,
        "acc": 0.6026666666666667,
        "acc_k": 0.6026666666666667,
        "ece": 0.26803290375073746,
        "sel_acc80": 0.68,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.2716589001643595,
        "brier": 0.5893011552599684,
        "jsd": 0.2249425030779627,
        "acc": 0.6026666666666667,
        "acc_k": 0.6026666666666667,
        "ece": 0.2633645134369532,
        "sel_acc80": 0.68,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "systemone_lite_hard": {
      "raw": {
        "nll": 0.32642191182042274,
        "brier": 0.17189764452077014,
        "jsd": 0.06927031888838348,
        "acc": 0.8826666666666667,
        "acc_k": 0.8826666666666667,
        "ece": 0.04653552297751107,
        "sel_acc80": 0.9458333333333333,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 0.3206730075344011,
        "brier": 0.1710769966178729,
        "jsd": 0.06952373056541901,
        "acc": 0.8826666666666667,
        "acc_k": 0.8826666666666667,
        "ece": 0.0421845922271411,
        "sel_acc80": 0.9458333333333333,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "tree_choice_cap": {
      "raw": {
        "nll": 1.8059283410666016,
        "brier": 0.6318870635104122,
        "jsd": 0.31713011149335896,
        "acc": 0.5375,
        "acc_k": 0.6,
        "ece": 0.11325502035518487,
        "sel_acc80": 0.6197916666666666,
        "coverage": 0.8,
        "false_abstain": 0.2,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.7984774021118792,
        "brier": 0.6330959581547211,
        "jsd": 0.31947562577323346,
        "acc": 0.5375,
        "acc_k": 0.6,
        "ece": 0.11445001408250796,
        "sel_acc80": 0.6215277777777778,
        "coverage": 0.7972222222222223,
        "false_abstain": 0.20277777777777778,
        "null_recall": "n/a"
      }
    },
    "typed_decisions_test": {
      "raw": {
        "nll": 2.00088336147219,
        "brier": 0.4425575284541666,
        "jsd": 0.18983895937760947,
        "acc": 0.422,
        "acc_k": 0.422,
        "ece": 0.33466433302561444,
        "sel_acc80": 0.44083333333333335,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 1.9460634838656419,
        "brier": 0.43500238428936305,
        "jsd": 0.18668195642568758,
        "acc": 0.422,
        "acc_k": 0.422,
        "ece": 0.3275858606696128,
        "sel_acc80": 0.4425,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    },
    "typed_decisions_train": {
      "raw": {
        "nll": 2.3080436786322527,
        "brier": 0.4904933688193668,
        "jsd": 0.2051707895984692,
        "acc": 0.35933333333333334,
        "acc_k": 0.35933333333333334,
        "ece": 0.44574949810902287,
        "sel_acc80": 0.3641666666666667,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      },
      "scaled": {
        "nll": 2.239424416856678,
        "brier": 0.4801745735180739,
        "jsd": 0.2006978346623256,
        "acc": 0.35933333333333334,
        "acc_k": 0.35933333333333334,
        "ece": 0.4376716514229774,
        "sel_acc80": 0.365,
        "coverage": 1.0,
        "false_abstain": 0.0,
        "null_recall": "n/a"
      }
    }
  }
}