| { |
| "T": 1.03979841848149, |
| "val_nll": 0.3844288446342489, |
| "args": { |
| "name": "nc_v3_tap20_wf", |
| "backbone": "Qwen/Qwen3-1.7B-Base", |
| "lora_layers": 8, |
| "lora_r": 16, |
| "steps": 12000, |
| "bs": 64, |
| "lr": 0.0003, |
| "lora_lr": 0.0001, |
| "seed": 0, |
| "no_hybrid": false, |
| "no_cand_null": false, |
| "no_null": false, |
| "null": "factored", |
| "hard_only": false, |
| "mix": "full", |
| "eval_every": 6000, |
| "val_every": 1000, |
| "ckpt_every": 2000, |
| "eval_bs": 128, |
| "device": "auto", |
| "wandb": true, |
| "hf_repo": "guychuk/pcdm-runs", |
| "data": "data_v5", |
| "extra_data": "data_kb,data_wf,data_wf_hf", |
| "eval_cap": 1500, |
| "family_weights": "E:0.35,K:0.25,W:0.40", |
| "bucket_map": null, |
| "smoke": false, |
| "eval_limit": 0, |
| "tap_layer": 20, |
| "zscore": true, |
| "eval_only": false, |
| "dump_logits": null, |
| "tower_d": 512, |
| "tower_layers": 2, |
| "tower_heads": 8, |
| "joint": false, |
| "listwise": false, |
| "extra_tap": 0, |
| "init_from": null, |
| "readout": "native", |
| "nc_head": "n3", |
| "nc_render": "letters_nonull", |
| "perm_lambda": 0.0, |
| "no_shuffle": false, |
| "zero_shot": false, |
| "shots": 0, |
| "cand_encoder": "backbone", |
| "head": "mlp", |
| "z_dim": 128, |
| "z_probes": 8, |
| "tiny_layers": 2, |
| "distill_alpha": 1.0, |
| "distill_beta": 0.0, |
| "distill_T": 2.0, |
| "delta_gamma": 0.0 |
| }, |
| "eval": { |
| "anli_test": { |
| "raw": { |
| "nll": 1.102941199093527, |
| "brier": 0.6654224947329161, |
| "jsd": 0.2776569035848503, |
| "acc": 0.480625, |
| "acc_k": 0.480625, |
| "ece": 0.19827818479388953, |
| "sel_acc80": 0.5078125, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.0888644635622768, |
| "brier": 0.6598136085192928, |
| "jsd": 0.2771962879481138, |
| "acc": 0.480625, |
| "acc_k": 0.480625, |
| "ece": 0.18998124125413596, |
| "sel_acc80": 0.506640625, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "banking77_k": { |
| "raw": { |
| "nll": 0.8803650682647327, |
| "brier": 0.4301744118516424, |
| "jsd": 0.18126308231810428, |
| "acc": 0.7012353706111834, |
| "acc_k": 0.8533685601056803, |
| "ece": 0.12095656268230806, |
| "sel_acc80": 0.7704185290532304, |
| "conf_wrong": 0.08898847631241998, |
| "coverage": 0.640442132639792, |
| "false_abstain": 0.10501981505944517, |
| "null_recall": 0.6062740076824584, |
| "auroc_null": 0.869071762144864, |
| "auroc_conf": 0.8650941617037398 |
| }, |
| "scaled": { |
| "nll": 0.8744278074702166, |
| "brier": 0.42846344428894817, |
| "jsd": 0.18141129837781403, |
| "acc": 0.7009102730819246, |
| "acc_k": 0.8533685601056803, |
| "ece": 0.11870673383127112, |
| "sel_acc80": 0.7712312068264933, |
| "conf_wrong": 0.08770806658130602, |
| "coverage": 0.6397919375812744, |
| "false_abstain": 0.10634081902245707, |
| "null_recall": 0.6062740076824584, |
| "auroc_null": 0.869071762144864, |
| "auroc_conf": 0.8651785215919028 |
| } |
| }, |
| "banking77_test": { |
| "raw": { |
| "nll": 1.8943291566507845, |
| "brier": 0.593270217263453, |
| "jsd": 0.2957812292956273, |
| "acc": 0.5718465539661899, |
| "acc_k": 0.6089076723016905, |
| "ece": 0.09358625776184087, |
| "sel_acc80": 0.653799268590004, |
| "coverage": 0.881664499349805, |
| "false_abstain": 0.11833550065019506, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.8612821539463003, |
| "brier": 0.5919693859966786, |
| "jsd": 0.2973040180361969, |
| "acc": 0.5711963589076723, |
| "acc_k": 0.6089076723016905, |
| "ece": 0.08643925216131311, |
| "sel_acc80": 0.6525802519301097, |
| "coverage": 0.8780884265279584, |
| "false_abstain": 0.12191157347204161, |
| "null_recall": "n/a" |
| } |
| }, |
| "boolq_val": { |
| "raw": { |
| "nll": 0.39380937671763266, |
| "brier": 0.24073265857614617, |
| "jsd": 0.10124721371819573, |
| "acc": 0.8314984709480122, |
| "acc_k": 0.8318042813455657, |
| "ece": 0.0429865891415774, |
| "sel_acc80": 0.8937308868501529, |
| "coverage": 0.9996941896024465, |
| "false_abstain": 0.0003058103975535168, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.39024824040639783, |
| "brier": 0.2397877640930743, |
| "jsd": 0.10181249185984385, |
| "acc": 0.8314984709480122, |
| "acc_k": 0.8318042813455657, |
| "ece": 0.038267658646318155, |
| "sel_acc80": 0.8937308868501529, |
| "coverage": 0.9996941896024465, |
| "false_abstain": 0.0003058103975535168, |
| "null_recall": "n/a" |
| } |
| }, |
| "chaos_mnli": { |
| "raw": { |
| "nll": 1.1710537505967527, |
| "brier": 0.22073145417462792, |
| "jsd": 0.08313512961435864 |
| }, |
| "scaled": { |
| "nll": 1.1500571436778484, |
| "brier": 0.21457783233844271, |
| "jsd": 0.08064832731516772 |
| } |
| }, |
| "clinc_heldout": { |
| "raw": { |
| "nll": 0.27074162635707527, |
| "brier": 0.11428612065096902, |
| "jsd": 0.0543765164648202, |
| "acc": 0.9242761692650334, |
| "acc_k": 0.9621380846325167, |
| "ece": 0.01579013417053861, |
| "sel_acc80": 0.9833101529902643, |
| "coverage": 0.9443207126948775, |
| "false_abstain": 0.0556792873051225, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.2684241498104494, |
| "brier": 0.11447972908114325, |
| "jsd": 0.054635113203736436, |
| "acc": 0.9242761692650334, |
| "acc_k": 0.9621380846325167, |
| "ece": 0.013791607257252527, |
| "sel_acc80": 0.9833101529902643, |
| "coverage": 0.9443207126948775, |
| "false_abstain": 0.0556792873051225, |
| "null_recall": "n/a" |
| } |
| }, |
| "clinc_k": { |
| "raw": { |
| "nll": 0.311515750792269, |
| "brier": 0.16297286687242818, |
| "jsd": 0.07596730831547199, |
| "acc": 0.8914118705035972, |
| "acc_k": 0.9633437639696022, |
| "ece": 0.016932064215083087, |
| "sel_acc80": 0.9530767069401517, |
| "conf_wrong": 0.028493894165535955, |
| "coverage": 0.5476618705035972, |
| "false_abstain": 0.05185516316495306, |
| "null_recall": 0.8575305291723202, |
| "auroc_null": 0.968572628384877, |
| "auroc_conf": 0.9670774829069189 |
| }, |
| "scaled": { |
| "nll": 0.31090634881130236, |
| "brier": 0.16266923363280875, |
| "jsd": 0.07609676101977222, |
| "acc": 0.8920863309352518, |
| "acc_k": 0.9633437639696022, |
| "ece": 0.016195901120148588, |
| "sel_acc80": 0.9530767069401517, |
| "conf_wrong": 0.028493894165535955, |
| "coverage": 0.5469874100719425, |
| "false_abstain": 0.05185516316495306, |
| "null_recall": 0.858887381275441, |
| "auroc_null": 0.968572628384877, |
| "auroc_conf": 0.9670253196164098 |
| } |
| }, |
| "clinc_oos": { |
| "raw": { |
| "nll": 0.3819562982125116, |
| "brier": 0.18383285615491454, |
| "jsd": 0.11433253299812943, |
| "acc": 0.9009009009009009, |
| "ece": 0.1304006703623064, |
| "sel_acc80": 0.93625, |
| "conf_wrong": 0.006006006006006006, |
| "coverage": 0.09909909909909909, |
| "false_abstain": "n/a", |
| "null_recall": 0.9009009009009009 |
| }, |
| "scaled": { |
| "nll": 0.3819562920724052, |
| "brier": 0.18188391253101058, |
| "jsd": 0.11433253017955829, |
| "acc": 0.9029029029029029, |
| "ece": 0.13308967943783395, |
| "sel_acc80": 0.9375, |
| "conf_wrong": 0.006006006006006006, |
| "coverage": 0.09709709709709713, |
| "false_abstain": "n/a", |
| "null_recall": 0.9029029029029029 |
| } |
| }, |
| "clinc_test": { |
| "raw": { |
| "nll": 0.9257443901513311, |
| "brier": 0.36654857219284437, |
| "jsd": 0.1835993970985268, |
| "acc": 0.7360611510791367, |
| "acc_k": 0.8223920863309353, |
| "ece": 0.04578179257030756, |
| "sel_acc80": 0.8328182073616184, |
| "coverage": 0.835431654676259, |
| "false_abstain": 0.164568345323741, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.920197700698808, |
| "brier": 0.36788022046321867, |
| "jsd": 0.1850340584565983, |
| "acc": 0.7335881294964028, |
| "acc_k": 0.8223920863309353, |
| "ece": 0.04579438925084237, |
| "sel_acc80": 0.8316942961506041, |
| "coverage": 0.8322841726618705, |
| "false_abstain": 0.1677158273381295, |
| "null_recall": "n/a" |
| } |
| }, |
| "cse_banking77": { |
| "raw": { |
| "nll": 0.9572221236913963, |
| "brier": 0.3772673010367225, |
| "jsd": 0.16592351634826877, |
| "acc": 0.7, |
| "acc_k": 0.764, |
| "ece": 0.1161508621672789, |
| "sel_acc80": 0.7754166666666666, |
| "conf_wrong": 0.098, |
| "coverage": 0.8089999999999999, |
| "false_abstain": 0.1016, |
| "null_recall": 0.638, |
| "auroc_null": 0.8789352, |
| "auroc_conf": 0.8513904000000001 |
| }, |
| "scaled": { |
| "nll": 0.9426415598509017, |
| "brier": 0.375676489733989, |
| "jsd": 0.16591535574509875, |
| "acc": 0.7, |
| "acc_k": 0.764, |
| "ece": 0.11323217004537582, |
| "sel_acc80": 0.7754166666666666, |
| "conf_wrong": 0.096, |
| "coverage": 0.808, |
| "false_abstain": 0.1024, |
| "null_recall": 0.64, |
| "auroc_null": 0.8789352, |
| "auroc_conf": 0.8510016 |
| }, |
| "cse": { |
| "add_irr/p_irr": 0.0005661446037615407, |
| "add_irr/dlo_top2": 0.2631000002575834, |
| "remove_gold/dp_null": 0.4630419385624409, |
| "dup/mass_err": 0.034684130056172505, |
| "dup/slot_gap": 0.1855905814659261, |
| "dup/acc_dedup": 0.838, |
| "reorder/max_dp": 0.08897487102333913, |
| "near_dup/cluster_mass_err": 0.03278506057004932 |
| } |
| }, |
| "cse_clinc": { |
| "raw": { |
| "nll": 0.32437873855368415, |
| "brier": 0.13432812964214, |
| "jsd": 0.06377881461172775, |
| "acc": 0.861178091629349, |
| "acc_k": 0.8977980889073536, |
| "ece": 0.0228896729501822, |
| "sel_acc80": 0.9466207490314249, |
| "conf_wrong": 0.03024193548387097, |
| "coverage": 0.8256975542542198, |
| "false_abstain": 0.0402991275446614, |
| "null_recall": 0.8245967741935484, |
| "auroc_null": 0.9744327700121956, |
| "auroc_conf": 0.959220921505823 |
| }, |
| "scaled": { |
| "nll": 0.322856150605895, |
| "brier": 0.1338994864013624, |
| "jsd": 0.06387929138342104, |
| "acc": 0.8615225628660007, |
| "acc_k": 0.8977980889073536, |
| "ece": 0.021907006774406955, |
| "sel_acc80": 0.9461902712010332, |
| "conf_wrong": 0.03024193548387097, |
| "coverage": 0.825353083017568, |
| "false_abstain": 0.0402991275446614, |
| "null_recall": 0.8266129032258065, |
| "auroc_null": 0.9744327700121956, |
| "auroc_conf": 0.9589512108500743 |
| }, |
| "cse": { |
| "add_irr/p_irr": 0.0006255071544984961, |
| "add_irr/dlo_top2": 0.3369036140496365, |
| "remove_gold/dp_null": 0.7144053382439192, |
| "dup/mass_err": 0.01891620019684563, |
| "dup/slot_gap": 0.1731909674514862, |
| "dup/acc_dedup": 0.9758064516129032, |
| "reorder/max_dp": 0.035958222781689086, |
| "near_dup/cluster_mass_err": 0.01875680701157674 |
| } |
| }, |
| "cse_hwu64": { |
| "raw": { |
| "nll": 0.5650941517704147, |
| "brier": 0.2537955674753089, |
| "jsd": 0.11948498837404782, |
| "acc": 0.773, |
| "acc_k": 0.822, |
| "ece": 0.04737755128741265, |
| "sel_acc80": 0.84625, |
| "conf_wrong": 0.058, |
| "coverage": 0.8406666666666667, |
| "false_abstain": 0.0548, |
| "null_recall": 0.682, |
| "auroc_null": 0.9259464000000002, |
| "auroc_conf": 0.8916696 |
| }, |
| "scaled": { |
| "nll": 0.5615799890226735, |
| "brier": 0.2529652757233588, |
| "jsd": 0.11981138093082586, |
| "acc": 0.773, |
| "acc_k": 0.822, |
| "ece": 0.044626818845669436, |
| "sel_acc80": 0.8466666666666667, |
| "conf_wrong": 0.058, |
| "coverage": 0.8406666666666667, |
| "false_abstain": 0.0548, |
| "null_recall": 0.682, |
| "auroc_null": 0.9259464000000002, |
| "auroc_conf": 0.8913199999999999 |
| }, |
| "cse": { |
| "add_irr/p_irr": 0.009121834453501867, |
| "add_irr/dlo_top2": 0.2421149150189948, |
| "remove_gold/dp_null": 0.5026223030890106, |
| "dup/mass_err": 0.036702267746052814, |
| "dup/slot_gap": 0.1844224565153616, |
| "dup/acc_dedup": 0.94, |
| "reorder/max_dp": 0.06679525845381432, |
| "near_dup/cluster_mass_err": 0.0402173605489788 |
| } |
| }, |
| "cse_snli": { |
| "raw": { |
| "nll": 0.42645938467085154, |
| "brier": 0.17942825051131092, |
| "jsd": 0.09235424546372771, |
| "acc": 0.8393124065769806, |
| "acc_k": 0.8474264705882353, |
| "ece": 0.04809902380088343, |
| "sel_acc80": 0.9093881363848669, |
| "conf_wrong": 0.03, |
| "coverage": 0.8497757847533632, |
| "false_abstain": 0.0, |
| "null_recall": 0.804, |
| "auroc_null": 0.9997564338235294, |
| "auroc_conf": 0.9327977941176472 |
| }, |
| "scaled": { |
| "nll": 0.4289722985328097, |
| "brier": 0.1800702618260962, |
| "jsd": 0.09380953611654914, |
| "acc": 0.8396860986547086, |
| "acc_k": 0.8474264705882353, |
| "ece": 0.053872445317038144, |
| "sel_acc80": 0.9093881363848669, |
| "conf_wrong": 0.028, |
| "coverage": 0.8494020926756353, |
| "false_abstain": 0.0, |
| "null_recall": 0.806, |
| "auroc_null": 0.9997564338235294, |
| "auroc_conf": 0.9316098345588235 |
| }, |
| "cse": { |
| "add_irr/p_irr": 0.017130310322203912, |
| "add_irr/dlo_top2": 0.16061721221456687, |
| "remove_gold/dp_null": 0.7290901585564826, |
| "dup/mass_err": 0.048134207372087984, |
| "dup/slot_gap": 0.07666136570906383, |
| "dup/acc_dedup": 0.906, |
| "reorder/max_dp": 0.019010702397325076, |
| "near_dup/cluster_mass_err": 0.05454888817257597 |
| } |
| }, |
| "hwu64_test": { |
| "raw": { |
| "nll": 0.9872592959129582, |
| "brier": 0.37264704437162927, |
| "jsd": 0.20387588981527172, |
| "acc": 0.7360594795539034, |
| "acc_k": 0.7685873605947955, |
| "ece": 0.050404875778132685, |
| "sel_acc80": 0.8362369337979094, |
| "coverage": 0.9237918215613383, |
| "false_abstain": 0.0762081784386617, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.9838593010731094, |
| "brier": 0.374179300802023, |
| "jsd": 0.20625670639465926, |
| "acc": 0.7351301115241635, |
| "acc_k": 0.7685873605947955, |
| "ece": 0.05403944545297376, |
| "sel_acc80": 0.8385598141695703, |
| "coverage": 0.9191449814126395, |
| "false_abstain": 0.08085501858736059, |
| "null_recall": "n/a" |
| } |
| }, |
| "ksweep_banking77": { |
| "raw": { |
| "nll": 0.9074376456631574, |
| "brier": 0.4173011333165641, |
| "jsd": 0.18435439268294618, |
| "acc": 0.7067857142857142, |
| "acc_k": 0.835, |
| "ece": 0.10016975336841179, |
| "sel_acc80": 0.7852678571428572, |
| "conf_wrong": 0.055357142857142855, |
| "coverage": 0.6191071428571429, |
| "false_abstain": 0.11857142857142858, |
| "null_recall": 0.6432142857142857, |
| "auroc_null": 0.8660471938775509, |
| "auroc_conf": 0.8641072704081633 |
| }, |
| "scaled": { |
| "nll": 0.9013350930664579, |
| "brier": 0.41534687241103474, |
| "jsd": 0.18468721347220082, |
| "acc": 0.7075, |
| "acc_k": 0.835, |
| "ece": 0.09552091195381116, |
| "sel_acc80": 0.7866071428571428, |
| "conf_wrong": 0.055, |
| "coverage": 0.6182142857142857, |
| "false_abstain": 0.11892857142857143, |
| "null_recall": 0.6446428571428572, |
| "auroc_null": 0.8660471938775509, |
| "auroc_conf": 0.8642051020408164 |
| }, |
| "ksweep": { |
| "K2/p_null_absent": 0.9124126675825391, |
| "K2/p_null_present": 0.21334632581547794, |
| "K2/acc": 0.975, |
| "K2/auroc": 0.9396500000000001, |
| "K3/p_null_absent": 0.8384780986316036, |
| "K3/p_null_present": 0.15296061804300165, |
| "K3/acc": 0.9475, |
| "K3/auroc": 0.94301875, |
| "K5/p_null_absent": 0.6920988737579319, |
| "K5/p_null_present": 0.10306336762551155, |
| "K5/acc": 0.9325, |
| "K5/auroc": 0.92894375, |
| "K10/p_null_absent": 0.5897006903094006, |
| "K10/p_null_present": 0.13136198390784556, |
| "K10/acc": 0.8875, |
| "K10/auroc": 0.8729625, |
| "K20/p_null_absent": 0.5052241023955867, |
| "K20/p_null_present": 0.15151537593046668, |
| "K20/acc": 0.8175, |
| "K20/auroc": 0.8350124999999999, |
| "K50/p_null_absent": 0.3282962662074715, |
| "K50/p_null_present": 0.1271415389876347, |
| "K50/acc": 0.6775, |
| "K50/auroc": 0.7611312499999999, |
| "K77/p_null_absent": 0.2986328311264515, |
| "K77/p_null_present": 0.14399044846883044, |
| "K77/acc": 0.6075, |
| "K77/auroc": 0.727425, |
| "p_null_absent_range": 0.6137798364560876 |
| } |
| }, |
| "ksweep_clinc": { |
| "raw": { |
| "nll": 0.45495889803216055, |
| "brier": 0.2303747224801235, |
| "jsd": 0.10721702841363247, |
| "acc": 0.8410087719298246, |
| "acc_k": 0.931390977443609, |
| "ece": 0.021296209616021692, |
| "sel_acc80": 0.9091620986687549, |
| "conf_wrong": 0.03477443609022556, |
| "coverage": 0.5596804511278195, |
| "false_abstain": 0.07957393483709273, |
| "null_recall": 0.8010651629072681, |
| "auroc_null": 0.9493543146713903, |
| "auroc_conf": 0.943741814592873 |
| }, |
| "scaled": { |
| "nll": 0.4534248016144343, |
| "brier": 0.22990815748137056, |
| "jsd": 0.10749978171784527, |
| "acc": 0.8416353383458647, |
| "acc_k": 0.931390977443609, |
| "ece": 0.021236134109491114, |
| "sel_acc80": 0.9095536413469069, |
| "conf_wrong": 0.03320802005012531, |
| "coverage": 0.5584273182957393, |
| "false_abstain": 0.08051378446115288, |
| "null_recall": 0.8026315789473685, |
| "auroc_null": 0.9493543146713903, |
| "auroc_conf": 0.9435183843851482 |
| }, |
| "ksweep": { |
| "K2/p_null_absent": 0.9052965747044284, |
| "K2/p_null_present": 0.05470474093286318, |
| "K2/acc": 0.9974937343358395, |
| "K2/auroc": 0.9891332340877256, |
| "K3/p_null_absent": 0.8784392188938642, |
| "K3/p_null_present": 0.042013873441291844, |
| "K3/acc": 0.9924812030075187, |
| "K3/auroc": 0.9859108925195194, |
| "K5/p_null_absent": 0.8487086221536349, |
| "K5/p_null_present": 0.04932880515136326, |
| "K5/acc": 0.9849624060150376, |
| "K5/auroc": 0.9836370374557948, |
| "K10/p_null_absent": 0.7949157231020505, |
| "K10/p_null_present": 0.07082589385284982, |
| "K10/acc": 0.9774436090225563, |
| "K10/auroc": 0.9739700127511762, |
| "K20/p_null_absent": 0.7415750534238672, |
| "K20/p_null_present": 0.11202508829274964, |
| "K20/acc": 0.9598997493734336, |
| "K20/auroc": 0.946331995402039, |
| "K50/p_null_absent": 0.6288358126008383, |
| "K50/p_null_present": 0.1512641449672471, |
| "K50/acc": 0.9022556390977443, |
| "K50/auroc": 0.9079025885515795, |
| "K100/p_null_absent": 0.5329392300918698, |
| "K100/p_null_present": 0.16651480242535285, |
| "K100/acc": 0.8571428571428571, |
| "K100/auroc": 0.8676453037355293, |
| "K150/p_null_absent": 0.5002122879224388, |
| "K150/p_null_present": 0.21401167929611334, |
| "K150/acc": 0.7794486215538847, |
| "K150/auroc": 0.8178717470367649, |
| "p_null_absent_range": 0.4050842867819896 |
| } |
| }, |
| "mmlu_cf": { |
| "raw": { |
| "nll": 1.9737032157151875, |
| "brier": 0.7070673220421707, |
| "jsd": 0.39682694280253844, |
| "acc": 0.3486666666666667, |
| "acc_k": 0.358, |
| "ece": 0.08559551896651585, |
| "sel_acc80": 0.3908333333333333, |
| "coverage": 0.978, |
| "false_abstain": 0.022, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.9647858546383499, |
| "brier": 0.7052778805649923, |
| "jsd": 0.3981142799732584, |
| "acc": 0.3486666666666667, |
| "acc_k": 0.358, |
| "ece": 0.07654446144898731, |
| "sel_acc80": 0.39166666666666666, |
| "coverage": 0.978, |
| "false_abstain": 0.022, |
| "null_recall": "n/a" |
| }, |
| "cf": { |
| "orig/acc": 0.376, |
| "orig/among_k": 0.376, |
| "remove3/acc": 0.372, |
| "remove3/among_k": 0.424, |
| "add3_unrel/acc": 0.344, |
| "add3_unrel/among_k": 0.348, |
| "replace_hard/acc": 0.384, |
| "replace_hard/among_k": 0.384, |
| "add_neardup/acc": 0.248, |
| "add_neardup/among_k": 0.248, |
| "reorder/acc": 0.368, |
| "reorder/among_k": 0.368, |
| "add3_unrel/dlo_top2": 0.2423946739889807, |
| "reorder/max_dp": 0.13986547412350775, |
| "add_neardup/slot_gap": 0.17452707809314597, |
| "add_neardup/cluster_mass_err": 0.10785954518744256 |
| } |
| }, |
| "mmlu_cf_teacher": { |
| "raw": { |
| "nll": 2.0121815929819125, |
| "brier": 0.7285061878659447, |
| "jsd": 0.4095139954925811, |
| "acc": 0.3173611111111111, |
| "acc_k": 0.3279166666666667, |
| "ece": 0.08924061194165714, |
| "sel_acc80": 0.3581597222222222, |
| "coverage": 0.97875, |
| "false_abstain": 0.02125, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 2.003204512134324, |
| "brier": 0.7264348569473305, |
| "jsd": 0.4105225353643012, |
| "acc": 0.3173611111111111, |
| "acc_k": 0.3279166666666667, |
| "ece": 0.08019677360335156, |
| "sel_acc80": 0.3585069444444444, |
| "coverage": 0.97875, |
| "false_abstain": 0.02125, |
| "null_recall": "n/a" |
| }, |
| "cf": { |
| "orig/acc": 0.33, |
| "orig/among_k": 0.3308333333333333, |
| "orig/kl_teacher": 0.3248038523416908, |
| "remove3/acc": 0.35, |
| "remove3/among_k": 0.4066666666666667, |
| "remove3/kl_teacher": 0.9387059597443529, |
| "add3_unrel/acc": 0.32083333333333336, |
| "add3_unrel/among_k": 0.32166666666666666, |
| "add3_unrel/kl_teacher": 0.5062704442603856, |
| "replace_hard/acc": 0.33666666666666667, |
| "replace_hard/among_k": 0.34, |
| "replace_hard/kl_teacher": 0.39902467457106233, |
| "add_neardup/acc": 0.2225, |
| "add_neardup/among_k": 0.2225, |
| "add_neardup/kl_teacher": 0.3746501627036028, |
| "reorder/acc": 0.3441666666666667, |
| "reorder/among_k": 0.3458333333333333, |
| "reorder/kl_teacher": 0.3114207331236338, |
| "remove3/delta_mae": 1.1855408093066204, |
| "remove3/delta_teacher_abs": 1.135840148062828, |
| "add3_unrel/delta_mae": 0.7900708989092671, |
| "add3_unrel/delta_teacher_abs": 0.7390631803199154, |
| "replace_hard/delta_mae": 0.7568652725640471, |
| "replace_hard/delta_teacher_abs": 0.6681882120617108, |
| "add_neardup/delta_mae": 0.74848264109364, |
| "add_neardup/delta_teacher_abs": 0.7305883395327554, |
| "reorder/delta_mae": 1.027615587327876, |
| "reorder/delta_teacher_abs": 0.7763704285195197, |
| "add3_unrel/dlo_top2": 0.22777400764209627, |
| "reorder/max_dp": 0.1413952209048466, |
| "add_neardup/slot_gap": 0.1525056871988272, |
| "add_neardup/cluster_mass_err": 0.09664878458231972 |
| } |
| }, |
| "mmlu_choicesonly": { |
| "raw": { |
| "nll": 2.2140372835626967, |
| "brier": 0.8731933106846517, |
| "jsd": 0.4800525530271047, |
| "acc": 0.22333333333333333, |
| "acc_k": 0.22333333333333333, |
| "ece": 0.09060597602277995, |
| "sel_acc80": 0.24375, |
| "coverage": 0.9975, |
| "false_abstain": 0.0025, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 2.2027042994257227, |
| "brier": 0.8707815808273419, |
| "jsd": 0.4804888539877334, |
| "acc": 0.22333333333333333, |
| "acc_k": 0.22333333333333333, |
| "ece": 0.08494451654454074, |
| "sel_acc80": 0.24375, |
| "coverage": 0.9975, |
| "false_abstain": 0.0025, |
| "null_recall": "n/a" |
| } |
| }, |
| "mmlu_pro": { |
| "raw": { |
| "nll": 1.900452030855777, |
| "brier": 0.7831113370273955, |
| "jsd": 0.4215778910382713, |
| "acc": 0.33, |
| "acc_k": 0.3308333333333333, |
| "ece": 0.06917534026006857, |
| "sel_acc80": 0.3697916666666667, |
| "coverage": 0.9941666666666666, |
| "false_abstain": 0.005833333333333334, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.8933466892965118, |
| "brier": 0.7815541704992409, |
| "jsd": 0.42284726531134265, |
| "acc": 0.33, |
| "acc_k": 0.3308333333333333, |
| "ece": 0.06307571604847909, |
| "sel_acc80": 0.3697916666666667, |
| "coverage": 0.9941666666666666, |
| "false_abstain": 0.005833333333333334, |
| "null_recall": "n/a" |
| } |
| }, |
| "mmlu_shuffledq": { |
| "raw": { |
| "nll": 2.40635102023092, |
| "brier": 0.92227431248264, |
| "jsd": 0.49396211895175013, |
| "acc": 0.2075, |
| "acc_k": 0.21166666666666667, |
| "ece": 0.1442431898104648, |
| "sel_acc80": 0.22395833333333334, |
| "coverage": 0.9775, |
| "false_abstain": 0.0225, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 2.3869170155851944, |
| "brier": 0.9183096395248436, |
| "jsd": 0.4940199949436655, |
| "acc": 0.2075, |
| "acc_k": 0.21166666666666667, |
| "ece": 0.1356223477423191, |
| "sel_acc80": 0.22395833333333334, |
| "coverage": 0.9775, |
| "false_abstain": 0.0225, |
| "null_recall": "n/a" |
| } |
| }, |
| "mnli_val": { |
| "raw": { |
| "nll": 0.38822534996826497, |
| "brier": 0.21632864323315787, |
| "jsd": 0.10595431629698053, |
| "acc": 0.8493197278911565, |
| "acc_k": 0.8493197278911565, |
| "ece": 0.022966662576409422, |
| "sel_acc80": 0.9209183673469388, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.39044109534972526, |
| "brier": 0.21707419982724052, |
| "jsd": 0.10778225595049809, |
| "acc": 0.8493197278911565, |
| "acc_k": 0.8493197278911565, |
| "ece": 0.027150291964715845, |
| "sel_acc80": 0.9204931972789115, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "ng20_test": { |
| "raw": { |
| "nll": 1.6372119960226885, |
| "brier": 0.6259482143658505, |
| "jsd": 0.3162359946431149, |
| "acc": 0.5235816814764184, |
| "acc_k": 0.5696514012303486, |
| "ece": 0.08290197773494705, |
| "sel_acc80": 0.6006493506493507, |
| "coverage": 0.8281613123718387, |
| "false_abstain": 0.17183868762816132, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.622403194321037, |
| "brier": 0.6257776708518196, |
| "jsd": 0.31806354999429126, |
| "acc": 0.5216678058783322, |
| "acc_k": 0.5696514012303486, |
| "ece": 0.07770895541162419, |
| "sel_acc80": 0.5979152426520847, |
| "coverage": 0.82365003417635, |
| "false_abstain": 0.17634996582365003, |
| "null_recall": "n/a" |
| } |
| }, |
| "null_irrq_intent": { |
| "raw": { |
| "nll": 0.2980190814605242, |
| "brier": 0.14585620980287914, |
| "jsd": 0.08084519615677742, |
| "acc": 0.8998988877654196, |
| "acc_k": 0.9642857142857143, |
| "ece": 0.05771066426206286, |
| "sel_acc80": 0.9665192672141504, |
| "conf_wrong": 0.001002004008016032, |
| "coverage": 0.5399393326592518, |
| "false_abstain": 0.044897959183673466, |
| "null_recall": 0.8677354709418837, |
| "auroc_null": 0.9771737352255532, |
| "auroc_conf": 0.9805253363870599 |
| }, |
| "scaled": { |
| "nll": 0.29709066455898037, |
| "brier": 0.14508696242491342, |
| "jsd": 0.0809374973561244, |
| "acc": 0.9009100101112234, |
| "acc_k": 0.9642857142857143, |
| "ece": 0.05521615729474199, |
| "sel_acc80": 0.967150979153506, |
| "conf_wrong": 0.0, |
| "coverage": 0.5384226491405459, |
| "false_abstain": 0.04591836734693878, |
| "null_recall": 0.8697394789579158, |
| "auroc_null": 0.9771737352255532, |
| "auroc_conf": 0.9806715471759847 |
| } |
| }, |
| "null_irrq_squad": { |
| "raw": { |
| "nll": 0.15154312186284882, |
| "brier": 0.08788833430991315, |
| "jsd": 0.03980749293912364, |
| "acc": 0.9385, |
| "acc_k": 0.995, |
| "ece": 0.009013604998588616, |
| "sel_acc80": 0.989375, |
| "conf_wrong": 0.017, |
| "coverage": 0.5445, |
| "false_abstain": 0.015, |
| "null_recall": 0.896, |
| "auroc_null": 0.9931030000000001, |
| "auroc_conf": 0.9928480000000001 |
| }, |
| "scaled": { |
| "nll": 0.1514942799887926, |
| "brier": 0.08774260814706673, |
| "jsd": 0.039821659736051646, |
| "acc": 0.9385, |
| "acc_k": 0.995, |
| "ece": 0.009389410480856907, |
| "sel_acc80": 0.989375, |
| "conf_wrong": 0.017, |
| "coverage": 0.5445, |
| "false_abstain": 0.015, |
| "null_recall": 0.896, |
| "auroc_null": 0.9931030000000001, |
| "auroc_conf": 0.9928390000000001 |
| } |
| }, |
| "null_nearmiss_banking77": { |
| "raw": { |
| "nll": 0.9520833679756119, |
| "brier": 0.4805285036043141, |
| "jsd": 0.20084081249467256, |
| "acc": 0.665, |
| "acc_k": 0.878, |
| "ece": 0.14005471447110177, |
| "sel_acc80": 0.72875, |
| "conf_wrong": 0.104, |
| "coverage": 0.7010000000000001, |
| "false_abstain": 0.094, |
| "null_recall": 0.504, |
| "auroc_null": 0.854664, |
| "auroc_conf": 0.8435440000000001 |
| }, |
| "scaled": { |
| "nll": 0.9474953750672853, |
| "brier": 0.4787477860806431, |
| "jsd": 0.20112935951834626, |
| "acc": 0.665, |
| "acc_k": 0.878, |
| "ece": 0.13741894581913947, |
| "sel_acc80": 0.73125, |
| "conf_wrong": 0.102, |
| "coverage": 0.7010000000000001, |
| "false_abstain": 0.094, |
| "null_recall": 0.504, |
| "auroc_null": 0.854664, |
| "auroc_conf": 0.843312 |
| } |
| }, |
| "null_nearmiss_clinc": { |
| "raw": { |
| "nll": 0.6851653943413197, |
| "brier": 0.37972759426437286, |
| "jsd": 0.14847968896737876, |
| "acc": 0.7467071935157041, |
| "acc_k": 0.9373737373737374, |
| "ece": 0.13139047720511998, |
| "sel_acc80": 0.8063291139240506, |
| "conf_wrong": 0.1565040650406504, |
| "coverage": 0.6757852077001013, |
| "false_abstain": 0.052525252525252523, |
| "null_recall": 0.5975609756097561, |
| "auroc_null": 0.9133653609263365, |
| "auroc_conf": 0.882996632996633 |
| }, |
| "scaled": { |
| "nll": 0.6842461953661407, |
| "brier": 0.3792214233262601, |
| "jsd": 0.14869438900479817, |
| "acc": 0.7467071935157041, |
| "acc_k": 0.9373737373737374, |
| "ece": 0.13018060612219567, |
| "sel_acc80": 0.8063291139240506, |
| "conf_wrong": 0.15447154471544716, |
| "coverage": 0.6757852077001013, |
| "false_abstain": 0.052525252525252523, |
| "null_recall": 0.5975609756097561, |
| "auroc_null": 0.9133653609263365, |
| "auroc_conf": 0.8821630943582163 |
| } |
| }, |
| "null_nearmiss_hwu64": { |
| "raw": { |
| "nll": 0.7480553781771985, |
| "brier": 0.42575202811506824, |
| "jsd": 0.1816428992506284, |
| "acc": 0.689, |
| "acc_k": 0.906, |
| "ece": 0.11218150165677071, |
| "sel_acc80": 0.75875, |
| "conf_wrong": 0.086, |
| "coverage": 0.723, |
| "false_abstain": 0.05, |
| "null_recall": 0.504, |
| "auroc_null": 0.893416, |
| "auroc_conf": 0.864184 |
| }, |
| "scaled": { |
| "nll": 0.747809683498297, |
| "brier": 0.42425010959360987, |
| "jsd": 0.18204554811066134, |
| "acc": 0.69, |
| "acc_k": 0.906, |
| "ece": 0.1076480211019516, |
| "sel_acc80": 0.75875, |
| "conf_wrong": 0.086, |
| "coverage": 0.722, |
| "false_abstain": 0.05, |
| "null_recall": 0.506, |
| "auroc_null": 0.893416, |
| "auroc_conf": 0.8633120000000001 |
| } |
| }, |
| "snli_null": { |
| "raw": { |
| "nll": 0.38093771683344796, |
| "brier": 0.21158908219324113, |
| "jsd": 0.09917117969867743, |
| "acc": 0.8548807376227701, |
| "acc_k": 0.9408906882591093, |
| "ece": 0.009903888708127223, |
| "sel_acc80": 0.9210921843687375, |
| "conf_wrong": 0.03215561730845574, |
| "coverage": 0.5908999799559029, |
| "false_abstain": 0.022672064777327937, |
| "null_recall": 0.7880111155220325, |
| "auroc_null": 0.9758374009350796, |
| "auroc_conf": 0.9598274168947578 |
| }, |
| "scaled": { |
| "nll": 0.38155742627478745, |
| "brier": 0.21133330351090482, |
| "jsd": 0.09963839430183072, |
| "acc": 0.8552816195630387, |
| "acc_k": 0.9408906882591093, |
| "ece": 0.011238936343062077, |
| "sel_acc80": 0.9215931863727455, |
| "conf_wrong": 0.030964668519253672, |
| "coverage": 0.5904990980156344, |
| "false_abstain": 0.022672064777327937, |
| "null_recall": 0.7888050813815006, |
| "auroc_null": 0.9758374009350796, |
| "auroc_conf": 0.9593634129602873 |
| } |
| }, |
| "snli_test": { |
| "raw": { |
| "nll": 0.2944368899492289, |
| "brier": 0.16127000903644761, |
| "jsd": 0.08039721279595598, |
| "acc": 0.8935658448586891, |
| "acc_k": 0.8935658448586891, |
| "ece": 0.025348633677095857, |
| "sel_acc80": 0.9521543086172345, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.2968152009716646, |
| "brier": 0.16198921686136677, |
| "jsd": 0.08209697670114373, |
| "acc": 0.8935658448586891, |
| "acc_k": 0.8935658448586891, |
| "ece": 0.030635995678730602, |
| "sel_acc80": 0.9519038076152304, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "snli_test_hyponly": { |
| "raw": { |
| "nll": 1.5253299458289893, |
| "brier": 0.8161567119018235, |
| "jsd": 0.32832325568109827, |
| "acc": 0.41130487071557426, |
| "acc_k": 0.41130487071557426, |
| "ece": 0.314992772150575, |
| "sel_acc80": 0.43361723446893785, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.4912990926911411, |
| "brier": 0.8073056544838173, |
| "jsd": 0.3266241508995174, |
| "acc": 0.41130487071557426, |
| "acc_k": 0.41130487071557426, |
| "ece": 0.30664448151251045, |
| "sel_acc80": 0.4338677354709419, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "snli_test_paraphrase": { |
| "raw": { |
| "nll": 0.3621301663516111, |
| "brier": 0.19344863996217362, |
| "jsd": 0.1055394463756449, |
| "acc": 0.8817398276207656, |
| "acc_k": 0.8817398276207656, |
| "ece": 0.08257404655534728, |
| "sel_acc80": 0.9408817635270541, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.36799802624918665, |
| "brier": 0.19604129055167072, |
| "jsd": 0.1080423642559439, |
| "acc": 0.8817398276207656, |
| "acc_k": 0.8817398276207656, |
| "ece": 0.09054662749891076, |
| "sel_acc80": 0.9411322645290581, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "snli_test_qpara": { |
| "raw": { |
| "nll": 0.3062878039158672, |
| "brier": 0.16650863974613417, |
| "jsd": 0.08552545369825479, |
| "acc": 0.8856569709127382, |
| "acc_k": 0.8856569709127382, |
| "ece": 0.037319196367574685, |
| "sel_acc80": 0.9555137844611529, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.3097593663004104, |
| "brier": 0.16754294958887214, |
| "jsd": 0.08746037039837277, |
| "acc": 0.8856569709127382, |
| "acc_k": 0.8856569709127382, |
| "ece": 0.043105250301069334, |
| "sel_acc80": 0.9555137844611529, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "snli_test_soft": { |
| "raw": { |
| "nll": 0.5461843753820834, |
| "brier": 0.10901544554920943, |
| "jsd": 0.05977115001837822, |
| "acc": 0.8935658448586891, |
| "acc_k": 0.8935658448586891, |
| "ece": 0.025348633677095857, |
| "sel_acc80": 0.9521543086172345, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.5389270214964142, |
| "brier": 0.10806343070784269, |
| "jsd": 0.060134176388955525, |
| "acc": 0.8935658448586891, |
| "acc_k": 0.8935658448586891, |
| "ece": 0.030635995678730602, |
| "sel_acc80": 0.9519038076152304, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "squad_null": { |
| "raw": { |
| "nll": 0.08111459801249543, |
| "brier": 0.04076323473737811, |
| "jsd": 0.01734145744096447, |
| "acc": 0.97375, |
| "acc_k": 0.996, |
| "ece": 0.008793617211282249, |
| "sel_acc80": 0.99625, |
| "conf_wrong": 0.017, |
| "coverage": 0.51075, |
| "false_abstain": 0.014, |
| "null_recall": 0.9645, |
| "auroc_null": 0.9950402500000001, |
| "auroc_conf": 0.9949386250000001 |
| }, |
| "scaled": { |
| "nll": 0.08106301545108649, |
| "brier": 0.040767455693852005, |
| "jsd": 0.017360626133039504, |
| "acc": 0.97375, |
| "acc_k": 0.996, |
| "ece": 0.008595976613461967, |
| "sel_acc80": 0.99625, |
| "conf_wrong": 0.017, |
| "coverage": 0.51075, |
| "false_abstain": 0.014, |
| "null_recall": 0.9645, |
| "auroc_null": 0.9950402500000001, |
| "auroc_conf": 0.994932 |
| } |
| }, |
| "trec_coarse": { |
| "raw": { |
| "nll": 1.5918411787616937, |
| "brier": 0.6349207986106156, |
| "jsd": 0.29208722584601526, |
| "acc": 0.532, |
| "acc_k": 0.584, |
| "ece": 0.13663677060604096, |
| "sel_acc80": 0.5825, |
| "coverage": 0.86, |
| "false_abstain": 0.14, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.5655300627415383, |
| "brier": 0.6341792940622437, |
| "jsd": 0.2930151878513147, |
| "acc": 0.532, |
| "acc_k": 0.584, |
| "ece": 0.1302748074531555, |
| "sel_acc80": 0.58, |
| "coverage": 0.858, |
| "false_abstain": 0.142, |
| "null_recall": "n/a" |
| } |
| }, |
| "trec_fine": { |
| "raw": { |
| "nll": 2.169659186192131, |
| "brier": 0.8538986718630976, |
| "jsd": 0.3937861872099405, |
| "acc": 0.376, |
| "acc_k": 0.5, |
| "ece": 0.24738949897885326, |
| "sel_acc80": 0.3925, |
| "coverage": 0.6679999999999999, |
| "false_abstain": 0.332, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 2.143684417143925, |
| "brier": 0.8532065630013449, |
| "jsd": 0.39535413252242657, |
| "acc": 0.372, |
| "acc_k": 0.5, |
| "ece": 0.24531754493713384, |
| "sel_acc80": 0.3925, |
| "coverage": 0.6579999999999999, |
| "false_abstain": 0.342, |
| "null_recall": "n/a" |
| } |
| }, |
| "unli_test": { |
| "raw": { |
| "nll": 0.5318298036808603, |
| "brier": 0.13167920018417006, |
| "jsd": 0.04981357317021074, |
| "auroc_null": 0.8890171083529017, |
| "auroc_conf": 0.8890171083529017 |
| }, |
| "scaled": { |
| "nll": 0.5318297999376992, |
| "brier": 0.13167919956989954, |
| "jsd": 0.04981357268943865, |
| "auroc_null": 0.8890171083529017, |
| "auroc_conf": 0.8890171083529017 |
| } |
| }, |
| "data_kb_val": { |
| "raw": { |
| "nll": 0.8995894797702431, |
| "brier": 0.46785703357109987, |
| "jsd": 0.23309134218604935, |
| "acc": 0.6293333333333333, |
| "acc_k": 0.609704641350211, |
| "ece": 0.02164344120025636, |
| "sel_acc80": 0.7133333333333334, |
| "conf_wrong": 0.01282051282051282, |
| "coverage": 0.9486666666666667, |
| "false_abstain": 0.0, |
| "null_recall": 0.9871794871794872, |
| "auroc_null": 0.9999639366727975, |
| "auroc_conf": 0.9893613184752426 |
| }, |
| "scaled": { |
| "nll": 0.8988669709475279, |
| "brier": 0.4678002285972357, |
| "jsd": 0.23442374370611735, |
| "acc": 0.6293333333333333, |
| "acc_k": 0.609704641350211, |
| "ece": 0.0223244069914023, |
| "sel_acc80": 0.7125, |
| "conf_wrong": 0.01282051282051282, |
| "coverage": 0.9486666666666667, |
| "false_abstain": 0.0, |
| "null_recall": 0.9871794871794872, |
| "auroc_null": 0.9999639366727975, |
| "auroc_conf": 0.9892982076526381 |
| } |
| }, |
| "truthfulqa_mc1": { |
| "raw": { |
| "nll": 2.3583552921727957, |
| "brier": 0.9532752488732227, |
| "jsd": 0.43745611493995573, |
| "acc": 0.2668298653610771, |
| "acc_k": 0.3268053855569155, |
| "ece": 0.28699413596708306, |
| "sel_acc80": 0.28440366972477066, |
| "coverage": 0.8837209302325582, |
| "false_abstain": 0.11627906976744186, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 2.330985705103918, |
| "brier": 0.9483202788516172, |
| "jsd": 0.4370409987040793, |
| "acc": 0.2668298653610771, |
| "acc_k": 0.3268053855569155, |
| "ece": 0.2787222843457611, |
| "sel_acc80": 0.28593272171253825, |
| "coverage": 0.8837209302325582, |
| "false_abstain": 0.11627906976744186, |
| "null_recall": "n/a" |
| } |
| }, |
| "data_wf_val": { |
| "raw": { |
| "nll": 0.24610411725258516, |
| "brier": 0.1394640994048586, |
| "jsd": 0.0622050245204808, |
| "acc": 0.8986666666666666, |
| "acc_k": 0.8991138377641446, |
| "ece": 0.014609658221403754, |
| "sel_acc80": 0.9733333333333334, |
| "conf_wrong": 0.030303030303030304, |
| "coverage": 0.9806666666666667, |
| "false_abstain": 0.0, |
| "null_recall": 0.8787878787878788, |
| "auroc_null": 0.9997727789138832, |
| "auroc_conf": 0.9778356158724256 |
| }, |
| "scaled": { |
| "nll": 0.24495817350751345, |
| "brier": 0.1391821391803038, |
| "jsd": 0.06268002385019018, |
| "acc": 0.8986666666666666, |
| "acc_k": 0.8991138377641446, |
| "ece": 0.013083436906337762, |
| "sel_acc80": 0.9733333333333334, |
| "conf_wrong": 0.030303030303030304, |
| "coverage": 0.9806666666666667, |
| "false_abstain": 0.0, |
| "null_recall": 0.8787878787878788, |
| "auroc_null": 0.9997727789138832, |
| "auroc_conf": 0.9776910206358059 |
| } |
| }, |
| "wf_heldout_choice": { |
| "raw": { |
| "nll": 0.04069819362514963, |
| "brier": 0.018002550660832672, |
| "jsd": 0.008412281696821153, |
| "acc": 0.9893333333333333, |
| "acc_k": 0.9943382873319179, |
| "ece": 0.005535807013511662, |
| "sel_acc80": 1.0, |
| "conf_wrong": 0.06896551724137931, |
| "coverage": 0.9473333333333334, |
| "false_abstain": 0.0, |
| "null_recall": 0.9080459770114943, |
| "auroc_null": 0.9994468441646127, |
| "auroc_conf": 0.9956154265400916 |
| }, |
| "scaled": { |
| "nll": 0.04119968250350716, |
| "brier": 0.018094206464215678, |
| "jsd": 0.008601874984231025, |
| "acc": 0.9893333333333333, |
| "acc_k": 0.9943382873319179, |
| "ece": 0.005908609946568721, |
| "sel_acc80": 1.0, |
| "conf_wrong": 0.06896551724137931, |
| "coverage": 0.9473333333333334, |
| "false_abstain": 0.0, |
| "null_recall": 0.9080459770114943, |
| "auroc_null": 0.9994468441646127, |
| "auroc_conf": 0.9953957911348643 |
| } |
| }, |
| "wf_heldout_noul": { |
| "raw": { |
| "nll": 0.7866961798391291, |
| "brier": 0.4496349124070911, |
| "jsd": 0.1659911488117415, |
| "acc": 0.6986666666666667, |
| "acc_k": 0.6986666666666667, |
| "ece": 0.18171938677628838, |
| "sel_acc80": 0.7525, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.7674072662661018, |
| "brier": 0.44586403670437996, |
| "jsd": 0.16535243468099545, |
| "acc": 0.6986666666666667, |
| "acc_k": 0.6986666666666667, |
| "ece": 0.1772376935084661, |
| "sel_acc80": 0.7525, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "wf_heldout_score": { |
| "raw": { |
| "nll": 2.0987200270240858, |
| "brier": 0.8091494680533715, |
| "jsd": 0.29950320499754934, |
| "acc": 0.49733333333333335, |
| "acc_k": 0.49733333333333335, |
| "ece": 0.37649274045228964, |
| "sel_acc80": 0.5458333333333333, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 2.0312639152806957, |
| "brier": 0.8016462098572795, |
| "jsd": 0.2979000494751449, |
| "acc": 0.49733333333333335, |
| "acc_k": 0.49733333333333335, |
| "ece": 0.37102700203657146, |
| "sel_acc80": 0.5458333333333333, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "wf_heldout_style": { |
| "raw": { |
| "nll": 0.24614928496086558, |
| "brier": 0.1391935848872391, |
| "jsd": 0.06184622570106476, |
| "acc": 0.8993333333333333, |
| "acc_k": 0.8977657413676371, |
| "ece": 0.01664232387145353, |
| "sel_acc80": 0.9683333333333334, |
| "conf_wrong": 0.0, |
| "coverage": 0.9846666666666667, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0 |
| }, |
| "scaled": { |
| "nll": 0.24471071542524464, |
| "brier": 0.13901082740892692, |
| "jsd": 0.06236480920470213, |
| "acc": 0.8993333333333333, |
| "acc_k": 0.8977657413676371, |
| "ece": 0.013723565757274658, |
| "sel_acc80": 0.9683333333333334, |
| "conf_wrong": 0.0, |
| "coverage": 0.9846666666666667, |
| "false_abstain": 0.0, |
| "null_recall": 1.0, |
| "auroc_null": 1.0, |
| "auroc_conf": 1.0 |
| } |
| }, |
| "wf_rubric_flip": { |
| "raw": { |
| "nll": 0.5307086168197724, |
| "brier": 0.26619245796638785, |
| "jsd": 0.09958348690925925, |
| "acc": 0.8273333333333334, |
| "acc_k": 0.8273333333333334, |
| "ece": 0.10746072290341058, |
| "sel_acc80": 0.9, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.5168277089308791, |
| "brier": 0.26401031532355146, |
| "jsd": 0.09929868841268231, |
| "acc": 0.8273333333333334, |
| "acc_k": 0.8273333333333334, |
| "ece": 0.10272792806227998, |
| "sel_acc80": 0.9, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "flip": { |
| "n_pairs": 750, |
| "flip_rate": 0.7466666666666667, |
| "both_correct": 0.6946666666666667, |
| "acc": 0.8273333333333334 |
| } |
| }, |
| "wf_rubric_shuffled": { |
| "raw": { |
| "nll": 4.5102297803204126, |
| "brier": 1.0442621780841999, |
| "jsd": 0.3776491722774307, |
| "acc": 0.39866666666666667, |
| "acc_k": 0.39866666666666667, |
| "ece": 0.4885187878410022, |
| "sel_acc80": 0.405, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 4.348945097725256, |
| "brier": 1.0389491271745737, |
| "jsd": 0.3764887276833897, |
| "acc": 0.39866666666666667, |
| "acc_k": 0.39866666666666667, |
| "ece": 0.4842325220704079, |
| "sel_acc80": 0.4041666666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "data_wf_hf_val": { |
| "raw": { |
| "nll": 0.010585208018264583, |
| "brier": 0.005168010358705758, |
| "jsd": 0.0020250369475977915, |
| "acc": 0.9966666666666667, |
| "acc_k": 0.9966666666666667, |
| "ece": 0.0028440783818563034, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.010354351768417104, |
| "brier": 0.005121358381636529, |
| "jsd": 0.002035180293716168, |
| "acc": 0.9966666666666667, |
| "acc_k": 0.9966666666666667, |
| "ece": 0.002598859508832256, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "cua_s1_forms_demo": { |
| "raw": { |
| "nll": 9.68217684130784e-05, |
| "brier": 2.42991571051238e-06, |
| "jsd": 3.3502051815451166e-05, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 9.620737056348361e-05, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.0001166535209764421, |
| "brier": 3.334119317415377e-06, |
| "jsd": 4.0353015412178415e-05, |
| "acc": 1.0, |
| "acc_k": 1.0, |
| "ece": 0.00011580908785058242, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "cua_s1_forms_test": { |
| "raw": { |
| "nll": 0.010497363812569337, |
| "brier": 0.005111427197274268, |
| "jsd": 0.0020249838705119843, |
| "acc": 0.996, |
| "acc_k": 0.996, |
| "ece": 0.0026506466865538827, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.010267529708334153, |
| "brier": 0.00506279966705414, |
| "jsd": 0.002029791549827035, |
| "acc": 0.996, |
| "acc_k": 0.996, |
| "ece": 0.002581955194473174, |
| "sel_acc80": 1.0, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "jev4b_adversarial": { |
| "raw": { |
| "nll": 0.6563798084706536, |
| "brier": 0.2509300255617882, |
| "jsd": 0.09856897364784455, |
| "acc": 0.8433333333333334, |
| "acc_k": 0.8433333333333334, |
| "ece": 0.09749235555529595, |
| "sel_acc80": 0.9229166666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.6374143261836251, |
| "brier": 0.24906241688616718, |
| "jsd": 0.09839928565661872, |
| "acc": 0.8433333333333334, |
| "acc_k": 0.8433333333333334, |
| "ece": 0.09604769155383108, |
| "sel_acc80": 0.9229166666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "jevlogs_triage": { |
| "raw": { |
| "nll": 0.9845623682584176, |
| "brier": 0.6729167311320402, |
| "jsd": 0.2490643395451554, |
| "acc": 0.49733333333333335, |
| "acc_k": 0.49733333333333335, |
| "ece": 0.27361580948034925, |
| "sel_acc80": 0.5175, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.9654136745276661, |
| "brier": 0.6654463383788486, |
| "jsd": 0.2473301509447621, |
| "acc": 0.49733333333333335, |
| "acc_k": 0.49733333333333335, |
| "ece": 0.2673972271680832, |
| "sel_acc80": 0.5175, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "mind2web_choice": { |
| "raw": { |
| "nll": 1.6608707584399627, |
| "brier": 0.7200349551249486, |
| "jsd": 0.32339959282045455, |
| "acc": 0.422, |
| "acc_k": 0.422, |
| "ece": 0.2274525083303452, |
| "sel_acc80": 0.4841666666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.6333868224230361, |
| "brier": 0.7136883533617026, |
| "jsd": 0.3232182688768001, |
| "acc": 0.422, |
| "acc_k": 0.422, |
| "ece": 0.2181771738330523, |
| "sel_acc80": 0.48583333333333334, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "pagerduty_trigger": { |
| "raw": { |
| "nll": 1.3089463058665416, |
| "brier": 0.5937149639931447, |
| "jsd": 0.225649696977532, |
| "acc": 0.6026666666666667, |
| "acc_k": 0.6026666666666667, |
| "ece": 0.26803290375073746, |
| "sel_acc80": 0.68, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.2716589001643595, |
| "brier": 0.5893011552599684, |
| "jsd": 0.2249425030779627, |
| "acc": 0.6026666666666667, |
| "acc_k": 0.6026666666666667, |
| "ece": 0.2633645134369532, |
| "sel_acc80": 0.68, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "systemone_lite_hard": { |
| "raw": { |
| "nll": 0.32642191182042274, |
| "brier": 0.17189764452077014, |
| "jsd": 0.06927031888838348, |
| "acc": 0.8826666666666667, |
| "acc_k": 0.8826666666666667, |
| "ece": 0.04653552297751107, |
| "sel_acc80": 0.9458333333333333, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 0.3206730075344011, |
| "brier": 0.1710769966178729, |
| "jsd": 0.06952373056541901, |
| "acc": 0.8826666666666667, |
| "acc_k": 0.8826666666666667, |
| "ece": 0.0421845922271411, |
| "sel_acc80": 0.9458333333333333, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "tree_choice_cap": { |
| "raw": { |
| "nll": 1.8059283410666016, |
| "brier": 0.6318870635104122, |
| "jsd": 0.31713011149335896, |
| "acc": 0.5375, |
| "acc_k": 0.6, |
| "ece": 0.11325502035518487, |
| "sel_acc80": 0.6197916666666666, |
| "coverage": 0.8, |
| "false_abstain": 0.2, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.7984774021118792, |
| "brier": 0.6330959581547211, |
| "jsd": 0.31947562577323346, |
| "acc": 0.5375, |
| "acc_k": 0.6, |
| "ece": 0.11445001408250796, |
| "sel_acc80": 0.6215277777777778, |
| "coverage": 0.7972222222222223, |
| "false_abstain": 0.20277777777777778, |
| "null_recall": "n/a" |
| } |
| }, |
| "typed_decisions_test": { |
| "raw": { |
| "nll": 2.00088336147219, |
| "brier": 0.4425575284541666, |
| "jsd": 0.18983895937760947, |
| "acc": 0.422, |
| "acc_k": 0.422, |
| "ece": 0.33466433302561444, |
| "sel_acc80": 0.44083333333333335, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 1.9460634838656419, |
| "brier": 0.43500238428936305, |
| "jsd": 0.18668195642568758, |
| "acc": 0.422, |
| "acc_k": 0.422, |
| "ece": 0.3275858606696128, |
| "sel_acc80": 0.4425, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| }, |
| "typed_decisions_train": { |
| "raw": { |
| "nll": 2.3080436786322527, |
| "brier": 0.4904933688193668, |
| "jsd": 0.2051707895984692, |
| "acc": 0.35933333333333334, |
| "acc_k": 0.35933333333333334, |
| "ece": 0.44574949810902287, |
| "sel_acc80": 0.3641666666666667, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| }, |
| "scaled": { |
| "nll": 2.239424416856678, |
| "brier": 0.4801745735180739, |
| "jsd": 0.2006978346623256, |
| "acc": 0.35933333333333334, |
| "acc_k": 0.35933333333333334, |
| "ece": 0.4376716514229774, |
| "sel_acc80": 0.365, |
| "coverage": 1.0, |
| "false_abstain": 0.0, |
| "null_recall": "n/a" |
| } |
| } |
| } |
| } |
