Invalid JSON:Unexpected token 'N', ..."acc@0.9": NaN
"... is not valid JSON
| { | |
| "latest": "v0007", | |
| "versions": { | |
| "v0007": { | |
| "version": "v0007", | |
| "parent": null, | |
| "kind": "lmhead", | |
| "backbone": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "max_len": 1024, | |
| "created": "2026-09-17T15:58:07+00:00", | |
| "tasks": [ | |
| "ag_news", | |
| "banking77", | |
| "boolq", | |
| "clinc150", | |
| "commonsense_qa", | |
| "copa", | |
| "dbpedia", | |
| "emotion", | |
| "facts", | |
| "fits", | |
| "imdb", | |
| "jailbreak", | |
| "massive_intent", | |
| "mnli", | |
| "mrpc", | |
| "openbookqa", | |
| "paws", | |
| "qnli", | |
| "read", | |
| "rte", | |
| "sciq", | |
| "sst2", | |
| "swag", | |
| "tweet_emoji", | |
| "tweet_hate", | |
| "tweet_irony", | |
| "tweet_offensive", | |
| "tweet_sentiment", | |
| "yahoo", | |
| "yelp", | |
| "anli", | |
| "winogrande", | |
| "hellaswag", | |
| "race", | |
| "scitail", | |
| "qqp", | |
| "stsb", | |
| "toxic", | |
| "stance_abortion", | |
| "stance_atheism", | |
| "stance_feminist", | |
| "stance_hillary", | |
| "match", | |
| "goemo_soft", | |
| "reason", | |
| "formality", | |
| "politeness", | |
| "strategyqa", | |
| "vitaminc", | |
| "ruletaker", | |
| "proofwriter", | |
| "folio", | |
| "logiqa", | |
| "tracie", | |
| "temporal_nli", | |
| "piqa", | |
| "siqa", | |
| "clutrr", | |
| "gsm8k", | |
| "svamp", | |
| "aqua" | |
| ], | |
| "trained_on": [ | |
| "ag_news", | |
| "banking77", | |
| "boolq", | |
| "clinc150", | |
| "commonsense_qa", | |
| "copa", | |
| "dbpedia", | |
| "emotion", | |
| "facts", | |
| "fits", | |
| "imdb", | |
| "jailbreak", | |
| "massive_intent", | |
| "mnli", | |
| "mrpc", | |
| "openbookqa", | |
| "paws", | |
| "qnli", | |
| "read", | |
| "rte", | |
| "sciq", | |
| "sst2", | |
| "swag", | |
| "tweet_emoji", | |
| "tweet_hate", | |
| "tweet_irony", | |
| "tweet_offensive", | |
| "tweet_sentiment", | |
| "yahoo", | |
| "yelp", | |
| "anli", | |
| "winogrande", | |
| "hellaswag", | |
| "race", | |
| "scitail", | |
| "qqp", | |
| "stsb", | |
| "toxic", | |
| "stance_abortion", | |
| "stance_atheism", | |
| "stance_feminist", | |
| "stance_hillary", | |
| "match", | |
| "goemo_soft", | |
| "reason", | |
| "formality", | |
| "politeness", | |
| "strategyqa", | |
| "vitaminc", | |
| "ruletaker", | |
| "proofwriter", | |
| "folio", | |
| "logiqa", | |
| "tracie", | |
| "temporal_nli", | |
| "piqa", | |
| "siqa", | |
| "clutrr", | |
| "gsm8k", | |
| "svamp", | |
| "aqua" | |
| ], | |
| "holdout": [ | |
| "probe", | |
| "bbh", | |
| "cola", | |
| "wic", | |
| "subj", | |
| "spam", | |
| "counterfactual", | |
| "cb", | |
| "arc_challenge", | |
| "stance_climate", | |
| "trec", | |
| "sst5", | |
| "fin_sentiment", | |
| "arc_easy", | |
| "newsgroups" | |
| ], | |
| "steps": 12118, | |
| "train_examples": 158246, | |
| "args": { | |
| "cmd": "lmtrain", | |
| "lora_r": 16, | |
| "loss": "mix", | |
| "max_per_task": 3000, | |
| "epochs": 1, | |
| "lr": 0.0001, | |
| "anchor": 0.1 | |
| }, | |
| "metrics": { | |
| "ag_news": { | |
| "n": 300, | |
| "acc": 0.9233333333333333, | |
| "nll": 0.23956205062784391, | |
| "brier": 0.12588630912802798, | |
| "ece": 0.03565445333719247, | |
| "mean_conf": 0.9105748584866524, | |
| "cov@0.5": 0.9833333333333333, | |
| "acc@0.5": 0.9288135593220339, | |
| "cov@0.7": 0.88, | |
| "acc@0.7": 0.9545454545454546, | |
| "cov@0.9": 0.77, | |
| "acc@0.9": 0.9653679653679653 | |
| }, | |
| "boolq": { | |
| "n": 300, | |
| "acc": 0.8766666666666667, | |
| "nll": 0.33278360864483053, | |
| "brier": 0.19905847383131203, | |
| "ece": 0.048838321963946066, | |
| "mean_conf": 0.8610335459311803, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.8766666666666667, | |
| "cov@0.7": 0.8866666666666667, | |
| "acc@0.7": 0.9060150375939849, | |
| "cov@0.9": 0.4666666666666667, | |
| "acc@0.9": 0.9642857142857143 | |
| }, | |
| "commonsense_qa": { | |
| "n": 300, | |
| "acc": 0.7866666666666666, | |
| "nll": 0.5812667453454432, | |
| "brier": 0.2960317921716136, | |
| "ece": 0.05657371540864312, | |
| "mean_conf": 0.7944276158014933, | |
| "cov@0.5": 0.8933333333333333, | |
| "acc@0.5": 0.832089552238806, | |
| "cov@0.7": 0.7, | |
| "acc@0.7": 0.9047619047619048, | |
| "cov@0.9": 0.42333333333333334, | |
| "acc@0.9": 0.952755905511811 | |
| }, | |
| "copa": { | |
| "n": 200, | |
| "acc": 0.93, | |
| "nll": 0.17489093500636219, | |
| "brier": 0.10265215895968784, | |
| "ece": 0.03722041517496112, | |
| "mean_conf": 0.8977384361624717, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.93, | |
| "cov@0.7": 0.9, | |
| "acc@0.7": 0.9611111111111111, | |
| "cov@0.9": 0.69, | |
| "acc@0.9": 1.0 | |
| }, | |
| "dbpedia": { | |
| "n": 300, | |
| "acc": 0.97, | |
| "nll": 0.12029726431718998, | |
| "brier": 0.0467073057529959, | |
| "ece": 0.011912721196810366, | |
| "mean_conf": 0.9797261367241542, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.97, | |
| "cov@0.7": 0.9866666666666667, | |
| "acc@0.7": 0.9797297297297297, | |
| "cov@0.9": 0.9766666666666667, | |
| "acc@0.9": 0.9829351535836177 | |
| }, | |
| "emotion": { | |
| "n": 300, | |
| "acc": 0.7766666666666666, | |
| "nll": 0.6417595775959655, | |
| "brier": 0.31279347659514983, | |
| "ece": 0.07161758591731388, | |
| "mean_conf": 0.8046423467993736, | |
| "cov@0.5": 0.92, | |
| "acc@0.5": 0.8043478260869565, | |
| "cov@0.7": 0.7233333333333334, | |
| "acc@0.7": 0.9032258064516129, | |
| "cov@0.9": 0.4266666666666667, | |
| "acc@0.9": 0.9609375 | |
| }, | |
| "facts": { | |
| "n": 54, | |
| "acc": 0.9629629629629629, | |
| "nll": 0.10233185567308967, | |
| "brier": 0.0580309563229181, | |
| "ece": 0.03901768503365692, | |
| "mean_conf": 0.985795874286581, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.9629629629629629, | |
| "cov@0.7": 1.0, | |
| "acc@0.7": 0.9629629629629629, | |
| "cov@0.9": 0.9444444444444444, | |
| "acc@0.9": 1.0 | |
| }, | |
| "fits": { | |
| "n": 214, | |
| "acc": 0.8598130841121495, | |
| "nll": 0.4789074549575723, | |
| "brier": 0.09971697154134483, | |
| "ece": 0.08341867177285883, | |
| "mean_conf": 0.818978947456752, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.8598130841121495, | |
| "cov@0.7": 0.8037383177570093, | |
| "acc@0.7": 0.936046511627907, | |
| "cov@0.9": 0.3317757009345794, | |
| "acc@0.9": 1.0 | |
| }, | |
| "imdb": { | |
| "n": 300, | |
| "acc": 0.95, | |
| "nll": 0.1449115194526972, | |
| "brier": 0.07315853850730186, | |
| "ece": 0.028431474765141816, | |
| "mean_conf": 0.9752282838026682, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.95, | |
| "cov@0.7": 0.9833333333333333, | |
| "acc@0.7": 0.9627118644067797, | |
| "cov@0.9": 0.95, | |
| "acc@0.9": 0.9754385964912281 | |
| }, | |
| "jailbreak": { | |
| "n": 200, | |
| "acc": 0.98, | |
| "nll": 0.07703215111456595, | |
| "brier": 0.03552832502947578, | |
| "ece": 0.014975480735301977, | |
| "mean_conf": 0.972049820125103, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.98, | |
| "cov@0.7": 0.995, | |
| "acc@0.7": 0.9798994974874372, | |
| "cov@0.9": 0.965, | |
| "acc@0.9": 0.9896373056994818 | |
| }, | |
| "mnli": { | |
| "n": 300, | |
| "acc": 0.8466666666666667, | |
| "nll": 0.44322894987837685, | |
| "brier": 0.24040505319501904, | |
| "ece": 0.05213867117961251, | |
| "mean_conf": 0.8880125307043394, | |
| "cov@0.5": 0.9933333333333333, | |
| "acc@0.5": 0.8489932885906041, | |
| "cov@0.7": 0.9, | |
| "acc@0.7": 0.8814814814814815, | |
| "cov@0.9": 0.64, | |
| "acc@0.9": 0.9427083333333334 | |
| }, | |
| "mrpc": { | |
| "n": 200, | |
| "acc": 0.83, | |
| "nll": 0.3825738710855869, | |
| "brier": 0.2414001594036636, | |
| "ece": 0.0455596360564232, | |
| "mean_conf": 0.8177212104201317, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.83, | |
| "cov@0.7": 0.745, | |
| "acc@0.7": 0.9060402684563759, | |
| "cov@0.9": 0.415, | |
| "acc@0.9": 0.9518072289156626 | |
| }, | |
| "openbookqa": { | |
| "n": 300, | |
| "acc": 0.8166666666666667, | |
| "nll": 0.4884890429825752, | |
| "brier": 0.26006302091594946, | |
| "ece": 0.06033645391464233, | |
| "mean_conf": 0.8382209448019663, | |
| "cov@0.5": 0.9266666666666666, | |
| "acc@0.5": 0.8525179856115108, | |
| "cov@0.7": 0.7966666666666666, | |
| "acc@0.7": 0.899581589958159, | |
| "cov@0.9": 0.52, | |
| "acc@0.9": 0.9743589743589743 | |
| }, | |
| "paws": { | |
| "n": 300, | |
| "acc": 0.93, | |
| "nll": 0.2171823905014709, | |
| "brier": 0.11854569746905995, | |
| "ece": 0.028286268909772223, | |
| "mean_conf": 0.9089529289801915, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.93, | |
| "cov@0.7": 0.9166666666666666, | |
| "acc@0.7": 0.9490909090909091, | |
| "cov@0.9": 0.7366666666666667, | |
| "acc@0.9": 0.9638009049773756 | |
| }, | |
| "qnli": { | |
| "n": 300, | |
| "acc": 0.88, | |
| "nll": 0.29461164414824, | |
| "brier": 0.1746824483593742, | |
| "ece": 0.03399742662906641, | |
| "mean_conf": 0.9018590325117111, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.88, | |
| "cov@0.7": 0.92, | |
| "acc@0.7": 0.9166666666666666, | |
| "cov@0.9": 0.6933333333333334, | |
| "acc@0.9": 0.9423076923076923 | |
| }, | |
| "read": { | |
| "n": 300, | |
| "acc": 1.0, | |
| "nll": 0.013479419595217349, | |
| "brier": 0.0006464961621803141, | |
| "ece": 0.01327262739340459, | |
| "mean_conf": 0.9867273726065954, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 1.0, | |
| "cov@0.7": 1.0, | |
| "acc@0.7": 1.0, | |
| "cov@0.9": 0.9966666666666667, | |
| "acc@0.9": 1.0 | |
| }, | |
| "rte": { | |
| "n": 200, | |
| "acc": 0.875, | |
| "nll": 0.2619901953560894, | |
| "brier": 0.1655817686780098, | |
| "ece": 0.07393659174442292, | |
| "mean_conf": 0.8983567506074905, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.875, | |
| "cov@0.7": 0.915, | |
| "acc@0.7": 0.912568306010929, | |
| "cov@0.9": 0.69, | |
| "acc@0.9": 0.9855072463768116 | |
| }, | |
| "sciq": { | |
| "n": 300, | |
| "acc": 0.9833333333333333, | |
| "nll": 0.0542967460035165, | |
| "brier": 0.02836314450778487, | |
| "ece": 0.013889081676801088, | |
| "mean_conf": 0.9804451249043147, | |
| "cov@0.5": 0.9966666666666667, | |
| "acc@0.5": 0.9866220735785953, | |
| "cov@0.7": 0.99, | |
| "acc@0.7": 0.9865319865319865, | |
| "cov@0.9": 0.95, | |
| "acc@0.9": 0.9929824561403509 | |
| }, | |
| "sst2": { | |
| "n": 300, | |
| "acc": 0.94, | |
| "nll": 0.15217635479920968, | |
| "brier": 0.08669829778686115, | |
| "ece": 0.02306454201539363, | |
| "mean_conf": 0.9453726333379745, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.94, | |
| "cov@0.7": 0.9633333333333334, | |
| "acc@0.7": 0.9584775086505191, | |
| "cov@0.9": 0.82, | |
| "acc@0.9": 0.991869918699187 | |
| }, | |
| "swag": { | |
| "n": 300, | |
| "acc": 0.7666666666666667, | |
| "nll": 0.7234309196216375, | |
| "brier": 0.3695786167661231, | |
| "ece": 0.06823263516028721, | |
| "mean_conf": 0.7794082881013552, | |
| "cov@0.5": 0.91, | |
| "acc@0.5": 0.7875457875457875, | |
| "cov@0.7": 0.69, | |
| "acc@0.7": 0.8405797101449275, | |
| "cov@0.9": 0.3333333333333333, | |
| "acc@0.9": 0.9 | |
| }, | |
| "tweet_emoji": { | |
| "n": 300, | |
| "acc": 0.24333333333333335, | |
| "nll": 2.5063694445877984, | |
| "brier": 0.83829218239292, | |
| "ece": 0.08510932529966037, | |
| "mean_conf": 0.2803319871922334, | |
| "cov@0.5": 0.12, | |
| "acc@0.5": 0.8055555555555556, | |
| "cov@0.7": 0.07666666666666666, | |
| "acc@0.7": 0.9130434782608695, | |
| "cov@0.9": 0.0033333333333333335, | |
| "acc@0.9": 1.0 | |
| }, | |
| "tweet_hate": { | |
| "n": 300, | |
| "acc": 0.7166666666666667, | |
| "nll": 0.5309249597827629, | |
| "brier": 0.36006795343339115, | |
| "ece": 0.09840704739093784, | |
| "mean_conf": 0.8124431739250819, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.7166666666666667, | |
| "cov@0.7": 0.7866666666666666, | |
| "acc@0.7": 0.7966101694915254, | |
| "cov@0.9": 0.32666666666666666, | |
| "acc@0.9": 0.9285714285714286 | |
| }, | |
| "tweet_irony": { | |
| "n": 300, | |
| "acc": 0.71, | |
| "nll": 0.5534378644031082, | |
| "brier": 0.37644439644653505, | |
| "ece": 0.03676844378312428, | |
| "mean_conf": 0.7351771769920985, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.71, | |
| "cov@0.7": 0.5866666666666667, | |
| "acc@0.7": 0.8068181818181818, | |
| "cov@0.9": 0.12333333333333334, | |
| "acc@0.9": 0.972972972972973 | |
| }, | |
| "tweet_offensive": { | |
| "n": 300, | |
| "acc": 0.7866666666666666, | |
| "nll": 0.47620018437820133, | |
| "brier": 0.3128923872633037, | |
| "ece": 0.05815195361773175, | |
| "mean_conf": 0.8035398570696513, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.7866666666666666, | |
| "cov@0.7": 0.76, | |
| "acc@0.7": 0.8421052631578947, | |
| "cov@0.9": 0.3333333333333333, | |
| "acc@0.9": 0.93 | |
| }, | |
| "tweet_sentiment": { | |
| "n": 300, | |
| "acc": 0.7266666666666667, | |
| "nll": 0.6050564579947201, | |
| "brier": 0.36215817406899686, | |
| "ece": 0.07843278278907143, | |
| "mean_conf": 0.7284325797359149, | |
| "score_mae": 0.3644068883561219, | |
| "cov@0.5": 0.96, | |
| "acc@0.5": 0.7291666666666666, | |
| "cov@0.7": 0.5333333333333333, | |
| "acc@0.7": 0.875, | |
| "cov@0.9": 0.16666666666666666, | |
| "acc@0.9": 0.92 | |
| }, | |
| "yahoo": { | |
| "n": 300, | |
| "acc": 0.7166666666666667, | |
| "nll": 0.8738976731967112, | |
| "brier": 0.3950192633049044, | |
| "ece": 0.092816769828399, | |
| "mean_conf": 0.8022240548829238, | |
| "cov@0.5": 0.9133333333333333, | |
| "acc@0.5": 0.7627737226277372, | |
| "cov@0.7": 0.7466666666666667, | |
| "acc@0.7": 0.8348214285714286, | |
| "cov@0.9": 0.4166666666666667, | |
| "acc@0.9": 0.92 | |
| }, | |
| "yelp": { | |
| "n": 300, | |
| "acc": 0.71, | |
| "nll": 0.731893961859198, | |
| "brier": 0.42142868253511284, | |
| "ece": 0.09561944127082822, | |
| "mean_conf": 0.7473884936173757, | |
| "score_mae": 0.3743417001541093, | |
| "cov@0.5": 0.9533333333333334, | |
| "acc@0.5": 0.7237762237762237, | |
| "cov@0.7": 0.6266666666666667, | |
| "acc@0.7": 0.7712765957446809, | |
| "cov@0.9": 0.19, | |
| "acc@0.9": 0.9649122807017544 | |
| }, | |
| "anli": { | |
| "n": 300, | |
| "acc": 0.5366666666666666, | |
| "nll": 1.1646257930212216, | |
| "brier": 0.675594648318733, | |
| "ece": 0.223523634771506, | |
| "mean_conf": 0.7592115387320518, | |
| "cov@0.5": 0.9433333333333334, | |
| "acc@0.5": 0.5547703180212014, | |
| "cov@0.7": 0.65, | |
| "acc@0.7": 0.558974358974359, | |
| "cov@0.9": 0.23, | |
| "acc@0.9": 0.5507246376811594 | |
| }, | |
| "winogrande": { | |
| "n": 300, | |
| "acc": 0.7966666666666666, | |
| "nll": 0.46445836318766226, | |
| "brier": 0.3013142673360519, | |
| "ece": 0.07452347179253899, | |
| "mean_conf": 0.8619708905617396, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.7966666666666666, | |
| "cov@0.7": 0.87, | |
| "acc@0.7": 0.8275862068965517, | |
| "cov@0.9": 0.55, | |
| "acc@0.9": 0.9030303030303031 | |
| }, | |
| "hellaswag": { | |
| "n": 300, | |
| "acc": 0.8566666666666667, | |
| "nll": 0.3762581950318828, | |
| "brier": 0.19513976820616458, | |
| "ece": 0.05684188375870386, | |
| "mean_conf": 0.8465098922451337, | |
| "cov@0.5": 0.9366666666666666, | |
| "acc@0.5": 0.9074733096085409, | |
| "cov@0.7": 0.7833333333333333, | |
| "acc@0.7": 0.9446808510638298, | |
| "cov@0.9": 0.5533333333333333, | |
| "acc@0.9": 0.9819277108433735 | |
| }, | |
| "race": { | |
| "n": 300, | |
| "acc": 0.7933333333333333, | |
| "nll": 0.5311142158904583, | |
| "brier": 0.2739426106933522, | |
| "ece": 0.0815848172704379, | |
| "mean_conf": 0.8720341417193412, | |
| "cov@0.5": 0.93, | |
| "acc@0.5": 0.8422939068100358, | |
| "cov@0.7": 0.8466666666666667, | |
| "acc@0.7": 0.8818897637795275, | |
| "cov@0.9": 0.6566666666666666, | |
| "acc@0.9": 0.934010152284264 | |
| }, | |
| "scitail": { | |
| "n": 300, | |
| "acc": 0.97, | |
| "nll": 0.1026227491797548, | |
| "brier": 0.05202258655692825, | |
| "ece": 0.028236998518308055, | |
| "mean_conf": 0.950906420747439, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.97, | |
| "cov@0.7": 0.9833333333333333, | |
| "acc@0.7": 0.9728813559322034, | |
| "cov@0.9": 0.8466666666666667, | |
| "acc@0.9": 0.9921259842519685 | |
| }, | |
| "qqp": { | |
| "n": 300, | |
| "acc": 0.88, | |
| "nll": 0.2769447782152307, | |
| "brier": 0.17137568112639565, | |
| "ece": 0.045283984939257296, | |
| "mean_conf": 0.8734472642342249, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.88, | |
| "cov@0.7": 0.8533333333333334, | |
| "acc@0.7": 0.93359375, | |
| "cov@0.9": 0.5766666666666667, | |
| "acc@0.9": 0.9710982658959537 | |
| }, | |
| "stsb": { | |
| "n": 287, | |
| "acc": 0.6167247386759582, | |
| "nll": 1.0364299467274243, | |
| "brier": 0.2999134000942182, | |
| "ece": 0.07439108956150894, | |
| "mean_conf": 0.5469927265461314, | |
| "score_mae": 0.5417481492766754, | |
| "cov@0.5": 0.6306620209059234, | |
| "acc@0.5": 0.6850828729281768, | |
| "cov@0.7": 0.09407665505226481, | |
| "acc@0.7": 0.8518518518518519, | |
| "cov@0.9": 0.0, | |
| "acc@0.9": NaN | |
| }, | |
| "toxic": { | |
| "n": 300, | |
| "acc": 0.84, | |
| "nll": 0.34017511751074586, | |
| "brier": 0.21106674918538404, | |
| "ece": 0.04502722958723705, | |
| "mean_conf": 0.8586645072698593, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.84, | |
| "cov@0.7": 0.88, | |
| "acc@0.7": 0.8977272727272727, | |
| "cov@0.9": 0.52, | |
| "acc@0.9": 0.9615384615384616 | |
| }, | |
| "stance_abortion": { | |
| "n": 66, | |
| "acc": 0.8636363636363636, | |
| "nll": 0.44344493585724276, | |
| "brier": 0.23815874340787538, | |
| "ece": 0.16279316354881634, | |
| "mean_conf": 0.7459786872972142, | |
| "cov@0.5": 0.8636363636363636, | |
| "acc@0.5": 0.8947368421052632, | |
| "cov@0.7": 0.5909090909090909, | |
| "acc@0.7": 0.9230769230769231, | |
| "cov@0.9": 0.22727272727272727, | |
| "acc@0.9": 1.0 | |
| }, | |
| "stance_atheism": { | |
| "n": 52, | |
| "acc": 0.7115384615384616, | |
| "nll": 0.6112422593374254, | |
| "brier": 0.37005177211571516, | |
| "ece": 0.12039663585332726, | |
| "mean_conf": 0.7864993226069671, | |
| "cov@0.5": 0.9423076923076923, | |
| "acc@0.5": 0.7346938775510204, | |
| "cov@0.7": 0.6538461538461539, | |
| "acc@0.7": 0.8235294117647058, | |
| "cov@0.9": 0.3269230769230769, | |
| "acc@0.9": 1.0 | |
| }, | |
| "stance_feminist": { | |
| "n": 67, | |
| "acc": 0.7313432835820896, | |
| "nll": 0.7303124558726009, | |
| "brier": 0.4342147904610734, | |
| "ece": 0.1691147155726134, | |
| "mean_conf": 0.7381730840277316, | |
| "cov@0.5": 0.9402985074626866, | |
| "acc@0.5": 0.746031746031746, | |
| "cov@0.7": 0.5522388059701493, | |
| "acc@0.7": 0.7027027027027027, | |
| "cov@0.9": 0.19402985074626866, | |
| "acc@0.9": 0.8461538461538461 | |
| }, | |
| "stance_hillary": { | |
| "n": 69, | |
| "acc": 0.7246376811594203, | |
| "nll": 0.6620274924672167, | |
| "brier": 0.3935280964850548, | |
| "ece": 0.09731849552928537, | |
| "mean_conf": 0.7686952758526456, | |
| "cov@0.5": 0.9565217391304348, | |
| "acc@0.5": 0.7424242424242424, | |
| "cov@0.7": 0.6666666666666666, | |
| "acc@0.7": 0.8043478260869565, | |
| "cov@0.9": 0.18840579710144928, | |
| "acc@0.9": 0.7692307692307693 | |
| }, | |
| "match": { | |
| "n": 300, | |
| "acc": 0.99, | |
| "nll": 0.03724694484885282, | |
| "brier": 0.016051036461267973, | |
| "ece": 0.009268070856730138, | |
| "mean_conf": 0.9877331558863321, | |
| "cov@0.5": 0.9966666666666667, | |
| "acc@0.5": 0.9933110367892977, | |
| "cov@0.7": 0.9966666666666667, | |
| "acc@0.7": 0.9933110367892977, | |
| "cov@0.9": 0.99, | |
| "acc@0.9": 0.9932659932659933 | |
| }, | |
| "reason": { | |
| "n": 300, | |
| "acc": 0.9466666666666667, | |
| "nll": 0.17577242359452994, | |
| "brier": 0.09125921721706085, | |
| "ece": 0.04738917231559757, | |
| "mean_conf": 0.9024230941136678, | |
| "cov@0.5": 0.9833333333333333, | |
| "acc@0.5": 0.9559322033898305, | |
| "cov@0.7": 0.88, | |
| "acc@0.7": 0.9772727272727273, | |
| "cov@0.9": 0.7733333333333333, | |
| "acc@0.9": 0.9913793103448276 | |
| }, | |
| "formality": { | |
| "n": 300, | |
| "acc": 0.6066666666666667, | |
| "nll": 0.9962800608084179, | |
| "brier": 0.25308033830347887, | |
| "ece": 0.07329747378826143, | |
| "mean_conf": 0.5359433833758036, | |
| "score_mae": 0.47120167226336584, | |
| "cov@0.5": 0.6866666666666666, | |
| "acc@0.5": 0.6699029126213593, | |
| "cov@0.7": 0.0033333333333333335, | |
| "acc@0.7": 1.0, | |
| "cov@0.9": 0.0, | |
| "acc@0.9": NaN | |
| }, | |
| "politeness": { | |
| "n": 300, | |
| "acc": 0.8633333333333333, | |
| "nll": 0.3753622173094621, | |
| "brier": 0.18929635618156296, | |
| "ece": 0.026931450863679252, | |
| "mean_conf": 0.8615820496280988, | |
| "score_mae": 0.21370556724568207, | |
| "cov@0.5": 0.9633333333333334, | |
| "acc@0.5": 0.8858131487889274, | |
| "cov@0.7": 0.8266666666666667, | |
| "acc@0.7": 0.9435483870967742, | |
| "cov@0.9": 0.5766666666666667, | |
| "acc@0.9": 0.9826589595375722 | |
| }, | |
| "strategyqa": { | |
| "n": 200, | |
| "acc": 0.67, | |
| "nll": 0.5862248309774405, | |
| "brier": 0.40554963519042464, | |
| "ece": 0.05940887540578843, | |
| "mean_conf": 0.6801840284466744, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.67, | |
| "cov@0.7": 0.385, | |
| "acc@0.7": 0.8051948051948052, | |
| "cov@0.9": 0.07, | |
| "acc@0.9": 1.0 | |
| }, | |
| "vitaminc": { | |
| "n": 300, | |
| "acc": 0.82, | |
| "nll": 0.5214132864032636, | |
| "brier": 0.2848927857191196, | |
| "ece": 0.03640172024567924, | |
| "mean_conf": 0.8344497634967168, | |
| "cov@0.5": 0.9666666666666667, | |
| "acc@0.5": 0.8275862068965517, | |
| "cov@0.7": 0.83, | |
| "acc@0.7": 0.8634538152610441, | |
| "cov@0.9": 0.4533333333333333, | |
| "acc@0.9": 0.9338235294117647 | |
| }, | |
| "ruletaker": { | |
| "n": 300, | |
| "acc": 0.8066666666666666, | |
| "nll": 0.3848169850830573, | |
| "brier": 0.25008683944182986, | |
| "ece": 0.030118134220441215, | |
| "mean_conf": 0.812346151471138, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.8066666666666666, | |
| "cov@0.7": 0.7133333333333334, | |
| "acc@0.7": 0.9018691588785047, | |
| "cov@0.9": 0.43666666666666665, | |
| "acc@0.9": 0.9618320610687023 | |
| }, | |
| "proofwriter": { | |
| "n": 300, | |
| "acc": 0.7933333333333333, | |
| "nll": 0.4490651794863921, | |
| "brier": 0.2722325912094693, | |
| "ece": 0.06963174422581991, | |
| "mean_conf": 0.8091626433531444, | |
| "cov@0.5": 0.98, | |
| "acc@0.5": 0.7993197278911565, | |
| "cov@0.7": 0.7633333333333333, | |
| "acc@0.7": 0.8777292576419214, | |
| "cov@0.9": 0.37333333333333335, | |
| "acc@0.9": 0.9910714285714286 | |
| }, | |
| "folio": { | |
| "n": 200, | |
| "acc": 0.625, | |
| "nll": 0.7944451580787752, | |
| "brier": 0.4657608423671937, | |
| "ece": 0.11621890529990193, | |
| "mean_conf": 0.6586561058461666, | |
| "cov@0.5": 0.835, | |
| "acc@0.5": 0.688622754491018, | |
| "cov@0.7": 0.375, | |
| "acc@0.7": 0.8533333333333334, | |
| "cov@0.9": 0.06, | |
| "acc@0.9": 0.75 | |
| }, | |
| "logiqa": { | |
| "n": 300, | |
| "acc": 0.5833333333333334, | |
| "nll": 0.6570202462503513, | |
| "brier": 0.46499107764281516, | |
| "ece": 0.061852243741353355, | |
| "mean_conf": 0.6316624116897583, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.5833333333333334, | |
| "cov@0.7": 0.23666666666666666, | |
| "acc@0.7": 0.7323943661971831, | |
| "cov@0.9": 0.013333333333333334, | |
| "acc@0.9": 0.75 | |
| }, | |
| "tracie": { | |
| "n": 200, | |
| "acc": 0.77, | |
| "nll": 0.5158300579084877, | |
| "brier": 0.3394760687221474, | |
| "ece": 0.07212192535400391, | |
| "mean_conf": 0.70532983481884, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.77, | |
| "cov@0.7": 0.49, | |
| "acc@0.7": 0.8673469387755102, | |
| "cov@0.9": 0.03, | |
| "acc@0.9": 1.0 | |
| }, | |
| "temporal_nli": { | |
| "n": 300, | |
| "acc": 0.8633333333333333, | |
| "nll": 0.3496475835064699, | |
| "brier": 0.20055396242126278, | |
| "ece": 0.07610381027062735, | |
| "mean_conf": 0.7972539271910986, | |
| "cov@0.5": 0.9866666666666667, | |
| "acc@0.5": 0.8614864864864865, | |
| "cov@0.7": 0.7666666666666667, | |
| "acc@0.7": 0.9217391304347826, | |
| "cov@0.9": 0.26666666666666666, | |
| "acc@0.9": 1.0 | |
| }, | |
| "piqa": { | |
| "n": 300, | |
| "acc": 0.8166666666666667, | |
| "nll": 0.36417624160320555, | |
| "brier": 0.23331890879921213, | |
| "ece": 0.038980930248896296, | |
| "mean_conf": 0.8269510519504547, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.8166666666666667, | |
| "cov@0.7": 0.7633333333333333, | |
| "acc@0.7": 0.9126637554585153, | |
| "cov@0.9": 0.43666666666666665, | |
| "acc@0.9": 0.9618320610687023 | |
| }, | |
| "siqa": { | |
| "n": 300, | |
| "acc": 0.8233333333333334, | |
| "nll": 0.4337533873205736, | |
| "brier": 0.24543190225066103, | |
| "ece": 0.034700063069661474, | |
| "mean_conf": 0.8447816316286723, | |
| "cov@0.5": 0.9633333333333334, | |
| "acc@0.5": 0.8408304498269896, | |
| "cov@0.7": 0.7833333333333333, | |
| "acc@0.7": 0.9148936170212766, | |
| "cov@0.9": 0.5533333333333333, | |
| "acc@0.9": 0.9457831325301205 | |
| }, | |
| "clutrr": { | |
| "n": 300, | |
| "acc": 0.6133333333333333, | |
| "nll": 0.9127365656335344, | |
| "brier": 0.4770832175168861, | |
| "ece": 0.04643365234136578, | |
| "mean_conf": 0.5962887792785962, | |
| "cov@0.5": 0.72, | |
| "acc@0.5": 0.6944444444444444, | |
| "cov@0.7": 0.21333333333333335, | |
| "acc@0.7": 0.9375, | |
| "cov@0.9": 0.08333333333333333, | |
| "acc@0.9": 1.0 | |
| }, | |
| "gsm8k": { | |
| "n": 300, | |
| "acc": 0.74, | |
| "nll": 0.589682709113467, | |
| "brier": 0.32688956856451207, | |
| "ece": 0.06247660587231318, | |
| "mean_conf": 0.7070875727136929, | |
| "cov@0.5": 0.7866666666666666, | |
| "acc@0.5": 0.826271186440678, | |
| "cov@0.7": 0.53, | |
| "acc@0.7": 0.9308176100628931, | |
| "cov@0.9": 0.25, | |
| "acc@0.9": 1.0 | |
| }, | |
| "svamp": { | |
| "n": 100, | |
| "acc": 0.63, | |
| "nll": 0.8085613738920417, | |
| "brier": 0.4651299440112998, | |
| "ece": 0.12156755775213245, | |
| "mean_conf": 0.601172327697277, | |
| "cov@0.5": 0.68, | |
| "acc@0.5": 0.75, | |
| "cov@0.7": 0.32, | |
| "acc@0.7": 0.875, | |
| "cov@0.9": 0.11, | |
| "acc@0.9": 1.0 | |
| }, | |
| "aqua": { | |
| "n": 253, | |
| "acc": 0.383399209486166, | |
| "nll": 1.467006593256583, | |
| "brier": 0.7354768193410129, | |
| "ece": 0.05388568453637978, | |
| "mean_conf": 0.3621667535173092, | |
| "cov@0.5": 0.09486166007905138, | |
| "acc@0.5": 0.6666666666666666, | |
| "cov@0.7": 0.019762845849802372, | |
| "acc@0.7": 0.4, | |
| "cov@0.9": 0.0, | |
| "acc@0.9": NaN | |
| } | |
| }, | |
| "unseen_test_uncalibrated": { | |
| "probe": { | |
| "n": 97, | |
| "acc": 0.8969072164948454, | |
| "nll": 0.20691493444770823, | |
| "brier": 0.12394450383288905, | |
| "ece": 0.06851052070401381, | |
| "mean_conf": 0.9103295477395205, | |
| "score_mae": 0.18648642087646294, | |
| "cov@0.5": 0.9896907216494846, | |
| "acc@0.5": 0.90625, | |
| "cov@0.7": 0.9072164948453608, | |
| "acc@0.7": 0.9545454545454546, | |
| "cov@0.9": 0.7628865979381443, | |
| "acc@0.9": 0.9864864864864865, | |
| "families": { | |
| "desc": [ | |
| 13, | |
| 15 | |
| ], | |
| "negation": [ | |
| 10, | |
| 10 | |
| ], | |
| "logic": [ | |
| 14, | |
| 17 | |
| ], | |
| "score": [ | |
| 10, | |
| 12 | |
| ], | |
| "json": [ | |
| 6, | |
| 6 | |
| ], | |
| "taxonomy": [ | |
| 14, | |
| 14 | |
| ], | |
| "plausible": [ | |
| 4, | |
| 4 | |
| ], | |
| "twist": [ | |
| 3, | |
| 3 | |
| ], | |
| "time": [ | |
| 4, | |
| 5 | |
| ], | |
| "intent": [ | |
| 2, | |
| 3 | |
| ], | |
| "compare": [ | |
| 3, | |
| 4 | |
| ], | |
| "criteria": [ | |
| 4, | |
| 4 | |
| ] | |
| } | |
| }, | |
| "bbh": { | |
| "n": 1000, | |
| "acc": 0.517, | |
| "nll": 1.108523054891869, | |
| "brier": 0.6035352775259032, | |
| "ece": 0.07363909149914981, | |
| "mean_conf": 0.5724366051629186, | |
| "cov@0.5": 0.64, | |
| "acc@0.5": 0.6140625, | |
| "cov@0.7": 0.273, | |
| "acc@0.7": 0.6959706959706959, | |
| "cov@0.9": 0.071, | |
| "acc@0.9": 0.7887323943661971 | |
| }, | |
| "cola": { | |
| "n": 1000, | |
| "acc": 0.75, | |
| "nll": 0.5104252819118784, | |
| "brier": 0.33587838373719203, | |
| "ece": 0.0569284417629242, | |
| "mean_conf": 0.7190280594825744, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.75, | |
| "cov@0.7": 0.58, | |
| "acc@0.7": 0.8448275862068966, | |
| "cov@0.9": 0.013, | |
| "acc@0.9": 1.0 | |
| }, | |
| "wic": { | |
| "n": 638, | |
| "acc": 0.5909090909090909, | |
| "nll": 0.69998100116263, | |
| "brier": 0.4982001306496651, | |
| "ece": 0.09791882723850147, | |
| "mean_conf": 0.6873193471969855, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.5909090909090909, | |
| "cov@0.7": 0.44357366771159873, | |
| "acc@0.7": 0.6537102473498233, | |
| "cov@0.9": 0.0219435736677116, | |
| "acc@0.9": 0.7142857142857143 | |
| }, | |
| "subj": { | |
| "n": 1000, | |
| "acc": 0.682, | |
| "nll": 0.5917236264696343, | |
| "brier": 0.4104362382942676, | |
| "ece": 0.1212564522027969, | |
| "mean_conf": 0.7977630772590637, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.682, | |
| "cov@0.7": 0.733, | |
| "acc@0.7": 0.7517053206002728, | |
| "cov@0.9": 0.306, | |
| "acc@0.9": 0.9215686274509803 | |
| }, | |
| "spam": { | |
| "n": 1000, | |
| "acc": 0.749, | |
| "nll": 0.47590900971852373, | |
| "brier": 0.32175875624611966, | |
| "ece": 0.10708969771862033, | |
| "mean_conf": 0.8200223511457443, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.749, | |
| "cov@0.7": 0.816, | |
| "acc@0.7": 0.803921568627451, | |
| "cov@0.9": 0.353, | |
| "acc@0.9": 0.9773371104815864 | |
| }, | |
| "counterfactual": { | |
| "n": 1000, | |
| "acc": 0.82, | |
| "nll": 0.4564823726332652, | |
| "brier": 0.2849392428188435, | |
| "ece": 0.10961719477176668, | |
| "mean_conf": 0.7216410273313523, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.82, | |
| "cov@0.7": 0.617, | |
| "acc@0.7": 0.9141004862236629, | |
| "cov@0.9": 0.01, | |
| "acc@0.9": 0.8 | |
| }, | |
| "cb": { | |
| "n": 56, | |
| "acc": 0.875, | |
| "nll": 0.3821062575477204, | |
| "brier": 0.19684515831431662, | |
| "ece": 0.11659146206719534, | |
| "mean_conf": 0.8077813791377204, | |
| "cov@0.5": 0.9821428571428571, | |
| "acc@0.5": 0.8909090909090909, | |
| "cov@0.7": 0.75, | |
| "acc@0.7": 0.9761904761904762, | |
| "cov@0.9": 0.35714285714285715, | |
| "acc@0.9": 1.0 | |
| }, | |
| "arc_challenge": { | |
| "n": 1000, | |
| "acc": 0.79, | |
| "nll": 0.5784666197380147, | |
| "brier": 0.3084519084178791, | |
| "ece": 0.04053584739565846, | |
| "mean_conf": 0.8078768512308597, | |
| "cov@0.5": 0.897, | |
| "acc@0.5": 0.8249721293199554, | |
| "cov@0.7": 0.719, | |
| "acc@0.7": 0.8873435326842837, | |
| "cov@0.9": 0.46, | |
| "acc@0.9": 0.9586956521739131 | |
| }, | |
| "stance_climate": { | |
| "n": 169, | |
| "acc": 0.7100591715976331, | |
| "nll": 0.8004923563430824, | |
| "brier": 0.43626973420217074, | |
| "ece": 0.08912066789068414, | |
| "mean_conf": 0.7112431993498605, | |
| "cov@0.5": 0.863905325443787, | |
| "acc@0.5": 0.726027397260274, | |
| "cov@0.7": 0.5502958579881657, | |
| "acc@0.7": 0.8709677419354839, | |
| "cov@0.9": 0.1301775147928994, | |
| "acc@0.9": 0.9545454545454546 | |
| }, | |
| "trec": { | |
| "n": 500, | |
| "acc": 0.728, | |
| "nll": 0.7789360883597788, | |
| "brier": 0.3915759426435931, | |
| "ece": 0.055817239046096784, | |
| "mean_conf": 0.7733147183656692, | |
| "cov@0.5": 0.89, | |
| "acc@0.5": 0.7797752808988764, | |
| "cov@0.7": 0.668, | |
| "acc@0.7": 0.8323353293413174, | |
| "cov@0.9": 0.362, | |
| "acc@0.9": 0.8895027624309392 | |
| }, | |
| "sst5": { | |
| "n": 1000, | |
| "acc": 0.569, | |
| "nll": 0.9569207478808627, | |
| "brier": 0.5500160496510278, | |
| "ece": 0.051933742076158515, | |
| "mean_conf": 0.5590169258415699, | |
| "score_mae": 0.5127075865020743, | |
| "cov@0.5": 0.684, | |
| "acc@0.5": 0.6198830409356725, | |
| "cov@0.7": 0.117, | |
| "acc@0.7": 0.7094017094017094, | |
| "cov@0.9": 0.005, | |
| "acc@0.9": 0.6 | |
| }, | |
| "fin_sentiment": { | |
| "n": 1000, | |
| "acc": 0.821, | |
| "nll": 0.45971791800530354, | |
| "brier": 0.26788516663028156, | |
| "ece": 0.06331040862202642, | |
| "mean_conf": 0.7591086620986461, | |
| "cov@0.5": 0.981, | |
| "acc@0.5": 0.8287461773700305, | |
| "cov@0.7": 0.684, | |
| "acc@0.7": 0.9078947368421053, | |
| "cov@0.9": 0.139, | |
| "acc@0.9": 0.9640287769784173 | |
| }, | |
| "arc_easy": { | |
| "n": 1000, | |
| "acc": 0.893, | |
| "nll": 0.30681700111668636, | |
| "brier": 0.16114664739945947, | |
| "ece": 0.017386740416288377, | |
| "mean_conf": 0.8903764767348766, | |
| "cov@0.5": 0.959, | |
| "acc@0.5": 0.9113660062565172, | |
| "cov@0.7": 0.864, | |
| "acc@0.7": 0.9444444444444444, | |
| "cov@0.9": 0.704, | |
| "acc@0.9": 0.9758522727272727 | |
| }, | |
| "newsgroups": { | |
| "n": 1000, | |
| "acc": 0.645, | |
| "nll": 1.2438821946531702, | |
| "brier": 0.4654980387667391, | |
| "ece": 0.0510300065651536, | |
| "mean_conf": 0.6572305353507399, | |
| "cov@0.5": 0.683, | |
| "acc@0.5": 0.8272327964860908, | |
| "cov@0.7": 0.5, | |
| "acc@0.7": 0.9, | |
| "cov@0.9": 0.246, | |
| "acc@0.9": 0.9878048780487805 | |
| } | |
| }, | |
| "summary": { | |
| "mean_acc": 0.7922933763477235, | |
| "mean_ece": 0.06318428710662419 | |
| }, | |
| "history": [], | |
| "temperature": 1.0194810628890991, | |
| "temperature_by_k": { | |
| "2-2": 0.9909, | |
| "3-5": 1.0355, | |
| "6-20": 1.0193 | |
| }, | |
| "calibration": { | |
| "before": { | |
| "n": 10608, | |
| "acc": 0.7931749622926093, | |
| "nll": 0.5081595549816602, | |
| "brier": 0.26480504720767034, | |
| "ece": 0.010474447911776612, | |
| "mean_conf": 0.8014159178505982, | |
| "score_mae": 0.3837145484503708, | |
| "cov@0.5": 0.9200603318250377, | |
| "acc@0.5": 0.8326844262295082, | |
| "cov@0.7": 0.7074849170437406, | |
| "acc@0.7": 0.9077948034643571, | |
| "cov@0.9": 0.4572963800904977, | |
| "acc@0.9": 0.9647495361781077 | |
| }, | |
| "after": { | |
| "n": 10608, | |
| "acc": 0.7931749622926093, | |
| "nll": 0.5079944898473756, | |
| "brier": 0.2647027577519989, | |
| "ece": 0.009998726121221617, | |
| "mean_conf": 0.7984406564077068, | |
| "score_mae": 0.3859622411504242, | |
| "cov@0.5": 0.9168552036199095, | |
| "acc@0.5": 0.8339502364795394, | |
| "cov@0.7": 0.7034313725490197, | |
| "acc@0.7": 0.908201554543018, | |
| "cov@0.9": 0.4518288084464555, | |
| "acc@0.9": 0.9670352597538077 | |
| }, | |
| "tasks": [ | |
| "ag_news", | |
| "banking77", | |
| "boolq", | |
| "clinc150", | |
| "commonsense_qa", | |
| "copa", | |
| "dbpedia", | |
| "emotion", | |
| "facts", | |
| "fits", | |
| "imdb", | |
| "jailbreak", | |
| "massive_intent", | |
| "mnli", | |
| "mrpc", | |
| "openbookqa", | |
| "paws", | |
| "qnli", | |
| "read", | |
| "rte", | |
| "sciq", | |
| "sst2", | |
| "swag", | |
| "tweet_emoji", | |
| "tweet_hate", | |
| "tweet_irony", | |
| "tweet_offensive", | |
| "tweet_sentiment", | |
| "yahoo", | |
| "yelp", | |
| "anli", | |
| "winogrande", | |
| "hellaswag", | |
| "race", | |
| "scitail", | |
| "qqp", | |
| "stsb", | |
| "toxic", | |
| "stance_abortion", | |
| "stance_atheism", | |
| "stance_feminist", | |
| "stance_hillary", | |
| "match", | |
| "goemo_soft", | |
| "reason", | |
| "formality", | |
| "politeness", | |
| "strategyqa", | |
| "vitaminc", | |
| "ruletaker", | |
| "proofwriter", | |
| "folio", | |
| "logiqa", | |
| "tracie", | |
| "temporal_nli", | |
| "piqa", | |
| "siqa", | |
| "clutrr", | |
| "gsm8k", | |
| "svamp", | |
| "aqua" | |
| ] | |
| } | |
| }, | |
| "v0007-spam": { | |
| "version": "v0007-spam", | |
| "parent": "v0007", | |
| "parents": [ | |
| "v0007" | |
| ], | |
| "kind": "lmhead", | |
| "backbone": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "max_len": 1024, | |
| "created": "2026-09-18T06:05:30+00:00", | |
| "tasks": [ | |
| "spam" | |
| ], | |
| "trained_on": [ | |
| "spam" | |
| ], | |
| "holdout": [ | |
| "probe", | |
| "sst2", | |
| "mnli", | |
| "cola" | |
| ], | |
| "steps": 589, | |
| "train_examples": 4000, | |
| "args": { | |
| "cmd": "lmtrain", | |
| "lora_r": 8, | |
| "loss": "mix", | |
| "max_per_task": 4000, | |
| "epochs": 1, | |
| "lr": 0.0001, | |
| "anchor": 0.1 | |
| }, | |
| "metrics": { | |
| "spam": { | |
| "n": 300, | |
| "acc": 0.99, | |
| "nll": 0.04094129466820647, | |
| "brier": 0.01309215253075488, | |
| "ece": 0.031613957881927515, | |
| "mean_conf": 0.9694881041844686, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.99, | |
| "cov@0.7": 0.9933333333333333, | |
| "acc@0.7": 0.9966442953020134, | |
| "cov@0.9": 0.9666666666666667, | |
| "acc@0.9": 1.0 | |
| } | |
| }, | |
| "unseen_test_uncalibrated": { | |
| "probe": { | |
| "n": 97, | |
| "acc": 0.8969072164948454, | |
| "nll": 0.21773648728926945, | |
| "brier": 0.1273214505148998, | |
| "ece": 0.07348626329726782, | |
| "mean_conf": 0.9161194676590949, | |
| "score_mae": 0.15704563955659978, | |
| "cov@0.5": 0.9896907216494846, | |
| "acc@0.5": 0.90625, | |
| "cov@0.7": 0.8969072164948454, | |
| "acc@0.7": 0.9540229885057471, | |
| "cov@0.9": 0.7938144329896907, | |
| "acc@0.9": 0.987012987012987, | |
| "families": { | |
| "desc": [ | |
| 13, | |
| 15 | |
| ], | |
| "negation": [ | |
| 10, | |
| 10 | |
| ], | |
| "logic": [ | |
| 13, | |
| 17 | |
| ], | |
| "score": [ | |
| 11, | |
| 12 | |
| ], | |
| "json": [ | |
| 6, | |
| 6 | |
| ], | |
| "taxonomy": [ | |
| 14, | |
| 14 | |
| ], | |
| "plausible": [ | |
| 4, | |
| 4 | |
| ], | |
| "twist": [ | |
| 3, | |
| 3 | |
| ], | |
| "time": [ | |
| 4, | |
| 5 | |
| ], | |
| "intent": [ | |
| 2, | |
| 3 | |
| ], | |
| "compare": [ | |
| 3, | |
| 4 | |
| ], | |
| "criteria": [ | |
| 4, | |
| 4 | |
| ] | |
| } | |
| }, | |
| "sst2": { | |
| "n": 872, | |
| "acc": 0.9575688073394495, | |
| "nll": 0.1396011574813463, | |
| "brier": 0.07172239838956931, | |
| "ece": 0.014292597087151384, | |
| "mean_conf": 0.9680821417121712, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.9575688073394495, | |
| "cov@0.7": 0.9793577981651376, | |
| "acc@0.7": 0.9637002341920374, | |
| "cov@0.9": 0.9438073394495413, | |
| "acc@0.9": 0.9732685297691372 | |
| }, | |
| "mnli": { | |
| "n": 1000, | |
| "acc": 0.859, | |
| "nll": 0.3888388354725572, | |
| "brier": 0.21223603757591059, | |
| "ece": 0.04788112017512322, | |
| "mean_conf": 0.8856889481842518, | |
| "cov@0.5": 0.987, | |
| "acc@0.5": 0.8662613981762918, | |
| "cov@0.7": 0.909, | |
| "acc@0.7": 0.8954895489548955, | |
| "cov@0.9": 0.655, | |
| "acc@0.9": 0.9587786259541985 | |
| }, | |
| "cola": { | |
| "n": 1000, | |
| "acc": 0.759, | |
| "nll": 0.5066963657737495, | |
| "brier": 0.33443985155856215, | |
| "ece": 0.0523103475570679, | |
| "mean_conf": 0.791339822769165, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.759, | |
| "cov@0.7": 0.764, | |
| "acc@0.7": 0.8167539267015707, | |
| "cov@0.9": 0.207, | |
| "acc@0.9": 0.9516908212560387 | |
| } | |
| }, | |
| "summary": { | |
| "mean_acc": 0.99, | |
| "mean_ece": 0.031613957881927515 | |
| }, | |
| "history": [] | |
| }, | |
| "v0007-support": { | |
| "version": "v0007-support", | |
| "parent": "v0007", | |
| "parents": [ | |
| "v0007" | |
| ], | |
| "kind": "lmhead", | |
| "backbone": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "max_len": 1024, | |
| "created": "2026-09-18T12:52:37+00:00", | |
| "tasks": [ | |
| "banking77", | |
| "clinc150", | |
| "massive_intent" | |
| ], | |
| "trained_on": [ | |
| "banking77", | |
| "clinc150", | |
| "massive_intent" | |
| ], | |
| "holdout": [ | |
| "trec" | |
| ], | |
| "steps": 1067, | |
| "train_examples": 12000, | |
| "args": { | |
| "cmd": "lmtrain", | |
| "lora_r": 8, | |
| "loss": "mix", | |
| "max_per_task": 4000, | |
| "epochs": 1, | |
| "lr": 0.0001, | |
| "anchor": 0.1 | |
| }, | |
| "metrics": {}, | |
| "unseen_test_uncalibrated": { | |
| "trec": { | |
| "n": 500, | |
| "acc": 0.758, | |
| "nll": 0.767611707548029, | |
| "brier": 0.37113630112335455, | |
| "ece": 0.08773053616285323, | |
| "mean_conf": 0.8239709965586662, | |
| "cov@0.5": 0.924, | |
| "acc@0.5": 0.7835497835497836, | |
| "cov@0.7": 0.762, | |
| "acc@0.7": 0.8293963254593176, | |
| "cov@0.9": 0.474, | |
| "acc@0.9": 0.8734177215189873 | |
| } | |
| }, | |
| "summary": { | |
| "mean_acc": null, | |
| "mean_ece": null | |
| }, | |
| "history": [] | |
| }, | |
| "v0007-safety": { | |
| "version": "v0007-safety", | |
| "parent": "v0007", | |
| "parents": [ | |
| "v0007" | |
| ], | |
| "kind": "lmhead", | |
| "backbone": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "max_len": 1024, | |
| "created": "2026-09-18T13:21:25+00:00", | |
| "tasks": [ | |
| "jailbreak", | |
| "toxic", | |
| "tweet_hate", | |
| "tweet_offensive" | |
| ], | |
| "trained_on": [ | |
| "jailbreak", | |
| "toxic", | |
| "tweet_hate", | |
| "tweet_offensive" | |
| ], | |
| "holdout": [], | |
| "steps": 607, | |
| "train_examples": 8344, | |
| "args": { | |
| "cmd": "lmtrain", | |
| "lora_r": 8, | |
| "loss": "mix", | |
| "max_per_task": 2500, | |
| "epochs": 1, | |
| "lr": 0.0001, | |
| "anchor": 0.1 | |
| }, | |
| "metrics": { | |
| "jailbreak": { | |
| "n": 200, | |
| "acc": 0.98, | |
| "nll": 0.06724073947718683, | |
| "brier": 0.03142695878535079, | |
| "ece": 0.008043854534626017, | |
| "mean_conf": 0.988043854534626, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.98, | |
| "cov@0.7": 0.995, | |
| "acc@0.7": 0.9849246231155779, | |
| "cov@0.9": 0.99, | |
| "acc@0.9": 0.98989898989899 | |
| }, | |
| "toxic": { | |
| "n": 300, | |
| "acc": 0.8666666666666667, | |
| "nll": 0.3084903371258136, | |
| "brier": 0.19258679203988513, | |
| "ece": 0.03767327169577279, | |
| "mean_conf": 0.8952765788634618, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.8666666666666667, | |
| "cov@0.7": 0.8966666666666666, | |
| "acc@0.7": 0.9033457249070632, | |
| "cov@0.9": 0.6766666666666666, | |
| "acc@0.9": 0.9655172413793104 | |
| }, | |
| "tweet_hate": { | |
| "n": 300, | |
| "acc": 0.7433333333333333, | |
| "nll": 0.5153975753819442, | |
| "brier": 0.3425311312237358, | |
| "ece": 0.08356260061264038, | |
| "mean_conf": 0.8240628039836884, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.7433333333333333, | |
| "cov@0.7": 0.78, | |
| "acc@0.7": 0.811965811965812, | |
| "cov@0.9": 0.39, | |
| "acc@0.9": 0.9145299145299145 | |
| }, | |
| "tweet_offensive": { | |
| "n": 300, | |
| "acc": 0.7866666666666666, | |
| "nll": 0.4662258180163463, | |
| "brier": 0.3043113272015309, | |
| "ece": 0.053681320548057555, | |
| "mean_conf": 0.8303003575404485, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.7866666666666666, | |
| "cov@0.7": 0.7933333333333333, | |
| "acc@0.7": 0.8403361344537815, | |
| "cov@0.9": 0.4533333333333333, | |
| "acc@0.9": 0.9191176470588235 | |
| } | |
| }, | |
| "unseen_test_uncalibrated": {}, | |
| "summary": { | |
| "mean_acc": 0.8441666666666666, | |
| "mean_ece": 0.04574026184777419 | |
| }, | |
| "history": [] | |
| }, | |
| "v0007-reason": { | |
| "version": "v0007-reason", | |
| "parent": "v0007", | |
| "parents": [ | |
| "v0007" | |
| ], | |
| "kind": "lmhead", | |
| "backbone": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "max_len": 1024, | |
| "created": "2026-09-22T05:53:47+00:00", | |
| "tasks": [ | |
| "traps", | |
| "reason", | |
| "skills" | |
| ], | |
| "trained_on": [ | |
| "traps", | |
| "reason", | |
| "skills" | |
| ], | |
| "holdout": [ | |
| "probe" | |
| ], | |
| "steps": 3144, | |
| "train_examples": 49321, | |
| "args": { | |
| "cmd": "lmtrain", | |
| "lora_r": 16, | |
| "loss": "mix", | |
| "max_per_task": 22000, | |
| "epochs": 1, | |
| "lr": 0.0001, | |
| "anchor": 0.1 | |
| }, | |
| "metrics": { | |
| "traps": { | |
| "n": 300, | |
| "acc": 0.9466666666666667, | |
| "nll": 0.20538718146178303, | |
| "brier": 0.06905301894228903, | |
| "ece": 0.04965091367562613, | |
| "mean_conf": 0.92676440179348, | |
| "score_mae": 0.47862898526946085, | |
| "cov@0.5": 0.9866666666666667, | |
| "acc@0.5": 0.9493243243243243, | |
| "cov@0.7": 0.9466666666666667, | |
| "acc@0.7": 0.971830985915493, | |
| "cov@0.9": 0.8433333333333334, | |
| "acc@0.9": 0.9920948616600791 | |
| }, | |
| "reason": { | |
| "n": 300, | |
| "acc": 0.98, | |
| "nll": 0.046410952195114846, | |
| "brier": 0.025619819302402664, | |
| "ece": 0.025865860184033737, | |
| "mean_conf": 0.9718742754062016, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.98, | |
| "cov@0.7": 0.97, | |
| "acc@0.7": 0.9896907216494846, | |
| "cov@0.9": 0.95, | |
| "acc@0.9": 1.0 | |
| }, | |
| "skills": { | |
| "n": 300, | |
| "acc": 0.9033333333333333, | |
| "nll": 0.20629042731885439, | |
| "brier": 0.11973303077943658, | |
| "ece": 0.061399669448534644, | |
| "mean_conf": 0.8790679361422856, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.9033333333333333, | |
| "cov@0.7": 0.83, | |
| "acc@0.7": 0.9759036144578314, | |
| "cov@0.9": 0.7166666666666667, | |
| "acc@0.9": 1.0 | |
| } | |
| }, | |
| "unseen_test_uncalibrated": { | |
| "probe": { | |
| "n": 97, | |
| "acc": 0.9381443298969072, | |
| "nll": 0.1893354459761264, | |
| "brier": 0.09341768317288839, | |
| "ece": 0.04105151006855912, | |
| "mean_conf": 0.9446589375279614, | |
| "score_mae": 0.18023038718577786, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.9381443298969072, | |
| "cov@0.7": 0.9381443298969072, | |
| "acc@0.7": 0.967032967032967, | |
| "cov@0.9": 0.865979381443299, | |
| "acc@0.9": 0.9761904761904762, | |
| "families": { | |
| "desc": [ | |
| 14, | |
| 15 | |
| ], | |
| "negation": [ | |
| 10, | |
| 10 | |
| ], | |
| "logic": [ | |
| 15, | |
| 17 | |
| ], | |
| "score": [ | |
| 11, | |
| 12 | |
| ], | |
| "json": [ | |
| 6, | |
| 6 | |
| ], | |
| "taxonomy": [ | |
| 14, | |
| 14 | |
| ], | |
| "plausible": [ | |
| 4, | |
| 4 | |
| ], | |
| "twist": [ | |
| 3, | |
| 3 | |
| ], | |
| "time": [ | |
| 4, | |
| 5 | |
| ], | |
| "intent": [ | |
| 2, | |
| 3 | |
| ], | |
| "compare": [ | |
| 4, | |
| 4 | |
| ], | |
| "criteria": [ | |
| 4, | |
| 4 | |
| ] | |
| } | |
| } | |
| }, | |
| "summary": { | |
| "mean_acc": 0.9433333333333334, | |
| "mean_ece": 0.04563881443606484 | |
| }, | |
| "history": [] | |
| }, | |
| "v0007-language": { | |
| "version": "v0007-language", | |
| "parent": "v0007", | |
| "parents": [ | |
| "v0007" | |
| ], | |
| "kind": "lmhead", | |
| "backbone": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "max_len": 1024, | |
| "created": "2026-09-22T05:59:59+00:00", | |
| "tasks": [ | |
| "language" | |
| ], | |
| "trained_on": [ | |
| "language" | |
| ], | |
| "holdout": [], | |
| "steps": 625, | |
| "train_examples": 10000, | |
| "args": { | |
| "cmd": "lmtrain", | |
| "lora_r": 8, | |
| "loss": "mix", | |
| "max_per_task": 10000, | |
| "epochs": 1, | |
| "lr": 0.0001, | |
| "anchor": 0.1 | |
| }, | |
| "metrics": { | |
| "language": { | |
| "n": 300, | |
| "acc": 1.0, | |
| "nll": 0.11204298576288962, | |
| "brier": 0.0022332283333394826, | |
| "ece": 0.05060336291790009, | |
| "mean_conf": 0.9493966370820999, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 1.0, | |
| "cov@0.7": 1.0, | |
| "acc@0.7": 1.0, | |
| "cov@0.9": 0.99, | |
| "acc@0.9": 1.0 | |
| } | |
| }, | |
| "unseen_test_uncalibrated": {}, | |
| "summary": { | |
| "mean_acc": 1.0, | |
| "mean_ece": 0.05060336291790009 | |
| }, | |
| "history": [] | |
| }, | |
| "v0007-sentiment": { | |
| "version": "v0007-sentiment", | |
| "parent": "v0007", | |
| "parents": [ | |
| "v0007" | |
| ], | |
| "kind": "lmhead", | |
| "backbone": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "max_len": 1024, | |
| "created": "2026-09-22T06:27:25+00:00", | |
| "tasks": [ | |
| "sst2", | |
| "sst5", | |
| "yelp", | |
| "tweet_sentiment", | |
| "emotion", | |
| "goemo_soft", | |
| "tweet_irony", | |
| "imdb", | |
| "formality", | |
| "politeness", | |
| "sarcasm" | |
| ], | |
| "trained_on": [ | |
| "sst2", | |
| "sst5", | |
| "yelp", | |
| "tweet_sentiment", | |
| "emotion", | |
| "goemo_soft", | |
| "tweet_irony", | |
| "imdb", | |
| "formality", | |
| "politeness", | |
| "sarcasm" | |
| ], | |
| "holdout": [ | |
| "fin_sentiment", | |
| "counterfactual" | |
| ], | |
| "steps": 1782, | |
| "train_examples": 22000, | |
| "args": { | |
| "cmd": "lmtrain", | |
| "lora_r": 8, | |
| "loss": "mix", | |
| "max_per_task": 2000, | |
| "epochs": 1, | |
| "lr": 0.0001, | |
| "anchor": 0.1 | |
| }, | |
| "metrics": { | |
| "sst2": { | |
| "n": 300, | |
| "acc": 0.9466666666666667, | |
| "nll": 0.1605975828277054, | |
| "brier": 0.0888536530799026, | |
| "ece": 0.027998056411743223, | |
| "mean_conf": 0.9642687650521596, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.9466666666666667, | |
| "cov@0.7": 0.9766666666666667, | |
| "acc@0.7": 0.9556313993174061, | |
| "cov@0.9": 0.9066666666666666, | |
| "acc@0.9": 0.9632352941176471 | |
| }, | |
| "sst5": { | |
| "n": 300, | |
| "acc": 0.5766666666666667, | |
| "nll": 1.030410210134918, | |
| "brier": 0.5736773828010426, | |
| "ece": 0.05870265493790308, | |
| "mean_conf": 0.5829293202360472, | |
| "score_mae": 0.5457962888351175, | |
| "cov@0.5": 0.79, | |
| "acc@0.5": 0.6075949367088608, | |
| "cov@0.7": 0.11666666666666667, | |
| "acc@0.7": 0.7142857142857143, | |
| "cov@0.9": 0.0, | |
| "acc@0.9": NaN | |
| }, | |
| "yelp": { | |
| "n": 300, | |
| "acc": 0.69, | |
| "nll": 0.7307452479655708, | |
| "brier": 0.4235077727191108, | |
| "ece": 0.08372766455014549, | |
| "mean_conf": 0.7693662059307098, | |
| "score_mae": 0.36723209824798686, | |
| "cov@0.5": 0.9533333333333334, | |
| "acc@0.5": 0.7027972027972028, | |
| "cov@0.7": 0.69, | |
| "acc@0.7": 0.782608695652174, | |
| "cov@0.9": 0.21333333333333335, | |
| "acc@0.9": 0.921875 | |
| }, | |
| "tweet_sentiment": { | |
| "n": 300, | |
| "acc": 0.7233333333333334, | |
| "nll": 0.6324473977408053, | |
| "brier": 0.37319421960896293, | |
| "ece": 0.061867911020914726, | |
| "mean_conf": 0.7788027099768321, | |
| "score_mae": 0.3486348866733412, | |
| "cov@0.5": 0.99, | |
| "acc@0.5": 0.7239057239057239, | |
| "cov@0.7": 0.69, | |
| "acc@0.7": 0.8115942028985508, | |
| "cov@0.9": 0.2733333333333333, | |
| "acc@0.9": 0.9024390243902439 | |
| }, | |
| "emotion": { | |
| "n": 300, | |
| "acc": 0.78, | |
| "nll": 0.6305815624709215, | |
| "brier": 0.3066913792466958, | |
| "ece": 0.07204169511795044, | |
| "mean_conf": 0.8520416951179505, | |
| "cov@0.5": 0.96, | |
| "acc@0.5": 0.7951388888888888, | |
| "cov@0.7": 0.8166666666666667, | |
| "acc@0.7": 0.8612244897959184, | |
| "cov@0.9": 0.55, | |
| "acc@0.9": 0.9575757575757575 | |
| }, | |
| "tweet_irony": { | |
| "n": 300, | |
| "acc": 0.7366666666666667, | |
| "nll": 0.5502091578371892, | |
| "brier": 0.3718147638320287, | |
| "ece": 0.08492387334505717, | |
| "mean_conf": 0.7882708374659221, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.7366666666666667, | |
| "cov@0.7": 0.74, | |
| "acc@0.7": 0.7837837837837838, | |
| "cov@0.9": 0.23333333333333334, | |
| "acc@0.9": 0.9142857142857143 | |
| }, | |
| "imdb": { | |
| "n": 300, | |
| "acc": 0.9633333333333334, | |
| "nll": 0.13351794521006682, | |
| "brier": 0.060667604680717205, | |
| "ece": 0.025028558770815598, | |
| "mean_conf": 0.9825289577245713, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.9633333333333334, | |
| "cov@0.7": 0.99, | |
| "acc@0.7": 0.9696969696969697, | |
| "cov@0.9": 0.9633333333333334, | |
| "acc@0.9": 0.9792387543252595 | |
| }, | |
| "formality": { | |
| "n": 300, | |
| "acc": 0.6133333333333333, | |
| "nll": 0.9954126704919857, | |
| "brier": 0.25669049533824123, | |
| "ece": 0.061814626157283774, | |
| "mean_conf": 0.5946429490049681, | |
| "score_mae": 0.4740935998460433, | |
| "cov@0.5": 0.89, | |
| "acc@0.5": 0.6179775280898876, | |
| "cov@0.7": 0.06666666666666667, | |
| "acc@0.7": 0.75, | |
| "cov@0.9": 0.0, | |
| "acc@0.9": NaN | |
| }, | |
| "politeness": { | |
| "n": 300, | |
| "acc": 0.8566666666666667, | |
| "nll": 0.3775658668930503, | |
| "brier": 0.19322453000823264, | |
| "ece": 0.03797433485587436, | |
| "mean_conf": 0.8931960561871528, | |
| "score_mae": 0.2005545631381392, | |
| "cov@0.5": 0.9766666666666667, | |
| "acc@0.5": 0.8703071672354948, | |
| "cov@0.7": 0.87, | |
| "acc@0.7": 0.9233716475095786, | |
| "cov@0.9": 0.6633333333333333, | |
| "acc@0.9": 0.9748743718592965 | |
| }, | |
| "sarcasm": { | |
| "n": 300, | |
| "acc": 0.8766666666666667, | |
| "nll": 0.45201081925126335, | |
| "brier": 0.10946457478491955, | |
| "ece": 0.08731486479441326, | |
| "mean_conf": 0.8211214067538579, | |
| "score_mae": 0.4119935893134519, | |
| "cov@0.5": 0.94, | |
| "acc@0.5": 0.8971631205673759, | |
| "cov@0.7": 0.7633333333333333, | |
| "acc@0.7": 0.9606986899563319, | |
| "cov@0.9": 0.49, | |
| "acc@0.9": 0.9931972789115646 | |
| } | |
| }, | |
| "unseen_test_uncalibrated": { | |
| "fin_sentiment": { | |
| "n": 1000, | |
| "acc": 0.807, | |
| "nll": 0.4482609479897583, | |
| "brier": 0.2712111023947865, | |
| "ece": 0.025465373486280456, | |
| "mean_conf": 0.8211387673318387, | |
| "cov@0.5": 0.992, | |
| "acc@0.5": 0.8104838709677419, | |
| "cov@0.7": 0.799, | |
| "acc@0.7": 0.8685857321652065, | |
| "cov@0.9": 0.369, | |
| "acc@0.9": 0.9376693766937669 | |
| }, | |
| "counterfactual": { | |
| "n": 1000, | |
| "acc": 0.855, | |
| "nll": 0.3709632075470155, | |
| "brier": 0.2216712187729024, | |
| "ece": 0.07320722198486329, | |
| "mean_conf": 0.7820079625844956, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.855, | |
| "cov@0.7": 0.748, | |
| "acc@0.7": 0.9251336898395722, | |
| "cov@0.9": 0.135, | |
| "acc@0.9": 0.9703703703703703 | |
| } | |
| }, | |
| "summary": { | |
| "mean_acc": 0.7763333333333333, | |
| "mean_ece": 0.06013942399621011 | |
| }, | |
| "history": [] | |
| }, | |
| "v0007-typed": { | |
| "version": "v0007-typed", | |
| "parent": "v0007", | |
| "parents": [ | |
| "v0007" | |
| ], | |
| "kind": "lmhead", | |
| "backbone": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "max_len": 1024, | |
| "created": "2026-09-22T11:39:50+00:00", | |
| "tasks": [ | |
| "typed_decisions" | |
| ], | |
| "trained_on": [ | |
| "typed_decisions" | |
| ], | |
| "holdout": [], | |
| "steps": 1352, | |
| "train_examples": 5400, | |
| "args": { | |
| "cmd": "lmtrain", | |
| "lora_r": 16, | |
| "loss": "mix", | |
| "max_per_task": 6000, | |
| "epochs": 2, | |
| "lr": 0.0001, | |
| "anchor": 0.1 | |
| }, | |
| "metrics": { | |
| "typed_decisions": { | |
| "n": 300, | |
| "acc": 0.85, | |
| "nll": 0.7897254475627804, | |
| "brier": 0.04430931889174507, | |
| "ece": 0.21966197381416955, | |
| "mean_conf": 0.6374534544348717, | |
| "score_mae": 0.34401526501434937, | |
| "cov@0.5": 0.76, | |
| "acc@0.5": 0.9342105263157895, | |
| "cov@0.7": 0.30666666666666664, | |
| "acc@0.7": 0.9891304347826086, | |
| "cov@0.9": 0.13666666666666666, | |
| "acc@0.9": 1.0 | |
| } | |
| }, | |
| "unseen_test_uncalibrated": {}, | |
| "summary": { | |
| "mean_acc": 0.85, | |
| "mean_ece": 0.21966197381416955 | |
| }, | |
| "history": [] | |
| }, | |
| "v0007-math": { | |
| "version": "v0007-math", | |
| "parent": "v0007", | |
| "parents": [ | |
| "v0007" | |
| ], | |
| "kind": "lmhead", | |
| "backbone": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "max_len": 1024, | |
| "created": "2026-09-22T13:55:28+00:00", | |
| "tasks": [ | |
| "math", | |
| "gsm8k", | |
| "svamp", | |
| "aqua" | |
| ], | |
| "trained_on": [ | |
| "math", | |
| "gsm8k", | |
| "svamp", | |
| "aqua" | |
| ], | |
| "holdout": [], | |
| "steps": 3684, | |
| "train_examples": 57771, | |
| "args": { | |
| "cmd": "lmtrain", | |
| "lora_r": 32, | |
| "loss": "mix", | |
| "max_per_task": 30000, | |
| "epochs": 1, | |
| "lr": 0.0001, | |
| "anchor": 0.1 | |
| }, | |
| "metrics": { | |
| "math": { | |
| "n": 300, | |
| "acc": 0.8733333333333333, | |
| "nll": 0.2491237256136509, | |
| "brier": 0.15517219901605317, | |
| "ece": 0.04117123444875081, | |
| "mean_conf": 0.8545713822046915, | |
| "cov@0.5": 1.0, | |
| "acc@0.5": 0.8733333333333333, | |
| "cov@0.7": 0.7533333333333333, | |
| "acc@0.7": 0.9690265486725663, | |
| "cov@0.9": 0.7066666666666667, | |
| "acc@0.9": 1.0 | |
| }, | |
| "gsm8k": { | |
| "n": 300, | |
| "acc": 0.8166666666666667, | |
| "nll": 0.4634518215859158, | |
| "brier": 0.24914535577158609, | |
| "ece": 0.08174108674128854, | |
| "mean_conf": 0.7950625959038734, | |
| "cov@0.5": 0.9, | |
| "acc@0.5": 0.8666666666666667, | |
| "cov@0.7": 0.7, | |
| "acc@0.7": 0.9238095238095239, | |
| "cov@0.9": 0.4266666666666667, | |
| "acc@0.9": 0.9921875 | |
| }, | |
| "svamp": { | |
| "n": 100, | |
| "acc": 0.68, | |
| "nll": 0.7193946922798429, | |
| "brier": 0.41086004277348953, | |
| "ece": 0.07071352988481525, | |
| "mean_conf": 0.7261623141169548, | |
| "cov@0.5": 0.83, | |
| "acc@0.5": 0.7349397590361446, | |
| "cov@0.7": 0.58, | |
| "acc@0.7": 0.8275862068965517, | |
| "cov@0.9": 0.2, | |
| "acc@0.9": 1.0 | |
| }, | |
| "aqua": { | |
| "n": 253, | |
| "acc": 0.391304347826087, | |
| "nll": 1.414271918680051, | |
| "brier": 0.7115495715758425, | |
| "ece": 0.0654898358899143, | |
| "mean_conf": 0.3998582398467384, | |
| "cov@0.5": 0.20553359683794467, | |
| "acc@0.5": 0.6538461538461539, | |
| "cov@0.7": 0.06324110671936758, | |
| "acc@0.7": 0.8125, | |
| "cov@0.9": 0.007905138339920948, | |
| "acc@0.9": 1.0 | |
| } | |
| }, | |
| "unseen_test_uncalibrated": {}, | |
| "summary": { | |
| "mean_acc": 0.6903260869565218, | |
| "mean_ece": 0.06477892174119222 | |
| }, | |
| "history": [] | |
| } | |
| } | |
| } |