{ "latest": "v0007", "versions": { "v0007": { "version": "v0007", "parent": null, "kind": "lmhead", "backbone": "Qwen/Qwen2.5-1.5B-Instruct", "max_len": 1024, "created": "2026-09-17T15:58:07+00:00", "tasks": [ "ag_news", "banking77", "boolq", "clinc150", "commonsense_qa", "copa", "dbpedia", "emotion", "facts", "fits", "imdb", "jailbreak", "massive_intent", "mnli", "mrpc", "openbookqa", "paws", "qnli", "read", "rte", "sciq", "sst2", "swag", "tweet_emoji", "tweet_hate", "tweet_irony", "tweet_offensive", "tweet_sentiment", "yahoo", "yelp", "anli", "winogrande", "hellaswag", "race", "scitail", "qqp", "stsb", "toxic", "stance_abortion", "stance_atheism", "stance_feminist", "stance_hillary", "match", "goemo_soft", "reason", "formality", "politeness", "strategyqa", "vitaminc", "ruletaker", "proofwriter", "folio", "logiqa", "tracie", "temporal_nli", "piqa", "siqa", "clutrr", "gsm8k", "svamp", "aqua" ], "trained_on": [ "ag_news", "banking77", "boolq", "clinc150", "commonsense_qa", "copa", "dbpedia", "emotion", "facts", "fits", "imdb", "jailbreak", "massive_intent", "mnli", "mrpc", "openbookqa", "paws", "qnli", "read", "rte", "sciq", "sst2", "swag", "tweet_emoji", "tweet_hate", "tweet_irony", "tweet_offensive", "tweet_sentiment", "yahoo", "yelp", "anli", "winogrande", "hellaswag", "race", "scitail", "qqp", "stsb", "toxic", "stance_abortion", "stance_atheism", "stance_feminist", "stance_hillary", "match", "goemo_soft", "reason", "formality", "politeness", "strategyqa", "vitaminc", "ruletaker", "proofwriter", "folio", "logiqa", "tracie", "temporal_nli", "piqa", "siqa", "clutrr", "gsm8k", "svamp", "aqua" ], "holdout": [ "probe", "bbh", "cola", "wic", "subj", "spam", "counterfactual", "cb", "arc_challenge", "stance_climate", "trec", "sst5", "fin_sentiment", "arc_easy", "newsgroups" ], "steps": 12118, "train_examples": 158246, "args": { "cmd": "lmtrain", "lora_r": 16, "loss": "mix", "max_per_task": 3000, "epochs": 1, "lr": 0.0001, "anchor": 0.1 }, "metrics": { "ag_news": { "n": 300, "acc": 0.9233333333333333, "nll": 0.23956205062784391, "brier": 0.12588630912802798, "ece": 0.03565445333719247, "mean_conf": 0.9105748584866524, "cov@0.5": 0.9833333333333333, "acc@0.5": 0.9288135593220339, "cov@0.7": 0.88, "acc@0.7": 0.9545454545454546, "cov@0.9": 0.77, "acc@0.9": 0.9653679653679653 }, "boolq": { "n": 300, "acc": 0.8766666666666667, "nll": 0.33278360864483053, "brier": 0.19905847383131203, "ece": 0.048838321963946066, "mean_conf": 0.8610335459311803, "cov@0.5": 1.0, "acc@0.5": 0.8766666666666667, "cov@0.7": 0.8866666666666667, "acc@0.7": 0.9060150375939849, "cov@0.9": 0.4666666666666667, "acc@0.9": 0.9642857142857143 }, "commonsense_qa": { "n": 300, "acc": 0.7866666666666666, "nll": 0.5812667453454432, "brier": 0.2960317921716136, "ece": 0.05657371540864312, "mean_conf": 0.7944276158014933, "cov@0.5": 0.8933333333333333, "acc@0.5": 0.832089552238806, "cov@0.7": 0.7, "acc@0.7": 0.9047619047619048, "cov@0.9": 0.42333333333333334, "acc@0.9": 0.952755905511811 }, "copa": { "n": 200, "acc": 0.93, "nll": 0.17489093500636219, "brier": 0.10265215895968784, "ece": 0.03722041517496112, "mean_conf": 0.8977384361624717, "cov@0.5": 1.0, "acc@0.5": 0.93, "cov@0.7": 0.9, "acc@0.7": 0.9611111111111111, "cov@0.9": 0.69, "acc@0.9": 1.0 }, "dbpedia": { "n": 300, "acc": 0.97, "nll": 0.12029726431718998, "brier": 0.0467073057529959, "ece": 0.011912721196810366, "mean_conf": 0.9797261367241542, "cov@0.5": 1.0, "acc@0.5": 0.97, "cov@0.7": 0.9866666666666667, "acc@0.7": 0.9797297297297297, "cov@0.9": 0.9766666666666667, "acc@0.9": 0.9829351535836177 }, "emotion": { "n": 300, "acc": 0.7766666666666666, "nll": 0.6417595775959655, "brier": 0.31279347659514983, "ece": 0.07161758591731388, "mean_conf": 0.8046423467993736, "cov@0.5": 0.92, "acc@0.5": 0.8043478260869565, "cov@0.7": 0.7233333333333334, "acc@0.7": 0.9032258064516129, "cov@0.9": 0.4266666666666667, "acc@0.9": 0.9609375 }, "facts": { "n": 54, "acc": 0.9629629629629629, "nll": 0.10233185567308967, "brier": 0.0580309563229181, "ece": 0.03901768503365692, "mean_conf": 0.985795874286581, "cov@0.5": 1.0, "acc@0.5": 0.9629629629629629, "cov@0.7": 1.0, "acc@0.7": 0.9629629629629629, "cov@0.9": 0.9444444444444444, "acc@0.9": 1.0 }, "fits": { "n": 214, "acc": 0.8598130841121495, "nll": 0.4789074549575723, "brier": 0.09971697154134483, "ece": 0.08341867177285883, "mean_conf": 0.818978947456752, "cov@0.5": 1.0, "acc@0.5": 0.8598130841121495, "cov@0.7": 0.8037383177570093, "acc@0.7": 0.936046511627907, "cov@0.9": 0.3317757009345794, "acc@0.9": 1.0 }, "imdb": { "n": 300, "acc": 0.95, "nll": 0.1449115194526972, "brier": 0.07315853850730186, "ece": 0.028431474765141816, "mean_conf": 0.9752282838026682, "cov@0.5": 1.0, "acc@0.5": 0.95, "cov@0.7": 0.9833333333333333, "acc@0.7": 0.9627118644067797, "cov@0.9": 0.95, "acc@0.9": 0.9754385964912281 }, "jailbreak": { "n": 200, "acc": 0.98, "nll": 0.07703215111456595, "brier": 0.03552832502947578, "ece": 0.014975480735301977, "mean_conf": 0.972049820125103, "cov@0.5": 1.0, "acc@0.5": 0.98, "cov@0.7": 0.995, "acc@0.7": 0.9798994974874372, "cov@0.9": 0.965, "acc@0.9": 0.9896373056994818 }, "mnli": { "n": 300, "acc": 0.8466666666666667, "nll": 0.44322894987837685, "brier": 0.24040505319501904, "ece": 0.05213867117961251, "mean_conf": 0.8880125307043394, "cov@0.5": 0.9933333333333333, "acc@0.5": 0.8489932885906041, "cov@0.7": 0.9, "acc@0.7": 0.8814814814814815, "cov@0.9": 0.64, "acc@0.9": 0.9427083333333334 }, "mrpc": { "n": 200, "acc": 0.83, "nll": 0.3825738710855869, "brier": 0.2414001594036636, "ece": 0.0455596360564232, "mean_conf": 0.8177212104201317, "cov@0.5": 1.0, "acc@0.5": 0.83, "cov@0.7": 0.745, "acc@0.7": 0.9060402684563759, "cov@0.9": 0.415, "acc@0.9": 0.9518072289156626 }, "openbookqa": { "n": 300, "acc": 0.8166666666666667, "nll": 0.4884890429825752, "brier": 0.26006302091594946, "ece": 0.06033645391464233, "mean_conf": 0.8382209448019663, "cov@0.5": 0.9266666666666666, "acc@0.5": 0.8525179856115108, "cov@0.7": 0.7966666666666666, "acc@0.7": 0.899581589958159, "cov@0.9": 0.52, "acc@0.9": 0.9743589743589743 }, "paws": { "n": 300, "acc": 0.93, "nll": 0.2171823905014709, "brier": 0.11854569746905995, "ece": 0.028286268909772223, "mean_conf": 0.9089529289801915, "cov@0.5": 1.0, "acc@0.5": 0.93, "cov@0.7": 0.9166666666666666, "acc@0.7": 0.9490909090909091, "cov@0.9": 0.7366666666666667, "acc@0.9": 0.9638009049773756 }, "qnli": { "n": 300, "acc": 0.88, "nll": 0.29461164414824, "brier": 0.1746824483593742, "ece": 0.03399742662906641, "mean_conf": 0.9018590325117111, "cov@0.5": 1.0, "acc@0.5": 0.88, "cov@0.7": 0.92, "acc@0.7": 0.9166666666666666, "cov@0.9": 0.6933333333333334, "acc@0.9": 0.9423076923076923 }, "read": { "n": 300, "acc": 1.0, "nll": 0.013479419595217349, "brier": 0.0006464961621803141, "ece": 0.01327262739340459, "mean_conf": 0.9867273726065954, "cov@0.5": 1.0, "acc@0.5": 1.0, "cov@0.7": 1.0, "acc@0.7": 1.0, "cov@0.9": 0.9966666666666667, "acc@0.9": 1.0 }, "rte": { "n": 200, "acc": 0.875, "nll": 0.2619901953560894, "brier": 0.1655817686780098, "ece": 0.07393659174442292, "mean_conf": 0.8983567506074905, "cov@0.5": 1.0, "acc@0.5": 0.875, "cov@0.7": 0.915, "acc@0.7": 0.912568306010929, "cov@0.9": 0.69, "acc@0.9": 0.9855072463768116 }, "sciq": { "n": 300, "acc": 0.9833333333333333, "nll": 0.0542967460035165, "brier": 0.02836314450778487, "ece": 0.013889081676801088, "mean_conf": 0.9804451249043147, "cov@0.5": 0.9966666666666667, "acc@0.5": 0.9866220735785953, "cov@0.7": 0.99, "acc@0.7": 0.9865319865319865, "cov@0.9": 0.95, "acc@0.9": 0.9929824561403509 }, "sst2": { "n": 300, "acc": 0.94, "nll": 0.15217635479920968, "brier": 0.08669829778686115, "ece": 0.02306454201539363, "mean_conf": 0.9453726333379745, "cov@0.5": 1.0, "acc@0.5": 0.94, "cov@0.7": 0.9633333333333334, "acc@0.7": 0.9584775086505191, "cov@0.9": 0.82, "acc@0.9": 0.991869918699187 }, "swag": { "n": 300, "acc": 0.7666666666666667, "nll": 0.7234309196216375, "brier": 0.3695786167661231, "ece": 0.06823263516028721, "mean_conf": 0.7794082881013552, "cov@0.5": 0.91, "acc@0.5": 0.7875457875457875, "cov@0.7": 0.69, "acc@0.7": 0.8405797101449275, "cov@0.9": 0.3333333333333333, "acc@0.9": 0.9 }, "tweet_emoji": { "n": 300, "acc": 0.24333333333333335, "nll": 2.5063694445877984, "brier": 0.83829218239292, "ece": 0.08510932529966037, "mean_conf": 0.2803319871922334, "cov@0.5": 0.12, "acc@0.5": 0.8055555555555556, "cov@0.7": 0.07666666666666666, "acc@0.7": 0.9130434782608695, "cov@0.9": 0.0033333333333333335, "acc@0.9": 1.0 }, "tweet_hate": { "n": 300, "acc": 0.7166666666666667, "nll": 0.5309249597827629, "brier": 0.36006795343339115, "ece": 0.09840704739093784, "mean_conf": 0.8124431739250819, "cov@0.5": 1.0, "acc@0.5": 0.7166666666666667, "cov@0.7": 0.7866666666666666, "acc@0.7": 0.7966101694915254, "cov@0.9": 0.32666666666666666, "acc@0.9": 0.9285714285714286 }, "tweet_irony": { "n": 300, "acc": 0.71, "nll": 0.5534378644031082, "brier": 0.37644439644653505, "ece": 0.03676844378312428, "mean_conf": 0.7351771769920985, "cov@0.5": 1.0, "acc@0.5": 0.71, "cov@0.7": 0.5866666666666667, "acc@0.7": 0.8068181818181818, "cov@0.9": 0.12333333333333334, "acc@0.9": 0.972972972972973 }, "tweet_offensive": { "n": 300, "acc": 0.7866666666666666, "nll": 0.47620018437820133, "brier": 0.3128923872633037, "ece": 0.05815195361773175, "mean_conf": 0.8035398570696513, "cov@0.5": 1.0, "acc@0.5": 0.7866666666666666, "cov@0.7": 0.76, "acc@0.7": 0.8421052631578947, "cov@0.9": 0.3333333333333333, "acc@0.9": 0.93 }, "tweet_sentiment": { "n": 300, "acc": 0.7266666666666667, "nll": 0.6050564579947201, "brier": 0.36215817406899686, "ece": 0.07843278278907143, "mean_conf": 0.7284325797359149, "score_mae": 0.3644068883561219, "cov@0.5": 0.96, "acc@0.5": 0.7291666666666666, "cov@0.7": 0.5333333333333333, "acc@0.7": 0.875, "cov@0.9": 0.16666666666666666, "acc@0.9": 0.92 }, "yahoo": { "n": 300, "acc": 0.7166666666666667, "nll": 0.8738976731967112, "brier": 0.3950192633049044, "ece": 0.092816769828399, "mean_conf": 0.8022240548829238, "cov@0.5": 0.9133333333333333, "acc@0.5": 0.7627737226277372, "cov@0.7": 0.7466666666666667, "acc@0.7": 0.8348214285714286, "cov@0.9": 0.4166666666666667, "acc@0.9": 0.92 }, "yelp": { "n": 300, "acc": 0.71, "nll": 0.731893961859198, "brier": 0.42142868253511284, "ece": 0.09561944127082822, "mean_conf": 0.7473884936173757, "score_mae": 0.3743417001541093, "cov@0.5": 0.9533333333333334, "acc@0.5": 0.7237762237762237, "cov@0.7": 0.6266666666666667, "acc@0.7": 0.7712765957446809, "cov@0.9": 0.19, "acc@0.9": 0.9649122807017544 }, "anli": { "n": 300, "acc": 0.5366666666666666, "nll": 1.1646257930212216, "brier": 0.675594648318733, "ece": 0.223523634771506, "mean_conf": 0.7592115387320518, "cov@0.5": 0.9433333333333334, "acc@0.5": 0.5547703180212014, "cov@0.7": 0.65, "acc@0.7": 0.558974358974359, "cov@0.9": 0.23, "acc@0.9": 0.5507246376811594 }, "winogrande": { "n": 300, "acc": 0.7966666666666666, "nll": 0.46445836318766226, "brier": 0.3013142673360519, "ece": 0.07452347179253899, "mean_conf": 0.8619708905617396, "cov@0.5": 1.0, "acc@0.5": 0.7966666666666666, "cov@0.7": 0.87, "acc@0.7": 0.8275862068965517, "cov@0.9": 0.55, "acc@0.9": 0.9030303030303031 }, "hellaswag": { "n": 300, "acc": 0.8566666666666667, "nll": 0.3762581950318828, "brier": 0.19513976820616458, "ece": 0.05684188375870386, "mean_conf": 0.8465098922451337, "cov@0.5": 0.9366666666666666, "acc@0.5": 0.9074733096085409, "cov@0.7": 0.7833333333333333, "acc@0.7": 0.9446808510638298, "cov@0.9": 0.5533333333333333, "acc@0.9": 0.9819277108433735 }, "race": { "n": 300, "acc": 0.7933333333333333, "nll": 0.5311142158904583, "brier": 0.2739426106933522, "ece": 0.0815848172704379, "mean_conf": 0.8720341417193412, "cov@0.5": 0.93, "acc@0.5": 0.8422939068100358, "cov@0.7": 0.8466666666666667, "acc@0.7": 0.8818897637795275, "cov@0.9": 0.6566666666666666, "acc@0.9": 0.934010152284264 }, "scitail": { "n": 300, "acc": 0.97, "nll": 0.1026227491797548, "brier": 0.05202258655692825, "ece": 0.028236998518308055, "mean_conf": 0.950906420747439, "cov@0.5": 1.0, "acc@0.5": 0.97, "cov@0.7": 0.9833333333333333, "acc@0.7": 0.9728813559322034, "cov@0.9": 0.8466666666666667, "acc@0.9": 0.9921259842519685 }, "qqp": { "n": 300, "acc": 0.88, "nll": 0.2769447782152307, "brier": 0.17137568112639565, "ece": 0.045283984939257296, "mean_conf": 0.8734472642342249, "cov@0.5": 1.0, "acc@0.5": 0.88, "cov@0.7": 0.8533333333333334, "acc@0.7": 0.93359375, "cov@0.9": 0.5766666666666667, "acc@0.9": 0.9710982658959537 }, "stsb": { "n": 287, "acc": 0.6167247386759582, "nll": 1.0364299467274243, "brier": 0.2999134000942182, "ece": 0.07439108956150894, "mean_conf": 0.5469927265461314, "score_mae": 0.5417481492766754, "cov@0.5": 0.6306620209059234, "acc@0.5": 0.6850828729281768, "cov@0.7": 0.09407665505226481, "acc@0.7": 0.8518518518518519, "cov@0.9": 0.0, "acc@0.9": NaN }, "toxic": { "n": 300, "acc": 0.84, "nll": 0.34017511751074586, "brier": 0.21106674918538404, "ece": 0.04502722958723705, "mean_conf": 0.8586645072698593, "cov@0.5": 1.0, "acc@0.5": 0.84, "cov@0.7": 0.88, "acc@0.7": 0.8977272727272727, "cov@0.9": 0.52, "acc@0.9": 0.9615384615384616 }, "stance_abortion": { "n": 66, "acc": 0.8636363636363636, "nll": 0.44344493585724276, "brier": 0.23815874340787538, "ece": 0.16279316354881634, "mean_conf": 0.7459786872972142, "cov@0.5": 0.8636363636363636, "acc@0.5": 0.8947368421052632, "cov@0.7": 0.5909090909090909, "acc@0.7": 0.9230769230769231, "cov@0.9": 0.22727272727272727, "acc@0.9": 1.0 }, "stance_atheism": { "n": 52, "acc": 0.7115384615384616, "nll": 0.6112422593374254, "brier": 0.37005177211571516, "ece": 0.12039663585332726, "mean_conf": 0.7864993226069671, "cov@0.5": 0.9423076923076923, "acc@0.5": 0.7346938775510204, "cov@0.7": 0.6538461538461539, "acc@0.7": 0.8235294117647058, "cov@0.9": 0.3269230769230769, "acc@0.9": 1.0 }, "stance_feminist": { "n": 67, "acc": 0.7313432835820896, "nll": 0.7303124558726009, "brier": 0.4342147904610734, "ece": 0.1691147155726134, "mean_conf": 0.7381730840277316, "cov@0.5": 0.9402985074626866, "acc@0.5": 0.746031746031746, "cov@0.7": 0.5522388059701493, "acc@0.7": 0.7027027027027027, "cov@0.9": 0.19402985074626866, "acc@0.9": 0.8461538461538461 }, "stance_hillary": { "n": 69, "acc": 0.7246376811594203, "nll": 0.6620274924672167, "brier": 0.3935280964850548, "ece": 0.09731849552928537, "mean_conf": 0.7686952758526456, "cov@0.5": 0.9565217391304348, "acc@0.5": 0.7424242424242424, "cov@0.7": 0.6666666666666666, "acc@0.7": 0.8043478260869565, "cov@0.9": 0.18840579710144928, "acc@0.9": 0.7692307692307693 }, "match": { "n": 300, "acc": 0.99, "nll": 0.03724694484885282, "brier": 0.016051036461267973, "ece": 0.009268070856730138, "mean_conf": 0.9877331558863321, "cov@0.5": 0.9966666666666667, "acc@0.5": 0.9933110367892977, "cov@0.7": 0.9966666666666667, "acc@0.7": 0.9933110367892977, "cov@0.9": 0.99, "acc@0.9": 0.9932659932659933 }, "reason": { "n": 300, "acc": 0.9466666666666667, "nll": 0.17577242359452994, "brier": 0.09125921721706085, "ece": 0.04738917231559757, "mean_conf": 0.9024230941136678, "cov@0.5": 0.9833333333333333, "acc@0.5": 0.9559322033898305, "cov@0.7": 0.88, "acc@0.7": 0.9772727272727273, "cov@0.9": 0.7733333333333333, "acc@0.9": 0.9913793103448276 }, "formality": { "n": 300, "acc": 0.6066666666666667, "nll": 0.9962800608084179, "brier": 0.25308033830347887, "ece": 0.07329747378826143, "mean_conf": 0.5359433833758036, "score_mae": 0.47120167226336584, "cov@0.5": 0.6866666666666666, "acc@0.5": 0.6699029126213593, "cov@0.7": 0.0033333333333333335, "acc@0.7": 1.0, "cov@0.9": 0.0, "acc@0.9": NaN }, "politeness": { "n": 300, "acc": 0.8633333333333333, "nll": 0.3753622173094621, "brier": 0.18929635618156296, "ece": 0.026931450863679252, "mean_conf": 0.8615820496280988, "score_mae": 0.21370556724568207, "cov@0.5": 0.9633333333333334, "acc@0.5": 0.8858131487889274, "cov@0.7": 0.8266666666666667, "acc@0.7": 0.9435483870967742, "cov@0.9": 0.5766666666666667, "acc@0.9": 0.9826589595375722 }, "strategyqa": { "n": 200, "acc": 0.67, "nll": 0.5862248309774405, "brier": 0.40554963519042464, "ece": 0.05940887540578843, "mean_conf": 0.6801840284466744, "cov@0.5": 1.0, "acc@0.5": 0.67, "cov@0.7": 0.385, "acc@0.7": 0.8051948051948052, "cov@0.9": 0.07, "acc@0.9": 1.0 }, "vitaminc": { "n": 300, "acc": 0.82, "nll": 0.5214132864032636, "brier": 0.2848927857191196, "ece": 0.03640172024567924, "mean_conf": 0.8344497634967168, "cov@0.5": 0.9666666666666667, "acc@0.5": 0.8275862068965517, "cov@0.7": 0.83, "acc@0.7": 0.8634538152610441, "cov@0.9": 0.4533333333333333, "acc@0.9": 0.9338235294117647 }, "ruletaker": { "n": 300, "acc": 0.8066666666666666, "nll": 0.3848169850830573, "brier": 0.25008683944182986, "ece": 0.030118134220441215, "mean_conf": 0.812346151471138, "cov@0.5": 1.0, "acc@0.5": 0.8066666666666666, "cov@0.7": 0.7133333333333334, "acc@0.7": 0.9018691588785047, "cov@0.9": 0.43666666666666665, "acc@0.9": 0.9618320610687023 }, "proofwriter": { "n": 300, "acc": 0.7933333333333333, "nll": 0.4490651794863921, "brier": 0.2722325912094693, "ece": 0.06963174422581991, "mean_conf": 0.8091626433531444, "cov@0.5": 0.98, "acc@0.5": 0.7993197278911565, "cov@0.7": 0.7633333333333333, "acc@0.7": 0.8777292576419214, "cov@0.9": 0.37333333333333335, "acc@0.9": 0.9910714285714286 }, "folio": { "n": 200, "acc": 0.625, "nll": 0.7944451580787752, "brier": 0.4657608423671937, "ece": 0.11621890529990193, "mean_conf": 0.6586561058461666, "cov@0.5": 0.835, "acc@0.5": 0.688622754491018, "cov@0.7": 0.375, "acc@0.7": 0.8533333333333334, "cov@0.9": 0.06, "acc@0.9": 0.75 }, "logiqa": { "n": 300, "acc": 0.5833333333333334, "nll": 0.6570202462503513, "brier": 0.46499107764281516, "ece": 0.061852243741353355, "mean_conf": 0.6316624116897583, "cov@0.5": 1.0, "acc@0.5": 0.5833333333333334, "cov@0.7": 0.23666666666666666, "acc@0.7": 0.7323943661971831, "cov@0.9": 0.013333333333333334, "acc@0.9": 0.75 }, "tracie": { "n": 200, "acc": 0.77, "nll": 0.5158300579084877, "brier": 0.3394760687221474, "ece": 0.07212192535400391, "mean_conf": 0.70532983481884, "cov@0.5": 1.0, "acc@0.5": 0.77, "cov@0.7": 0.49, "acc@0.7": 0.8673469387755102, "cov@0.9": 0.03, "acc@0.9": 1.0 }, "temporal_nli": { "n": 300, "acc": 0.8633333333333333, "nll": 0.3496475835064699, "brier": 0.20055396242126278, "ece": 0.07610381027062735, "mean_conf": 0.7972539271910986, "cov@0.5": 0.9866666666666667, "acc@0.5": 0.8614864864864865, "cov@0.7": 0.7666666666666667, "acc@0.7": 0.9217391304347826, "cov@0.9": 0.26666666666666666, "acc@0.9": 1.0 }, "piqa": { "n": 300, "acc": 0.8166666666666667, "nll": 0.36417624160320555, "brier": 0.23331890879921213, "ece": 0.038980930248896296, "mean_conf": 0.8269510519504547, "cov@0.5": 1.0, "acc@0.5": 0.8166666666666667, "cov@0.7": 0.7633333333333333, "acc@0.7": 0.9126637554585153, "cov@0.9": 0.43666666666666665, "acc@0.9": 0.9618320610687023 }, "siqa": { "n": 300, "acc": 0.8233333333333334, "nll": 0.4337533873205736, "brier": 0.24543190225066103, "ece": 0.034700063069661474, "mean_conf": 0.8447816316286723, "cov@0.5": 0.9633333333333334, "acc@0.5": 0.8408304498269896, "cov@0.7": 0.7833333333333333, "acc@0.7": 0.9148936170212766, "cov@0.9": 0.5533333333333333, "acc@0.9": 0.9457831325301205 }, "clutrr": { "n": 300, "acc": 0.6133333333333333, "nll": 0.9127365656335344, "brier": 0.4770832175168861, "ece": 0.04643365234136578, "mean_conf": 0.5962887792785962, "cov@0.5": 0.72, "acc@0.5": 0.6944444444444444, "cov@0.7": 0.21333333333333335, "acc@0.7": 0.9375, "cov@0.9": 0.08333333333333333, "acc@0.9": 1.0 }, "gsm8k": { "n": 300, "acc": 0.74, "nll": 0.589682709113467, "brier": 0.32688956856451207, "ece": 0.06247660587231318, "mean_conf": 0.7070875727136929, "cov@0.5": 0.7866666666666666, "acc@0.5": 0.826271186440678, "cov@0.7": 0.53, "acc@0.7": 0.9308176100628931, "cov@0.9": 0.25, "acc@0.9": 1.0 }, "svamp": { "n": 100, "acc": 0.63, "nll": 0.8085613738920417, "brier": 0.4651299440112998, "ece": 0.12156755775213245, "mean_conf": 0.601172327697277, "cov@0.5": 0.68, "acc@0.5": 0.75, "cov@0.7": 0.32, "acc@0.7": 0.875, "cov@0.9": 0.11, "acc@0.9": 1.0 }, "aqua": { "n": 253, "acc": 0.383399209486166, "nll": 1.467006593256583, "brier": 0.7354768193410129, "ece": 0.05388568453637978, "mean_conf": 0.3621667535173092, "cov@0.5": 0.09486166007905138, "acc@0.5": 0.6666666666666666, "cov@0.7": 0.019762845849802372, "acc@0.7": 0.4, "cov@0.9": 0.0, "acc@0.9": NaN } }, "unseen_test_uncalibrated": { "probe": { "n": 97, "acc": 0.8969072164948454, "nll": 0.20691493444770823, "brier": 0.12394450383288905, "ece": 0.06851052070401381, "mean_conf": 0.9103295477395205, "score_mae": 0.18648642087646294, "cov@0.5": 0.9896907216494846, "acc@0.5": 0.90625, "cov@0.7": 0.9072164948453608, "acc@0.7": 0.9545454545454546, "cov@0.9": 0.7628865979381443, "acc@0.9": 0.9864864864864865, "families": { "desc": [ 13, 15 ], "negation": [ 10, 10 ], "logic": [ 14, 17 ], "score": [ 10, 12 ], "json": [ 6, 6 ], "taxonomy": [ 14, 14 ], "plausible": [ 4, 4 ], "twist": [ 3, 3 ], "time": [ 4, 5 ], "intent": [ 2, 3 ], "compare": [ 3, 4 ], "criteria": [ 4, 4 ] } }, "bbh": { "n": 1000, "acc": 0.517, "nll": 1.108523054891869, "brier": 0.6035352775259032, "ece": 0.07363909149914981, "mean_conf": 0.5724366051629186, "cov@0.5": 0.64, "acc@0.5": 0.6140625, "cov@0.7": 0.273, "acc@0.7": 0.6959706959706959, "cov@0.9": 0.071, "acc@0.9": 0.7887323943661971 }, "cola": { "n": 1000, "acc": 0.75, "nll": 0.5104252819118784, "brier": 0.33587838373719203, "ece": 0.0569284417629242, "mean_conf": 0.7190280594825744, "cov@0.5": 1.0, "acc@0.5": 0.75, "cov@0.7": 0.58, "acc@0.7": 0.8448275862068966, "cov@0.9": 0.013, "acc@0.9": 1.0 }, "wic": { "n": 638, "acc": 0.5909090909090909, "nll": 0.69998100116263, "brier": 0.4982001306496651, "ece": 0.09791882723850147, "mean_conf": 0.6873193471969855, "cov@0.5": 1.0, "acc@0.5": 0.5909090909090909, "cov@0.7": 0.44357366771159873, "acc@0.7": 0.6537102473498233, "cov@0.9": 0.0219435736677116, "acc@0.9": 0.7142857142857143 }, "subj": { "n": 1000, "acc": 0.682, "nll": 0.5917236264696343, "brier": 0.4104362382942676, "ece": 0.1212564522027969, "mean_conf": 0.7977630772590637, "cov@0.5": 1.0, "acc@0.5": 0.682, "cov@0.7": 0.733, "acc@0.7": 0.7517053206002728, "cov@0.9": 0.306, "acc@0.9": 0.9215686274509803 }, "spam": { "n": 1000, "acc": 0.749, "nll": 0.47590900971852373, "brier": 0.32175875624611966, "ece": 0.10708969771862033, "mean_conf": 0.8200223511457443, "cov@0.5": 1.0, "acc@0.5": 0.749, "cov@0.7": 0.816, "acc@0.7": 0.803921568627451, "cov@0.9": 0.353, "acc@0.9": 0.9773371104815864 }, "counterfactual": { "n": 1000, "acc": 0.82, "nll": 0.4564823726332652, "brier": 0.2849392428188435, "ece": 0.10961719477176668, "mean_conf": 0.7216410273313523, "cov@0.5": 1.0, "acc@0.5": 0.82, "cov@0.7": 0.617, "acc@0.7": 0.9141004862236629, "cov@0.9": 0.01, "acc@0.9": 0.8 }, "cb": { "n": 56, "acc": 0.875, "nll": 0.3821062575477204, "brier": 0.19684515831431662, "ece": 0.11659146206719534, "mean_conf": 0.8077813791377204, "cov@0.5": 0.9821428571428571, "acc@0.5": 0.8909090909090909, "cov@0.7": 0.75, "acc@0.7": 0.9761904761904762, "cov@0.9": 0.35714285714285715, "acc@0.9": 1.0 }, "arc_challenge": { "n": 1000, "acc": 0.79, "nll": 0.5784666197380147, "brier": 0.3084519084178791, "ece": 0.04053584739565846, "mean_conf": 0.8078768512308597, "cov@0.5": 0.897, "acc@0.5": 0.8249721293199554, "cov@0.7": 0.719, "acc@0.7": 0.8873435326842837, "cov@0.9": 0.46, "acc@0.9": 0.9586956521739131 }, "stance_climate": { "n": 169, "acc": 0.7100591715976331, "nll": 0.8004923563430824, "brier": 0.43626973420217074, "ece": 0.08912066789068414, "mean_conf": 0.7112431993498605, "cov@0.5": 0.863905325443787, "acc@0.5": 0.726027397260274, "cov@0.7": 0.5502958579881657, "acc@0.7": 0.8709677419354839, "cov@0.9": 0.1301775147928994, "acc@0.9": 0.9545454545454546 }, "trec": { "n": 500, "acc": 0.728, "nll": 0.7789360883597788, "brier": 0.3915759426435931, "ece": 0.055817239046096784, "mean_conf": 0.7733147183656692, "cov@0.5": 0.89, "acc@0.5": 0.7797752808988764, "cov@0.7": 0.668, "acc@0.7": 0.8323353293413174, "cov@0.9": 0.362, "acc@0.9": 0.8895027624309392 }, "sst5": { "n": 1000, "acc": 0.569, "nll": 0.9569207478808627, "brier": 0.5500160496510278, "ece": 0.051933742076158515, "mean_conf": 0.5590169258415699, "score_mae": 0.5127075865020743, "cov@0.5": 0.684, "acc@0.5": 0.6198830409356725, "cov@0.7": 0.117, "acc@0.7": 0.7094017094017094, "cov@0.9": 0.005, "acc@0.9": 0.6 }, "fin_sentiment": { "n": 1000, "acc": 0.821, "nll": 0.45971791800530354, "brier": 0.26788516663028156, "ece": 0.06331040862202642, "mean_conf": 0.7591086620986461, "cov@0.5": 0.981, "acc@0.5": 0.8287461773700305, "cov@0.7": 0.684, "acc@0.7": 0.9078947368421053, "cov@0.9": 0.139, "acc@0.9": 0.9640287769784173 }, "arc_easy": { "n": 1000, "acc": 0.893, "nll": 0.30681700111668636, "brier": 0.16114664739945947, "ece": 0.017386740416288377, "mean_conf": 0.8903764767348766, "cov@0.5": 0.959, "acc@0.5": 0.9113660062565172, "cov@0.7": 0.864, "acc@0.7": 0.9444444444444444, "cov@0.9": 0.704, "acc@0.9": 0.9758522727272727 }, "newsgroups": { "n": 1000, "acc": 0.645, "nll": 1.2438821946531702, "brier": 0.4654980387667391, "ece": 0.0510300065651536, "mean_conf": 0.6572305353507399, "cov@0.5": 0.683, "acc@0.5": 0.8272327964860908, "cov@0.7": 0.5, "acc@0.7": 0.9, "cov@0.9": 0.246, "acc@0.9": 0.9878048780487805 } }, "summary": { "mean_acc": 0.7922933763477235, "mean_ece": 0.06318428710662419 }, "history": [], "temperature": 1.0194810628890991, "temperature_by_k": { "2-2": 0.9909, "3-5": 1.0355, "6-20": 1.0193 }, "calibration": { "before": { "n": 10608, "acc": 0.7931749622926093, "nll": 0.5081595549816602, "brier": 0.26480504720767034, "ece": 0.010474447911776612, "mean_conf": 0.8014159178505982, "score_mae": 0.3837145484503708, "cov@0.5": 0.9200603318250377, "acc@0.5": 0.8326844262295082, "cov@0.7": 0.7074849170437406, "acc@0.7": 0.9077948034643571, "cov@0.9": 0.4572963800904977, "acc@0.9": 0.9647495361781077 }, "after": { "n": 10608, "acc": 0.7931749622926093, "nll": 0.5079944898473756, "brier": 0.2647027577519989, "ece": 0.009998726121221617, "mean_conf": 0.7984406564077068, "score_mae": 0.3859622411504242, "cov@0.5": 0.9168552036199095, "acc@0.5": 0.8339502364795394, "cov@0.7": 0.7034313725490197, "acc@0.7": 0.908201554543018, "cov@0.9": 0.4518288084464555, "acc@0.9": 0.9670352597538077 }, "tasks": [ "ag_news", "banking77", "boolq", "clinc150", "commonsense_qa", "copa", "dbpedia", "emotion", "facts", "fits", "imdb", "jailbreak", "massive_intent", "mnli", "mrpc", "openbookqa", "paws", "qnli", "read", "rte", "sciq", "sst2", "swag", "tweet_emoji", "tweet_hate", "tweet_irony", "tweet_offensive", "tweet_sentiment", "yahoo", "yelp", "anli", "winogrande", "hellaswag", "race", "scitail", "qqp", "stsb", "toxic", "stance_abortion", "stance_atheism", "stance_feminist", "stance_hillary", "match", "goemo_soft", "reason", "formality", "politeness", "strategyqa", "vitaminc", "ruletaker", "proofwriter", "folio", "logiqa", "tracie", "temporal_nli", "piqa", "siqa", "clutrr", "gsm8k", "svamp", "aqua" ] } }, "v0007-spam": { "version": "v0007-spam", "parent": "v0007", "parents": [ "v0007" ], "kind": "lmhead", "backbone": "Qwen/Qwen2.5-1.5B-Instruct", "max_len": 1024, "created": "2026-09-18T06:05:30+00:00", "tasks": [ "spam" ], "trained_on": [ "spam" ], "holdout": [ "probe", "sst2", "mnli", "cola" ], "steps": 589, "train_examples": 4000, "args": { "cmd": "lmtrain", "lora_r": 8, "loss": "mix", "max_per_task": 4000, "epochs": 1, "lr": 0.0001, "anchor": 0.1 }, "metrics": { "spam": { "n": 300, "acc": 0.99, "nll": 0.04094129466820647, "brier": 0.01309215253075488, "ece": 0.031613957881927515, "mean_conf": 0.9694881041844686, "cov@0.5": 1.0, "acc@0.5": 0.99, "cov@0.7": 0.9933333333333333, "acc@0.7": 0.9966442953020134, "cov@0.9": 0.9666666666666667, "acc@0.9": 1.0 } }, "unseen_test_uncalibrated": { "probe": { "n": 97, "acc": 0.8969072164948454, "nll": 0.21773648728926945, "brier": 0.1273214505148998, "ece": 0.07348626329726782, "mean_conf": 0.9161194676590949, "score_mae": 0.15704563955659978, "cov@0.5": 0.9896907216494846, "acc@0.5": 0.90625, "cov@0.7": 0.8969072164948454, "acc@0.7": 0.9540229885057471, "cov@0.9": 0.7938144329896907, "acc@0.9": 0.987012987012987, "families": { "desc": [ 13, 15 ], "negation": [ 10, 10 ], "logic": [ 13, 17 ], "score": [ 11, 12 ], "json": [ 6, 6 ], "taxonomy": [ 14, 14 ], "plausible": [ 4, 4 ], "twist": [ 3, 3 ], "time": [ 4, 5 ], "intent": [ 2, 3 ], "compare": [ 3, 4 ], "criteria": [ 4, 4 ] } }, "sst2": { "n": 872, "acc": 0.9575688073394495, "nll": 0.1396011574813463, "brier": 0.07172239838956931, "ece": 0.014292597087151384, "mean_conf": 0.9680821417121712, "cov@0.5": 1.0, "acc@0.5": 0.9575688073394495, "cov@0.7": 0.9793577981651376, "acc@0.7": 0.9637002341920374, "cov@0.9": 0.9438073394495413, "acc@0.9": 0.9732685297691372 }, "mnli": { "n": 1000, "acc": 0.859, "nll": 0.3888388354725572, "brier": 0.21223603757591059, "ece": 0.04788112017512322, "mean_conf": 0.8856889481842518, "cov@0.5": 0.987, "acc@0.5": 0.8662613981762918, "cov@0.7": 0.909, "acc@0.7": 0.8954895489548955, "cov@0.9": 0.655, "acc@0.9": 0.9587786259541985 }, "cola": { "n": 1000, "acc": 0.759, "nll": 0.5066963657737495, "brier": 0.33443985155856215, "ece": 0.0523103475570679, "mean_conf": 0.791339822769165, "cov@0.5": 1.0, "acc@0.5": 0.759, "cov@0.7": 0.764, "acc@0.7": 0.8167539267015707, "cov@0.9": 0.207, "acc@0.9": 0.9516908212560387 } }, "summary": { "mean_acc": 0.99, "mean_ece": 0.031613957881927515 }, "history": [] }, "v0007-support": { "version": "v0007-support", "parent": "v0007", "parents": [ "v0007" ], "kind": "lmhead", "backbone": "Qwen/Qwen2.5-1.5B-Instruct", "max_len": 1024, "created": "2026-09-18T12:52:37+00:00", "tasks": [ "banking77", "clinc150", "massive_intent" ], "trained_on": [ "banking77", "clinc150", "massive_intent" ], "holdout": [ "trec" ], "steps": 1067, "train_examples": 12000, "args": { "cmd": "lmtrain", "lora_r": 8, "loss": "mix", "max_per_task": 4000, "epochs": 1, "lr": 0.0001, "anchor": 0.1 }, "metrics": {}, "unseen_test_uncalibrated": { "trec": { "n": 500, "acc": 0.758, "nll": 0.767611707548029, "brier": 0.37113630112335455, "ece": 0.08773053616285323, "mean_conf": 0.8239709965586662, "cov@0.5": 0.924, "acc@0.5": 0.7835497835497836, "cov@0.7": 0.762, "acc@0.7": 0.8293963254593176, "cov@0.9": 0.474, "acc@0.9": 0.8734177215189873 } }, "summary": { "mean_acc": null, "mean_ece": null }, "history": [] }, "v0007-safety": { "version": "v0007-safety", "parent": "v0007", "parents": [ "v0007" ], "kind": "lmhead", "backbone": "Qwen/Qwen2.5-1.5B-Instruct", "max_len": 1024, "created": "2026-09-18T13:21:25+00:00", "tasks": [ "jailbreak", "toxic", "tweet_hate", "tweet_offensive" ], "trained_on": [ "jailbreak", "toxic", "tweet_hate", "tweet_offensive" ], "holdout": [], "steps": 607, "train_examples": 8344, "args": { "cmd": "lmtrain", "lora_r": 8, "loss": "mix", "max_per_task": 2500, "epochs": 1, "lr": 0.0001, "anchor": 0.1 }, "metrics": { "jailbreak": { "n": 200, "acc": 0.98, "nll": 0.06724073947718683, "brier": 0.03142695878535079, "ece": 0.008043854534626017, "mean_conf": 0.988043854534626, "cov@0.5": 1.0, "acc@0.5": 0.98, "cov@0.7": 0.995, "acc@0.7": 0.9849246231155779, "cov@0.9": 0.99, "acc@0.9": 0.98989898989899 }, "toxic": { "n": 300, "acc": 0.8666666666666667, "nll": 0.3084903371258136, "brier": 0.19258679203988513, "ece": 0.03767327169577279, "mean_conf": 0.8952765788634618, "cov@0.5": 1.0, "acc@0.5": 0.8666666666666667, "cov@0.7": 0.8966666666666666, "acc@0.7": 0.9033457249070632, "cov@0.9": 0.6766666666666666, "acc@0.9": 0.9655172413793104 }, "tweet_hate": { "n": 300, "acc": 0.7433333333333333, "nll": 0.5153975753819442, "brier": 0.3425311312237358, "ece": 0.08356260061264038, "mean_conf": 0.8240628039836884, "cov@0.5": 1.0, "acc@0.5": 0.7433333333333333, "cov@0.7": 0.78, "acc@0.7": 0.811965811965812, "cov@0.9": 0.39, "acc@0.9": 0.9145299145299145 }, "tweet_offensive": { "n": 300, "acc": 0.7866666666666666, "nll": 0.4662258180163463, "brier": 0.3043113272015309, "ece": 0.053681320548057555, "mean_conf": 0.8303003575404485, "cov@0.5": 1.0, "acc@0.5": 0.7866666666666666, "cov@0.7": 0.7933333333333333, "acc@0.7": 0.8403361344537815, "cov@0.9": 0.4533333333333333, "acc@0.9": 0.9191176470588235 } }, "unseen_test_uncalibrated": {}, "summary": { "mean_acc": 0.8441666666666666, "mean_ece": 0.04574026184777419 }, "history": [] }, "v0007-reason": { "version": "v0007-reason", "parent": "v0007", "parents": [ "v0007" ], "kind": "lmhead", "backbone": "Qwen/Qwen2.5-1.5B-Instruct", "max_len": 1024, "created": "2026-09-22T05:53:47+00:00", "tasks": [ "traps", "reason", "skills" ], "trained_on": [ "traps", "reason", "skills" ], "holdout": [ "probe" ], "steps": 3144, "train_examples": 49321, "args": { "cmd": "lmtrain", "lora_r": 16, "loss": "mix", "max_per_task": 22000, "epochs": 1, "lr": 0.0001, "anchor": 0.1 }, "metrics": { "traps": { "n": 300, "acc": 0.9466666666666667, "nll": 0.20538718146178303, "brier": 0.06905301894228903, "ece": 0.04965091367562613, "mean_conf": 0.92676440179348, "score_mae": 0.47862898526946085, "cov@0.5": 0.9866666666666667, "acc@0.5": 0.9493243243243243, "cov@0.7": 0.9466666666666667, "acc@0.7": 0.971830985915493, "cov@0.9": 0.8433333333333334, "acc@0.9": 0.9920948616600791 }, "reason": { "n": 300, "acc": 0.98, "nll": 0.046410952195114846, "brier": 0.025619819302402664, "ece": 0.025865860184033737, "mean_conf": 0.9718742754062016, "cov@0.5": 1.0, "acc@0.5": 0.98, "cov@0.7": 0.97, "acc@0.7": 0.9896907216494846, "cov@0.9": 0.95, "acc@0.9": 1.0 }, "skills": { "n": 300, "acc": 0.9033333333333333, "nll": 0.20629042731885439, "brier": 0.11973303077943658, "ece": 0.061399669448534644, "mean_conf": 0.8790679361422856, "cov@0.5": 1.0, "acc@0.5": 0.9033333333333333, "cov@0.7": 0.83, "acc@0.7": 0.9759036144578314, "cov@0.9": 0.7166666666666667, "acc@0.9": 1.0 } }, "unseen_test_uncalibrated": { "probe": { "n": 97, "acc": 0.9381443298969072, "nll": 0.1893354459761264, "brier": 0.09341768317288839, "ece": 0.04105151006855912, "mean_conf": 0.9446589375279614, "score_mae": 0.18023038718577786, "cov@0.5": 1.0, "acc@0.5": 0.9381443298969072, "cov@0.7": 0.9381443298969072, "acc@0.7": 0.967032967032967, "cov@0.9": 0.865979381443299, "acc@0.9": 0.9761904761904762, "families": { "desc": [ 14, 15 ], "negation": [ 10, 10 ], "logic": [ 15, 17 ], "score": [ 11, 12 ], "json": [ 6, 6 ], "taxonomy": [ 14, 14 ], "plausible": [ 4, 4 ], "twist": [ 3, 3 ], "time": [ 4, 5 ], "intent": [ 2, 3 ], "compare": [ 4, 4 ], "criteria": [ 4, 4 ] } } }, "summary": { "mean_acc": 0.9433333333333334, "mean_ece": 0.04563881443606484 }, "history": [] }, "v0007-language": { "version": "v0007-language", "parent": "v0007", "parents": [ "v0007" ], "kind": "lmhead", "backbone": "Qwen/Qwen2.5-1.5B-Instruct", "max_len": 1024, "created": "2026-09-22T05:59:59+00:00", "tasks": [ "language" ], "trained_on": [ "language" ], "holdout": [], "steps": 625, "train_examples": 10000, "args": { "cmd": "lmtrain", "lora_r": 8, "loss": "mix", "max_per_task": 10000, "epochs": 1, "lr": 0.0001, "anchor": 0.1 }, "metrics": { "language": { "n": 300, "acc": 1.0, "nll": 0.11204298576288962, "brier": 0.0022332283333394826, "ece": 0.05060336291790009, "mean_conf": 0.9493966370820999, "cov@0.5": 1.0, "acc@0.5": 1.0, "cov@0.7": 1.0, "acc@0.7": 1.0, "cov@0.9": 0.99, "acc@0.9": 1.0 } }, "unseen_test_uncalibrated": {}, "summary": { "mean_acc": 1.0, "mean_ece": 0.05060336291790009 }, "history": [] }, "v0007-sentiment": { "version": "v0007-sentiment", "parent": "v0007", "parents": [ "v0007" ], "kind": "lmhead", "backbone": "Qwen/Qwen2.5-1.5B-Instruct", "max_len": 1024, "created": "2026-09-22T06:27:25+00:00", "tasks": [ "sst2", "sst5", "yelp", "tweet_sentiment", "emotion", "goemo_soft", "tweet_irony", "imdb", "formality", "politeness", "sarcasm" ], "trained_on": [ "sst2", "sst5", "yelp", "tweet_sentiment", "emotion", "goemo_soft", "tweet_irony", "imdb", "formality", "politeness", "sarcasm" ], "holdout": [ "fin_sentiment", "counterfactual" ], "steps": 1782, "train_examples": 22000, "args": { "cmd": "lmtrain", "lora_r": 8, "loss": "mix", "max_per_task": 2000, "epochs": 1, "lr": 0.0001, "anchor": 0.1 }, "metrics": { "sst2": { "n": 300, "acc": 0.9466666666666667, "nll": 0.1605975828277054, "brier": 0.0888536530799026, "ece": 0.027998056411743223, "mean_conf": 0.9642687650521596, "cov@0.5": 1.0, "acc@0.5": 0.9466666666666667, "cov@0.7": 0.9766666666666667, "acc@0.7": 0.9556313993174061, "cov@0.9": 0.9066666666666666, "acc@0.9": 0.9632352941176471 }, "sst5": { "n": 300, "acc": 0.5766666666666667, "nll": 1.030410210134918, "brier": 0.5736773828010426, "ece": 0.05870265493790308, "mean_conf": 0.5829293202360472, "score_mae": 0.5457962888351175, "cov@0.5": 0.79, "acc@0.5": 0.6075949367088608, "cov@0.7": 0.11666666666666667, "acc@0.7": 0.7142857142857143, "cov@0.9": 0.0, "acc@0.9": NaN }, "yelp": { "n": 300, "acc": 0.69, "nll": 0.7307452479655708, "brier": 0.4235077727191108, "ece": 0.08372766455014549, "mean_conf": 0.7693662059307098, "score_mae": 0.36723209824798686, "cov@0.5": 0.9533333333333334, "acc@0.5": 0.7027972027972028, "cov@0.7": 0.69, "acc@0.7": 0.782608695652174, "cov@0.9": 0.21333333333333335, "acc@0.9": 0.921875 }, "tweet_sentiment": { "n": 300, "acc": 0.7233333333333334, "nll": 0.6324473977408053, "brier": 0.37319421960896293, "ece": 0.061867911020914726, "mean_conf": 0.7788027099768321, "score_mae": 0.3486348866733412, "cov@0.5": 0.99, "acc@0.5": 0.7239057239057239, "cov@0.7": 0.69, "acc@0.7": 0.8115942028985508, "cov@0.9": 0.2733333333333333, "acc@0.9": 0.9024390243902439 }, "emotion": { "n": 300, "acc": 0.78, "nll": 0.6305815624709215, "brier": 0.3066913792466958, "ece": 0.07204169511795044, "mean_conf": 0.8520416951179505, "cov@0.5": 0.96, "acc@0.5": 0.7951388888888888, "cov@0.7": 0.8166666666666667, "acc@0.7": 0.8612244897959184, "cov@0.9": 0.55, "acc@0.9": 0.9575757575757575 }, "tweet_irony": { "n": 300, "acc": 0.7366666666666667, "nll": 0.5502091578371892, "brier": 0.3718147638320287, "ece": 0.08492387334505717, "mean_conf": 0.7882708374659221, "cov@0.5": 1.0, "acc@0.5": 0.7366666666666667, "cov@0.7": 0.74, "acc@0.7": 0.7837837837837838, "cov@0.9": 0.23333333333333334, "acc@0.9": 0.9142857142857143 }, "imdb": { "n": 300, "acc": 0.9633333333333334, "nll": 0.13351794521006682, "brier": 0.060667604680717205, "ece": 0.025028558770815598, "mean_conf": 0.9825289577245713, "cov@0.5": 1.0, "acc@0.5": 0.9633333333333334, "cov@0.7": 0.99, "acc@0.7": 0.9696969696969697, "cov@0.9": 0.9633333333333334, "acc@0.9": 0.9792387543252595 }, "formality": { "n": 300, "acc": 0.6133333333333333, "nll": 0.9954126704919857, "brier": 0.25669049533824123, "ece": 0.061814626157283774, "mean_conf": 0.5946429490049681, "score_mae": 0.4740935998460433, "cov@0.5": 0.89, "acc@0.5": 0.6179775280898876, "cov@0.7": 0.06666666666666667, "acc@0.7": 0.75, "cov@0.9": 0.0, "acc@0.9": NaN }, "politeness": { "n": 300, "acc": 0.8566666666666667, "nll": 0.3775658668930503, "brier": 0.19322453000823264, "ece": 0.03797433485587436, "mean_conf": 0.8931960561871528, "score_mae": 0.2005545631381392, "cov@0.5": 0.9766666666666667, "acc@0.5": 0.8703071672354948, "cov@0.7": 0.87, "acc@0.7": 0.9233716475095786, "cov@0.9": 0.6633333333333333, "acc@0.9": 0.9748743718592965 }, "sarcasm": { "n": 300, "acc": 0.8766666666666667, "nll": 0.45201081925126335, "brier": 0.10946457478491955, "ece": 0.08731486479441326, "mean_conf": 0.8211214067538579, "score_mae": 0.4119935893134519, "cov@0.5": 0.94, "acc@0.5": 0.8971631205673759, "cov@0.7": 0.7633333333333333, "acc@0.7": 0.9606986899563319, "cov@0.9": 0.49, "acc@0.9": 0.9931972789115646 } }, "unseen_test_uncalibrated": { "fin_sentiment": { "n": 1000, "acc": 0.807, "nll": 0.4482609479897583, "brier": 0.2712111023947865, "ece": 0.025465373486280456, "mean_conf": 0.8211387673318387, "cov@0.5": 0.992, "acc@0.5": 0.8104838709677419, "cov@0.7": 0.799, "acc@0.7": 0.8685857321652065, "cov@0.9": 0.369, "acc@0.9": 0.9376693766937669 }, "counterfactual": { "n": 1000, "acc": 0.855, "nll": 0.3709632075470155, "brier": 0.2216712187729024, "ece": 0.07320722198486329, "mean_conf": 0.7820079625844956, "cov@0.5": 1.0, "acc@0.5": 0.855, "cov@0.7": 0.748, "acc@0.7": 0.9251336898395722, "cov@0.9": 0.135, "acc@0.9": 0.9703703703703703 } }, "summary": { "mean_acc": 0.7763333333333333, "mean_ece": 0.06013942399621011 }, "history": [] }, "v0007-typed": { "version": "v0007-typed", "parent": "v0007", "parents": [ "v0007" ], "kind": "lmhead", "backbone": "Qwen/Qwen2.5-1.5B-Instruct", "max_len": 1024, "created": "2026-09-22T11:39:50+00:00", "tasks": [ "typed_decisions" ], "trained_on": [ "typed_decisions" ], "holdout": [], "steps": 1352, "train_examples": 5400, "args": { "cmd": "lmtrain", "lora_r": 16, "loss": "mix", "max_per_task": 6000, "epochs": 2, "lr": 0.0001, "anchor": 0.1 }, "metrics": { "typed_decisions": { "n": 300, "acc": 0.85, "nll": 0.7897254475627804, "brier": 0.04430931889174507, "ece": 0.21966197381416955, "mean_conf": 0.6374534544348717, "score_mae": 0.34401526501434937, "cov@0.5": 0.76, "acc@0.5": 0.9342105263157895, "cov@0.7": 0.30666666666666664, "acc@0.7": 0.9891304347826086, "cov@0.9": 0.13666666666666666, "acc@0.9": 1.0 } }, "unseen_test_uncalibrated": {}, "summary": { "mean_acc": 0.85, "mean_ece": 0.21966197381416955 }, "history": [] }, "v0007-math": { "version": "v0007-math", "parent": "v0007", "parents": [ "v0007" ], "kind": "lmhead", "backbone": "Qwen/Qwen2.5-1.5B-Instruct", "max_len": 1024, "created": "2026-09-22T13:55:28+00:00", "tasks": [ "math", "gsm8k", "svamp", "aqua" ], "trained_on": [ "math", "gsm8k", "svamp", "aqua" ], "holdout": [], "steps": 3684, "train_examples": 57771, "args": { "cmd": "lmtrain", "lora_r": 32, "loss": "mix", "max_per_task": 30000, "epochs": 1, "lr": 0.0001, "anchor": 0.1 }, "metrics": { "math": { "n": 300, "acc": 0.8733333333333333, "nll": 0.2491237256136509, "brier": 0.15517219901605317, "ece": 0.04117123444875081, "mean_conf": 0.8545713822046915, "cov@0.5": 1.0, "acc@0.5": 0.8733333333333333, "cov@0.7": 0.7533333333333333, "acc@0.7": 0.9690265486725663, "cov@0.9": 0.7066666666666667, "acc@0.9": 1.0 }, "gsm8k": { "n": 300, "acc": 0.8166666666666667, "nll": 0.4634518215859158, "brier": 0.24914535577158609, "ece": 0.08174108674128854, "mean_conf": 0.7950625959038734, "cov@0.5": 0.9, "acc@0.5": 0.8666666666666667, "cov@0.7": 0.7, "acc@0.7": 0.9238095238095239, "cov@0.9": 0.4266666666666667, "acc@0.9": 0.9921875 }, "svamp": { "n": 100, "acc": 0.68, "nll": 0.7193946922798429, "brier": 0.41086004277348953, "ece": 0.07071352988481525, "mean_conf": 0.7261623141169548, "cov@0.5": 0.83, "acc@0.5": 0.7349397590361446, "cov@0.7": 0.58, "acc@0.7": 0.8275862068965517, "cov@0.9": 0.2, "acc@0.9": 1.0 }, "aqua": { "n": 253, "acc": 0.391304347826087, "nll": 1.414271918680051, "brier": 0.7115495715758425, "ece": 0.0654898358899143, "mean_conf": 0.3998582398467384, "cov@0.5": 0.20553359683794467, "acc@0.5": 0.6538461538461539, "cov@0.7": 0.06324110671936758, "acc@0.7": 0.8125, "cov@0.9": 0.007905138339920948, "acc@0.9": 1.0 } }, "unseen_test_uncalibrated": {}, "summary": { "mean_acc": 0.6903260869565218, "mean_ece": 0.06477892174119222 }, "history": [] } } }