{
 "model": "decider-4b-v2.1-tr-lora",
 "path": "/workspace/laya/decider_run/model",
 "revision": "eb5fbdfc9448473ec25e399882912863afbdb70e",
 "fp8_layers": 0,
 "limit": 0,
 "skipped": {},
 "seconds": 3595,
 "report": {
  "all": {
   "n": 101716,
   "accuracy": 0.7962,
   "brier": 0.2653,
   "nll": 0.53,
   "ece": 0.0203
  },
  "source/bev_tr": {
   "n": 46320,
   "accuracy": 0.8382,
   "brier": 0.2215,
   "nll": 0.3964,
   "ece": 0.0092
  },
  "source/bev_tr/choice": {
   "n": 18808,
   "accuracy": 0.8465,
   "brier": 0.214,
   "nll": 0.3982,
   "ece": 0.009
  },
  "source/bev_tr/noul": {
   "n": 17572,
   "accuracy": 0.9029,
   "brier": 0.1396,
   "nll": 0.2281,
   "ece": 0.0099
  },
  "source/bev_tr/score": {
   "n": 9940,
   "accuracy": 0.708,
   "brier": 0.3805,
   "nll": 0.6904,
   "ece": 0.018
  },
  "source/bev_tr_en": {
   "n": 5766,
   "accuracy": 0.7542,
   "brier": 0.3138,
   "nll": 0.5674,
   "ece": 0.0126
  },
  "source/bev_tr_en/choice": {
   "n": 2386,
   "accuracy": 0.7812,
   "brier": 0.3036,
   "nll": 0.5762,
   "ece": 0.0289
  },
  "source/bev_tr_en/noul": {
   "n": 2117,
   "accuracy": 0.8734,
   "brier": 0.1665,
   "nll": 0.2696,
   "ece": 0.0262
  },
  "source/bev_tr_en/score": {
   "n": 1263,
   "accuracy": 0.5036,
   "brier": 0.5799,
   "nll": 1.0501,
   "ece": 0.0373
  },
  "source/beyazperde_tr": {
   "n": 5000,
   "accuracy": 0.6336,
   "brier": 0.4769,
   "nll": 0.8667,
   "ece": 0.0402
  },
  "source/beyazperde_tr/score": {
   "n": 5000,
   "accuracy": 0.6336,
   "brier": 0.4769,
   "nll": 0.8667,
   "ece": 0.0402
  },
  "source/jev_bench_tr": {
   "n": 22773,
   "accuracy": 0.7078,
   "brier": 0.3407,
   "nll": 0.821,
   "ece": 0.051
  },
  "source/jev_bench_tr/choice": {
   "n": 9599,
   "accuracy": 0.7316,
   "brier": 0.2873,
   "nll": 0.9497,
   "ece": 0.0593
  },
  "source/jev_bench_tr/noul": {
   "n": 7174,
   "accuracy": 0.8246,
   "brier": 0.2226,
   "nll": 0.4021,
   "ece": 0.0458
  },
  "source/jev_bench_tr/score": {
   "n": 6000,
   "accuracy": 0.5298,
   "brier": 0.5675,
   "nll": 1.1159,
   "ece": 0.0453
  },
  "source/jev_bench_tr_en": {
   "n": 2000,
   "accuracy": 0.7705,
   "brier": 0.2689,
   "nll": 0.6936,
   "ece": 0.0351
  },
  "source/jev_bench_tr_en/choice": {
   "n": 839,
   "accuracy": 0.7926,
   "brier": 0.2156,
   "nll": 0.8206,
   "ece": 0.0399
  },
  "source/jev_bench_tr_en/noul": {
   "n": 635,
   "accuracy": 0.8945,
   "brier": 0.1434,
   "nll": 0.2821,
   "ece": 0.0208
  },
  "source/jev_bench_tr_en/score": {
   "n": 526,
   "accuracy": 0.5856,
   "brier": 0.5053,
   "nll": 0.988,
   "ece": 0.0867
  },
  "source/massive_tr": {
   "n": 2974,
   "accuracy": 0.8705,
   "brier": 0.1941,
   "nll": 0.5568,
   "ece": 0.0362
  },
  "source/massive_tr/choice": {
   "n": 2974,
   "accuracy": 0.8705,
   "brier": 0.1941,
   "nll": 0.5568,
   "ece": 0.0362
  },
  "source/squad_tr": {
   "n": 11873,
   "accuracy": 0.8619,
   "brier": 0.2028,
   "nll": 0.3322,
   "ece": 0.0168
  },
  "source/squad_tr/noul": {
   "n": 11873,
   "accuracy": 0.8619,
   "brier": 0.2028,
   "nll": 0.3322,
   "ece": 0.0168
  },
  "source/xnli_tr": {
   "n": 5010,
   "accuracy": 0.8315,
   "brier": 0.2488,
   "nll": 0.4509,
   "ece": 0.0151
  },
  "source/xnli_tr/choice": {
   "n": 5010,
   "accuracy": 0.8315,
   "brier": 0.2488,
   "nll": 0.4509,
   "ece": 0.0151
  },
  "type/choice": {
   "n": 39616,
   "accuracy": 0.8135,
   "brier": 0.2401,
   "nll": 0.5701,
   "ece": 0.0214
  },
  "type/noul": {
   "n": 39371,
   "accuracy": 0.8746,
   "brier": 0.1753,
   "nll": 0.2943,
   "ece": 0.0177
  },
  "type/score": {
   "n": 22729,
   "accuracy": 0.6304,
   "brier": 0.465,
   "nll": 0.8684,
   "ece": 0.0235
  },
  "workflow/bev_tr/Browser interaction": {
   "n": 286,
   "accuracy": 0.8497,
   "brier": 0.1897,
   "nll": 0.3352,
   "ece": 0.0394
  },
  "workflow/bev_tr/Civic and safety operations": {
   "n": 261,
   "accuracy": 0.7701,
   "brier": 0.3154,
   "nll": 0.6527,
   "ece": 0.0616
  },
  "workflow/bev_tr/Contract evidence": {
   "n": 320,
   "accuracy": 0.8375,
   "brier": 0.2302,
   "nll": 0.4253,
   "ece": 0.0491
  },
  "workflow/bev_tr/Engagement and ranking": {
   "n": 82,
   "accuracy": 0.6463,
   "brier": 0.4851,
   "nll": 0.6918,
   "ece": 0.1104
  },
  "workflow/bev_tr/Financial reporting and banking": {
   "n": 1740,
   "accuracy": 0.8264,
   "brier": 0.2371,
   "nll": 0.4219,
   "ece": 0.0215
  },
  "workflow/bev_tr/Game-state decisions": {
   "n": 2039,
   "accuracy": 0.8794,
   "brier": 0.1475,
   "nll": 0.2267,
   "ece": 0.0145
  },
  "workflow/bev_tr/Reading comprehension": {
   "n": 1422,
   "accuracy": 0.8643,
   "brier": 0.1945,
   "nll": 0.3188,
   "ece": 0.0163
  },
  "workflow/bev_tr/Response preference and quality": {
   "n": 3176,
   "accuracy": 0.6382,
   "brier": 0.4731,
   "nll": 0.8563,
   "ece": 0.0269
  },
  "workflow/bev_tr/Retail, product, and shopping": {
   "n": 6601,
   "accuracy": 0.9365,
   "brier": 0.0922,
   "nll": 0.1653,
   "ece": 0.0122
  },
  "workflow/bev_tr/Scientific and paper understanding": {
   "n": 508,
   "accuracy": 0.8346,
   "brier": 0.248,
   "nll": 0.4421,
   "ece": 0.0278
  },
  "workflow/bev_tr/Sensory quality rating": {
   "n": 260,
   "accuracy": 0.4,
   "brier": 0.6968,
   "nll": 1.3241,
   "ece": 0.0533
  },
  "workflow/bev_tr/Sentiment, emotion, and moderation": {
   "n": 11526,
   "accuracy": 0.8647,
   "brier": 0.1892,
   "nll": 0.3338,
   "ece": 0.0152
  },
  "workflow/bev_tr/Software engineering and code": {
   "n": 2039,
   "accuracy": 0.7229,
   "brier": 0.3732,
   "nll": 0.7028,
   "ece": 0.0368
  },
  "workflow/bev_tr/Software security": {
   "n": 2240,
   "accuracy": 0.6879,
   "brier": 0.4086,
   "nll": 0.7059,
   "ece": 0.0458
  },
  "workflow/bev_tr/Spam detection": {
   "n": 271,
   "accuracy": 0.9779,
   "brier": 0.0284,
   "nll": 0.0569,
   "ece": 0.0159
  },
  "workflow/bev_tr/Spatial and logical reasoning": {
   "n": 8384,
   "accuracy": 0.7832,
   "brier": 0.3054,
   "nll": 0.5545,
   "ece": 0.0198
  },
  "workflow/bev_tr/Support and intent routing": {
   "n": 2858,
   "accuracy": 0.971,
   "brier": 0.0462,
   "nll": 0.087,
   "ece": 0.011
  },
  "workflow/bev_tr/Tool and workflow decisions": {
   "n": 2307,
   "accuracy": 0.9866,
   "brier": 0.0197,
   "nll": 0.0423,
   "ece": 0.0068
  },
  "workflow/bev_tr_en/Browser interaction": {
   "n": 39,
   "accuracy": 0.7692,
   "brier": 0.2566,
   "nll": 0.4157,
   "ece": 0.1178
  },
  "workflow/bev_tr_en/Civic and safety operations": {
   "n": 31,
   "accuracy": 0.5484,
   "brier": 0.5519,
   "nll": 0.9665,
   "ece": 0.2049
  },
  "workflow/bev_tr_en/Contract evidence": {
   "n": 49,
   "accuracy": 0.7959,
   "brier": 0.3148,
   "nll": 0.5499,
   "ece": 0.1107
  },
  "workflow/bev_tr_en/Engagement and ranking": {
   "n": 6,
   "accuracy": 0.6667,
   "brier": 0.4327,
   "nll": 0.6113,
   "ece": 0.278
  },
  "workflow/bev_tr_en/Financial reporting and banking": {
   "n": 196,
   "accuracy": 0.6531,
   "brier": 0.4856,
   "nll": 0.975,
   "ece": 0.1033
  },
  "workflow/bev_tr_en/Game-state decisions": {
   "n": 303,
   "accuracy": 0.835,
   "brier": 0.2434,
   "nll": 0.4021,
   "ece": 0.0929
  },
  "workflow/bev_tr_en/Reading comprehension": {
   "n": 168,
   "accuracy": 0.9048,
   "brier": 0.1584,
   "nll": 0.2737,
   "ece": 0.0486
  },
  "workflow/bev_tr_en/Response preference and quality": {
   "n": 389,
   "accuracy": 0.5604,
   "brier": 0.5324,
   "nll": 0.9589,
   "ece": 0.0574
  },
  "workflow/bev_tr_en/Retail, product, and shopping": {
   "n": 902,
   "accuracy": 0.6896,
   "brier": 0.3378,
   "nll": 0.5532,
   "ece": 0.1003
  },
  "workflow/bev_tr_en/Scientific and paper understanding": {
   "n": 66,
   "accuracy": 0.7424,
   "brier": 0.3035,
   "nll": 0.5148,
   "ece": 0.1262
  },
  "workflow/bev_tr_en/Sensory quality rating": {
   "n": 37,
   "accuracy": 0.2432,
   "brier": 0.7863,
   "nll": 1.5804,
   "ece": 0.1299
  },
  "workflow/bev_tr_en/Sentiment, emotion, and moderation": {
   "n": 1439,
   "accuracy": 0.8457,
   "brier": 0.202,
   "nll": 0.3659,
   "ece": 0.0443
  },
  "workflow/bev_tr_en/Software engineering and code": {
   "n": 260,
   "accuracy": 0.7077,
   "brier": 0.3893,
   "nll": 0.7283,
   "ece": 0.076
  },
  "workflow/bev_tr_en/Software security": {
   "n": 234,
   "accuracy": 0.5556,
   "brier": 0.5142,
   "nll": 0.8403,
   "ece": 0.0889
  },
  "workflow/bev_tr_en/Spam detection": {
   "n": 23,
   "accuracy": 1.0,
   "brier": 0.0001,
   "nll": 0.006,
   "ece": 0.006
  },
  "workflow/bev_tr_en/Spatial and logical reasoning": {
   "n": 1042,
   "accuracy": 0.6823,
   "brier": 0.4407,
   "nll": 0.8513,
   "ece": 0.0635
  },
  "workflow/bev_tr_en/Support and intent routing": {
   "n": 310,
   "accuracy": 0.9581,
   "brier": 0.0834,
   "nll": 0.1668,
   "ece": 0.0519
  },
  "workflow/bev_tr_en/Tool and workflow decisions": {
   "n": 272,
   "accuracy": 0.9779,
   "brier": 0.0379,
   "nll": 0.074,
   "ece": 0.0203
  },
  "workflow/beyazperde_tr": {
   "n": 5000,
   "accuracy": 0.6336,
   "brier": 0.4769,
   "nll": 0.8667,
   "ece": 0.0402
  },
  "workflow/jev_bench_tr/arc_challenge": {
   "n": 1000,
   "accuracy": 0.833,
   "brier": 0.2373,
   "nll": 0.452,
   "ece": 0.0305
  },
  "workflow/jev_bench_tr/banking77": {
   "n": 1000,
   "accuracy": 0.81,
   "brier": 0.2667,
   "nll": 0.6704,
   "ece": 0.0496
  },
  "workflow/jev_bench_tr/boolq": {
   "n": 1000,
   "accuracy": 0.877,
   "brier": 0.1889,
   "nll": 0.3099,
   "ece": 0.0233
  },
  "workflow/jev_bench_tr/chaosnli": {
   "n": 1599,
   "accuracy": 0.6373,
   "brier": 0.2039,
   "nll": 1.1669,
   "ece": 0.1133
  },
  "workflow/jev_bench_tr/civil_comments": {
   "n": 2000,
   "accuracy": 0.863,
   "brier": 0.1263,
   "nll": 0.3793,
   "ece": 0.0208
  },
  "workflow/jev_bench_tr/clinc150": {
   "n": 1000,
   "accuracy": 0.873,
   "brier": 0.1986,
   "nll": 0.6114,
   "ece": 0.0333
  },
  "workflow/jev_bench_tr/fever_evidence": {
   "n": 1000,
   "accuracy": 0.926,
   "brier": 0.1063,
   "nll": 0.1935,
   "ece": 0.0218
  },
  "workflow/jev_bench_tr/go_emotions": {
   "n": 1000,
   "accuracy": 0.496,
   "brier": 0.3399,
   "nll": 2.3234,
   "ece": 0.1412
  },
  "workflow/jev_bench_tr/helpsteer2_helpfulness": {
   "n": 1000,
   "accuracy": 0.403,
   "brier": 0.7142,
   "nll": 1.3625,
   "ece": 0.1558
  },
  "workflow/jev_bench_tr/helpsteer2_verbosity": {
   "n": 1000,
   "accuracy": 0.661,
   "brier": 0.5088,
   "nll": 0.9137,
   "ece": 0.1524
  },
  "workflow/jev_bench_tr/ledgar": {
   "n": 1000,
   "accuracy": 0.678,
   "brier": 0.4474,
   "nll": 1.2384,
   "ece": 0.0563
  },
  "workflow/jev_bench_tr/massive": {
   "n": 1000,
   "accuracy": 0.867,
   "brier": 0.1986,
   "nll": 0.5556,
   "ece": 0.0434
  },
  "workflow/jev_bench_tr/measuring_hate_speech": {
   "n": 1000,
   "accuracy": 0.534,
   "brier": 0.3917,
   "nll": 1.1279,
   "ece": 0.1391
  },
  "workflow/jev_bench_tr/mmlu": {
   "n": 1000,
   "accuracy": 0.615,
   "brier": 0.5026,
   "nll": 0.9645,
   "ece": 0.0798
  },
  "workflow/jev_bench_tr/mnli": {
   "n": 1000,
   "accuracy": 0.832,
   "brier": 0.2406,
   "nll": 0.4346,
   "ece": 0.0289
  },
  "workflow/jev_bench_tr/paws": {
   "n": 1000,
   "accuracy": 0.694,
   "brier": 0.4599,
   "nll": 0.7411,
   "ece": 0.1698
  },
  "workflow/jev_bench_tr/sms_spam": {
   "n": 800,
   "accuracy": 0.9587,
   "brier": 0.0616,
   "nll": 0.1232,
   "ece": 0.0165
  },
  "workflow/jev_bench_tr/sst5": {
   "n": 1000,
   "accuracy": 0.428,
   "brier": 0.6809,
   "nll": 1.2517,
   "ece": 0.0952
  },
  "workflow/jev_bench_tr/strategyqa_closed": {
   "n": 687,
   "accuracy": 0.5429,
   "brier": 0.5218,
   "nll": 0.7334,
   "ece": 0.1441
  },
  "workflow/jev_bench_tr/strategyqa_grounded": {
   "n": 687,
   "accuracy": 0.8049,
   "brier": 0.2641,
   "nll": 0.4067,
   "ece": 0.0331
  },
  "workflow/jev_bench_tr/stsb": {
   "n": 1000,
   "accuracy": 0.51,
   "brier": 0.634,
   "nll": 1.2248,
   "ece": 0.0975
  },
  "workflow/jev_bench_tr/yelp5": {
   "n": 1000,
   "accuracy": 0.643,
   "brier": 0.4752,
   "nll": 0.8146,
   "ece": 0.0633
  },
  "workflow/jev_bench_tr_en/arc_challenge": {
   "n": 91,
   "accuracy": 0.9121,
   "brier": 0.1091,
   "nll": 0.1997,
   "ece": 0.0625
  },
  "workflow/jev_bench_tr_en/banking77": {
   "n": 71,
   "accuracy": 0.8592,
   "brier": 0.2199,
   "nll": 0.5531,
   "ece": 0.0805
  },
  "workflow/jev_bench_tr_en/boolq": {
   "n": 80,
   "accuracy": 0.9,
   "brier": 0.1214,
   "nll": 0.2017,
   "ece": 0.0737
  },
  "workflow/jev_bench_tr_en/chaosnli": {
   "n": 151,
   "accuracy": 0.7086,
   "brier": 0.1814,
   "nll": 1.1407,
   "ece": 0.1201
  },
  "workflow/jev_bench_tr_en/civil_comments": {
   "n": 171,
   "accuracy": 0.8655,
   "brier": 0.14,
   "nll": 0.3978,
   "ece": 0.0587
  },
  "workflow/jev_bench_tr_en/clinc150": {
   "n": 96,
   "accuracy": 0.8854,
   "brier": 0.1594,
   "nll": 0.3886,
   "ece": 0.0521
  },
  "workflow/jev_bench_tr_en/fever_evidence": {
   "n": 91,
   "accuracy": 0.967,
   "brier": 0.0561,
   "nll": 0.1315,
   "ece": 0.0348
  },
  "workflow/jev_bench_tr_en/go_emotions": {
   "n": 93,
   "accuracy": 0.5269,
   "brier": 0.323,
   "nll": 2.3535,
   "ece": 0.1604
  },
  "workflow/jev_bench_tr_en/helpsteer2_helpfulness": {
   "n": 89,
   "accuracy": 0.4831,
   "brier": 0.6573,
   "nll": 1.1603,
   "ece": 0.1857
  },
  "workflow/jev_bench_tr_en/helpsteer2_verbosity": {
   "n": 79,
   "accuracy": 0.7089,
   "brier": 0.4661,
   "nll": 0.8944,
   "ece": 0.1493
  },
  "workflow/jev_bench_tr_en/ledgar": {
   "n": 70,
   "accuracy": 0.7857,
   "brier": 0.3543,
   "nll": 0.8834,
   "ece": 0.1722
  },
  "workflow/jev_bench_tr_en/massive": {
   "n": 95,
   "accuracy": 0.8526,
   "brier": 0.207,
   "nll": 0.73,
   "ece": 0.0683
  },
  "workflow/jev_bench_tr_en/measuring_hate_speech": {
   "n": 96,
   "accuracy": 0.5729,
   "brier": 0.3906,
   "nll": 1.1441,
   "ece": 0.187
  },
  "workflow/jev_bench_tr_en/mmlu": {
   "n": 80,
   "accuracy": 0.725,
   "brier": 0.3559,
   "nll": 0.6658,
   "ece": 0.071
  },
  "workflow/jev_bench_tr_en/mnli": {
   "n": 92,
   "accuracy": 0.9348,
   "brier": 0.1053,
   "nll": 0.1974,
   "ece": 0.0927
  },
  "workflow/jev_bench_tr_en/paws": {
   "n": 80,
   "accuracy": 0.8375,
   "brier": 0.2238,
   "nll": 0.3598,
   "ece": 0.0984
  },
  "workflow/jev_bench_tr_en/sms_spam": {
   "n": 82,
   "accuracy": 1.0,
   "brier": 0.0055,
   "nll": 0.0164,
   "ece": 0.0147
  },
  "workflow/jev_bench_tr_en/sst5": {
   "n": 73,
   "accuracy": 0.5205,
   "brier": 0.5669,
   "nll": 0.9347,
   "ece": 0.0965
  },
  "workflow/jev_bench_tr_en/strategyqa_closed": {
   "n": 60,
   "accuracy": 0.8333,
   "brier": 0.3338,
   "nll": 0.5123,
   "ece": 0.1264
  },
  "workflow/jev_bench_tr_en/strategyqa_grounded": {
   "n": 71,
   "accuracy": 0.8592,
   "brier": 0.1958,
   "nll": 0.3115,
   "ece": 0.0442
  },
  "workflow/jev_bench_tr_en/stsb": {
   "n": 90,
   "accuracy": 0.5111,
   "brier": 0.577,
   "nll": 1.098,
   "ece": 0.1376
  },
  "workflow/jev_bench_tr_en/yelp5": {
   "n": 99,
   "accuracy": 0.7071,
   "brier": 0.4005,
   "nll": 0.6957,
   "ece": 0.1338
  },
  "workflow/massive_tr": {
   "n": 2974,
   "accuracy": 0.8705,
   "brier": 0.1941,
   "nll": 0.5568,
   "ece": 0.0362
  },
  "workflow/squad_tr": {
   "n": 11873,
   "accuracy": 0.8619,
   "brier": 0.2028,
   "nll": 0.3322,
   "ece": 0.0168
  },
  "workflow/xnli_tr": {
   "n": 5010,
   "accuracy": 0.8315,
   "brier": 0.2488,
   "nll": 0.4509,
   "ece": 0.0151
  },
  "squad_tr/answerability": {
   "auroc": 0.9397,
   "acc_answerable": 0.9379,
   "acc_unanswerable": 0.783
  }
 },
 "train_key": "411bcda4c97e6b5b"
}