{
 "model": "decider-4b-v2.1",
 "path": "Mapika/decider-4b",
 "revision": "eb5fbdfc9448473ec25e399882912863afbdb70e",
 "skipped": {},
 "seconds": 4083,
 "report": {
  "all": {
   "n": 101716,
   "accuracy": 0.689,
   "brier": 0.3876,
   "nll": 0.7511,
   "ece": 0.0282
  },
  "source/bev_tr": {
   "n": 46320,
   "accuracy": 0.6687,
   "brier": 0.416,
   "nll": 0.7659,
   "ece": 0.0272
  },
  "source/bev_tr/choice": {
   "n": 18808,
   "accuracy": 0.6687,
   "brier": 0.4305,
   "nll": 0.853,
   "ece": 0.069
  },
  "source/bev_tr/noul": {
   "n": 17572,
   "accuracy": 0.8119,
   "brier": 0.2643,
   "nll": 0.4119,
   "ece": 0.0192
  },
  "source/bev_tr/score": {
   "n": 9940,
   "accuracy": 0.4157,
   "brier": 0.6566,
   "nll": 1.227,
   "ece": 0.0427
  },
  "source/bev_tr_en": {
   "n": 5766,
   "accuracy": 0.7197,
   "brier": 0.352,
   "nll": 0.653,
   "ece": 0.0206
  },
  "source/bev_tr_en/choice": {
   "n": 2386,
   "accuracy": 0.715,
   "brier": 0.3711,
   "nll": 0.7288,
   "ece": 0.0509
  },
  "source/bev_tr_en/noul": {
   "n": 2117,
   "accuracy": 0.8635,
   "brier": 0.1836,
   "nll": 0.3012,
   "ece": 0.0466
  },
  "source/bev_tr_en/score": {
   "n": 1263,
   "accuracy": 0.4877,
   "brier": 0.5983,
   "nll": 1.0994,
   "ece": 0.036
  },
  "source/beyazperde_tr": {
   "n": 5000,
   "accuracy": 0.5204,
   "brier": 0.5888,
   "nll": 1.0806,
   "ece": 0.0503
  },
  "source/beyazperde_tr/score": {
   "n": 5000,
   "accuracy": 0.5204,
   "brier": 0.5888,
   "nll": 1.0806,
   "ece": 0.0503
  },
  "source/jev_bench_tr": {
   "n": 22773,
   "accuracy": 0.71,
   "brier": 0.3406,
   "nll": 0.8272,
   "ece": 0.0308
  },
  "source/jev_bench_tr/choice": {
   "n": 9599,
   "accuracy": 0.7209,
   "brier": 0.3008,
   "nll": 1.002,
   "ece": 0.059
  },
  "source/jev_bench_tr/noul": {
   "n": 7174,
   "accuracy": 0.8433,
   "brier": 0.2132,
   "nll": 0.386,
   "ece": 0.0263
  },
  "source/jev_bench_tr/score": {
   "n": 6000,
   "accuracy": 0.533,
   "brier": 0.5565,
   "nll": 1.0753,
   "ece": 0.0604
  },
  "source/jev_bench_tr_en": {
   "n": 2000,
   "accuracy": 0.768,
   "brier": 0.2677,
   "nll": 0.6802,
   "ece": 0.0338
  },
  "source/jev_bench_tr_en/choice": {
   "n": 839,
   "accuracy": 0.7926,
   "brier": 0.2158,
   "nll": 0.8053,
   "ece": 0.0562
  },
  "source/jev_bench_tr_en/noul": {
   "n": 635,
   "accuracy": 0.8866,
   "brier": 0.147,
   "nll": 0.2887,
   "ece": 0.0317
  },
  "source/jev_bench_tr_en/score": {
   "n": 526,
   "accuracy": 0.5856,
   "brier": 0.4964,
   "nll": 0.9534,
   "ece": 0.0678
  },
  "source/massive_tr": {
   "n": 2974,
   "accuracy": 0.8285,
   "brier": 0.2533,
   "nll": 0.7069,
   "ece": 0.0362
  },
  "source/massive_tr/choice": {
   "n": 2974,
   "accuracy": 0.8285,
   "brier": 0.2533,
   "nll": 0.7069,
   "ece": 0.0362
  },
  "source/squad_tr": {
   "n": 11873,
   "accuracy": 0.6832,
   "brier": 0.4016,
   "nll": 0.585,
   "ece": 0.0488
  },
  "source/squad_tr/noul": {
   "n": 11873,
   "accuracy": 0.6832,
   "brier": 0.4016,
   "nll": 0.585,
   "ece": 0.0488
  },
  "source/xnli_tr": {
   "n": 5010,
   "accuracy": 0.8128,
   "brier": 0.2741,
   "nll": 0.4991,
   "ece": 0.0356
  },
  "source/xnli_tr/choice": {
   "n": 5010,
   "accuracy": 0.8128,
   "brier": 0.2741,
   "nll": 0.4991,
   "ece": 0.0356
  },
  "type/choice": {
   "n": 39616,
   "accuracy": 0.717,
   "brier": 0.3579,
   "nll": 0.8249,
   "ece": 0.0563
  },
  "type/noul": {
   "n": 39371,
   "accuracy": 0.7828,
   "brier": 0.2902,
   "nll": 0.4514,
   "ece": 0.0073
  },
  "type/score": {
   "n": 22729,
   "accuracy": 0.4776,
   "brier": 0.6083,
   "nll": 1.1413,
   "ece": 0.0321
  },
  "workflow/bev_tr/Browser interaction": {
   "n": 286,
   "accuracy": 0.7622,
   "brier": 0.3497,
   "nll": 0.6694,
   "ece": 0.0703
  },
  "workflow/bev_tr/Civic and safety operations": {
   "n": 261,
   "accuracy": 0.4598,
   "brier": 0.7236,
   "nll": 1.6077,
   "ece": 0.0844
  },
  "workflow/bev_tr/Contract evidence": {
   "n": 320,
   "accuracy": 0.7,
   "brier": 0.4117,
   "nll": 0.7334,
   "ece": 0.0947
  },
  "workflow/bev_tr/Engagement and ranking": {
   "n": 82,
   "accuracy": 0.5732,
   "brier": 0.4994,
   "nll": 0.6958,
   "ece": 0.1553
  },
  "workflow/bev_tr/Financial reporting and banking": {
   "n": 1740,
   "accuracy": 0.573,
   "brier": 0.5624,
   "nll": 1.0865,
   "ece": 0.0942
  },
  "workflow/bev_tr/Game-state decisions": {
   "n": 2039,
   "accuracy": 0.7013,
   "brier": 0.4137,
   "nll": 0.6803,
   "ece": 0.0833
  },
  "workflow/bev_tr/Reading comprehension": {
   "n": 1422,
   "accuracy": 0.8601,
   "brier": 0.2171,
   "nll": 0.3537,
   "ece": 0.0343
  },
  "workflow/bev_tr/Response preference and quality": {
   "n": 3176,
   "accuracy": 0.5107,
   "brier": 0.5648,
   "nll": 1.0151,
   "ece": 0.0319
  },
  "workflow/bev_tr/Retail, product, and shopping": {
   "n": 6601,
   "accuracy": 0.6946,
   "brier": 0.3684,
   "nll": 0.6556,
   "ece": 0.0808
  },
  "workflow/bev_tr/Scientific and paper understanding": {
   "n": 508,
   "accuracy": 0.6713,
   "brier": 0.4589,
   "nll": 0.7734,
   "ece": 0.0884
  },
  "workflow/bev_tr/Sensory quality rating": {
   "n": 260,
   "accuracy": 0.2692,
   "brier": 0.8003,
   "nll": 1.6129,
   "ece": 0.1629
  },
  "workflow/bev_tr/Sentiment, emotion, and moderation": {
   "n": 11526,
   "accuracy": 0.727,
   "brier": 0.3386,
   "nll": 0.594,
   "ece": 0.0549
  },
  "workflow/bev_tr/Software engineering and code": {
   "n": 2039,
   "accuracy": 0.6709,
   "brier": 0.428,
   "nll": 0.8088,
   "ece": 0.0432
  },
  "workflow/bev_tr/Software security": {
   "n": 2240,
   "accuracy": 0.5161,
   "brier": 0.573,
   "nll": 0.9591,
   "ece": 0.034
  },
  "workflow/bev_tr/Spam detection": {
   "n": 271,
   "accuracy": 0.9852,
   "brier": 0.0443,
   "nll": 0.1076,
   "ece": 0.0481
  },
  "workflow/bev_tr/Spatial and logical reasoning": {
   "n": 8384,
   "accuracy": 0.5081,
   "brier": 0.621,
   "nll": 1.2295,
   "ece": 0.1028
  },
  "workflow/bev_tr/Support and intent routing": {
   "n": 2858,
   "accuracy": 0.8656,
   "brier": 0.1927,
   "nll": 0.3617,
   "ece": 0.023
  },
  "workflow/bev_tr/Tool and workflow decisions": {
   "n": 2307,
   "accuracy": 0.9515,
   "brier": 0.0784,
   "nll": 0.1804,
   "ece": 0.0202
  },
  "workflow/bev_tr_en/Browser interaction": {
   "n": 39,
   "accuracy": 0.7436,
   "brier": 0.332,
   "nll": 0.5797,
   "ece": 0.2032
  },
  "workflow/bev_tr_en/Civic and safety operations": {
   "n": 31,
   "accuracy": 0.2903,
   "brier": 0.7323,
   "nll": 1.3491,
   "ece": 0.2817
  },
  "workflow/bev_tr_en/Contract evidence": {
   "n": 49,
   "accuracy": 0.7959,
   "brier": 0.3294,
   "nll": 0.592,
   "ece": 0.1754
  },
  "workflow/bev_tr_en/Engagement and ranking": {
   "n": 6,
   "accuracy": 0.5,
   "brier": 0.5541,
   "nll": 0.7408,
   "ece": 0.2987
  },
  "workflow/bev_tr_en/Financial reporting and banking": {
   "n": 196,
   "accuracy": 0.6173,
   "brier": 0.5212,
   "nll": 1.0502,
   "ece": 0.1278
  },
  "workflow/bev_tr_en/Game-state decisions": {
   "n": 303,
   "accuracy": 0.7987,
   "brier": 0.2917,
   "nll": 0.477,
   "ece": 0.1155
  },
  "workflow/bev_tr_en/Reading comprehension": {
   "n": 168,
   "accuracy": 0.8988,
   "brier": 0.1561,
   "nll": 0.2729,
   "ece": 0.0442
  },
  "workflow/bev_tr_en/Response preference and quality": {
   "n": 389,
   "accuracy": 0.5733,
   "brier": 0.5404,
   "nll": 0.9819,
   "ece": 0.0858
  },
  "workflow/bev_tr_en/Retail, product, and shopping": {
   "n": 902,
   "accuracy": 0.6641,
   "brier": 0.3671,
   "nll": 0.6347,
   "ece": 0.0989
  },
  "workflow/bev_tr_en/Scientific and paper understanding": {
   "n": 66,
   "accuracy": 0.6818,
   "brier": 0.4373,
   "nll": 0.6896,
   "ece": 0.1184
  },
  "workflow/bev_tr_en/Sensory quality rating": {
   "n": 37,
   "accuracy": 0.2973,
   "brier": 0.8005,
   "nll": 1.645,
   "ece": 0.026
  },
  "workflow/bev_tr_en/Sentiment, emotion, and moderation": {
   "n": 1439,
   "accuracy": 0.8367,
   "brier": 0.2179,
   "nll": 0.4039,
   "ece": 0.0738
  },
  "workflow/bev_tr_en/Software engineering and code": {
   "n": 260,
   "accuracy": 0.6615,
   "brier": 0.4328,
   "nll": 0.7957,
   "ece": 0.0837
  },
  "workflow/bev_tr_en/Software security": {
   "n": 234,
   "accuracy": 0.4829,
   "brier": 0.5552,
   "nll": 0.9091,
   "ece": 0.0714
  },
  "workflow/bev_tr_en/Spam detection": {
   "n": 23,
   "accuracy": 1.0,
   "brier": 0.014,
   "nll": 0.0504,
   "ece": 0.0463
  },
  "workflow/bev_tr_en/Spatial and logical reasoning": {
   "n": 1042,
   "accuracy": 0.5979,
   "brier": 0.5265,
   "nll": 1.0567,
   "ece": 0.0797
  },
  "workflow/bev_tr_en/Support and intent routing": {
   "n": 310,
   "accuracy": 0.9032,
   "brier": 0.1216,
   "nll": 0.2252,
   "ece": 0.0401
  },
  "workflow/bev_tr_en/Tool and workflow decisions": {
   "n": 272,
   "accuracy": 0.9669,
   "brier": 0.0568,
   "nll": 0.1371,
   "ece": 0.0194
  },
  "workflow/beyazperde_tr": {
   "n": 5000,
   "accuracy": 0.5204,
   "brier": 0.5888,
   "nll": 1.0806,
   "ece": 0.0503
  },
  "workflow/jev_bench_tr/arc_challenge": {
   "n": 1000,
   "accuracy": 0.844,
   "brier": 0.2238,
   "nll": 0.4245,
   "ece": 0.0291
  },
  "workflow/jev_bench_tr/banking77": {
   "n": 1000,
   "accuracy": 0.786,
   "brier": 0.3131,
   "nll": 0.8613,
   "ece": 0.0587
  },
  "workflow/jev_bench_tr/boolq": {
   "n": 1000,
   "accuracy": 0.862,
   "brier": 0.2002,
   "nll": 0.3298,
   "ece": 0.0267
  },
  "workflow/jev_bench_tr/chaosnli": {
   "n": 1599,
   "accuracy": 0.5935,
   "brier": 0.2505,
   "nll": 1.2798,
   "ece": 0.1838
  },
  "workflow/jev_bench_tr/civil_comments": {
   "n": 2000,
   "accuracy": 0.9375,
   "brier": 0.0852,
   "nll": 0.326,
   "ece": 0.1395
  },
  "workflow/jev_bench_tr/clinc150": {
   "n": 1000,
   "accuracy": 0.877,
   "brier": 0.2026,
   "nll": 0.6939,
   "ece": 0.0488
  },
  "workflow/jev_bench_tr/fever_evidence": {
   "n": 1000,
   "accuracy": 0.909,
   "brier": 0.1436,
   "nll": 0.2527,
   "ece": 0.0384
  },
  "workflow/jev_bench_tr/go_emotions": {
   "n": 1000,
   "accuracy": 0.486,
   "brier": 0.3056,
   "nll": 2.2315,
   "ece": 0.077
  },
  "workflow/jev_bench_tr/helpsteer2_helpfulness": {
   "n": 1000,
   "accuracy": 0.432,
   "brier": 0.7229,
   "nll": 1.3629,
   "ece": 0.191
  },
  "workflow/jev_bench_tr/helpsteer2_verbosity": {
   "n": 1000,
   "accuracy": 0.668,
   "brier": 0.4887,
   "nll": 0.8986,
   "ece": 0.1154
  },
  "workflow/jev_bench_tr/ledgar": {
   "n": 1000,
   "accuracy": 0.694,
   "brier": 0.4402,
   "nll": 1.2495,
   "ece": 0.0355
  },
  "workflow/jev_bench_tr/massive": {
   "n": 1000,
   "accuracy": 0.832,
   "brier": 0.2553,
   "nll": 0.7084,
   "ece": 0.0477
  },
  "workflow/jev_bench_tr/measuring_hate_speech": {
   "n": 1000,
   "accuracy": 0.46,
   "brier": 0.3972,
   "nll": 1.0576,
   "ece": 0.1811
  },
  "workflow/jev_bench_tr/mmlu": {
   "n": 1000,
   "accuracy": 0.631,
   "brier": 0.4863,
   "nll": 0.9305,
   "ece": 0.0703
  },
  "workflow/jev_bench_tr/mnli": {
   "n": 1000,
   "accuracy": 0.821,
   "brier": 0.2604,
   "nll": 0.472,
   "ece": 0.0469
  },
  "workflow/jev_bench_tr/paws": {
   "n": 1000,
   "accuracy": 0.665,
   "brier": 0.4888,
   "nll": 0.7479,
   "ece": 0.1876
  },
  "workflow/jev_bench_tr/sms_spam": {
   "n": 800,
   "accuracy": 0.9762,
   "brier": 0.0415,
   "nll": 0.088,
   "ece": 0.0262
  },
  "workflow/jev_bench_tr/sst5": {
   "n": 1000,
   "accuracy": 0.477,
   "brier": 0.624,
   "nll": 1.1233,
   "ece": 0.034
  },
  "workflow/jev_bench_tr/strategyqa_closed": {
   "n": 687,
   "accuracy": 0.5531,
   "brier": 0.4841,
   "nll": 0.6782,
   "ece": 0.0609
  },
  "workflow/jev_bench_tr/strategyqa_grounded": {
   "n": 687,
   "accuracy": 0.8413,
   "brier": 0.2337,
   "nll": 0.3645,
   "ece": 0.0358
  },
  "workflow/jev_bench_tr/stsb": {
   "n": 1000,
   "accuracy": 0.509,
   "brier": 0.6394,
   "nll": 1.2233,
   "ece": 0.0989
  },
  "workflow/jev_bench_tr/yelp5": {
   "n": 1000,
   "accuracy": 0.652,
   "brier": 0.467,
   "nll": 0.7862,
   "ece": 0.0543
  },
  "workflow/jev_bench_tr_en/arc_challenge": {
   "n": 91,
   "accuracy": 0.9451,
   "brier": 0.0923,
   "nll": 0.1834,
   "ece": 0.0425
  },
  "workflow/jev_bench_tr_en/banking77": {
   "n": 71,
   "accuracy": 0.8592,
   "brier": 0.2192,
   "nll": 0.5844,
   "ece": 0.104
  },
  "workflow/jev_bench_tr_en/boolq": {
   "n": 80,
   "accuracy": 0.9125,
   "brier": 0.1191,
   "nll": 0.202,
   "ece": 0.0567
  },
  "workflow/jev_bench_tr_en/chaosnli": {
   "n": 151,
   "accuracy": 0.6755,
   "brier": 0.2455,
   "nll": 1.3287,
   "ece": 0.1913
  },
  "workflow/jev_bench_tr_en/civil_comments": {
   "n": 171,
   "accuracy": 0.8538,
   "brier": 0.1343,
   "nll": 0.3876,
   "ece": 0.0761
  },
  "workflow/jev_bench_tr_en/clinc150": {
   "n": 96,
   "accuracy": 0.9062,
   "brier": 0.1405,
   "nll": 0.3832,
   "ece": 0.0887
  },
  "workflow/jev_bench_tr_en/fever_evidence": {
   "n": 91,
   "accuracy": 0.956,
   "brier": 0.0721,
   "nll": 0.1511,
   "ece": 0.0373
  },
  "workflow/jev_bench_tr_en/go_emotions": {
   "n": 93,
   "accuracy": 0.5161,
   "brier": 0.283,
   "nll": 2.1674,
   "ece": 0.1568
  },
  "workflow/jev_bench_tr_en/helpsteer2_helpfulness": {
   "n": 89,
   "accuracy": 0.427,
   "brier": 0.6556,
   "nll": 1.1478,
   "ece": 0.1033
  },
  "workflow/jev_bench_tr_en/helpsteer2_verbosity": {
   "n": 79,
   "accuracy": 0.7089,
   "brier": 0.4528,
   "nll": 0.8678,
   "ece": 0.1142
  },
  "workflow/jev_bench_tr_en/ledgar": {
   "n": 70,
   "accuracy": 0.8,
   "brier": 0.3384,
   "nll": 0.8433,
   "ece": 0.1181
  },
  "workflow/jev_bench_tr_en/massive": {
   "n": 95,
   "accuracy": 0.8105,
   "brier": 0.2327,
   "nll": 0.5822,
   "ece": 0.0781
  },
  "workflow/jev_bench_tr_en/measuring_hate_speech": {
   "n": 96,
   "accuracy": 0.5312,
   "brier": 0.3995,
   "nll": 1.0836,
   "ece": 0.1889
  },
  "workflow/jev_bench_tr_en/mmlu": {
   "n": 80,
   "accuracy": 0.7625,
   "brier": 0.3231,
   "nll": 0.603,
   "ece": 0.1016
  },
  "workflow/jev_bench_tr_en/mnli": {
   "n": 92,
   "accuracy": 0.9457,
   "brier": 0.0929,
   "nll": 0.1728,
   "ece": 0.0731
  },
  "workflow/jev_bench_tr_en/paws": {
   "n": 80,
   "accuracy": 0.825,
   "brier": 0.2561,
   "nll": 0.413,
   "ece": 0.0911
  },
  "workflow/jev_bench_tr_en/sms_spam": {
   "n": 82,
   "accuracy": 0.9634,
   "brier": 0.0418,
   "nll": 0.0741,
   "ece": 0.0379
  },
  "workflow/jev_bench_tr_en/sst5": {
   "n": 73,
   "accuracy": 0.589,
   "brier": 0.5294,
   "nll": 0.8771,
   "ece": 0.1161
  },
  "workflow/jev_bench_tr_en/strategyqa_closed": {
   "n": 60,
   "accuracy": 0.8,
   "brier": 0.3278,
   "nll": 0.5048,
   "ece": 0.0974
  },
  "workflow/jev_bench_tr_en/strategyqa_grounded": {
   "n": 71,
   "accuracy": 0.9014,
   "brier": 0.1506,
   "nll": 0.2501,
   "ece": 0.0844
  },
  "workflow/jev_bench_tr_en/stsb": {
   "n": 90,
   "accuracy": 0.5556,
   "brier": 0.5762,
   "nll": 1.0837,
   "ece": 0.167
  },
  "workflow/jev_bench_tr_en/yelp5": {
   "n": 99,
   "accuracy": 0.7071,
   "brier": 0.3851,
   "nll": 0.6587,
   "ece": 0.1091
  },
  "workflow/massive_tr": {
   "n": 2974,
   "accuracy": 0.8285,
   "brier": 0.2533,
   "nll": 0.7069,
   "ece": 0.0362
  },
  "workflow/squad_tr": {
   "n": 11873,
   "accuracy": 0.6832,
   "brier": 0.4016,
   "nll": 0.585,
   "ece": 0.0488
  },
  "workflow/xnli_tr": {
   "n": 5010,
   "accuracy": 0.8128,
   "brier": 0.2741,
   "nll": 0.4991,
   "ece": 0.0356
  },
  "squad_tr/answerability": {
   "auroc": 0.8093,
   "acc_answerable": 0.9178,
   "acc_unanswerable": 0.439
  }
 }
}