{ "model": "decider-4b-v2.1-tr-lora", "path": "hayriyigit/decider-4b-tr", "revision": "df9aacb9f4c9e4e64178176502f51ae4db271f26", "fp8_layers": 0, "limit": 0, "skipped": {}, "seconds": 579, "report": { "all": { "n": 16243, "accuracy": 0.8601, "brier": 0.2038, "nll": 0.3316, "ece": 0.0234 }, "source/hotpot_tr": { "n": 1500, "accuracy": 0.806, "brier": 0.2812, "nll": 0.4504, "ece": 0.0652 }, "source/hotpot_tr/noul": { "n": 1500, "accuracy": 0.806, "brier": 0.2812, "nll": 0.4504, "ece": 0.0652 }, "source/musique_tr": { "n": 500, "accuracy": 0.634, "brier": 0.523, "nll": 0.8011, "ece": 0.1863 }, "source/musique_tr/noul": { "n": 500, "accuracy": 0.634, "brier": 0.523, "nll": 0.8011, "ece": 0.1863 }, "source/squad_tr": { "n": 11873, "accuracy": 0.8617, "brier": 0.2028, "nll": 0.3322, "ece": 0.0159 }, "source/squad_tr/noul": { "n": 11873, "accuracy": 0.8617, "brier": 0.2028, "nll": 0.3322, "ece": 0.0159 }, "source/synth_tr": { "n": 1170, "accuracy": 0.906, "brier": 0.1319, "nll": 0.2137, "ece": 0.0275 }, "source/synth_tr/noul": { "n": 1170, "accuracy": 0.906, "brier": 0.1319, "nll": 0.2137, "ece": 0.0275 }, "source/wiki2_tr": { "n": 1200, "accuracy": 0.9617, "brier": 0.0538, "nll": 0.0966, "ece": 0.0221 }, "source/wiki2_tr/noul": { "n": 1200, "accuracy": 0.9617, "brier": 0.0538, "nll": 0.0966, "ece": 0.0221 }, "type/noul": { "n": 16243, "accuracy": 0.8601, "brier": 0.2038, "nll": 0.3316, "ece": 0.0234 }, "workflow/hotpot_tr/all_hops": { "n": 500, "accuracy": 0.616, "brier": 0.5436, "nll": 0.8223, "ece": 0.2049 }, "workflow/hotpot_tr/distractors_only": { "n": 500, "accuracy": 0.952, "brier": 0.0703, "nll": 0.1353, "ece": 0.0447 }, "workflow/hotpot_tr/missing_hop": { "n": 500, "accuracy": 0.85, "brier": 0.2297, "nll": 0.3936, "ece": 0.0512 }, "workflow/musique_tr/all_hops": { "n": 250, "accuracy": 0.36, "brier": 0.9114, "nll": 1.371, "ece": 0.4278 }, "workflow/musique_tr/missing_hop": { "n": 250, "accuracy": 0.908, "brier": 0.1345, "nll": 0.2311, "ece": 0.0685 }, "workflow/squad_tr/answerability": { "n": 11873, "accuracy": 0.8617, "brier": 0.2028, "nll": 0.3322, "ece": 0.0159 }, "workflow/synth_tr/synth_answerable": { "n": 352, "accuracy": 0.9972, "brier": 0.0062, "nll": 0.0253, "ece": 0.0232 }, "workflow/synth_tr/synth_eksik_oznitelik": { "n": 127, "accuracy": 0.9134, "brier": 0.1201, "nll": 0.193, "ece": 0.0363 }, "workflow/synth_tr/synth_kismi": { "n": 232, "accuracy": 0.8103, "brier": 0.2776, "nll": 0.4405, "ece": 0.0786 }, "workflow/synth_tr/synth_kurum": { "n": 113, "accuracy": 0.7788, "brier": 0.2835, "nll": 0.4126, "ece": 0.1197 }, "workflow/synth_tr/synth_k\u0131smi": { "n": 2, "accuracy": 1.0, "brier": 0.0007, "nll": 0.0162, "ece": 0.0161 }, "workflow/synth_tr/synth_varlik": { "n": 115, "accuracy": 0.913, "brier": 0.1032, "nll": 0.168, "ece": 0.0349 }, "workflow/synth_tr/synth_varl\u0131k": { "n": 3, "accuracy": 1.0, "brier": 0.0195, "nll": 0.0668, "ece": 0.0613 }, "workflow/synth_tr/synth_yer_facet": { "n": 113, "accuracy": 0.9292, "brier": 0.1065, "nll": 0.1809, "ece": 0.0365 }, "workflow/synth_tr/synth_zaman": { "n": 113, "accuracy": 0.9027, "brier": 0.1455, "nll": 0.2456, "ece": 0.0386 }, "workflow/wiki2_tr/other_entity_same_attribute": { "n": 600, "accuracy": 0.9367, "brier": 0.089, "nll": 0.1505, "ece": 0.0317 }, "workflow/wiki2_tr/same_entity": { "n": 600, "accuracy": 0.9867, "brier": 0.0186, "nll": 0.0426, "ece": 0.0192 }, "hotpot_tr/answerability": { "n": 1500, "acc_answerable": 0.618, "acc_unanswerable": 0.899, "auroc": 0.8713 }, "hotpot_tr/all_hops/answerability": { "n": 500, "acc_answerable": 0.618, "acc_unanswerable": null }, "hotpot_tr/distractors_only/answerability": { "n": 500, "acc_answerable": null, "acc_unanswerable": 0.952 }, "hotpot_tr/missing_hop/answerability": { "n": 500, "acc_answerable": null, "acc_unanswerable": 0.846 }, "musique_tr/answerability": { "n": 500, "acc_answerable": 0.376, "acc_unanswerable": 0.908, "auroc": 0.7513 }, "musique_tr/all_hops/answerability": { "n": 250, "acc_answerable": 0.376, "acc_unanswerable": null }, "musique_tr/missing_hop/answerability": { "n": 250, "acc_answerable": null, "acc_unanswerable": 0.908 }, "squad_tr/answerability": { "n": 11873, "acc_answerable": 0.9374, "acc_unanswerable": 0.7827, "auroc": 0.9398 }, "squad_tr/answerability/answerability": { "n": 11873, "acc_answerable": 0.9374, "acc_unanswerable": 0.7827, "auroc": 0.9398 }, "synth_tr/answerability": { "n": 1170, "acc_answerable": 0.9972, "acc_unanswerable": 0.8619, "auroc": 0.996 }, "synth_tr/synth_answerable/answerability": { "n": 352, "acc_answerable": 0.9972, "acc_unanswerable": null }, "synth_tr/synth_eksik_oznitelik/answerability": { "n": 127, "acc_answerable": null, "acc_unanswerable": 0.9134 }, "synth_tr/synth_kismi/answerability": { "n": 232, "acc_answerable": null, "acc_unanswerable": 0.7974 }, "synth_tr/synth_kurum/answerability": { "n": 113, "acc_answerable": null, "acc_unanswerable": 0.7788 }, "synth_tr/synth_k\u0131smi/answerability": { "n": 2, "acc_answerable": null, "acc_unanswerable": 1.0 }, "synth_tr/synth_varlik/answerability": { "n": 115, "acc_answerable": null, "acc_unanswerable": 0.913 }, "synth_tr/synth_varl\u0131k/answerability": { "n": 3, "acc_answerable": null, "acc_unanswerable": 1.0 }, "synth_tr/synth_yer_facet/answerability": { "n": 113, "acc_answerable": null, "acc_unanswerable": 0.9292 }, "synth_tr/synth_zaman/answerability": { "n": 113, "acc_answerable": null, "acc_unanswerable": 0.8938 }, "wiki2_tr/answerability": { "n": 1200, "acc_answerable": 0.9867, "acc_unanswerable": 0.935, "auroc": 0.997 }, "wiki2_tr/other_entity_same_attribute/answerability": { "n": 600, "acc_answerable": null, "acc_unanswerable": 0.935 }, "wiki2_tr/same_entity/answerability": { "n": 600, "acc_answerable": 0.9867, "acc_unanswerable": null } } }