{
 "model": "decider-4b-v2.1-tr-lora-tr-lora",
 "path": "/workspace/laya/answerability_run/model",
 "revision": "eb5fbdfc9448473ec25e399882912863afbdb70e",
 "fp8_layers": 0,
 "limit": 0,
 "skipped": {},
 "seconds": 502,
 "report": {
  "all": {
   "n": 16243,
   "accuracy": 0.8931,
   "brier": 0.1653,
   "nll": 0.2898,
   "ece": 0.0426
  },
  "source/hotpot_tr": {
   "n": 1500,
   "accuracy": 0.98,
   "brier": 0.032,
   "nll": 0.0605,
   "ece": 0.0063
  },
  "source/hotpot_tr/noul": {
   "n": 1500,
   "accuracy": 0.98,
   "brier": 0.032,
   "nll": 0.0605,
   "ece": 0.0063
  },
  "source/musique_tr": {
   "n": 500,
   "accuracy": 0.898,
   "brier": 0.1518,
   "nll": 0.2529,
   "ece": 0.0296
  },
  "source/musique_tr/noul": {
   "n": 500,
   "accuracy": 0.898,
   "brier": 0.1518,
   "nll": 0.2529,
   "ece": 0.0296
  },
  "source/squad_tr": {
   "n": 11873,
   "accuracy": 0.8614,
   "brier": 0.2146,
   "nll": 0.3759,
   "ece": 0.0572
  },
  "source/squad_tr/noul": {
   "n": 11873,
   "accuracy": 0.8614,
   "brier": 0.2146,
   "nll": 0.3759,
   "ece": 0.0572
  },
  "source/synth_tr": {
   "n": 1170,
   "accuracy": 0.9923,
   "brier": 0.0113,
   "nll": 0.023,
   "ece": 0.0038
  },
  "source/synth_tr/noul": {
   "n": 1170,
   "accuracy": 0.9923,
   "brier": 0.0113,
   "nll": 0.023,
   "ece": 0.0038
  },
  "source/wiki2_tr": {
   "n": 1200,
   "accuracy": 1.0,
   "brier": 0.0001,
   "nll": 0.0006,
   "ece": 0.0006
  },
  "source/wiki2_tr/noul": {
   "n": 1200,
   "accuracy": 1.0,
   "brier": 0.0001,
   "nll": 0.0006,
   "ece": 0.0006
  },
  "type/noul": {
   "n": 16243,
   "accuracy": 0.8931,
   "brier": 0.1653,
   "nll": 0.2898,
   "ece": 0.0426
  },
  "workflow/hotpot_tr/all_hops": {
   "n": 500,
   "accuracy": 0.97,
   "brier": 0.0527,
   "nll": 0.1011,
   "ece": 0.0126
  },
  "workflow/hotpot_tr/distractors_only": {
   "n": 500,
   "accuracy": 0.996,
   "brier": 0.0059,
   "nll": 0.01,
   "ece": 0.0065
  },
  "workflow/hotpot_tr/missing_hop": {
   "n": 500,
   "accuracy": 0.974,
   "brier": 0.0374,
   "nll": 0.0704,
   "ece": 0.0127
  },
  "workflow/musique_tr/all_hops": {
   "n": 250,
   "accuracy": 0.824,
   "brier": 0.2596,
   "nll": 0.4257,
   "ece": 0.0587
  },
  "workflow/musique_tr/missing_hop": {
   "n": 250,
   "accuracy": 0.972,
   "brier": 0.044,
   "nll": 0.0801,
   "ece": 0.0199
  },
  "workflow/squad_tr/answerability": {
   "n": 11873,
   "accuracy": 0.8614,
   "brier": 0.2146,
   "nll": 0.3759,
   "ece": 0.0572
  },
  "workflow/synth_tr/synth_answerable": {
   "n": 352,
   "accuracy": 0.983,
   "brier": 0.0226,
   "nll": 0.0439,
   "ece": 0.0118
  },
  "workflow/synth_tr/synth_eksik_oznitelik": {
   "n": 127,
   "accuracy": 0.9921,
   "brier": 0.0208,
   "nll": 0.0531,
   "ece": 0.014
  },
  "workflow/synth_tr/synth_kismi": {
   "n": 232,
   "accuracy": 0.9957,
   "brier": 0.0055,
   "nll": 0.0096,
   "ece": 0.0073
  },
  "workflow/synth_tr/synth_kurum": {
   "n": 113,
   "accuracy": 1.0,
   "brier": 0.0,
   "nll": 0.0005,
   "ece": 0.0005
  },
  "workflow/synth_tr/synth_kısmi": {
   "n": 2,
   "accuracy": 1.0,
   "brier": 0.0,
   "nll": 0.0001,
   "ece": 0.0001
  },
  "workflow/synth_tr/synth_varlik": {
   "n": 115,
   "accuracy": 0.9913,
   "brier": 0.0115,
   "nll": 0.016,
   "ece": 0.0056
  },
  "workflow/synth_tr/synth_varlık": {
   "n": 3,
   "accuracy": 1.0,
   "brier": 0.0,
   "nll": 0.0002,
   "ece": 0.0002
  },
  "workflow/synth_tr/synth_yer_facet": {
   "n": 113,
   "accuracy": 1.0,
   "brier": 0.0002,
   "nll": 0.0025,
   "ece": 0.0025
  },
  "workflow/synth_tr/synth_zaman": {
   "n": 113,
   "accuracy": 1.0,
   "brier": 0.0001,
   "nll": 0.0023,
   "ece": 0.0023
  },
  "workflow/wiki2_tr/other_entity_same_attribute": {
   "n": 600,
   "accuracy": 1.0,
   "brier": 0.0,
   "nll": 0.0002,
   "ece": 0.0002
  },
  "workflow/wiki2_tr/same_entity": {
   "n": 600,
   "accuracy": 1.0,
   "brier": 0.0003,
   "nll": 0.001,
   "ece": 0.0009
  },
  "hotpot_tr/answerability": {
   "n": 1500,
   "acc_answerable": 0.97,
   "acc_unanswerable": 0.984,
   "auroc": 0.997
  },
  "hotpot_tr/all_hops/answerability": {
   "n": 500,
   "acc_answerable": 0.97,
   "acc_unanswerable": null
  },
  "hotpot_tr/distractors_only/answerability": {
   "n": 500,
   "acc_answerable": null,
   "acc_unanswerable": 0.996
  },
  "hotpot_tr/missing_hop/answerability": {
   "n": 500,
   "acc_answerable": null,
   "acc_unanswerable": 0.972
  },
  "musique_tr/answerability": {
   "n": 500,
   "acc_answerable": 0.832,
   "acc_unanswerable": 0.972,
   "auroc": 0.9783
  },
  "musique_tr/all_hops/answerability": {
   "n": 250,
   "acc_answerable": 0.832,
   "acc_unanswerable": null
  },
  "musique_tr/missing_hop/answerability": {
   "n": 250,
   "acc_answerable": null,
   "acc_unanswerable": 0.972
  },
  "squad_tr/answerability": {
   "n": 11873,
   "acc_answerable": 0.9239,
   "acc_unanswerable": 0.7978,
   "auroc": 0.9346
  },
  "squad_tr/answerability/answerability": {
   "n": 11873,
   "acc_answerable": 0.9239,
   "acc_unanswerable": 0.7978,
   "auroc": 0.9346
  },
  "synth_tr/answerability": {
   "n": 1170,
   "acc_answerable": 0.983,
   "acc_unanswerable": 0.9963,
   "auroc": 0.9996
  },
  "synth_tr/synth_answerable/answerability": {
   "n": 352,
   "acc_answerable": 0.983,
   "acc_unanswerable": null
  },
  "synth_tr/synth_eksik_oznitelik/answerability": {
   "n": 127,
   "acc_answerable": null,
   "acc_unanswerable": 0.9921
  },
  "synth_tr/synth_kismi/answerability": {
   "n": 232,
   "acc_answerable": null,
   "acc_unanswerable": 0.9957
  },
  "synth_tr/synth_kurum/answerability": {
   "n": 113,
   "acc_answerable": null,
   "acc_unanswerable": 1.0
  },
  "synth_tr/synth_kısmi/answerability": {
   "n": 2,
   "acc_answerable": null,
   "acc_unanswerable": 1.0
  },
  "synth_tr/synth_varlik/answerability": {
   "n": 115,
   "acc_answerable": null,
   "acc_unanswerable": 0.9913
  },
  "synth_tr/synth_varlık/answerability": {
   "n": 3,
   "acc_answerable": null,
   "acc_unanswerable": 1.0
  },
  "synth_tr/synth_yer_facet/answerability": {
   "n": 113,
   "acc_answerable": null,
   "acc_unanswerable": 1.0
  },
  "synth_tr/synth_zaman/answerability": {
   "n": 113,
   "acc_answerable": null,
   "acc_unanswerable": 1.0
  },
  "wiki2_tr/answerability": {
   "n": 1200,
   "acc_answerable": 1.0,
   "acc_unanswerable": 1.0,
   "auroc": 1.0
  },
  "wiki2_tr/other_entity_same_attribute/answerability": {
   "n": 600,
   "acc_answerable": null,
   "acc_unanswerable": 1.0
  },
  "wiki2_tr/same_entity/answerability": {
   "n": 600,
   "acc_answerable": 1.0,
   "acc_unanswerable": null
  }
 },
 "train_key": "dc5d693207f19271"
}