{
 "reference": "BF16 source, layer-streamed",
 "format": "native generation format: every assistant </think> is followed by the newline-newline token; the decision is measured on the token after it",
 "agentic_ref": {
  "sequences": 96,
  "positions": 65719,
  "median_kl": 0.39468857645988464,
  "mean_kl": 1.053911805152893,
  "p90": 2.7632646560668945,
  "p95": 4.292669773101807,
  "p99": 9.689709663391113,
  "kl_max": 27.009990692138672,
  "top1_pct": 51.24697576043458,
  "top5_pct": 75.1335230298696,
  "top10_pct": 81.79065414872412,
  "text_ppl_q": 1896.9827880859375,
  "text_ppl_ref": 1744.8138427734375,
  "text_ppl_ratio": 1.0872116088867188,
  "text_nll_delta_mean": 0.08361628651618958,
  "text_nll_delta_sem": 0.006842579407496338,
  "text_positions_compared_pct": 100.0,
  "text_ppl_unbiased": true,
  "text_top1_acc_q_pct": 18.089137083644,
  "text_top1_acc_ref_pct": 18.764740790334606,
  "call_positions": 8544,
  "call_nll_q": 5.43035888671875,
  "call_nll_ref": 5.388685703277588,
  "call_nll_delta": 0.041673433035612106,
  "call_nll_delta_sem": 0.020003612910480514,
  "call_top1_acc_q_pct": 42.017790262172284,
  "call_top1_acc_ref_pct": 42.017790262172284,
  "call_p_correct_q_geomean": 0.004381523001939058,
  "call_p_correct_ref_geomean": 0.0045679728500545025,
  "per_domain": {
   "tool_conv": {
    "sequences": 96,
    "positions": 65719,
    "median_kl": 0.39468857645988464,
    "mean_kl": 1.053911805152893,
    "p90": 2.7632646560668945,
    "p95": 4.292669773101807,
    "p99": 9.689709663391113,
    "kl_max": 27.009990692138672,
    "top1_pct": 51.24697576043458,
    "top5_pct": 75.1335230298696,
    "top10_pct": 81.79065414872412,
    "text_ppl_q": 1896.9827880859375,
    "text_ppl_ref": 1744.8138427734375,
    "text_ppl_ratio": 1.0872116088867188,
    "text_nll_delta_mean": 0.08361628651618958,
    "text_nll_delta_sem": 0.006842579407496338,
    "text_positions_compared_pct": 100.0,
    "text_ppl_unbiased": true,
    "text_top1_acc_q_pct": 18.089137083644,
    "text_top1_acc_ref_pct": 18.764740790334606,
    "call_positions": 8544,
    "call_nll_q": 5.43035888671875,
    "call_nll_ref": 5.388685703277588,
    "call_nll_delta": 0.041673433035612106,
    "call_nll_delta_sem": 0.020003612910480514,
    "call_top1_acc_q_pct": 42.017790262172284,
    "call_top1_acc_ref_pct": 42.017790262172284,
    "call_p_correct_q_geomean": 0.004381523001939058,
    "call_p_correct_ref_geomean": 0.0045679728500545025
   }
  },
  "prefill_tok_per_s_full_forward": 507.23925530066384,
  "reference": "BF16 (layer-streamed source)",
  "decisions": {
   "call": {
    "n": 112,
    "argmax_agree_pct": 93.75,
    "call_answer_flips": 7,
    "reference_starts_tool_call": 100,
    "model_starts_tool_call": 101,
    "median_p_toolcall_ref": 0.8585168123245239,
    "median_p_toolcall_q": 0.8894042670726776,
    "min_p_toolcall_q": 0.13265414535999298,
    "max_p_toolcall_q": 0.9932858943939209
   },
   "answer": {
    "n": 96,
    "argmax_agree_pct": 88.54166666666667,
    "call_answer_flips": 0,
    "reference_starts_tool_call": 1,
    "model_starts_tool_call": 1,
    "median_p_toolcall_ref": 0.0023557787062600255,
    "median_p_toolcall_q": 0.005304558901116252,
    "min_p_toolcall_q": 0.00010572261817287654,
    "max_p_toolcall_q": 0.2591971457004547
   }
  }
 }
}