| { |
| "reference": "BF16 source, layer-streamed", |
| "format": "native generation format: every assistant </think> is followed by the newline-newline token; the decision is measured on the token after it", |
| "agentic_ref": { |
| "sequences": 96, |
| "positions": 65719, |
| "median_kl": 0.39468857645988464, |
| "mean_kl": 1.053911805152893, |
| "p90": 2.7632646560668945, |
| "p95": 4.292669773101807, |
| "p99": 9.689709663391113, |
| "kl_max": 27.009990692138672, |
| "top1_pct": 51.24697576043458, |
| "top5_pct": 75.1335230298696, |
| "top10_pct": 81.79065414872412, |
| "text_ppl_q": 1896.9827880859375, |
| "text_ppl_ref": 1744.8138427734375, |
| "text_ppl_ratio": 1.0872116088867188, |
| "text_nll_delta_mean": 0.08361628651618958, |
| "text_nll_delta_sem": 0.006842579407496338, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 18.089137083644, |
| "text_top1_acc_ref_pct": 18.764740790334606, |
| "call_positions": 8544, |
| "call_nll_q": 5.43035888671875, |
| "call_nll_ref": 5.388685703277588, |
| "call_nll_delta": 0.041673433035612106, |
| "call_nll_delta_sem": 0.020003612910480514, |
| "call_top1_acc_q_pct": 42.017790262172284, |
| "call_top1_acc_ref_pct": 42.017790262172284, |
| "call_p_correct_q_geomean": 0.004381523001939058, |
| "call_p_correct_ref_geomean": 0.0045679728500545025, |
| "per_domain": { |
| "tool_conv": { |
| "sequences": 96, |
| "positions": 65719, |
| "median_kl": 0.39468857645988464, |
| "mean_kl": 1.053911805152893, |
| "p90": 2.7632646560668945, |
| "p95": 4.292669773101807, |
| "p99": 9.689709663391113, |
| "kl_max": 27.009990692138672, |
| "top1_pct": 51.24697576043458, |
| "top5_pct": 75.1335230298696, |
| "top10_pct": 81.79065414872412, |
| "text_ppl_q": 1896.9827880859375, |
| "text_ppl_ref": 1744.8138427734375, |
| "text_ppl_ratio": 1.0872116088867188, |
| "text_nll_delta_mean": 0.08361628651618958, |
| "text_nll_delta_sem": 0.006842579407496338, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 18.089137083644, |
| "text_top1_acc_ref_pct": 18.764740790334606, |
| "call_positions": 8544, |
| "call_nll_q": 5.43035888671875, |
| "call_nll_ref": 5.388685703277588, |
| "call_nll_delta": 0.041673433035612106, |
| "call_nll_delta_sem": 0.020003612910480514, |
| "call_top1_acc_q_pct": 42.017790262172284, |
| "call_top1_acc_ref_pct": 42.017790262172284, |
| "call_p_correct_q_geomean": 0.004381523001939058, |
| "call_p_correct_ref_geomean": 0.0045679728500545025 |
| } |
| }, |
| "prefill_tok_per_s_full_forward": 507.23925530066384, |
| "reference": "BF16 (layer-streamed source)", |
| "decisions": { |
| "call": { |
| "n": 112, |
| "argmax_agree_pct": 93.75, |
| "call_answer_flips": 7, |
| "reference_starts_tool_call": 100, |
| "model_starts_tool_call": 101, |
| "median_p_toolcall_ref": 0.8585168123245239, |
| "median_p_toolcall_q": 0.8894042670726776, |
| "min_p_toolcall_q": 0.13265414535999298, |
| "max_p_toolcall_q": 0.9932858943939209 |
| }, |
| "answer": { |
| "n": 96, |
| "argmax_agree_pct": 88.54166666666667, |
| "call_answer_flips": 0, |
| "reference_starts_tool_call": 1, |
| "model_starts_tool_call": 1, |
| "median_p_toolcall_ref": 0.0023557787062600255, |
| "median_p_toolcall_q": 0.005304558901116252, |
| "min_p_toolcall_q": 0.00010572261817287654, |
| "max_p_toolcall_q": 0.2591971457004547 |
| } |
| } |
| } |
| } |
