| { |
| "reference": "BF16 source, layer-streamed", |
| "mode": "served (2048-token prefill chunks, runtime caches)", |
| "note": "text_* fields are the deviation from the reference on documents rendered inside user turns; they are a fidelity number, not a quality metric", |
| "klref": { |
| "sequences": 28, |
| "positions": 68370, |
| "median_kl": 0.1704166978597641, |
| "mean_kl": 1.0084285736083984, |
| "p90": 3.112718105316162, |
| "p95": 5.369931697845459, |
| "p99": 10.504664421081543, |
| "kl_max": 24.663476943969727, |
| "top1_pct": 64.3513236799766, |
| "top5_pct": 81.47140558724587, |
| "top10_pct": 85.58432060845401, |
| "text_ppl_q": 87.12694549560547, |
| "text_ppl_ref": 64.08563232421875, |
| "text_ppl_ratio": 1.3595391511917114, |
| "text_nll_delta_mean": 0.3071458041667938, |
| "text_nll_delta_sem": 0.008856940366766429, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 42.59616790990201, |
| "text_top1_acc_ref_pct": 44.99195553605382, |
| "per_domain": { |
| "academic_mc": { |
| "sequences": 2, |
| "positions": 3123, |
| "median_kl": 0.21307168900966644, |
| "mean_kl": 0.8185907602310181, |
| "p90": 2.195399522781372, |
| "p95": 4.076174259185791, |
| "p99": 9.240348815917969, |
| "kl_max": 15.55548095703125, |
| "top1_pct": 62.343900096061475, |
| "top5_pct": 83.09317963496638, |
| "top10_pct": 88.3445405059238, |
| "text_ppl_q": 121.92414093017578, |
| "text_ppl_ref": 83.09352111816406, |
| "text_ppl_ratio": 1.4673125743865967, |
| "text_nll_delta_mean": 0.3834325075149536, |
| "text_nll_delta_sem": 0.03734057597673582, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 35.510726865193725, |
| "text_top1_acc_ref_pct": 38.58469420429075 |
| }, |
| "agentic": { |
| "sequences": 4, |
| "positions": 7411, |
| "median_kl": 0.23808088898658752, |
| "mean_kl": 1.1584773063659668, |
| "p90": 3.5633037090301514, |
| "p95": 5.776463985443115, |
| "p99": 10.857256889343262, |
| "kl_max": 19.707765579223633, |
| "top1_pct": 60.38321414114155, |
| "top5_pct": 77.97868034003508, |
| "top10_pct": 82.52597490217245, |
| "text_ppl_q": 197.68115234375, |
| "text_ppl_ref": 144.7749786376953, |
| "text_ppl_ratio": 1.3654372692108154, |
| "text_nll_delta_mean": 0.3114747405052185, |
| "text_nll_delta_sem": 0.02703807786033071, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 38.15949264606665, |
| "text_top1_acc_ref_pct": 40.08905680744839 |
| }, |
| "chinese": { |
| "sequences": 2, |
| "positions": 3158, |
| "median_kl": 0.1715339720249176, |
| "mean_kl": 0.8395028710365295, |
| "p90": 2.2756786346435547, |
| "p95": 4.545201778411865, |
| "p99": 9.803081512451172, |
| "kl_max": 17.90928077697754, |
| "top1_pct": 65.13616212792907, |
| "top5_pct": 84.26219126029132, |
| "top10_pct": 88.22039265357822, |
| "text_ppl_q": 218.26402282714844, |
| "text_ppl_ref": 139.631103515625, |
| "text_ppl_ratio": 1.563148021697998, |
| "text_nll_delta_mean": 0.4467017352581024, |
| "text_nll_delta_sem": 0.04302379258395983, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 33.69221025965801, |
| "text_top1_acc_ref_pct": 35.718809373020896 |
| }, |
| "coding": { |
| "sequences": 5, |
| "positions": 8249, |
| "median_kl": 0.08199650049209595, |
| "mean_kl": 0.9424503445625305, |
| "p90": 3.0316593647003174, |
| "p95": 5.3630523681640625, |
| "p99": 10.516594886779785, |
| "kl_max": 19.948368072509766, |
| "top1_pct": 68.83258576797188, |
| "top5_pct": 83.64650260637653, |
| "top10_pct": 87.11358952600315, |
| "text_ppl_q": 56.27544021606445, |
| "text_ppl_ref": 48.28934097290039, |
| "text_ppl_ratio": 1.1653801202774048, |
| "text_nll_delta_mean": 0.1530473530292511, |
| "text_nll_delta_sem": 0.023818490760019757, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 47.957328160989206, |
| "text_top1_acc_ref_pct": 49.399927263910776 |
| }, |
| "cybersec": { |
| "sequences": 4, |
| "positions": 6653, |
| "median_kl": 0.05261153727769852, |
| "mean_kl": 0.7892881035804749, |
| "p90": 2.4670655727386475, |
| "p95": 4.8032917976379395, |
| "p99": 9.959638595581055, |
| "kl_max": 18.70598030090332, |
| "top1_pct": 73.01969036524876, |
| "top5_pct": 86.38208327070494, |
| "top10_pct": 89.07259882759658, |
| "text_ppl_q": 50.75177764892578, |
| "text_ppl_ref": 40.27240753173828, |
| "text_ppl_ratio": 1.2602119445800781, |
| "text_nll_delta_mean": 0.23127995431423187, |
| "text_nll_delta_sem": 0.024030208570325735, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 50.72899443859913, |
| "text_top1_acc_ref_pct": 52.72809258980911 |
| }, |
| "general": { |
| "sequences": 3, |
| "positions": 5311, |
| "median_kl": 0.2415456622838974, |
| "mean_kl": 1.014784336090088, |
| "p90": 3.076566457748413, |
| "p95": 5.115887641906738, |
| "p99": 10.112685203552246, |
| "kl_max": 16.833053588867188, |
| "top1_pct": 60.6477122952363, |
| "top5_pct": 80.64394652607795, |
| "top10_pct": 84.59800414234607, |
| "text_ppl_q": 191.92210388183594, |
| "text_ppl_ref": 122.87806701660156, |
| "text_ppl_ratio": 1.5618903636932373, |
| "text_nll_delta_mean": 0.44589686393737793, |
| "text_nll_delta_sem": 0.03573012642731573, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 36.057239691206924, |
| "text_top1_acc_ref_pct": 39.20165693842967 |
| }, |
| "long_agentic": { |
| "sequences": 2, |
| "positions": 10190, |
| "median_kl": 0.2484634965658188, |
| "mean_kl": 1.3893784284591675, |
| "p90": 4.4491071701049805, |
| "p95": 6.867280960083008, |
| "p99": 11.967666625976562, |
| "kl_max": 23.30169677734375, |
| "top1_pct": 58.96957801766438, |
| "top5_pct": 75.44651619234544, |
| "top10_pct": 79.96074582924436, |
| "text_ppl_q": 100.91949462890625, |
| "text_ppl_ref": 77.45440673828125, |
| "text_ppl_ratio": 1.3029543161392212, |
| "text_nll_delta_mean": 0.2646341919898987, |
| "text_nll_delta_sem": 0.027041340773589443, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 43.49362119725221, |
| "text_top1_acc_ref_pct": 45.37782139352306 |
| }, |
| "long_coding": { |
| "sequences": 2, |
| "positions": 9390, |
| "median_kl": 0.08500003814697266, |
| "mean_kl": 0.8798364400863647, |
| "p90": 2.7945637702941895, |
| "p95": 5.145983695983887, |
| "p99": 9.630831718444824, |
| "kl_max": 24.311695098876953, |
| "top1_pct": 69.86155484558041, |
| "top5_pct": 83.87646432374866, |
| "top10_pct": 87.39084132055378, |
| "text_ppl_q": 41.933414459228516, |
| "text_ppl_ref": 37.08015441894531, |
| "text_ppl_ratio": 1.1308856010437012, |
| "text_nll_delta_mean": 0.1230010911822319, |
| "text_nll_delta_sem": 0.02238255679146158, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 51.15015974440894, |
| "text_top1_acc_ref_pct": 52.51331203407881 |
| }, |
| "long_longctx": { |
| "sequences": 2, |
| "positions": 10990, |
| "median_kl": 0.22215455770492554, |
| "mean_kl": 0.8677619099617004, |
| "p90": 2.328246831893921, |
| "p95": 4.105473041534424, |
| "p99": 9.772666931152344, |
| "kl_max": 24.663476943969727, |
| "top1_pct": 63.366696997270246, |
| "top5_pct": 83.31210191082803, |
| "top10_pct": 87.96178343949045, |
| "text_ppl_q": 74.3593521118164, |
| "text_ppl_ref": 48.85396957397461, |
| "text_ppl_ratio": 1.5220731496810913, |
| "text_nll_delta_mean": 0.42007333040237427, |
| "text_nll_delta_sem": 0.020436424192156313, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 38.28935395814377, |
| "text_top1_acc_ref_pct": 42.029117379435846 |
| }, |
| "science": { |
| "sequences": 2, |
| "positions": 3895, |
| "median_kl": 0.3555840253829956, |
| "mean_kl": 1.227760672569275, |
| "p90": 3.644615650177002, |
| "p95": 5.74046516418457, |
| "p99": 10.63033390045166, |
| "kl_max": 21.231704711914062, |
| "top1_pct": 57.201540436457, |
| "top5_pct": 77.45827984595635, |
| "top10_pct": 82.84980744544288, |
| "text_ppl_q": 89.30754089355469, |
| "text_ppl_ref": 47.66631317138672, |
| "text_ppl_ratio": 1.8735977411270142, |
| "text_nll_delta_mean": 0.6278604865074158, |
| "text_nll_delta_sem": 0.038523879100658555, |
| "text_positions_compared_pct": 100.0, |
| "text_ppl_unbiased": true, |
| "text_top1_acc_q_pct": 36.79075738125802, |
| "text_top1_acc_ref_pct": 41.54043645699615 |
| } |
| }, |
| "prefill_tok_per_s_full_forward": 596.8083549490221, |
| "reference": "BF16 (layer-streamed source)" |
| }, |
| "peak_memory_gib": 104.80185685772449 |
| } |
