| { |
| "performance": { |
| "scope": "Single matched final IU4 two-arm block, not independent temporal mirrors or standalone per-patch effects. Fixed-token speed probes; no quality score. Baseline B2 already contains S5 and D0.", |
| "cases": [ |
| { |
| "case": "incident-r0", |
| "baseline_tg": 37.7877302722166, |
| "candidate_tg": 44.53075218801981, |
| "tg_pct": 17.84447456152458, |
| "baseline_pp": 824.2772548132926, |
| "candidate_pp": 889.1267211393995, |
| "cross_arm_text_exact": true, |
| "candidate_canonical_text_exact": true, |
| "prompt_work_comparable": true, |
| "baseline_timings": { |
| "cache_n": 0, |
| "prompt_n": 12960, |
| "prompt_ms": 15722.865, |
| "prompt_per_token_ms": 1.2131840277777777, |
| "prompt_per_second": 824.2772548132926, |
| "predicted_n": 512, |
| "predicted_ms": 13522.908, |
| "predicted_per_token_ms": 26.463616438356162, |
| "predicted_per_second": 37.7877302722166, |
| "draft_n": 469, |
| "draft_n_accepted": 353 |
| }, |
| "candidate_timings": { |
| "cache_n": 0, |
| "prompt_n": 12960, |
| "prompt_ms": 14576.1, |
| "prompt_per_token_ms": 1.124699074074074, |
| "prompt_per_second": 889.1267211393995, |
| "predicted_n": 512, |
| "predicted_ms": 11475.216, |
| "predicted_per_token_ms": 22.456391389432486, |
| "predicted_per_second": 44.53075218801981, |
| "draft_n": 469, |
| "draft_n_accepted": 353 |
| }, |
| "baseline_memory": { |
| "idle": { |
| "t": 1789602929.0682585, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 104068890624, |
| "ram_available_bytes": 30240776192, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789602958.3758624, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108090400768, |
| "ram_available_bytes": 26219266048, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 119, |
| "peak_ram_used_bytes": 108330250240, |
| "delta_ram_used_bytes": 4261359616 |
| }, |
| "candidate_memory": { |
| "idle": { |
| "t": 1789603503.0725048, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 104604065792, |
| "ram_available_bytes": 29705601024, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603529.186254, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108212645888, |
| "ram_available_bytes": 26097020928, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 106, |
| "peak_ram_used_bytes": 108486135808, |
| "delta_ram_used_bytes": 3882070016 |
| }, |
| "pp_pct": 7.867433675674573, |
| "ttfp_ms": { |
| "baseline": 15783.607006072998, |
| "candidate": 14637.105464935303, |
| "change_pct": -7.263875365729522 |
| }, |
| "total_ms": { |
| "baseline": 29307.98888206482, |
| "candidate": 26114.10355567932, |
| "change_pct": -10.897661177768537 |
| } |
| }, |
| { |
| "case": "A1-r0", |
| "baseline_tg": 39.79633018531356, |
| "candidate_tg": 43.83965208852103, |
| "tg_pct": 10.160037079749685, |
| "baseline_pp": 917.0627147153697, |
| "candidate_pp": 951.3390182805041, |
| "cross_arm_text_exact": true, |
| "candidate_canonical_text_exact": true, |
| "prompt_work_comparable": true, |
| "baseline_timings": { |
| "cache_n": 0, |
| "prompt_n": 5247, |
| "prompt_ms": 5721.528, |
| "prompt_per_token_ms": 1.0904379645511721, |
| "prompt_per_second": 917.0627147153697, |
| "predicted_n": 256, |
| "predicted_ms": 6407.626, |
| "predicted_per_token_ms": 25.127945098039216, |
| "predicted_per_second": 39.79633018531356, |
| "draft_n": 243, |
| "draft_n_accepted": 173 |
| }, |
| "candidate_timings": { |
| "cache_n": 0, |
| "prompt_n": 5247, |
| "prompt_ms": 5515.384, |
| "prompt_per_token_ms": 1.051149990470745, |
| "prompt_per_second": 951.3390182805041, |
| "predicted_n": 256, |
| "predicted_ms": 5816.652, |
| "predicted_per_token_ms": 22.8104, |
| "predicted_per_second": 43.83965208852103, |
| "draft_n": 243, |
| "draft_n_accepted": 173 |
| }, |
| "baseline_memory": { |
| "idle": { |
| "t": 1789602958.399876, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108089262080, |
| "ram_available_bytes": 26220404736, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789602970.6784027, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109161226240, |
| "ram_available_bytes": 25148440576, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 51, |
| "peak_ram_used_bytes": 109244203008, |
| "delta_ram_used_bytes": 1154940928 |
| }, |
| "candidate_memory": { |
| "idle": { |
| "t": 1789603529.2159796, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108211982336, |
| "ram_available_bytes": 26097684480, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603540.6942146, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109445541888, |
| "ram_available_bytes": 24864124928, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 47, |
| "peak_ram_used_bytes": 109457702912, |
| "delta_ram_used_bytes": 1245720576 |
| }, |
| "pp_pct": 3.7376182691903237, |
| "ttfp_ms": { |
| "baseline": 5869.6653842926025, |
| "candidate": 5659.878969192505, |
| "change_pct": -3.5740779306004833 |
| }, |
| "total_ms": { |
| "baseline": 12278.690814971924, |
| "candidate": 11478.629350662231, |
| "change_pct": -6.5158531668062185 |
| } |
| }, |
| { |
| "case": "B1-r0", |
| "baseline_tg": 41.541519445422345, |
| "candidate_tg": 45.189249920475774, |
| "tg_pct": 8.780926946704138, |
| "baseline_pp": 932.4374683123201, |
| "candidate_pp": 970.7405083150297, |
| "cross_arm_text_exact": true, |
| "candidate_canonical_text_exact": true, |
| "prompt_work_comparable": true, |
| "baseline_timings": { |
| "cache_n": 0, |
| "prompt_n": 5247, |
| "prompt_ms": 5627.187, |
| "prompt_per_token_ms": 1.0724579759862778, |
| "prompt_per_second": 932.4374683123201, |
| "predicted_n": 256, |
| "predicted_ms": 6138.437, |
| "predicted_per_token_ms": 24.072301960784312, |
| "predicted_per_second": 41.541519445422345, |
| "draft_n": 230, |
| "draft_n_accepted": 178 |
| }, |
| "candidate_timings": { |
| "cache_n": 0, |
| "prompt_n": 5247, |
| "prompt_ms": 5405.152, |
| "prompt_per_token_ms": 1.0301414141414142, |
| "prompt_per_second": 970.7405083150297, |
| "predicted_n": 256, |
| "predicted_ms": 5642.935, |
| "predicted_per_token_ms": 22.1291568627451, |
| "predicted_per_second": 45.189249920475774, |
| "draft_n": 230, |
| "draft_n_accepted": 178 |
| }, |
| "baseline_memory": { |
| "idle": { |
| "t": 1789602970.6939769, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109159923712, |
| "ram_available_bytes": 25149743104, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789602982.4689696, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109052878848, |
| "ram_available_bytes": 25256787968, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 48, |
| "peak_ram_used_bytes": 109159923712, |
| "delta_ram_used_bytes": 0 |
| }, |
| "candidate_memory": { |
| "idle": { |
| "t": 1789603540.7137277, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109446057984, |
| "ram_available_bytes": 24863608832, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603551.8952494, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108821299200, |
| "ram_available_bytes": 25488367616, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 46, |
| "peak_ram_used_bytes": 109446057984, |
| "delta_ram_used_bytes": 0 |
| }, |
| "pp_pct": 4.107840075542746, |
| "ttfp_ms": { |
| "baseline": 5635.688066482544, |
| "candidate": 5537.263631820679, |
| "change_pct": -1.7464492977748436 |
| }, |
| "total_ms": { |
| "baseline": 11775.2685546875, |
| "candidate": 11182.04641342163, |
| "change_pct": -5.037865068731017 |
| } |
| }, |
| { |
| "case": "A2-r0", |
| "baseline_tg": 40.035985285597874, |
| "candidate_tg": 44.3758330257479, |
| "tg_pct": null, |
| "baseline_pp": 929.8286807933757, |
| "candidate_pp": 39.90622038210206, |
| "cross_arm_text_exact": false, |
| "candidate_canonical_text_exact": true, |
| "prompt_work_comparable": false, |
| "baseline_timings": { |
| "cache_n": 0, |
| "prompt_n": 5247, |
| "prompt_ms": 5642.975, |
| "prompt_per_token_ms": 1.0754669334858016, |
| "prompt_per_second": 929.8286807933757, |
| "predicted_n": 256, |
| "predicted_ms": 6369.27, |
| "predicted_per_token_ms": 24.977529411764706, |
| "predicted_per_second": 40.035985285597874, |
| "draft_n": 242, |
| "draft_n_accepted": 174 |
| }, |
| "candidate_timings": { |
| "cache_n": 5243, |
| "prompt_n": 4, |
| "prompt_ms": 100.235, |
| "prompt_per_token_ms": 25.05875, |
| "prompt_per_second": 39.90622038210206, |
| "predicted_n": 256, |
| "predicted_ms": 5746.371, |
| "predicted_per_token_ms": 22.53478823529412, |
| "predicted_per_second": 44.3758330257479, |
| "draft_n": 243, |
| "draft_n_accepted": 173 |
| }, |
| "baseline_memory": { |
| "idle": { |
| "t": 1789602982.4912872, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109052764160, |
| "ram_available_bytes": 25256902656, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789602994.5130851, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109049774080, |
| "ram_available_bytes": 25259892736, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 49, |
| "peak_ram_used_bytes": 109101096960, |
| "delta_ram_used_bytes": 48332800 |
| }, |
| "candidate_memory": { |
| "idle": { |
| "t": 1789603551.9198458, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108821061632, |
| "ram_available_bytes": 25488605184, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603557.887965, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108725809152, |
| "ram_available_bytes": 25583857664, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 25, |
| "peak_ram_used_bytes": 108821061632, |
| "delta_ram_used_bytes": 0 |
| }, |
| "pp_pct": null, |
| "comparison_note": "Prompt restoration changes fresh-token count; use cached/fresh counts and first-token/total latency, not a PP throughput percentage.", |
| "ttfp_ms": { |
| "baseline": 5652.848482131958, |
| "candidate": 220.4749584197998, |
| "change_pct": -96.09975467913748 |
| }, |
| "total_ms": { |
| "baseline": 12022.283792495728, |
| "candidate": 5968.55354309082, |
| "change_pct": -50.354245115920705 |
| } |
| }, |
| { |
| "case": "B2-r0", |
| "baseline_tg": 41.30325203989474, |
| "candidate_tg": 46.83362455889154, |
| "tg_pct": 13.389678163005247, |
| "baseline_pp": 929.6082623341406, |
| "candidate_pp": 39.69947497444346, |
| "cross_arm_text_exact": true, |
| "candidate_canonical_text_exact": true, |
| "prompt_work_comparable": false, |
| "baseline_timings": { |
| "cache_n": 0, |
| "prompt_n": 5247, |
| "prompt_ms": 5644.313, |
| "prompt_per_token_ms": 1.0757219363445778, |
| "prompt_per_second": 929.6082623341406, |
| "predicted_n": 256, |
| "predicted_ms": 6173.848, |
| "predicted_per_token_ms": 24.21116862745098, |
| "predicted_per_second": 41.30325203989474, |
| "draft_n": 230, |
| "draft_n_accepted": 178 |
| }, |
| "candidate_timings": { |
| "cache_n": 5243, |
| "prompt_n": 4, |
| "prompt_ms": 100.757, |
| "prompt_per_token_ms": 25.18925, |
| "prompt_per_second": 39.69947497444346, |
| "predicted_n": 256, |
| "predicted_ms": 5444.806, |
| "predicted_per_token_ms": 21.35218039215686, |
| "predicted_per_second": 46.83362455889154, |
| "draft_n": 230, |
| "draft_n_accepted": 178 |
| }, |
| "baseline_memory": { |
| "idle": { |
| "t": 1789602994.5375905, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109053657088, |
| "ram_available_bytes": 25256009728, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603006.3658593, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109109821440, |
| "ram_available_bytes": 25199845376, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 49, |
| "peak_ram_used_bytes": 109136678912, |
| "delta_ram_used_bytes": 83021824 |
| }, |
| "candidate_memory": { |
| "idle": { |
| "t": 1789603557.9165292, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108725809152, |
| "ram_available_bytes": 25583857664, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603563.5847316, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108616736768, |
| "ram_available_bytes": 25692930048, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 24, |
| "peak_ram_used_bytes": 108725809152, |
| "delta_ram_used_bytes": 0 |
| }, |
| "pp_pct": null, |
| "comparison_note": "Prompt restoration changes fresh-token count; use cached/fresh counts and first-token/total latency, not a PP throughput percentage.", |
| "ttfp_ms": { |
| "baseline": 5652.911901473999, |
| "candidate": 221.79675102233887, |
| "change_pct": -96.07641592708165 |
| }, |
| "total_ms": { |
| "baseline": 11828.643798828125, |
| "candidate": 5668.706178665161, |
| "change_pct": -52.07644870304773 |
| } |
| }, |
| { |
| "case": "240k-r0", |
| "baseline_tg": 7.619547607925039, |
| "candidate_tg": 21.14235845035863, |
| "tg_pct": 177.47524575302353, |
| "baseline_pp": 751.1453522464128, |
| "candidate_pp": 761.2549339625284, |
| "cross_arm_text_exact": true, |
| "candidate_canonical_text_exact": true, |
| "prompt_work_comparable": true, |
| "baseline_timings": { |
| "cache_n": 0, |
| "prompt_n": 245760, |
| "prompt_ms": 327180.351, |
| "prompt_per_token_ms": 1.3313002563476564, |
| "prompt_per_second": 751.1453522464128, |
| "predicted_n": 512, |
| "predicted_ms": 67064.349, |
| "predicted_per_token_ms": 131.24138747553818, |
| "predicted_per_second": 7.619547607925039, |
| "draft_n": 623, |
| "draft_n_accepted": 302 |
| }, |
| "candidate_timings": { |
| "cache_n": 0, |
| "prompt_n": 245760, |
| "prompt_ms": 322835.346, |
| "prompt_per_token_ms": 1.3136203857421875, |
| "prompt_per_second": 761.2549339625284, |
| "predicted_n": 512, |
| "predicted_ms": 24169.489, |
| "predicted_per_token_ms": 47.29841291585127, |
| "predicted_per_second": 21.14235845035863, |
| "draft_n": 623, |
| "draft_n_accepted": 302 |
| }, |
| "baseline_memory": { |
| "idle": { |
| "t": 1789603006.3926795, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 109117845504, |
| "ram_available_bytes": 25191821312, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603400.972394, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 112018915328, |
| "ram_available_bytes": 22290751488, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 1578, |
| "peak_ram_used_bytes": 112572510208, |
| "delta_ram_used_bytes": 3454664704 |
| }, |
| "candidate_memory": { |
| "idle": { |
| "t": 1789603563.6150782, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 108618145792, |
| "ram_available_bytes": 25691521024, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603910.903359, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 112808243200, |
| "ram_available_bytes": 21501423616, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 1389, |
| "peak_ram_used_bytes": 113941381120, |
| "delta_ram_used_bytes": 5323235328 |
| }, |
| "pp_pct": 1.3458888730232266, |
| "ttfp_ms": { |
| "baseline": 327505.3644180298, |
| "candidate": 323100.4581451416, |
| "change_pct": -1.3449875182092397 |
| }, |
| "total_ms": { |
| "baseline": 394579.9479484558, |
| "candidate": 347288.6493206024, |
| "change_pct": -11.985226029283947 |
| } |
| }, |
| { |
| "case": "240k-append64-r0", |
| "baseline_tg": 7.505335140312534, |
| "candidate_tg": 20.784408456008457, |
| "tg_pct": 176.92845246005845, |
| "baseline_pp": 46.97452944301333, |
| "candidate_pp": 65.88719530805655, |
| "cross_arm_text_exact": true, |
| "candidate_canonical_text_exact": true, |
| "prompt_work_comparable": true, |
| "baseline_timings": { |
| "cache_n": 245756, |
| "prompt_n": 68, |
| "prompt_ms": 1447.593, |
| "prompt_per_token_ms": 21.288132352941176, |
| "prompt_per_second": 46.97452944301333, |
| "predicted_n": 512, |
| "predicted_ms": 68084.901, |
| "predicted_per_token_ms": 133.23855381604696, |
| "predicted_per_second": 7.505335140312534, |
| "draft_n": 635, |
| "draft_n_accepted": 299 |
| }, |
| "candidate_timings": { |
| "cache_n": 245756, |
| "prompt_n": 68, |
| "prompt_ms": 1032.067, |
| "prompt_per_token_ms": 15.177455882352941, |
| "prompt_per_second": 65.88719530805655, |
| "predicted_n": 512, |
| "predicted_ms": 24585.737, |
| "predicted_per_token_ms": 48.112988258317024, |
| "predicted_per_second": 20.784408456008457, |
| "draft_n": 635, |
| "draft_n_accepted": 299 |
| }, |
| "baseline_memory": { |
| "idle": { |
| "t": 1789603401.0131505, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 112027701248, |
| "ram_available_bytes": 22281965568, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603470.7465725, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 112231673856, |
| "ram_available_bytes": 22077992960, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 280, |
| "peak_ram_used_bytes": 112231673856, |
| "delta_ram_used_bytes": 203972608 |
| }, |
| "candidate_memory": { |
| "idle": { |
| "t": 1789603910.9404855, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 112810250240, |
| "ram_available_bytes": 21499416576, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "last": { |
| "t": 1789603936.7776108, |
| "ram_total_bytes": 134309666816, |
| "ram_used_bytes": 113007968256, |
| "ram_available_bytes": 21301698560, |
| "vram_total_bytes": null, |
| "vram_used_bytes": null, |
| "gtt_total_bytes": null, |
| "gtt_used_bytes": null |
| }, |
| "sample_count": 105, |
| "peak_ram_used_bytes": 113110175744, |
| "delta_ram_used_bytes": 299925504 |
| }, |
| "pp_pct": 40.26153340819927, |
| "ttfp_ms": { |
| "baseline": 1637.97926902771, |
| "candidate": 1232.0160865783691, |
| "change_pct": -24.784390750581174 |
| }, |
| "total_ms": { |
| "baseline": 69733.6814403534, |
| "candidate": 25837.472915649414, |
| "change_pct": -62.94835955599238 |
| } |
| } |
| ], |
| "source": "final-all-v1", |
| "candidate": "final-all-v1-pm4", |
| "baseline": "d0", |
| "all_candidate_canonical_text_exact": true |
| }, |
| "quality": { |
| "valid": true, |
| "tasks": 20, |
| "full_passes": 19, |
| "arithmetic_average": 98.5, |
| "canonical_score": { |
| "final-iu4-all-v1": { |
| "totalScore": 99, |
| "categories": [ |
| { |
| "id": "memory_recall", |
| "label": "Memory & Recall", |
| "score": 100, |
| "weight": 20 |
| }, |
| { |
| "id": "workspace_orchestration", |
| "label": "Workspace Orchestration", |
| "score": 100, |
| "weight": 20 |
| }, |
| { |
| "id": "skills_procedural_memory", |
| "label": "Skills & Procedural Memory", |
| "score": 100, |
| "weight": 20 |
| }, |
| { |
| "id": "scheduling_delivery", |
| "label": "Scheduling & Delivery", |
| "score": 100, |
| "weight": 20 |
| }, |
| { |
| "id": "delegation_recovery_boundaries", |
| "label": "Delegation, Recovery & Boundaries", |
| "score": 93, |
| "weight": 20 |
| } |
| ], |
| "summary": "Hermes passed most official scenarios, with a few verifier or model misses remaining." |
| } |
| }, |
| "first_samples_only": true, |
| "new_failed_scenarios_vs_both_history": [], |
| "retained_failure": "HA-17 score70, identical verifier outcome/details to both historical runs; merged delegation artifact wrong/missing", |
| "scope": "Single final candidate run; historical engines/seeds differ. No new failed scenario observed; no statistical quality-equivalence claim.", |
| "protocol_sha256": "6465f9465b78e9ca0b1456a8dfee2fd6df7e04681c7169dfefed9ebf9e025075", |
| "historical_comparison": { |
| "1": { |
| "HA-01": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-02": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-03": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-04": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-05": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-06": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-07": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-08": { |
| "old_score": 80, |
| "new_score": 100 |
| }, |
| "HA-09": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-10": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-11": { |
| "old_score": 50, |
| "new_score": 100 |
| }, |
| "HA-12": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-13": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-14": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-15": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-16": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-17": { |
| "old_score": 70, |
| "new_score": 70 |
| }, |
| "HA-18": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-19": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-20": { |
| "old_score": 100, |
| "new_score": 100 |
| } |
| }, |
| "2": { |
| "HA-01": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-02": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-03": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-04": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-05": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-06": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-07": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-08": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-09": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-10": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-11": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-12": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-13": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-14": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-15": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-16": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-17": { |
| "old_score": 70, |
| "new_score": 70 |
| }, |
| "HA-18": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-19": { |
| "old_score": 100, |
| "new_score": 100 |
| }, |
| "HA-20": { |
| "old_score": 100, |
| "new_score": 100 |
| } |
| } |
| } |
| }, |
| "he0_9": { |
| "completed_at": 1789608195.1082451, |
| "build": "final-all-v1-pm4", |
| "protocol": "HumanEval0-9 canonical prompts in retained local non-thinking greedy chat adapter; served MTP3, full262144 allocation, one first trajectory per task, natural EOS", |
| "weighted_decode_tokens_per_second": 60.35102655869076, |
| "minimum_case_tps": 56.09397436570597, |
| "maximum_case_tps": 64.76202653665553, |
| "generated_tokens": 1633, |
| "timed_decode_tokens": 1623, |
| "decode_seconds": 26.892666, |
| "summed_request_seconds": 36.057379722595215, |
| "drafted_tokens": 1260.0, |
| "accepted_tokens": 1221.0, |
| "acceptance_fraction": 0.969047619047619, |
| "valid_tasks": 10, |
| "quality_score_claim": false, |
| "fresh_prompt_cache_all_tasks": true, |
| "sqlite_rows_verified": 10, |
| "persistence_note": "Task0 completed in the first load; its metadata serialization failed after the full response and JSONL were saved. Corrected only command serialization, imported the original result, then generated tasks1-9 in a second identical load. No task was regenerated. The saved panel-resume panel_elapsed_seconds covers that resumed phase only; summed request/decode times include all10 tasks.", |
| "rows": [ |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 56.09397436570597, |
| "decode_seconds": 3.066283, |
| "decode_timed_tokens": 172, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 134, |
| "generated_tokens": 173, |
| "total_tokens": 307, |
| "request_wall_seconds": 4.12631893157959, |
| "ttfp_ms": 1057.2734710294753, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 173, |
| "tokens_evaluated": 134, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T011851Z-final-all-v1-pm4-HumanEval-0-api.raw", |
| "task": "HumanEval/0", |
| "recorder_exit": 1, |
| "persistence_recovered": true, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel/HumanEval-0/row-recovered.json", |
| "drafted_tokens": 135, |
| "accepted_tokens": 129 |
| }, |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 64.36654266122495, |
| "decode_seconds": 3.588821, |
| "decode_timed_tokens": 231, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 126, |
| "generated_tokens": 232, |
| "total_tokens": 358, |
| "request_wall_seconds": 5.121450185775757, |
| "ttfp_ms": 1530.2266778890043, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 232, |
| "tokens_evaluated": 126, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T012137Z-final-all-v1-pm4-HumanEval-1-api.raw", |
| "task": "HumanEval/1", |
| "recorder_exit": 0, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel-resume/HumanEval-1/row.json", |
| "drafted_tokens": 177.0, |
| "accepted_tokens": 174.0 |
| }, |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 63.59207801688104, |
| "decode_seconds": 1.509622, |
| "decode_timed_tokens": 96, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 95, |
| "generated_tokens": 97, |
| "total_tokens": 192, |
| "request_wall_seconds": 2.163600444793701, |
| "ttfp_ms": 651.7064250074327, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 97, |
| "tokens_evaluated": 95, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T012142Z-final-all-v1-pm4-HumanEval-2-api.raw", |
| "task": "HumanEval/2", |
| "recorder_exit": 0, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel-resume/HumanEval-2/row.json", |
| "drafted_tokens": 75.0, |
| "accepted_tokens": 74.0 |
| }, |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 64.76202653665553, |
| "decode_seconds": 2.408819, |
| "decode_timed_tokens": 156, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 129, |
| "generated_tokens": 157, |
| "total_tokens": 286, |
| "request_wall_seconds": 3.3317527770996094, |
| "ttfp_ms": 921.4452710002661, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 157, |
| "tokens_evaluated": 129, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T012144Z-final-all-v1-pm4-HumanEval-3-api.raw", |
| "task": "HumanEval/3", |
| "recorder_exit": 0, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel-resume/HumanEval-3/row.json", |
| "drafted_tokens": 120.0, |
| "accepted_tokens": 117.0 |
| }, |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 62.50495295944211, |
| "decode_seconds": 2.6877869999999997, |
| "decode_timed_tokens": 168, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 128, |
| "generated_tokens": 169, |
| "total_tokens": 297, |
| "request_wall_seconds": 3.5662331581115723, |
| "ttfp_ms": 875.4782220348716, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 169, |
| "tokens_evaluated": 128, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T012148Z-final-all-v1-pm4-HumanEval-4-api.raw", |
| "task": "HumanEval/4", |
| "recorder_exit": 0, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel-resume/HumanEval-4/row.json", |
| "drafted_tokens": 129.0, |
| "accepted_tokens": 126.0 |
| }, |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 57.36296811743736, |
| "decode_seconds": 2.6672260000000003, |
| "decode_timed_tokens": 153, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 104, |
| "generated_tokens": 154, |
| "total_tokens": 258, |
| "request_wall_seconds": 3.430769920349121, |
| "ttfp_ms": 761.1340200528502, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 154, |
| "tokens_evaluated": 104, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T012151Z-final-all-v1-pm4-HumanEval-5-api.raw", |
| "task": "HumanEval/5", |
| "recorder_exit": 0, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel-resume/HumanEval-5/row.json", |
| "drafted_tokens": 123.0, |
| "accepted_tokens": 115.0 |
| }, |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 58.35354247096843, |
| "decode_seconds": 3.650164, |
| "decode_timed_tokens": 213, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 124, |
| "generated_tokens": 214, |
| "total_tokens": 338, |
| "request_wall_seconds": 4.545674085617065, |
| "ttfp_ms": 893.1537920143455, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 214, |
| "tokens_evaluated": 124, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T012155Z-final-all-v1-pm4-HumanEval-6-api.raw", |
| "task": "HumanEval/6", |
| "recorder_exit": 0, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel-resume/HumanEval-6/row.json", |
| "drafted_tokens": 168.0, |
| "accepted_tokens": 160.0 |
| }, |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 59.95297742149265, |
| "decode_seconds": 1.851451, |
| "decode_timed_tokens": 111, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 104, |
| "generated_tokens": 112, |
| "total_tokens": 216, |
| "request_wall_seconds": 2.6149725914001465, |
| "ttfp_ms": 761.6006780881435, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 112, |
| "tokens_evaluated": 104, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T012200Z-final-all-v1-pm4-HumanEval-7-api.raw", |
| "task": "HumanEval/7", |
| "recorder_exit": 0, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel-resume/HumanEval-7/row.json", |
| "drafted_tokens": 84.0, |
| "accepted_tokens": 83.0 |
| }, |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 59.57582016045754, |
| "decode_seconds": 2.719224, |
| "decode_timed_tokens": 162, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 126, |
| "generated_tokens": 163, |
| "total_tokens": 289, |
| "request_wall_seconds": 3.6378886699676514, |
| "ttfp_ms": 915.6202138401568, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 163, |
| "tokens_evaluated": 126, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T012202Z-final-all-v1-pm4-HumanEval-8-api.raw", |
| "task": "HumanEval/8", |
| "recorder_exit": 0, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel-resume/HumanEval-8/row.json", |
| "drafted_tokens": 123.0, |
| "accepted_tokens": 122.0 |
| }, |
| { |
| "status": "complete", |
| "valid_speed_case": true, |
| "native_tg": 58.68910413087452, |
| "decode_seconds": 2.7432689999999997, |
| "decode_timed_tokens": 161, |
| "native_counting": "generated-minus-one", |
| "prompt_tokens": 110, |
| "generated_tokens": 162, |
| "total_tokens": 272, |
| "request_wall_seconds": 3.518718957901001, |
| "ttfp_ms": 773.098937002942, |
| "reasoning_block_detected": false, |
| "finish": { |
| "stop": true, |
| "tokens_predicted": 162, |
| "tokens_evaluated": 110, |
| "truncated": false, |
| "stop_type": "eos", |
| "stopping_word": "" |
| }, |
| "error": null, |
| "raw_output": "/home/halo/bench-results/llama/20260917T012206Z-final-all-v1-pm4-HumanEval-9-api.raw", |
| "task": "HumanEval/9", |
| "recorder_exit": 0, |
| "row": "/home/halo/flash-combined-20260916/humaneval-decode-20260917/panel-resume/HumanEval-9/row.json", |
| "drafted_tokens": 126.0, |
| "accepted_tokens": 121.0 |
| } |
| ] |
| }, |
| "boost": { |
| "finished_at": 1789608844.7464342, |
| "status": "SHORT_HE_SPEED_LEAD_ONE_QUOTE_STYLE_DIFFERENCE", |
| "baseline_tg": 60.35102655869076, |
| "kairic_tg": 64.06719136499001, |
| "change_pct": 6.157583421858304, |
| "baseline_request_seconds": 36.057379722595215, |
| "kairic_request_seconds": 33.86439895629883, |
| "request_time_pct": -6.081919382850121, |
| "output_exact_tasks": 9, |
| "tasks": 10, |
| "generated_tokens_each_arm": 1633, |
| "same_rendered_prompts": true, |
| "same_binary_and_environment": true, |
| "actual_lookup_admission": { |
| "observed": true, |
| "basis": "Native accepted-group mean exceeds4, which MTP depth3 alone cannot produce. Only additional proposer is ngram-mod24/64/64. This proves lookup contribution without changing log verbosity or running a profiler. Exact per-strategy proposal counts are not exposed at default verbosity.", |
| "mean_accept_lengths": [ |
| 7.86, |
| 5.95, |
| 8.91, |
| 8.67, |
| 7.95, |
| 6.33, |
| 6.21, |
| 8.54, |
| 8.05, |
| 6.48 |
| ] |
| }, |
| "per_task": [ |
| { |
| "task": "HumanEval/0", |
| "baseline_tg": 56.09397436570597, |
| "kairic_tg": 65.53199790297606, |
| "change_pct": 16.82537856158788, |
| "generated_tokens": 173, |
| "output_exact": true, |
| "baseline_output_sha256": "693c74127c0f8ff0d37eb26261623c8234dc8bbdb3897fcc539e9e4b41f792c8", |
| "candidate_output_sha256": "693c74127c0f8ff0d37eb26261623c8234dc8bbdb3897fcc539e9e4b41f792c8", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 7.86 |
| }, |
| { |
| "task": "HumanEval/1", |
| "baseline_tg": 64.36654266122495, |
| "kairic_tg": 68.07682956479455, |
| "change_pct": 5.76430976430975, |
| "generated_tokens": 232, |
| "output_exact": false, |
| "baseline_output_sha256": "7f29163271dac8208879c5409b5706123765b6589a54c62f155f04d98ce67ce1", |
| "candidate_output_sha256": "3c01ecdd3836f91e31519071b59322e2d106da6d3f0c617e13b9b5c20172bb96", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 5.95 |
| }, |
| { |
| "task": "HumanEval/2", |
| "baseline_tg": 63.59207801688104, |
| "kairic_tg": 63.10682802731999, |
| "change_pct": -0.7630667288970105, |
| "generated_tokens": 97, |
| "output_exact": true, |
| "baseline_output_sha256": "029220b47b8f0be3c68c840f5396b450f116e3c6da677a8f5d4853e808b098e4", |
| "candidate_output_sha256": "029220b47b8f0be3c68c840f5396b450f116e3c6da677a8f5d4853e808b098e4", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 8.91 |
| }, |
| { |
| "task": "HumanEval/3", |
| "baseline_tg": 64.76202653665553, |
| "kairic_tg": 70.95304954842477, |
| "change_pct": 9.559649910376278, |
| "generated_tokens": 157, |
| "output_exact": true, |
| "baseline_output_sha256": "6a4eae059484f334f3a2e1c38db1c246e9cde1563e4d6053d594e5285e123065", |
| "candidate_output_sha256": "6a4eae059484f334f3a2e1c38db1c246e9cde1563e4d6053d594e5285e123065", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 8.67 |
| }, |
| { |
| "task": "HumanEval/4", |
| "baseline_tg": 62.50495295944211, |
| "kairic_tg": 69.01277229235782, |
| "change_pct": 10.411685834142581, |
| "generated_tokens": 169, |
| "output_exact": true, |
| "baseline_output_sha256": "0c62e457d5759dd531027976692a157cd6425ad0ff0f8493a1c2370a9cfee9d5", |
| "candidate_output_sha256": "0c62e457d5759dd531027976692a157cd6425ad0ff0f8493a1c2370a9cfee9d5", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 7.95 |
| }, |
| { |
| "task": "HumanEval/5", |
| "baseline_tg": 57.36296811743736, |
| "kairic_tg": 60.504919484956815, |
| "change_pct": 5.477316587047998, |
| "generated_tokens": 154, |
| "output_exact": true, |
| "baseline_output_sha256": "2a816aebf493a710e524a7029f70cda237cce08b76ebaa5e61ae48c1c740d0ec", |
| "candidate_output_sha256": "2a816aebf493a710e524a7029f70cda237cce08b76ebaa5e61ae48c1c740d0ec", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 6.33 |
| }, |
| { |
| "task": "HumanEval/6", |
| "baseline_tg": 58.35354247096843, |
| "kairic_tg": 56.50370961443778, |
| "change_pct": -3.170043802312361, |
| "generated_tokens": 214, |
| "output_exact": true, |
| "baseline_output_sha256": "56b27b2443cbac3922e7cf433c03c49b6df95ae881ede6bc353c1f24d63068e5", |
| "candidate_output_sha256": "56b27b2443cbac3922e7cf433c03c49b6df95ae881ede6bc353c1f24d63068e5", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 6.21 |
| }, |
| { |
| "task": "HumanEval/7", |
| "baseline_tg": 59.95297742149265, |
| "kairic_tg": 61.393533656930714, |
| "change_pct": 2.40281016455679, |
| "generated_tokens": 112, |
| "output_exact": true, |
| "baseline_output_sha256": "c5cdf9b7531a6e47b9643d72359f7dec793e83c6b213642fbb46edc8bc18ab24", |
| "candidate_output_sha256": "c5cdf9b7531a6e47b9643d72359f7dec793e83c6b213642fbb46edc8bc18ab24", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 8.54 |
| }, |
| { |
| "task": "HumanEval/8", |
| "baseline_tg": 59.57582016045754, |
| "kairic_tg": 66.12844102608481, |
| "change_pct": 10.998792543650904, |
| "generated_tokens": 163, |
| "output_exact": true, |
| "baseline_output_sha256": "545a0ec81a61cb41f2193884ae8f45130beef02d62945cbc0e8a0ebd2c4789ed", |
| "candidate_output_sha256": "545a0ec81a61cb41f2193884ae8f45130beef02d62945cbc0e8a0ebd2c4789ed", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 8.05 |
| }, |
| { |
| "task": "HumanEval/9", |
| "baseline_tg": 58.68910413087452, |
| "kairic_tg": 61.81577899591978, |
| "change_pct": 5.32752188220984, |
| "generated_tokens": 162, |
| "output_exact": true, |
| "baseline_output_sha256": "8f21fb93f42d7923cf137a57cfa23c432bacc0e74be98c5faeac45cb0484b69c", |
| "candidate_output_sha256": "8f21fb93f42d7923cf137a57cfa23c432bacc0e74be98c5faeac45cb0484b69c", |
| "prompt_cache_n": 0, |
| "mean_accept_length": 6.48 |
| } |
| ], |
| "sqlite_rows_verified": 10, |
| "protocol": "One natural-EOS first response on HE0-9, greedy/non-thinking retained chat adapter, full262144 allocated context, MTP3 fallback; recent retained MTP-only control", |
| "limits": "One candidate load versus a recent retained control, no order-balanced repeats. Nine byte-exact outputs; HumanEval1 changes quote delimiters only and has an identical Python AST. No scored quality claim. This short non-thinking panel does not resolve the earlier long-copy/thinking trajectory concern. No production/profile change.", |
| "output_difference_review": [ |
| { |
| "task": "HumanEval/1", |
| "byte_exact": false, |
| "python_ast_identical": true, |
| "ast_sha256": "8a77fd245e2f4f6d9dbd755aa8ba0a87f711b220c6f987f55708b29af473a626", |
| "diff": "--- MTP-only\n+++ Boost+MTP\n@@ -11,7 +11,7 @@\n ['()', '(())', '(()())']\n \"\"\"\n # Remove all spaces\n- paren_string = paren_string.replace(\" \", \"\")\n+ paren_string = paren_string.replace(' ', '')\n \n result = []\n current_group = []\n", |
| "scope": "Parse-only syntactic comparison of generated code; no code execution or benchmark correctness score." |
| } |
| ], |
| "classification": "NARROW: promising for this short coding workload; general exactness/quality qualification remains open." |
| } |
| } |
|
|
