| { |
| "status": "complete", |
| "model": "Qwen/Qwen3.5-9B", |
| "device": "single P150", |
| "checkpoint": "New standalone mixed-BFP4/BFP8 TT-native checkpoint including all 15 original MTP tensors", |
| "runtime_image": "sha256:fe06066653f6c1dc3d68cf7c2ee6e31a7724aebec82f88ff7bf792b78a0b1b21", |
| "manifest_sha256": "34f774cf110e3ff94726171b52f6d462cb9b36651295b941efa85098ddb96ca2", |
| "target_tensors_byte_identical_to_previous_release": 475, |
| "mtp_source_tensors": 15, |
| "mtp_source_bytes": 486582864, |
| "native_reload": [ |
| { |
| "mode": "off", |
| "file": "equivalence.json", |
| "exact_logits_equal": true, |
| "tokens_compared": 1415 |
| }, |
| { |
| "mode": "on", |
| "file": "equivalence-mtp.json", |
| "exact_logits_equal": true, |
| "tokens_compared": 1415 |
| } |
| ], |
| "api": { |
| "behavior_passes": 11, |
| "long_context_passes": 1, |
| "request_isolation_passed": true |
| }, |
| "quality": { |
| "gsm8k": { |
| "accuracyPct": 96.04, |
| "avgLatencyMs": 7899, |
| "correct": 97, |
| "dataset": "gsm8k", |
| "declaredModel": "", |
| "effectiveModel": "", |
| "errors": 0, |
| "quantFormat": "ttnn-native", |
| "quantization": "mixed-bfp4-bfp8", |
| "questions": 101, |
| "scored": 101, |
| "scoring": "exact_match", |
| "shardIndex": 1 |
| }, |
| "arc_challenge": { |
| "accuracyPct": 94.25, |
| "avgLatencyMs": 218, |
| "correct": 754, |
| "dataset": "arc-challenge", |
| "declaredModel": "", |
| "effectiveModel": "", |
| "errors": 0, |
| "quantFormat": "ttnn-native", |
| "quantization": "mixed-bfp4-bfp8", |
| "questions": 800, |
| "scored": 800, |
| "scoring": "exact_match", |
| "shardIndex": 1 |
| } |
| }, |
| "speed": { |
| "fixed_prompt": { |
| "off_tok_s": 36.571853797115395, |
| "on_tok_s": 50.564418615171256, |
| "speedup_percent": 38.26047455969959, |
| "runs_per_mode": 5 |
| }, |
| "localmaxxing": { |
| "off_tok_s": 36.5, |
| "on_tok_s": 55.1, |
| "speedup_percent": 50.95890410958906, |
| "runs_per_mode": 3 |
| }, |
| "draft_acceptance": { |
| "accepted_per_run": 47, |
| "proposed_per_run": 80, |
| "percent": 58.75 |
| } |
| }, |
| "caveats": [ |
| "Greedy decoding, concurrency 1, thinking disabled.", |
| "MTP-on and MTP-off completions differ; reload equivalence is not cross-mode output equivalence.", |
| "Quality scores are complete declared LocalMaxxing shards, not full-dataset official scores.", |
| "Previous no-MTP quality scores also use an older runtime; comparison does not isolate MTP alone.", |
| "Nothing submitted publicly." |
| ], |
| "serving_restored": { |
| "base_url": "http://127.0.0.1:8000", |
| "health_status": 200, |
| "container": "qwen35-tt-bfp4-hf", |
| "image": "sha256:3c76d5443033f2245a3fc2edebdeb6c520ed6f392f3a081991fc9e1a27e1b95d", |
| "note": "Original endpoint restored unchanged; fixed new quant is packaged separately and defaults to MTP on." |
| } |
| } |
|
|
