{
  "environment": {
    "device": "Apple M2 Max",
    "unified_memory_gb": 64,
    "mlx": "0.32.0",
    "mlx_lm": "0.31.3",
    "mlx_vlm": "0.6.13"
  },
  "heldout_sparse_teacher_loss": {
    "records": 16,
    "rtn_q4_g64": 0.168036,
    "dwq_q4_g64": 0.093706,
    "dwq_relative_reduction": 0.44235
  },
  "mtp_greedy_probe": {
    "tokens": 128,
    "draft_block_size": 2,
    "exact_output_match": true,
    "accepted_draft_tokens": 0,
    "drafted_tokens": 128,
    "baseline_tokens_per_second": 21.5952,
    "mtp_tokens_per_second": 12.4998,
    "speedup_fraction": -0.421176,
    "baseline_peak_memory_gb": 18.7684,
    "mtp_peak_memory_gb": 19.4494
  },
  "runtime_smokes": {
    "text": {
      "response": "4",
      "peak_memory_gb": 18.2571,
      "pass": true
    },
    "vision": {
      "response": "Blue",
      "peak_memory_gb": 18.3426,
      "pass": true
    },
    "mtp_standalone_load": {
      "model_type": "Qwen3_5MTPDraftModel",
      "quantized_linear_modules": 8,
      "layout": "affine8_g128",
      "pass": true
    }
  },
  "template_ab": {
    "benchmark": "When2Call deterministic stratified MCQ",
    "rows": 96,
    "source_crack_template_correct": 42,
    "frog_v22_1_correct": 40,
    "selected": "source CRACK template"
  },
  "when2call": {
    "benchmark_repo": "NVIDIA/When2Call",
    "benchmark_revision": "ecc8d42388e91ab37e7e737d48e16e8ecea3d1dc",
    "dataset_repo": "nvidia/When2Call",
    "dataset_revision": "0582f7749df63a96fdc3070932e83e72396ace53",
    "dataset_crc32": "74792b16",
    "selection_crc32": "48efdbd6",
    "rows": 96,
    "seed": 20260816,
    "scoring": "teacher-forced choice log-likelihood",
    "results": {
      "source_mxfp8": {
        "correct": 38,
        "accuracy": 0.395833,
        "normalized_correct": 37,
        "peak_memory_gb": 30.6042
      },
      "rtn_q4_g64": {
        "correct": 40,
        "accuracy": 0.416667,
        "normalized_correct": 38,
        "peak_memory_gb": 18.0449
      },
      "dwq_q4_g64": {
        "correct": 42,
        "accuracy": 0.4375,
        "normalized_correct": 38,
        "peak_memory_gb": 18.0449
      }
    },
    "paired_dwq_minus_source": {
      "accuracy_delta": 0.041667,
      "bootstrap_95pct_ci": [0.010417, 0.083333],
      "dwq_only_correct": 4,
      "source_only_correct": 0,
      "mcnemar_two_sided_exact_p": 0.125
    },
    "paired_dwq_minus_rtn": {
      "accuracy_delta": 0.020833,
      "bootstrap_95pct_ci": [0.0, 0.052083],
      "dwq_only_correct": 2,
      "rtn_only_correct": 0,
      "mcnemar_two_sided_exact_p": 0.5
    },
    "limitations": "Small task-specific sample; not evidence of universal benchmark superiority."
  }
}