Correct and publish full MTP sweep

README.md

@@ -34,8 +34,10 @@ on one DGX Spark. They do not describe Inferact or RadixArk runtime performance.
 
 - The MTP load guard verified 31/31 expected tensors. The compact BF16 MTP file is
   4.86 GiB.
-- An MTP depth sweep selected `MTP=2`: 57.13 tok/s median versus 31.12 tok/s at
-  `MTP=0`, an 83.6% increase under the tested short-request method.
+- A single-stream MTP depth sweep selected `MTP=2`: 44.23 tok/s end-to-end,
+  including hidden reasoning and first-visible latency, versus 27.26 tok/s
+  at `MTP=0`, a 62.2% increase. Its separately measured visible-answer phase was
+  46.15 tok/s, with 1.324s to first visible content.
 - Native 262,144-token startup used 76.21 GiB for model loading and exposed 17.37
   GiB of BF16 KV, enough for 627,960 cached tokens (2.40 concurrent native windows).
 - Exact retrieval passed at 240,051 prompt tokens, with 116.80s TTFT and 2,055.17

benchmarks/common.py

@@ -63,7 +63,7 @@ def read_json(url: str, timeout: float) -> dict[str, Any]:
 
 
 def stream_completion(base_url: str, payload: dict[str, object], timeout: float) -> dict[str, Any]:
-    """Measure TTFT and decode rate from the first and last emitted delta."""
+    """Measure reasoning-aware TTFT, decode, visible-content, and wall rates."""
     request = urllib.request.Request(
         endpoint(base_url, "chat/completions"),
         data=json.dumps(payload).encode("utf-8"),
@@ -78,6 +78,8 @@ def stream_completion(base_url: str, payload: dict[str, object], timeout: float)
 
     first_delta: float | None = None
     last_delta: float | None = None
+    first_visible_delta: float | None = None
+    last_visible_delta: float | None = None
     usage: dict[str, Any] | None = None
     content_parts: list[str] = []
     with response:
@@ -93,25 +95,50 @@ def stream_completion(base_url: str, payload: dict[str, object], timeout: float)
                 usage = event["usage"]
             for choice in event.get("choices") or []:
                 delta = choice.get("delta") or {}
+                reasoning = delta.get("reasoning_content")
                 text = delta.get("content")
-                if isinstance(text, str) and text:
-                    content_parts.append(text)
+                has_reasoning = isinstance(reasoning, str) and bool(reasoning)
+                has_content = isinstance(text, str) and bool(text)
+                if has_reasoning or has_content:
                     now = time.perf_counter()
                     first_delta = now if first_delta is None else first_delta
                     last_delta = now
+                if has_content:
+                    content_parts.append(text)
+                    first_visible_delta = (
+                        now if first_visible_delta is None else first_visible_delta
+                    )
+                    last_visible_delta = now
     finished = time.perf_counter()
     if usage is None or first_delta is None:
         raise RuntimeError("stream ended without usage or final output")
     decode_seconds = max((last_delta or finished) - first_delta, 1e-9)
     completion_tokens = int(usage["completion_tokens"])
+    details = usage.get("completion_tokens_details") or {}
+    reasoning_tokens = int(details.get("reasoning_tokens") or 0)
+    visible_tokens = max(completion_tokens - reasoning_tokens, 0)
+    visible_seconds = (
+        max((last_visible_delta or finished) - first_visible_delta, 1e-9)
+        if first_visible_delta is not None
+        else None
+    )
     return {
         "content": "".join(content_parts),
         "usage": usage,
         "ttft_seconds": first_delta - started,
+        "time_to_first_visible_content_seconds": (
+            first_visible_delta - started if first_visible_delta is not None else None
+        ),
         "total_seconds": finished - started,
         "decode_seconds": decode_seconds,
         "prefill_tokens_per_second": int(usage["prompt_tokens"]) / max(first_delta - started, 1e-9),
         "decode_tokens_per_second": max(completion_tokens - 1, 0) / decode_seconds,
+        "visible_content_tokens_per_second": (
+            max(visible_tokens - 1, 0) / visible_seconds
+            if visible_seconds is not None
+            else None
+        ),
+        "end_to_end_tokens_per_second": completion_tokens / max(finished - started, 1e-9),
     }
 
 

docs/BENCHMARKS.md

@@ -10,31 +10,41 @@ also use the exact 31 BF16 MTP tensors from pinned
 `7b719225242aacd3dbd3f9407468c2ee9a9d2594`. Inferact and RadixArk are not
 runtime-qualified by these results.
 
-## BF16 MTP depth sweep
+## Single-stream BF16 MTP depth sweep
 
 The load guard verified 31/31 tensors before accepting requests. Each depth used a
-32,768-token server profile, one maximum sequence, 0.80 GPU-memory utilization, one
-128-token warm-up, then three fixed 256-token streamed samples. The table reports
-the median decode rate and TTFT. Thinking mode used medium reasoning effort,
+32,768-token server profile, `concurrency=1`, one maximum sequence, 0.80 GPU-memory
+utilization, one 128-token warm-up, then three fixed 256-token streamed samples.
+The primary rate divides all completion tokens—including hidden reasoning—by total
+request wall time, so it also includes latency before visible content. The next
+column is the median time from request start to the first visible content delta.
+Thinking mode used medium reasoning effort,
 `temperature=1.0`, `top_p=0.95`, `top_k=20`, `min_p=0.0`,
 `presence_penalty=0.0`, and `repetition_penalty=1.0`.
 
-| MTP depth | Median decode tok/s | Median TTFT | Aggregate acceptance |
+| MTP depth | Median end-to-end completion tok/s | Time to first visible content | Aggregate acceptance |
 | ---: | ---: | ---: | ---: |
-| 0 | 31.12 | 1.196s | n/a |
-| 1 | 39.16 | 1.206s | 305/463 (65.9%) |
-| **2** | **57.13** | 1.324s | **532/722 (73.7%)** |
-| 3 | 55.55 | 1.629s | 563/993 (56.7%) |
-| 4 | 51.04 | 1.123s | 611/1,148 (53.2%) |
-| 5 | 45.19 | 1.249s | 454/1,560 (29.1%) |
-| 6 | 44.43 | 1.429s | 516/1,230 (42.0%) |
-| 8 | 37.89 | 1.758s | 487/1,600 (30.4%) |
-| 10 | 36.32 | 1.648s | 642/2,150 (29.9%) |
-
-`MTP=2` is the qualified selection. It was 83.6% faster than `MTP=0` in this
-specific short-request measurement. Higher draft depths lost enough acceptance to
-cost more verification work than they saved. Depths five and above require the
-recipe's 48-token block alignment for the QSA speculative ring.
+| 0 | 27.26 | 1.196s | n/a |
+| 1 | 33.17 | 1.206s | 305/463 (65.9%) |
+| **2** | **44.23** | **1.324s** | **532/722 (73.7%)** |
+| 3 | 40.95 | 1.629s | 563/993 (56.7%) |
+| 4 | 41.84 | 1.123s | 611/1,148 (53.2%) |
+| 5 | 35.45 | 1.249s | 454/1,560 (29.1%) |
+| 6 | 35.71 | 1.429s | 516/1,230 (42.0%) |
+| 8 | 30.43 | 1.758s | 487/1,600 (30.4%) |
+| 10 | 28.31 | 1.648s | 642/2,150 (29.9%) |
+
+`MTP=2` is the qualified selection. It was 62.2% faster end-to-end than `MTP=0`
+in this single-stream short-request measurement. For MTP2, retained reasoning-token
+accounting also gives a separate median visible-answer phase rate of 46.15 tok/s,
+measured from the first through last visible content token. That rate excludes hidden
+reasoning and must not be combined with total completion-token counts.
+
+An earlier revision did combine all completion tokens with a visible-content-only
+time interval. That mixed metric has been superseded and is not comparable to either
+rate above. Higher draft depths lost enough acceptance to cost more verification
+work than they saved. Depths five and above require the recipe's 48-token block
+alignment for the QSA speculative ring.
 
 The machine-readable record is
 [`mtp-bf16-sweep.json`](../results/orcarouter/mtp-bf16-sweep.json).

results/orcarouter/mtp-bf16-sweep.json

@@ -5,6 +5,7 @@
   "mtp_source_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594",
   "runtime": "vLLM 0.1.dev20073+g8e685d198",
   "context_window": 32768,
+  "concurrency": 1,
   "max_num_seqs": 1,
   "gpu_memory_utilization": 0.8,
   "sampling": {
@@ -17,8 +18,14 @@
     "presence_penalty": 0.0,
     "repetition_penalty": 1.0
   },
-  "measurement": "median of three fixed 256-token streamed samples after a 128-token warm-up",
+  "measurement": "single-stream median of three fixed 256-token streamed samples after a 128-token warm-up",
+  "timing_definitions": {
+    "end_to_end_completion_tokens_per_second": "all completion tokens, including hidden reasoning, divided by total request wall time",
+    "time_to_first_visible_content_seconds": "request start through the first non-empty visible content delta",
+    "visible_content_tokens_per_second": "visible content tokens only, measured from the first through last visible content delta"
+  },
   "selected_depth": 2,
+  "selected_depth_end_to_end_gain_vs_mtp0": 0.6222,
   "mtp_overlay": {
     "dtype": "BF16",
     "loaded_tensors": 31,
@@ -26,15 +33,15 @@
     "compact_file_gib": 4.86
   },
   "results": [
-    {"depth": 0, "median_decode_tokens_per_second": 31.1235, "median_ttft_seconds": 1.1963, "acceptance": null},
-    {"depth": 1, "median_decode_tokens_per_second": 39.1602, "median_ttft_seconds": 1.20583, "accepted_tokens": 305, "draft_tokens": 463, "acceptance": 0.6587},
-    {"depth": 2, "median_decode_tokens_per_second": 57.1303, "median_ttft_seconds": 1.32445, "accepted_tokens": 532, "draft_tokens": 722, "acceptance": 0.7368},
-    {"depth": 3, "median_decode_tokens_per_second": 55.5512, "median_ttft_seconds": 1.62869, "accepted_tokens": 563, "draft_tokens": 993, "acceptance": 0.5670},
-    {"depth": 4, "median_decode_tokens_per_second": 51.0404, "median_ttft_seconds": 1.12255, "accepted_tokens": 611, "draft_tokens": 1148, "acceptance": 0.5322},
-    {"depth": 5, "median_decode_tokens_per_second": 45.1911, "median_ttft_seconds": 1.24926, "accepted_tokens": 454, "draft_tokens": 1560, "acceptance": 0.2910},
-    {"depth": 6, "median_decode_tokens_per_second": 44.4321, "median_ttft_seconds": 1.42869, "accepted_tokens": 516, "draft_tokens": 1230, "acceptance": 0.4195},
-    {"depth": 8, "median_decode_tokens_per_second": 37.8855, "median_ttft_seconds": 1.75837, "accepted_tokens": 487, "draft_tokens": 1600, "acceptance": 0.3044},
-    {"depth": 10, "median_decode_tokens_per_second": 36.3225, "median_ttft_seconds": 1.64792, "accepted_tokens": 642, "draft_tokens": 2150, "acceptance": 0.2986}
+    {"depth": 0, "median_end_to_end_completion_tokens_per_second": 27.2645, "median_time_to_first_visible_content_seconds": 1.1963, "acceptance": null},
+    {"depth": 1, "median_end_to_end_completion_tokens_per_second": 33.1749, "median_time_to_first_visible_content_seconds": 1.20583, "accepted_tokens": 305, "draft_tokens": 463, "acceptance": 0.6587},
+    {"depth": 2, "median_end_to_end_completion_tokens_per_second": 44.2286, "median_visible_content_tokens_per_second": 46.1523, "median_time_to_first_visible_content_seconds": 1.32445, "accepted_tokens": 532, "draft_tokens": 722, "acceptance": 0.7368},
+    {"depth": 3, "median_end_to_end_completion_tokens_per_second": 40.9544, "median_time_to_first_visible_content_seconds": 1.62869, "accepted_tokens": 563, "draft_tokens": 993, "acceptance": 0.5670},
+    {"depth": 4, "median_end_to_end_completion_tokens_per_second": 41.8389, "median_time_to_first_visible_content_seconds": 1.12255, "accepted_tokens": 611, "draft_tokens": 1148, "acceptance": 0.5322},
+    {"depth": 5, "median_end_to_end_completion_tokens_per_second": 35.4522, "median_time_to_first_visible_content_seconds": 1.24926, "accepted_tokens": 454, "draft_tokens": 1560, "acceptance": 0.2910},
+    {"depth": 6, "median_end_to_end_completion_tokens_per_second": 35.7146, "median_time_to_first_visible_content_seconds": 1.42869, "accepted_tokens": 516, "draft_tokens": 1230, "acceptance": 0.4195},
+    {"depth": 8, "median_end_to_end_completion_tokens_per_second": 30.4295, "median_time_to_first_visible_content_seconds": 1.75837, "accepted_tokens": 487, "draft_tokens": 1600, "acceptance": 0.3044},
+    {"depth": 10, "median_end_to_end_completion_tokens_per_second": 28.3075, "median_time_to_first_visible_content_seconds": 1.64792, "accepted_tokens": 642, "draft_tokens": 2150, "acceptance": 0.2986}
   ],
   "native_context_validation": {
     "max_model_len": 262144,

tests/test_benchmarks.py

@@ -12,10 +12,60 @@ ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT / "benchmarks"))
 
 from functional import validate  # noqa: E402
+import common  # noqa: E402
 import long_context  # noqa: E402
 import stability  # noqa: E402
 
 
+class _StreamResponse:
+    def __init__(self, lines: list[bytes]):
+        self._lines = lines
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        return None
+
+    def __iter__(self):
+        return iter(self._lines)
+
+
+class StreamCompletionMetricTests(unittest.TestCase):
+    def test_reasoning_and_visible_output_use_separate_token_clocks(self):
+        """Hidden tokens must never be divided by visible-content-only time."""
+        events = [
+            b'data: {"choices":[{"delta":{"reasoning_content":"think"}}]}\n',
+            b'data: {"choices":[{"delta":{"content":"A"}}]}\n',
+            b'data: {"choices":[{"delta":{"content":"B"}}]}\n',
+            (
+                b'data: {"choices":[],"usage":{"prompt_tokens":10,'
+                b'"completion_tokens":5,"completion_tokens_details":'
+                b'{"reasoning_tokens":2}}}\n'
+            ),
+            b'data: [DONE]\n',
+        ]
+        clock = [0.0, 1.0, 3.0, 4.0, 5.0]
+
+        with patch.object(
+            common.urllib.request,
+            "urlopen",
+            return_value=_StreamResponse(events),
+        ), patch.object(common.time, "perf_counter", side_effect=clock):
+            result = common.stream_completion(
+                "http://127.0.0.1:1/v1",
+                {"stream": True},
+                1.0,
+            )
+
+        self.assertEqual(result["content"], "AB")
+        self.assertEqual(result["ttft_seconds"], 1.0)
+        self.assertEqual(result["time_to_first_visible_content_seconds"], 3.0)
+        self.assertAlmostEqual(result["decode_tokens_per_second"], 4 / 3)
+        self.assertEqual(result["visible_content_tokens_per_second"], 2.0)
+        self.assertEqual(result["end_to_end_tokens_per_second"], 1.0)
+
+
 class FunctionalBenchmarkTests(unittest.TestCase):
     def test_code_validator_rejects_malicious_output_without_executing_it(self):
         """An extra payload after square must never be executed by validation."""

tests/test_publication_docs.py

@@ -93,7 +93,10 @@ class PublicationDocumentationTests(unittest.TestCase):
         self.assertIn("orca-uncensored-bf16-mtp", docs)
         self.assertIn("31/31", docs)
         self.assertIn("MTP=2", docs)
-        self.assertIn("57.13 tok/s", docs)
+        self.assertIn("44.23 tok/s", docs)
+        self.assertIn("46.15 tok/s", docs)
+        self.assertIn("single-stream", docs)
+        self.assertNotIn("57.13 tok/s", docs)
         self.assertIn("240,051 prompt tokens", docs)
         for stale_claim in (
             "requires `MTP=0`",
@@ -111,9 +114,29 @@ class PublicationDocumentationTests(unittest.TestCase):
             )
         )
         self.assertEqual(record["selected_depth"], 2)
+        self.assertEqual(record["concurrency"], 1)
         self.assertEqual([row["depth"] for row in record["results"]], [0, 1, 2, 3, 4, 5, 6, 8, 10])
         selected = next(row for row in record["results"] if row["depth"] == 2)
-        self.assertEqual(selected["median_decode_tokens_per_second"], 57.1303)
+        baseline = next(row for row in record["results"] if row["depth"] == 0)
+        self.assertEqual(
+            selected["median_end_to_end_completion_tokens_per_second"],
+            44.2286,
+        )
+        self.assertEqual(
+            selected["median_visible_content_tokens_per_second"],
+            46.1523,
+        )
+        self.assertEqual(
+            selected["median_time_to_first_visible_content_seconds"],
+            1.32445,
+        )
+        self.assertEqual(
+            baseline["median_end_to_end_completion_tokens_per_second"],
+            27.2645,
+        )
+        self.assertFalse(
+            [row for row in record["results"] if "median_decode_tokens_per_second" in row]
+        )
         self.assertEqual(record["mtp_overlay"]["loaded_tensors"], 31)
 
     def test_private_deployment_terms_are_absent(self) -> None: