Decision 2.0 package DEV2.0-2B (release) from 0ca621e0ecbb20e5ae1802b9f71f8dea801fa693-src_training_decision2

MODEL_MANIFEST.json

@@ -2,7 +2,7 @@
   "base": null,
   "builder": {
     "module_sha256": "bd498737bb797532fe74039088c45cfe2a75585d111034ee7d9bd04db838587c",
-    "source_commit": "539d2769fe4d13ae888dcef4cda8695cb4bafc9e"
+    "source_commit": "0ca621e0ecbb20e5ae1802b9f71f8dea801fa693"
   },
   "calibration": null,
   "card": {
@@ -11,14 +11,14 @@
       "assets/jevarena-v3-rank.svg": "26a36aa0569d9176357be3438320145c07cedbcc280d65864828c435b77ea17b",
       "assets/jevbench-public231-rank.svg": "7ed4bf852ef67c95c91ad8a85ba7faba53bdde0115f6a30d492dad2a7fb7ed4e"
     },
-    "readme_sha256": "3d679fe27250706a8f872910527bc6bd96fd660ce51c34bb1bd6502996acc61d"
+    "readme_sha256": "acbd4d50b6452339339e19613f2fc34681fad549709d7f2cefe84a80835bfe32"
   },
   "files_sha256": {
     "ATTRIBUTIONS.md": "69c8e5237dafeeaa1ec77c5a6636068bcb51c3054c07ce6688c7de691f0594e3",
     "LICENSE": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a",
     "LICENSES/Qwen3.5-2B-LICENSE.txt": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a",
     "NOTICE": "f82ed40c15223a2a5e93dbdd2931b48a86c2aab2c181838f3f6d7f2491a503ee",
-    "README.md": "3d679fe27250706a8f872910527bc6bd96fd660ce51c34bb1bd6502996acc61d",
+    "README.md": "acbd4d50b6452339339e19613f2fc34681fad549709d7f2cefe84a80835bfe32",
     "assets/DEV2.0-2B-owl-banner.png": "94008fafd629e5b8412e9c7b92d9dec07ff18a30e2cb973d4a2e43a8f927515a",
     "assets/jevarena-v3-model-task.svg": "5616473321481ad6afe45343adc863dfe4e97537b761833302c12bcdd75e102e",
     "assets/jevarena-v3-rank.svg": "26a36aa0569d9176357be3438320145c07cedbcc280d65864828c435b77ea17b",
@@ -40,7 +40,7 @@
     "decision2/_vendor/dev2model/lora.py": "7049e35e2a7bf50dd5888902cd0b7030d9bbd448d7d7aba8875ceb89aebff231",
     "decision2/_vendor/dev2model/source.py": "ef7b30171c4befb26163ea4d3d6e5add9dc602228a476d6f0ead9d40b450c7e3",
     "decision2/api.py": "3157c0509e8fb6c56038486f19b42ca09f44edcfce2ef32bc9ed5f4768829b0f",
-    "decision2/qwen.py": "59eb1605ece2066db3061530c2b687a667719202ea0c61ef88f748a3293ac49d",
+    "decision2/qwen.py": "97ba565f95d5a5ac9af6632e7397c05cfe9ab47fde68b56df19aabfa2d8fb187",
     "decision_config.json": "0b6c3429ee06032739d7386659c332ed7bb62d0b96ccddfcad4fad999e22cd4f",
     "decision_head.safetensors": "33b6541bb6636677eb91a4d8e06acd4db81152b11097088840796f11d49c707a",
     "evaluation/EVALUATION.md": "ecc747078712611e97ed56724e165a758bc3526eee5feb4a1f47c61b1dfedf32",
@@ -135,9 +135,9 @@
       "AutoModel": "modeling_decision2.Decision2Model"
     },
     "automap_source": {
-      "commit": "8e808244072cef8735b8f3fdcfb8487f36605c5c",
-      "content_manifest_sha256": "565576439331ef038c4b5c4794b867563bc6ad56964d19b0de948ed4f931f538",
-      "tree": "31010c6ec5e465b2c8d8567e994c9f1dfb571177"
+      "commit": "99432d1a7da5adbc70212df78ae7ebf7e50b41e4",
+      "content_manifest_sha256": "3bede2ca1fc0b805c167bf5ae6b2bd824a3b64d1b6ae98478d04add1cc1444ff",
+      "tree": "fbf37474b1ce2efd228b7f50803848c6f60ebbd2"
     },
     "custom_pipelines": {
       "decision": {
@@ -173,7 +173,7 @@
   },
   "repo_id": "llm-semantic-router/DEV2.0-2B",
   "runtime": {
-    "equivalence": "decision2/qwen.py loads this full checkpoint with the vendored training/model sources whose SHA-256 equal the scored adapter sources (checked at build time) and applies the per-item batching, BF16-backbone / FP32-head execution, raw probabilities (temperature 1; no calibration file) and answer normalization of v2.dec.infer_dec, which for this non-residual checkpoint wraps the same DecisionModel without extra readouts. The scored checkpoint (073bd1f2) stored every tensor in FP32; this package (v2.release.bf16_copy, receipt a5229ef1) stores its 186 Linear projection matrices in BF16 exactly as BF16 autocast rounds them and every other tensor bit for bit in FP32. From this revision the runtime holds the backbone's BF16-exact Linear weights in BF16, the values BF16 autocast multiplies with, instead of FP32 copies cast before every matmul; it was checked with 0 answer changes on every scored prompt and on mlx-diag (2,275). From this revision the package also ships 🤗 Transformers remote code (configuration_decision2.py, modeling_decision2.py, pipeline_decision2.py; config.json gains model_type, auto_map and custom_pipelines): AutoModel with trust_remote_code loads the package through this runtime, which now also refuses Transformers' remote-code prompt while loading the tokenizer; it was checked with 0 answer changes against the native runtime on every scored prompt and on mlx-diag (2,275). Checked on one GPU of the scoring node against the T = 1 predictions derived exactly from the sealed CAL698 predictions of every scored prompt (typed-final 1,600, css15 6,547, public231 231) and of the mlx-diag diagnostic (2,275) by release.sh --parity, with the scored run's persisted Triton autotune cache.",
+    "equivalence": "decision2/qwen.py loads this full checkpoint with the vendored training/model sources whose SHA-256 equal the scored adapter sources (checked at build time) and applies the per-item batching, BF16-backbone / FP32-head execution, raw probabilities (temperature 1; no calibration file) and answer normalization of v2.dec.infer_dec, which for this non-residual checkpoint wraps the same DecisionModel without extra readouts. The scored checkpoint (073bd1f2) stored every tensor in FP32; this package (v2.release.bf16_copy, receipt a5229ef1) stores its 186 Linear projection matrices in BF16 exactly as BF16 autocast rounds them and every other tensor bit for bit in FP32. From this revision the runtime holds the backbone's BF16-exact Linear weights in BF16, the values BF16 autocast multiplies with, instead of FP32 copies cast before every matmul; it was checked with 0 answer changes on every scored prompt and on mlx-diag (2,275). From this revision the package also ships 🤗 Transformers remote code (configuration_decision2.py, modeling_decision2.py, pipeline_decision2.py; config.json gains model_type, auto_map and custom_pipelines): AutoModel with trust_remote_code loads the package through this runtime, which now also refuses Transformers' remote-code prompt while loading the tokenizer; it was checked with 0 answer changes against the native runtime on every scored prompt and on mlx-diag (2,275). From this revision the runtime runs a request whose padded question batch would put more than 2**30 elements in a gated-delta q / k / v tensor (the FLA kernels' 32-bit offsets) as several GPU-sized batches, longest questions first; requests within that budget, every scored prompt among them, keep the single-batch path, and it was checked with 0 answer changes on every scored prompt and on mlx-diag (2,275). Checked on one GPU of the scoring node against the T = 1 predictions derived exactly from the sealed CAL698 predictions of every scored prompt (typed-final 1,600, css15 6,547, public231 231) and of the mlx-diag diagnostic (2,275) by release.sh --parity, with the scored run's persisted Triton autotune cache.",
     "requirements": {
       "causal-conv1d": "1.7.0 (GPU convolution kernels)",
       "flash-linear-attention": "0.5.2 (GPU gated-delta kernels)",
@@ -185,9 +185,9 @@
       "triton": "3.7.1 (persisted autotune cache)"
     },
     "runtime_source": {
-      "commit": "8e808244072cef8735b8f3fdcfb8487f36605c5c",
-      "content_manifest_sha256": "565576439331ef038c4b5c4794b867563bc6ad56964d19b0de948ed4f931f538",
-      "tree": "31010c6ec5e465b2c8d8567e994c9f1dfb571177"
+      "commit": "99432d1a7da5adbc70212df78ae7ebf7e50b41e4",
+      "content_manifest_sha256": "3bede2ca1fc0b805c167bf5ae6b2bd824a3b64d1b6ae98478d04add1cc1444ff",
+      "tree": "fbf37474b1ce2efd228b7f50803848c6f60ebbd2"
     },
     "scored_runtime_check": {
       "checked": {
@@ -267,9 +267,9 @@
     },
     "decision2/qwen.py": {
       "rewritten": false,
-      "sha256": "59eb1605ece2066db3061530c2b687a667719202ea0c61ef88f748a3293ac49d",
+      "sha256": "97ba565f95d5a5ac9af6632e7397c05cfe9ab47fde68b56df19aabfa2d8fb187",
       "source": "v2/release/runtime/qwen.py",
-      "source_sha256": "59eb1605ece2066db3061530c2b687a667719202ea0c61ef88f748a3293ac49d"
+      "source_sha256": "97ba565f95d5a5ac9af6632e7397c05cfe9ab47fde68b56df19aabfa2d8fb187"
     }
   },
   "schema": "dev2-package-manifest/1",

README.md

@@ -164,6 +164,7 @@ print(json.dumps(result["answers"], indent=2))
 - **Weights:** the uniform average of three full fine-tunes of Decision 1.0 Sol with the same recipe and start; the seeds differ only in data order, and each seed's checkpoint was chosen on a held-out selection set (updates 366, 741 and 638).
 - **Comparator licences:** Decider 2B's model card declares Apache-2.0 and This-That 1.2's declares MIT (it is adapted from Decider 2B); neither repository has a LICENSE file. Bosun v3.1 1.7B ships Apache-2.0 LICENSE and NOTICE files.
 - **Runtime update:** BF16-resident weights; answers unchanged; latency p50 24.1 → 23.5 ms, p95 24.8 → 23.7 ms; peak GPU memory 7.1 → 4.6 GiB (400 single requests, 346 input tokens on average, one AMD MI325X GPU).
+- **Runtime update:** very long multi-question requests are processed in GPU-sized batches (previously they could fail on extremely long inputs); answers unchanged.
 
 ### Training
 

decision2/qwen.py

@@ -7,10 +7,15 @@ records an import-only rewrite. A base-bound adapter's source files are pinned
 by repository revision and SHA-256; a supplied or downloaded copy is verified
 before use. GPU inference runs the backbone under BF16 autocast with its
 BF16-exact Linear weights held in BF16 and every other tensor, the head
-included, in FP32; CPU uses FP32.
+included, in FP32; CPU uses FP32. A request's questions run as one padded
+batch unless, on a GPU, that batch would put more than 2**30 elements in a
+gated-delta q / k / v tensor (``forward_token_budget``); then they run as
+several batches.
 A package whose manifest names a weight ``storage`` codec (bf16z) is first restored
 to exact safetensors files in a cache directory; the restored checkpoint must
-reproduce the scored identity.
+reproduce the scored identity. A checkpoint whose ``decision_config.json`` declares
+``readout: label_token`` is read through its tied LM head's label-token logits at
+the answer cue (vendored ``label_token.py``) instead of a candidate head.
 """
 
 from __future__ import annotations
@@ -98,6 +103,53 @@ def keep_linear_bf16(module: Any, torch: Any) -> dict[str, int]:
     return counts
 
 
+INT32_MAX = 2**31 - 1
+
+
+def forward_token_budget(config: Any) -> int | None:
+    """Most padded tokens one GPU forward may hold; None when nothing limits it.
+
+    The FLA gated-delta kernels of Qwen3.5-family backbones compute some
+    element offsets of their [batch, tokens, value heads, head dim] q / k / v
+    tensors in 32-bit integers (the L2-norm forward and the fused KKT-solve
+    kernel among them). A forward whose tensors exceed 2**31 - 1 elements reads
+    and writes the wrong memory for the later batch rows: wrong answers,
+    non-finite logits or a GPU memory fault. On DEV2.0-27B, forwards with
+    tensors between about 2**30 and 2**31 elements also hung or crashed the
+    process, so the budget keeps these tensors within 2**30 elements.
+    """
+    config = getattr(config, "text_config", None) or config
+    heads = getattr(config, "linear_num_value_heads", None)
+    if not heads:
+        return None
+    width = heads * max(config.linear_key_head_dim, config.linear_value_head_dim)
+    return (INT32_MAX // 2) // width
+
+
+def micro_batches(lengths: list[int], budget: int | None) -> list[list[int]]:
+    """Indices of each forward: one batch when its padded size fits the budget.
+
+    Otherwise the questions go longest first into batches whose padded size
+    (``collate`` pads to the longest, rounded up to 8) stays within the budget,
+    each batch listing its indices in request order.
+    """
+
+    def padded(length: int) -> int:
+        return -(-length // 8) * 8
+
+    if budget is None or padded(max(lengths)) * len(lengths) <= budget:
+        return [list(range(len(lengths)))]
+    order = sorted(range(len(lengths)), key=lambda i: (-lengths[i], i))
+    if padded(lengths[order[0]]) > budget:
+        raise ValueError("A single question exceeds the forward token budget")
+    groups: list[list[int]] = []
+    while order:
+        rows = budget // padded(lengths[order[0]])
+        groups.append(sorted(order[:rows]))
+        order = order[rows:]
+    return groups
+
+
 class QwenDecision:
     def __init__(
         self,
@@ -109,6 +161,8 @@ class QwenDecision:
         torch: Any,
         score_bias: dict[int, list[float]] | None = None,
         residency: dict[str, int] | None = None,
+        encode_fn: Any = None,
+        batch_tokens: int | None = None,
     ):
         self.model = model
         self.tokenizer = tokenizer
@@ -118,6 +172,8 @@ class QwenDecision:
         self.torch = torch
         self.score_bias = score_bias
         self.residency = residency
+        self.encode_fn = encode_fn
+        self.batch_tokens = batch_tokens
 
     @classmethod
     def load(
@@ -150,7 +206,12 @@ class QwenDecision:
             (model_root / "decision_config.json").read_text(encoding="utf-8")
         )
         residual = metadata.get("dec_residual") is not None
-        if residual:
+        label = metadata.get("readout") == "label_token"
+        if label:
+            from ._vendor.dev2model.label_token import label_fingerprint
+
+            identity = label_fingerprint(model_root, source)
+        elif residual:
             from ._vendor.dev2model.dec_model import dec_fingerprint
 
             identity = dec_fingerprint(model_root, source)
@@ -176,7 +237,16 @@ class QwenDecision:
             not torch.cuda.is_available() or not torch.cuda.is_bf16_supported()
         ):
             raise RuntimeError("A CUDA/ROCm BF16 GPU is required for GPU inference")
-        if residual:
+        encode_fn = None
+        if label:
+            from ._vendor.dev2model.label_token import (
+                encode_label,
+                load_label_checkpoint,
+            )
+
+            model, tokenizer = load_label_checkpoint(model_root, source)
+            encode_fn = encode_label
+        elif residual:
             from ._vendor.dev2model.dec_model import load_dec_checkpoint
 
             model, tokenizer = load_dec_checkpoint(model_root, source)
@@ -191,6 +261,11 @@ class QwenDecision:
         model = model.to(target).eval()
         if tokenizer.pad_token_id is None and tokenizer.eos_token_id is None:
             raise ValueError("Tokenizer needs a pad or EOS token")
+        batch_tokens = (
+            forward_token_budget(model.backbone.config)
+            if target.type == "cuda"
+            else None
+        )
         return cls(
             model,
             tokenizer,
@@ -200,6 +275,8 @@ class QwenDecision:
             torch,
             score_bias,
             residency,
+            encode_fn,
+            batch_tokens,
         )
 
     def parameter_count(self) -> int:
@@ -229,7 +306,7 @@ class QwenDecision:
         for qid, question in questions.items():
             try:
                 row = question_to_row(item, qid, question)
-                encoded = encode(row, self.tokenizer, self.cap)
+                encoded = (self.encode_fn or encode)(row, self.tokenizer, self.cap)
             except ValueError as exc:
                 reason = (
                     "max_length_exceeded"
@@ -250,21 +327,29 @@ class QwenDecision:
         pad_id = self.tokenizer.pad_token_id
         if pad_id is None:
             pad_id = self.tokenizer.eos_token_id
-        batch = {
-            key: value.to(self.device) if self.torch.is_tensor(value) else value
-            for key, value in collate(
-                [encoded for _, _, encoded in jobs], pad_id
-            ).items()
-        }
-        autocast = (
-            self.torch.autocast(device_type="cuda", dtype=self.torch.bfloat16)
-            if self.device.type == "cuda"
-            else nullcontext()
-        )
-        with self.torch.inference_mode(), autocast:
-            logits = self.model(**batch)
-        if len(logits) != len(jobs):
-            raise RuntimeError("Model returned the wrong number of question answers")
+        logits: list[Any] = [None] * len(jobs)
+        for group in micro_batches(
+            [len(encoded["ids"]) for _, _, encoded in jobs], self.batch_tokens
+        ):
+            batch = {
+                key: value.to(self.device) if self.torch.is_tensor(value) else value
+                for key, value in collate(
+                    [jobs[index][2] for index in group], pad_id
+                ).items()
+            }
+            autocast = (
+                self.torch.autocast(device_type="cuda", dtype=self.torch.bfloat16)
+                if self.device.type == "cuda"
+                else nullcontext()
+            )
+            with self.torch.inference_mode(), autocast:
+                output = self.model(**batch)
+            if len(output) != len(group):
+                raise RuntimeError(
+                    "Model returned the wrong number of question answers"
+                )
+            for index, values in zip(group, output):
+                logits[index] = values
         if self.score_bias is not None:
             from ._vendor.dev2model.score_bias import apply as apply_score_bias
         for (qid, row, encoded), values in zip(jobs, logits):