Release v1.3.1: faster SystemOne batching with exact output parity

RUNTIME-RELEASE.json

@@ -0,0 +1,30 @@
+{
+  "format": "decision-runtime-patch-v1",
+  "family": "Nox",
+  "release_tag": "v1.3.1",
+  "change_kind": "runtime_only",
+  "weights_revision": "ad089ad3a5dc9a7a21e6d96db546bb53e2212654",
+  "source_bundle_manifest_sha256": "92d7f5be5e1ef21ee682f574b35cc01de4dd8ab16b8aa6edf0dfbdfcfcfabba3",
+  "bundle_manifest_sha256": "83876db506b2d98e3e8ce7d34310b21f97bac30d4f5aef3371053798bff08830",
+  "changed_inference_files": [
+    "code/decision_api.py"
+  ],
+  "weights_tokenizer_prompts_calibration_profile_unchanged": true,
+  "default_public_parity": {
+    "requests": 2856,
+    "answers": 3160,
+    "fixtures": 58,
+    "raw_logits_probabilities_typed_outputs_exact": true,
+    "receipt_sha256": "946dba5c31074b63efa3223ede17857e5f9d1b7b3b44de08cd9add9ceece878c"
+  },
+  "timing": {
+    "receipt_sha256": "501f25efd1813b566d77e98278229440b7a3b4a2358def1462c0171a33e9f3be",
+    "shape": "distinct fixed-length questions; Q=32; 499 tokens per question",
+    "reference_ms": 419.124,
+    "candidate_ms": 414.199,
+    "same_measured_api_sha256": "273f6f10f22d5a68b8db34cfcbd35407fb43d8030f6d7cd188bcf118dd90a152",
+    "no_shared_state_neural_cache_claim": true
+  },
+  "capability_scores_unchanged": true,
+  "downloaded_package_offline_proof_required_before_promotion": true
+}

SERVING_OPTIMIZATION.json

@@ -0,0 +1,34 @@
+{
+  "format": "joint-serving-runtime-candidate-v1",
+  "family": "Nox",
+  "source_publication_revision": "ad089ad3a5dc9a7a21e6d96db546bb53e2212654",
+  "source_bundle_manifest_sha256": "92d7f5be5e1ef21ee682f574b35cc01de4dd8ab16b8aa6edf0dfbdfcfcfabba3",
+  "source_release_manifest_sha256": "3e11cc6011860ea245c53eb44b9da61a85d5feb5c6caa5da95428d63ede6d712",
+  "source_api_sha256": "1b068eccdffd3c3b67bfa52f8f526e6b482d92551c927668ad28a794767f8a40",
+  "candidate_api_sha256": "273f6f10f22d5a68b8db34cfcbd35407fb43d8030f6d7cd188bcf118dd90a152",
+  "changed_inference_files": [
+    "code/decision_api.py"
+  ],
+  "held_fixed": [
+    "weights",
+    "tokenizer",
+    "prompt",
+    "temperature",
+    "normalization_profile",
+    "BF16_backbone_FP32_head",
+    "batch8",
+    "input_limit16384",
+    "public_wrapper"
+  ],
+  "shared_state_neural_cache": false,
+  "cross_request_cache": false,
+  "timing_evidence": {
+    "analysis/decoder-joint-serving-v1/COMPLETED-PARITY.json": "4187fb76432eb69b263cb6d5ad55f10a09aef384fe405f3ad204acf1edc761b5",
+    "analysis/decoder-joint-timing-v1/INDEPENDENTLY-REVIEWED.json": "1b416c8556c2305ffd18bd1e3cd523964820a5e2a3029cf447f20b2829ba82f7",
+    "results/joint-serving-timing-v1/summary/SUMMARY.json": "501f25efd1813b566d77e98278229440b7a3b4a2358def1462c0171a33e9f3be",
+    "analysis/decoder4b/joint-timing-actual-review-v1/REVIEW.json": "e94fec9d9c6d487200ef7ea1cf82319b33c4836c08d83f0b856bf8590f2fbee6"
+  },
+  "candidate_default_entrypoint_proof_required": true,
+  "published": false,
+  "adoption_authorized": false
+}

bundle-manifest.json

@@ -1,6 +1,6 @@
 {
   "format": "research-pointer-bundle-v1",
-  "status": "profile-bound-candidate-awaiting-default-public-entrypoint-offline-proof",
+  "status": "joint-serving-candidate-awaiting-default-public-parity",
   "files": [
     {
       "file": "NORMALIZATION_RUNTIME.md",
@@ -12,6 +12,11 @@
       "bytes": 2621,
       "sha256": "fb76d9fc9147e15a678eb91dfc40d7039845a4eaebdecce041a5c9e76c3d3e0f"
     },
+    {
+      "file": "SERVING_OPTIMIZATION.json",
+      "bytes": 1548,
+      "sha256": "fe2fd8c3e046faec139b8685eed693db0e6aa5235ebe5f860bc9f6dd1888c8ad"
+    },
     {
       "file": "SOURCE_BUNDLE_MANIFEST.json",
       "bytes": 5637,
@@ -49,8 +54,8 @@
     },
     {
       "file": "code/decision_api.py",
-      "bytes": 8251,
-      "sha256": "1b068eccdffd3c3b67bfa52f8f526e6b482d92551c927668ad28a794767f8a40"
+      "bytes": 10952,
+      "sha256": "273f6f10f22d5a68b8db34cfcbd35407fb43d8030f6d7cd188bcf118dd90a152"
     },
     {
       "file": "code/decision_model.py",
@@ -216,7 +221,7 @@
     }
   ],
   "source_model_code_sha256": "d3e28489c09f3bd7130e2d43d92e0b5c4a08e25b09b21303904defb0ff1c3646",
-  "source_api_code_sha256": "1b068eccdffd3c3b67bfa52f8f526e6b482d92551c927668ad28a794767f8a40",
+  "source_api_code_sha256": "273f6f10f22d5a68b8db34cfcbd35407fb43d8030f6d7cd188bcf118dd90a152",
   "dev_sha256": "45b4cd46acc4be53b95c7ed8f333b3533972296a4d0557c7b8c48c9cab6ced61",
   "production_predictions_sha256": "ba311f5a6bc73182651bdf5c9744f0ed95921110501ddc93dcfd20a1d06bceff",
   "temperature_sha256": "69d80e5b215e2c1e6872f146fdb7ea5aa95c9fd678ef96208960d50e4e7b635d",
@@ -227,5 +232,7 @@
   "no_publication_performed": true,
   "source_bundle_manifest_sha256": "d437bc0149bcb8c9891fbb33c5abc4c2336c3a89ab6cfa0981da5a7f1d19f1f4",
   "normalization_profile_sha256": "be32858d15233e0a3fbee0e4257fb02be0b3439deee4eb9c3f61151df7b73850",
-  "public_wrapper_included": true
+  "public_wrapper_included": true,
+  "source_published_bundle_manifest_sha256": "92d7f5be5e1ef21ee682f574b35cc01de4dd8ab16b8aa6edf0dfbdfcfcfabba3",
+  "serving_optimization_sha256": "fe2fd8c3e046faec139b8685eed693db0e6aa5235ebe5f860bc9f6dd1888c8ad"
 }

code/decision_api.py

@@ -102,7 +102,7 @@ class DecisionEngine:
 
     def predict_rows(self, rows):
         import torch
-        encoded=[self.module.encode(row,self.tokenizer,self.max_length) for row in rows]
+        encoded=encode_request(rows,self.tokenizer,self.module,self.max_length)
         pad=self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
         records=[]
         with torch.inference_mode():
@@ -111,17 +111,28 @@ class DecisionEngine:
                 batch={key:value.to(self.device) if torch.is_tensor(value) else value
                        for key,value in self.module.collate(items,pad).items()}
                 with torch.autocast('cuda',dtype=torch.bfloat16):logits=self.model(**batch)
+                # Preserve each row's original float/temperature/softmax math,
+                # but defer host synchronization until the complete batch.
+                staged=[];transfers=[]
                 for row,item,values in zip(rows[start:start+self.batch_size],items,logits):
                     k=len(row['options']);values=values[:k].float()
                     temperature=self.temperatures.get(row['task_type'],1.)
-                    probabilities=(values/temperature).softmax(-1).tolist()
+                    probabilities=(values/temperature).softmax(-1)
+                    staged.append((row,item,k,temperature))
+                    transfers.extend((values,probabilities))
+                host_values=torch.cat(transfers).tolist()
+                offset=0
+                for row,item,k,temperature in staged:
+                    values=host_values[offset:offset+k]
+                    probabilities=host_values[offset+k:offset+2*k]
+                    offset+=2*k
                     answer=typed_answer(row,probabilities)
                     prediction=max(range(k),key=probabilities.__getitem__)
                     if row['task_type']=='noul':
                         chosen='true' if answer['noul']>=.5 else 'false'
                         prediction=[o['key'] for o in row['options']].index(chosen)
                     rec={'id':row['id'],'status':'ok','prediction':prediction,
-                         'probabilities':probabilities,'logits':values.tolist(),'temperature':temperature,
+                         'probabilities':probabilities,'logits':values,'temperature':temperature,
                          'native_contract':True,'truncated':False,'input_tokens':len(item['ids']),
                          'prompt_sha256':item['prompt_sha256'],'answer':answer}
                     if row['task_type']=='noul':rec['native_noul']=answer['noul']
@@ -137,3 +148,51 @@ class DecisionEngine:
         result=self.predict_rows(rows)
         return {'model':self.model_name,'answers':{r['id']:r['answer'] for r in result},
                 'usage':{'input_tokens':sum(r['input_tokens'] for r in result),'scored_questions':len(result)}}
+
+
+"""Experimental request-local exact-segment tokenization.
+
+Original encode/segments functions remain authoritative. Batch tokenize exact
+whole segments, never split a BPE prefix at a new boundary. No cross-request
+cache, GPU change, prompt change or change to the eight-row inference groups.
+"""
+
+
+class SegmentLookup:
+    def __init__(self, tokenizer, cache):
+        self.tokenizer, self.cache = tokenizer, cache
+
+    def encode(self, text, **kwargs):
+        if kwargs == {'add_special_tokens': False} and text in self.cache:
+            # Original encode extends its prefix list in place.
+            return list(self.cache[text])
+        return self.tokenizer.encode(text, **kwargs)
+
+
+def encode_request(rows, tokenizer, module, max_length=16384,
+                   max_cached_characters=8_000_000, segment_batch_size=64):
+    if max_cached_characters < 0 or segment_batch_size < 1:
+        raise ValueError('Invalid tokenizer resource bound')
+    unique = {}
+    characters = 0
+    for row in rows:
+        prefix, options, suffix = module.segments(row)
+        for segment in (prefix, *options, suffix):
+            if segment not in unique:
+                unique[segment] = None
+                characters += len(segment)
+                if characters > max_cached_characters:
+                    # Preserve the original behavior under the resource cap.
+                    return [module.encode(r, tokenizer, max_length) for r in rows]
+    strings = list(unique)
+    for start in range(0, len(strings), segment_batch_size):
+        batch = strings[start:start + segment_batch_size]
+        result = tokenizer(batch, add_special_tokens=False, padding=False,
+                           truncation=False, return_attention_mask=False,
+                           return_token_type_ids=False)['input_ids']
+        if len(result) != len(batch):
+            raise ValueError('Batch tokenizer output count differs')
+        for segment, ids in zip(batch, result):
+            unique[segment] = tuple(ids)
+    lookup = SegmentLookup(tokenizer, unique)
+    return [module.encode(row, lookup, max_length) for row in rows]

model-card-example.json

@@ -65,7 +65,7 @@
   "direct_engine_exact_response": true,
   "overflow_rejected": true,
   "overflow_message": "check: 40083 tokens exceeds max_length=16384; no truncation allowed",
-  "bundle_manifest_sha256": "92d7f5be5e1ef21ee682f574b35cc01de4dd8ab16b8aa6edf0dfbdfcfcfabba3",
+  "bundle_manifest_sha256": "83876db506b2d98e3e8ce7d34310b21f97bac30d4f5aef3371053798bff08830",
   "runtime": {
     "actual": {
       "torch": "2.12.0+git6bbd260",

release-manifest.json

@@ -1,12 +1,12 @@
 {
   "format": "decision-public-release-v1",
-  "status": "assembled-not-published",
-  "bundle_manifest_sha256": "92d7f5be5e1ef21ee682f574b35cc01de4dd8ab16b8aa6edf0dfbdfcfcfabba3",
-  "readiness_sha256": "8413ac6536011da3f9088af629e49eded82c90123bd238cbe5fe339f21448045",
+  "status": "runtime-patch-assembled-before-staged-download-proof",
+  "bundle_manifest_sha256": "83876db506b2d98e3e8ce7d34310b21f97bac30d4f5aef3371053798bff08830",
+  "readiness_sha256": "97c0c58b97685f95ad57a65b378f4e4f374ff3f7a7e3f7ef37c7960131981ffe",
   "model_card_sha256": "4ec3943c766b36eed886ac2713363dbcbc5b736b942c1e38f13cb1ab5f5758cd",
   "repo_id": "llm-semantic-router/Decision-1.0-Nox",
   "assembly_script_sha256": "7831ef5952b8256e585d4efa91cac4ce87738f4eb93c84b1cf1e7250def1f430",
-  "original_bundle_manifest_preserved": true,
+  "original_bundle_manifest_preserved": false,
   "files_exclude_this_manifest": true,
   "files": [
     {
@@ -54,6 +54,11 @@
       "bytes": 5001,
       "sha256": "4ec3943c766b36eed886ac2713363dbcbc5b736b942c1e38f13cb1ab5f5758cd"
     },
+    {
+      "file": "RUNTIME-RELEASE.json",
+      "bytes": 1264,
+      "sha256": "97c0c58b97685f95ad57a65b378f4e4f374ff3f7a7e3f7ef37c7960131981ffe"
+    },
     {
       "file": "RUNTIME.md",
       "bytes": 5205,
@@ -64,6 +69,11 @@
       "bytes": 2621,
       "sha256": "fb76d9fc9147e15a678eb91dfc40d7039845a4eaebdecce041a5c9e76c3d3e0f"
     },
+    {
+      "file": "SERVING_OPTIMIZATION.json",
+      "bytes": 1548,
+      "sha256": "fe2fd8c3e046faec139b8685eed693db0e6aa5235ebe5f860bc9f6dd1888c8ad"
+    },
     {
       "file": "SOURCE_BUNDLE_MANIFEST.json",
       "bytes": 5637,
@@ -261,8 +271,8 @@
     },
     {
       "file": "bundle-manifest.json",
-      "bytes": 7635,
-      "sha256": "92d7f5be5e1ef21ee682f574b35cc01de4dd8ab16b8aa6edf0dfbdfcfcfabba3"
+      "bytes": 7992,
+      "sha256": "83876db506b2d98e3e8ce7d34310b21f97bac30d4f5aef3371053798bff08830"
     },
     {
       "file": "chat_template.jinja",
@@ -271,8 +281,8 @@
     },
     {
       "file": "code/decision_api.py",
-      "bytes": 8251,
-      "sha256": "1b068eccdffd3c3b67bfa52f8f526e6b482d92551c927668ad28a794767f8a40"
+      "bytes": 10952,
+      "sha256": "273f6f10f22d5a68b8db34cfcbd35407fb43d8030f6d7cd188bcf118dd90a152"
     },
     {
       "file": "code/decision_model.py",
@@ -342,7 +352,7 @@
     {
       "file": "model-card-example.json",
       "bytes": 3775,
-      "sha256": "9d17a7495a0986d4850a966e282cd3d9e0a31130bdf08c0a9e60461795ce5502"
+      "sha256": "6bb34ba80660291be6c53005571a8a1d6546852bca186fff5da915019195d3a9"
     },
     {
       "file": "pyproject.toml",
@@ -492,5 +502,9 @@
     "MATERIALS.json": "copy",
     "metrics/semantic-consistency.json": "copy"
   },
-  "scope": "Inference weights, original numerical code, calibrated runtime metadata, wrapper, model card, license, attribution and approved assets only."
+  "scope": "Inference weights, original numerical code, calibrated runtime metadata, wrapper, model card, license, attribution and approved assets only.",
+  "release_tag": "v1.3.1",
+  "change_kind": "runtime_only",
+  "previous_main_revision": "3d42ac931bba5726374d9c898e5df70db44ffea1",
+  "previous_release_manifest_sha256": "c8a9ce88ed6312a7a822c0ef6d70f7ebceb06fadb323543fdfe494b8bdbba50d"
 }