serve v1.2: up to 588 options in one pass, per-model limit (max_one_pass in decision_config.json)

README.md

@@ -56,7 +56,9 @@ probabilities** over a bounded set of options. It does not generate text at deci
 1. **Prompt.** The state (evidence), the question (criterion) and the lettered options go into a structured
    prompt, following the SemIf format.
 2. **Readout.** The model reads the next-token logits **only over the option letters**. A softmax with a
-   fixed temperature gives the distribution.
+   fixed temperature gives the distribution. Options are labelled A–Z and then AA, AB, … (each label a single
+   token): up to 100 options are read in the same forward pass (serve v1.2, `max_one_pass` in
+   `decision_config.json`; beyond ~100 the 4B does better through a tournament whose final round is one pass).
 3. **Calibration.** Temperature T = 1 ships by default (see *Calibration*).
 4. **Training.** Distillation from a strong teacher (GLM-5.3-Flash, maximum reasoning effort) on:
    - generated and blindly labeled decision items;
@@ -88,7 +90,7 @@ python serve.py --model <MODEL_DIR> --vllm-url http://127.0.0.1:8001 --port 8000
 ```
 
 `serve_vllm.sh` runs `vllm serve` with the flags the readout needs: `--served-model-name decider`,
-`--enable-prefix-caching --mamba-cache-mode all`, `--logprobs-mode processed_logprobs --max-logprobs 32`,
+`--enable-prefix-caching --mamba-cache-mode align`, `--logprobs-mode processed_logprobs --max-logprobs 600`,
 `--dtype bfloat16` and `--max-model-len 16384`.
 
 ### HTTP API (TypeSafe-compatible)

SHA256SUMS

@@ -1,7 +1,7 @@
 2cf8c18cf63fd0e31addba3775a2bbec6f58aa31bb5bdfb549c640bd9b676611  LICENSE
 bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a  LICENSE-Qwen
 281b5c258f113cf0fd53df3bd20868eb8b3c63aef37e6496d1c172e468b3a073  NOTICE
-d62539a7029001cfd591cd48685eab7d498f2c9be87875b6ba3b7f027a00e9d7  README.md
+a657b2ff561a167d9652523606b6afc7fbe0b3b702d8563ce0e8e74682b55ef5  README.md
 08cbc176f11afd74bfd87ae095ad68665d659ad67d06c85782ffd4d77169f8b4  assets/chart_calibration.png
 f75c9334eff9de54322e655b0e6c22b1fcb776154ac0fc417ed87700f388a164  assets/chart_confidence.png
 f1cbab3e2e80d733ed844884eaddfdcde7779c67cbc9ca43fad3b50484780549  assets/chart_head_to_head.png
@@ -10,19 +10,19 @@ c6074e2c06ed7c2bde4b65d3292109b92f7e267800b22d40e755fb3fa9757234  assets/chart_l
 2f1645391d393a8fdf77661f765640ecb0f850fd1a16025ade22ea737a7b2686  calib.json
 a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715  chat_template.jinja
 ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670  config.json
-76a131c48890046ce740880052d4c892bcae8bc3a837e64a337e2756765a2cb6  decision_config.json
-f5ab755a8d4a92fda2dc4bc2cccdab733a9658ded5f2de034c6cc39ae0652dac  decision_core.py
-c51fb147cc1354d08186e8a561c3f5fae58e429db9aafa74681b04f960dad4ad  letter_adapter.py
+0facac31afd9c70af05b635f7f1bf41ae4940bd42262f5d13b93597375d15224  decision_config.json
+9ff8d754ce99c6539fc7f5bd88c10b196357c70124f59ed3d1bd1a0d3fdbb7d0  decision_core.py
+f39939e54c73b10933f81133971ba398f851cf623438abf4a54c12f2ad473b58  letter_adapter.py
 af1c02a2f57fbd2f41e0c11734ff38a5617d1d9f1cc3124c352d1b2e1d1f8391  local_demo.py
 d2903ce6ddb498b437a818b9c0c79816d9b523858438e312d9095da31ae4b291  local_demo_cases.py
 a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d  merges.txt
-de3d24440e60551e4b0acc05436d72b3617c5b1140ccfddc9d0344a235d9f569  mlx_decide.py
+b6b54867eaa2a9faf8d7f2e1d66fb2f720b11eceab1baf3a18dc23f9d5eb8564  mlx_decide.py
 81703b1d1c19b3a38c62b6269276787695643280055b768ed8f257a538db0c62  model.safetensors-00001-of-00002.safetensors
 3d34f2498429c7a1ccfe3f5c907c68cbd326569a2efdf26fd4ad5c2644423b45  model.safetensors-00002-of-00002.safetensors
 db0ac759fa4eb09c659191f75bc4122f3837168b4273802d0d15ac1138919dfd  model.safetensors.index.json
 27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516  preprocessor_config.json
-782cfe061913bf495e411054daca87129534c865304457ceb06b49a89ccd056b  serve.py
-87071395bbc169e36ba7bb63195f12027d63f36fab62cb5a3480e0c55ba6cf90  serve_vllm.sh
+0756bad2d64736b830b6908c23f2c20a830c16e60560d88a918cb9d6505dd966  serve.py
+9e8e362398735664f468bf7014f588f07a1e9abb3d369be2c9d8cd0200cb39a0  serve_vllm.sh
 5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42  tokenizer.json
 316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8  tokenizer_config.json
 7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13  video_preprocessor_config.json

decision_config.json

@@ -1,5 +1,6 @@
 {
  "prompt_version": "letter-v1-semif",
  "readout": "letter-logit",
- "calib": "calib.json"
-}
+ "calib": "calib.json",
+ "max_one_pass": 100
+}

decision_core.py

@@ -6,6 +6,8 @@ trained and evaluated.
   the SemIf format, MIT) or "ours" (plain text).
 - Temperature: fixed, or conditioned on observable signals (calib_fit.py).
 - Verify mode (optional): short reasoning with a token budget before reading the letter.
+- Option labels: A..Z, then the two-letter codes AA, AB, ... that are a single token (588 in all), so every option of
+  a question is read in the same forward pass. Up to 26 options the prompt is byte-identical to earlier versions.
 License: MIT.
 """
 from __future__ import annotations
@@ -15,6 +17,22 @@ import math
 import os
 
 LETTERS = [chr(65 + i) for i in range(26)]
+# Two-letter codes that the Qwen3.5 tokenizer (27B and 4B) splits into two tokens: skipped, because a label must be one
+# token for its probability to be read in the same forward pass as the others. letter_adapter checks this at load time.
+_SPLIT = frozenset(
+    "BQ BZ CJ CQ CZ DQ DZ EJ EY FJ FQ FV FZ GJ GK GQ GZ HJ IY JF JG JH JL JN JQ JW JX JY JZ KJ KQ KX KZ LH LJ LQ LW LX "
+    "LZ OJ OQ OY OZ PQ PZ QD QF QI QJ QK QO QV QW QX QY QZ RQ RZ TJ TQ UJ UO UQ UW VH VJ VQ VU VW VX VY VZ WJ WQ WU WV "
+    "WY WZ XG XJ XK XN XO XQ XU XV XW XZ YB YD YF YH YI YJ YK YQ YR YU YV YX ZB ZC ZD ZG ZJ ZK ZL ZM ZP ZQ ZS ZT ZU ZV"
+    .split())
+LABELS = LETTERS + [a + b for a in LETTERS for b in LETTERS if a + b not in _SPLIT]  # 588 labels, one token each
+# Most options read in one pass. Each model sets it in decision_config.json ("max_one_pass"): the one-pass readout was
+# measured against the tournament up to 151 options (27B better throughout; 4B better up to ~100, worse from ~120).
+MAX_ONE_PASS = len(LABELS)
+
+
+def set_max_one_pass(n: int | None) -> None:
+    global MAX_ONE_PASS
+    MAX_ONE_PASS = max(26, min(int(n), len(LABELS))) if n else len(LABELS)
 STYLE = os.environ.get("PROMPT_STYLE", "semif")
 if STYLE == "semif":
     SYSTEM = ("Apply the supplied criterion to the supplied evidence. Choose exactly one listed option. "
@@ -55,10 +73,10 @@ def render_user(state, question: dict, opts: list[tuple[str, str]]) -> str:
         noul = question.get("type") in ("noul", "boolean")
         shown = (lambda lab: {"yes": "true", "no": "false"}.get(lab, lab)) if noul else (lambda lab: lab)
         payload = {"evidence": state, "criterion": question.get("instructions", ""),
-                   "options": [{"letter": LETTERS[i], "description": f"{shown(lab)}: {desc}"}
+                   "options": [{"letter": LABELS[i], "description": f"{shown(lab)}: {desc}"}
                                for i, (lab, desc) in enumerate(opts)]}
         return json.dumps(payload, ensure_ascii=False)
-    ol = "\n".join(f"{LETTERS[i]}. {lab}: {desc}" for i, (lab, desc) in enumerate(opts))
+    ol = "\n".join(f"{LABELS[i]}. {lab}: {desc}" for i, (lab, desc) in enumerate(opts))
     return (f"State:\n{state_text(state)}\n\nQuestion: {question.get('instructions', '')}\n\n"
             f"Options:\n{ol}\n\nAnswer with the letter of the correct option.")
 
@@ -77,17 +95,23 @@ def temp_for(calib: dict | None, temp: float, n_tok: int, n_opts: int, qtype: st
     return math.exp(calib["b"] + sum(w * x for w, x in zip(calib["w"], f)))
 
 
-def tournament(dist_fn, opts, chunk: int = 20, keep: int = 2):
-    """More than 26 options: blocks of `chunk`; the top-`keep` of each block go to the final round (≤26)."""
-    if len(opts) <= 26:
+def tournament(dist_fn, opts, chunk: int | None = None, keep: int | None = None, limit: int | None = None):
+    """More options than one pass reads (`limit`, default MAX_ONE_PASS): blocks of `chunk` (default `limit`); the best
+    `keep` of each block (default: as many as fit in one final pass) go on to the next round.
+    v1.0/v1.1 behaviour: chunk=20, keep=2, limit=26."""
+    limit = limit or MAX_ONE_PASS
+    if len(opts) <= limit:
         return dist_fn(opts)
+    chunk = min(chunk or limit, limit)
+    n_blocks = math.ceil(len(opts) / chunk)
+    keep = keep or max(1, min(chunk // 2, limit // n_blocks))
     finalists, n_tok = [], 0
     for i in range(0, len(opts), chunk):
         block = opts[i:i + chunk]
         p, n = dist_fn(block)
         n_tok += n
         finalists += sorted(block, key=lambda o: -p[o[0]])[:keep]
-    pf, n = dist_fn(finalists[:26])
+    pf, n = tournament(dist_fn, finalists, chunk, keep, limit)  # the finalists fit in one pass, or play another round
     probs = {lab: 1e-4 for lab, _ in opts}
     probs.update(pf)
     z = sum(probs.values())

letter_adapter.py

@@ -1,5 +1,6 @@
 """Decision readout from letter logits (one forward pass per decision), used in evaluation (the JevBench harness and
-the evaluation suites) and by the server. Format/prompt come from decision_core (same as in training).
+the evaluation suites) and by the server. Format/prompt come from decision_core (same as in training). Options are
+labelled A..Z, AA, AB, ... (decision_core.LABELS, one token each): up to 588 options are read in one pass.
 
 Modes:
 - local (transformers): model_path = model path/repo (+ optional LoRA adapter);
@@ -13,13 +14,15 @@ import contextlib
 import copy
 import json
 import math
+import threading
 import time
 from dataclasses import dataclass, field
 from typing import Any, Optional
 
 import torch
 
-from decision_core import (LETTERS, PROMPT_VERSION, STYLE, SYSTEM, messages, options_of,  # noqa: F401
+import decision_core
+from decision_core import (LABELS, LETTERS, PROMPT_VERSION, STYLE, SYSTEM, messages, options_of,  # noqa: F401
                            render_user, state_text, temp_for, tournament)
 
 import os
@@ -69,43 +72,89 @@ class LetterAdapter:
         self.mix_w = mix_w if mix_w is not None else float(os.environ.get("MIX_W", "0.5"))
         self._jhead = None
         self._loaded = None
+        self._lp_cap = None  # --max-logprobs of the vLLM server, learned from its first refusal (None: not limited)
+        self._inflight, self._rlock = {}, threading.Lock()  # requests in flight per vLLM server
 
     # ---------------- loading ----------------
-    def _sglang(self):  # "sglang:<url>|<tokenizer>" or "vllm:<url>|<tokenizer>"
+    def _sglang(self):  # "sglang:<url>|<tokenizer>" or "vllm:<url>[,<url>...]|<tokenizer>"
         url, tok_path = self.model_path.split(":", 1)[1].split("|", 1)
-        return url.rstrip("/"), tok_path
+        return url.split(",")[0].strip().rstrip("/"), tok_path
+
+    @contextlib.contextmanager
+    def _replica(self):
+        """Several vLLM servers (e.g. one per GPU, comma-separated URLs): each request goes to the one with the fewest
+        requests in flight, and all of its calls stay there (its prefix cache holds the state)."""
+        urls = [u.strip().rstrip("/") for u in self.model_path.split(":", 1)[1].split("|", 1)[0].split(",")]
+        with self._rlock:
+            url = min(urls, key=lambda u: self._inflight.get(u, 0))
+            self._inflight[url] = self._inflight.get(url, 0) + 1
+        try:
+            yield url
+        finally:
+            with self._rlock:
+                self._inflight[url] -= 1
 
     @property
     def _is_vllm(self):
         return self.model_path.startswith("vllm:")
 
-    def _vllm_letters(self, texts, n_max):
-        """One /v1/completions call with all the prompts; allowed letters = A..(n_max); logprobs already normalized
-        over the letters only (server run with --logprobs-mode processed_logprobs). Returns (list of letter_logprobs,
-        n_tok)."""
+    def _vllm_letters(self, texts, n_max, url=None):
+        """One /v1/completions call with all the prompts; allowed labels = the first n_max; logprobs already
+        normalized over the allowed labels (server run with --logprobs-mode processed_logprobs). A server started with
+        a lower --max-logprobs (v1.0 and v1.1 used 32) only returns its top labels: the labels it leaves out share the
+        remaining probability evenly. Returns (list of label_logprobs, n_tok)."""
+        import urllib.error
         import urllib.request
-        url, _ = self._sglang()
+        url = url or self._sglang()[0]
         want = self._loaded[2][:n_max]
-        body = {"model": "decider", "prompt": texts, "max_tokens": 1, "temperature": 0.0, "logprobs": n_max,
-                "allowed_token_ids": want, "return_tokens_as_token_ids": True}
-        req = urllib.request.Request(f"{url}/v1/completions", data=json.dumps(body).encode(),
-                                     headers={"Content-Type": "application/json"})
-        with urllib.request.urlopen(req, timeout=900) as r:
-            d = json.loads(r.read())
+        while True:
+            n_lp = n_max if self._lp_cap is None else min(n_max, self._lp_cap)
+            body = {"model": "decider", "prompt": texts, "max_tokens": 1, "temperature": 0.0, "logprobs": n_lp,
+                    "allowed_token_ids": want, "return_tokens_as_token_ids": True}
+            req = urllib.request.Request(f"{url}/v1/completions", data=json.dumps(body).encode(),
+                                         headers={"Content-Type": "application/json"})
+            try:
+                with urllib.request.urlopen(req, timeout=900) as r:
+                    d = json.loads(r.read())
+                break
+            except urllib.error.HTTPError as e:
+                if e.code != 400 or n_lp <= 32 or "logprobs" not in e.read().decode(errors="replace"):
+                    raise
+                self._lp_cap = 32  # older server: read the top 32 labels and spread the rest
         out = [None] * len(texts)
         for ch in d["choices"]:
             top = ch["logprobs"]["top_logprobs"][0]
             lp = {int(k.split(":", 1)[1]) if k.startswith("token_id:") else k: v for k, v in top.items()}
+            rest = [i for i in want if i not in lp]
+            if rest:
+                tail = max(1.0 - sum(math.exp(v) for v in lp.values()), 1e-12)
+                lp.update({i: math.log(tail / len(rest)) for i in rest})
             out[ch["index"]] = lp
         n_tok = d.get("usage", {}).get("prompt_tokens", 0) // max(1, len(texts))
         return out, n_tok
 
+    def _warm_prefix(self, texts, url):
+        """vLLM does not share a prefix between the prompts of one call while it is being computed, so N questions about
+        one state would compute that state N times. One short call with their common prefix first puts its full cache
+        blocks in the prefix cache, and the N questions then read them. EIKOS_SHARED_PREFIX=0 turns this off."""
+        if len(texts) < 2 or os.environ.get("EIKOS_SHARED_PREFIX", "1") != "1":
+            return
+        pre = os.path.commonprefix(texts)
+        if len(pre) < int(os.environ.get("EIKOS_SHARED_MIN_CHARS", "3000")):  # less than ~1 cache block: nothing to gain
+            return
+        import urllib.request
+        body = {"model": "decider", "prompt": pre, "max_tokens": 1, "temperature": 0.0}
+        req = urllib.request.Request(f"{url}/v1/completions", data=json.dumps(body).encode(),
+                                     headers={"Content-Type": "application/json"})
+        with urllib.request.urlopen(req, timeout=900) as r:
+            r.read()
+
     @staticmethod
     def _letter_ids(tok):
         ids = []
-        for L in LETTERS:
+        for L in LABELS:
             t = tok.encode(L, add_special_tokens=False)
-            assert len(t) == 1, f"letter {L} is not a single token: {t}"
+            assert len(t) == 1, f"label {L} is not a single token: {t}"
             ids.append(t[0])
         assert len(set(ids)) == len(ids)
         return ids
@@ -197,7 +246,8 @@ class LetterAdapter:
                                                add_generation_prompt=True, enable_thinking=True)
             text = text + thought.split("</think>")[0].rstrip() + "\n</think>\n\n"
         if model is None and self._is_vllm:
-            (lp,), n_tok = self._vllm_letters([text], len(opts))
+            with self._replica() as url:
+                (lp,), n_tok = self._vllm_letters([text], len(opts), url)
             lg = torch.tensor([lp.get(i, -1e9) for i in want])
             return self._probs(lg, n_tok, opts, question), n_tok
         if model is None:
@@ -242,15 +292,17 @@ class LetterAdapter:
         """Parallel: several questions about the same state in a single GPU pass (batch, right-padding).
         items = [(question, opts), ...]. With vLLM or SGLang, all prompts go in one call and the server's prefix cache
         reuses the state. Falls back to the sequential path for the energy readout, verify mode, a single question or
-        >26 options."""
+        more options than one pass reads (decision_core.MAX_ONE_PASS)."""
         model, tok, let_ids = self.load()
         if (self._jhead is not None or self.verify_budget > 0 or len(items) == 1
-                or any(len(o) > len(LETTERS) for _, o in items)):
+                or any(len(o) > decision_core.MAX_ONE_PASS for _, o in items)):
             return [self.dist_any(state, q, o) for q, o in items]
         if model is None and self._is_vllm:  # vLLM: one call; the server's prefix cache reuses the state
             texts = [tok.apply_chat_template(messages(state, q, o), tokenize=False, add_generation_prompt=True,
                                              enable_thinking=False) for q, o in items]
-            lps, n_tok = self._vllm_letters(texts, max(len(o) for _, o in items))
+            with self._replica() as url:
+                self._warm_prefix(texts, url)
+                lps, n_tok = self._vllm_letters(texts, max(len(o) for _, o in items), url)
             return [(self._probs(torch.tensor([lp.get(i, -1e9) for i in let_ids[:len(o)]]), n_tok, o, q), n_tok)
                     for (q, o), lp in zip(items, lps)]
         if model is None:  # SGLang: one call with the list of prompts; the prefix cache processes the state once
@@ -311,7 +363,8 @@ class LetterAdapter:
         and the suffixes of the N questions run together in one batch. Same result as the full prompt."""
         import time as _t
         model, tok, let_ids = self.load()
-        if model is None or self._jhead is not None or self.verify_budget > 0 or any(len(o) > len(LETTERS) for _, o in items):
+        if (model is None or self._jhead is not None or self.verify_budget > 0
+                or any(len(o) > decision_core.MAX_ONE_PASS for _, o in items)):
             return self.dist_many(state, items)
         texts = [tok.apply_chat_template(messages(state, q, o), tokenize=False, add_generation_prompt=True,
                                          enable_thinking=False) for q, o in items]
@@ -355,7 +408,7 @@ class LetterAdapter:
                          "total_ms": round((_t.perf_counter() - t0) * 1000, 1)}
         return out
 
-    def dist_any(self, state, question, opts, chunk=20, keep=2):
+    def dist_any(self, state, question, opts, chunk=None, keep=None):
         return tournament(lambda o: self.dist(state, question, o), opts, chunk, keep)
 
     # ---------------- harness ----------------

mlx_decide.py

@@ -2,8 +2,8 @@
 PyTorch backend (letter_adapter), with no LoRA, no API and no internet. The common prefix (system + state) goes
 through the model once; the hybrid cache (attention KV + Gated DeltaNet states) is replicated and the N questions run
 together in one right-padded batch (real positions never see the padding: causal attention and the recurrence only
-look back). Questions with more than 26 options use the same multi-round tournament as the GPU backends
-(decision_core.tournament).
+look back). Options are labelled A..Z, AA, AB, ... (decision_core.LABELS, one token each), so up to 588 options are read
+in one pass; beyond that, the same tournament as the GPU backends (decision_core.tournament).
 usage: python mlx_decide.py <model_dir>        (demo; compares against PyTorch if COMPARE_TORCH=1)"""
 import copy
 import json
@@ -23,18 +23,21 @@ class MLXDecider:
     def __init__(self, path):
         cfg = json.load(open(os.path.join(path, "decision_config.json")))
         os.environ["PROMPT_STYLE"] = cfg["prompt_version"].rsplit("-", 1)[-1]
-        global LETTERS, messages, options_of, temp_for, tournament
-        from decision_core import LETTERS, messages, options_of, temp_for, tournament  # noqa: F401  (after PROMPT_STYLE is set)
+        global LABELS, messages, options_of, temp_for, tournament, dc
+        import decision_core as dc  # after PROMPT_STYLE is set
+        from decision_core import LABELS, messages, options_of, temp_for, tournament  # noqa: F401
+        dc.set_max_one_pass(cfg.get("max_one_pass"))
         cp = os.path.join(path, cfg.get("calib") or "calib.json")
         self.calib = json.load(open(cp)) if os.path.exists(cp) else None
         self.model, self.tok = mlx_load(path)
         lm = self.model.language_model if hasattr(self.model, "language_model") else self.model
         self.inner, self.lm = lm.model, lm
         self.let = []
-        for L in LETTERS:
+        for L in LABELS:
             t = self.tok.encode(L, add_special_tokens=False)
-            assert len(t) == 1, f"letter {L} is not a single token: {t}"
+            assert len(t) == 1, f"label {L} is not a single token: {t}"
             self.let.append(t[0])
+        assert len(set(self.let)) == len(self.let)
         self.let_idx = mx.array(self.let)
 
     def _ids(self, state, q, opts):
@@ -43,7 +46,7 @@ class MLXDecider:
         return self.tok.encode(text, add_special_tokens=False)
 
     def _letter_logits(self, h):
-        """h: (B, d) → logits (B, 26) over the letters only (tied weights: the embedding used as a linear layer)."""
+        """h: (B, d) → logits (B, 588) over the labels only (tied weights: the embedding used as a linear layer)."""
         if getattr(self.lm.args, "tie_word_embeddings", False):
             out = self.inner.embed_tokens.as_linear(h)
         else:
@@ -57,7 +60,7 @@ class MLXDecider:
         return {lab: p[i] for i, (lab, _) in enumerate(opts)}
 
     def dist(self, state, q, opts):
-        if len(opts) > 26:  # only 26 letters: blocks of 20, the top 2 of each block go to a final round
+        if len(opts) > dc.MAX_ONE_PASS:  # more than one pass reads: blocks, the best of each go to a final round
             return tournament(lambda o: self.dist(state, q, o), opts)
         ids = self._ids(state, q, opts)
         cache = self.lm.make_cache()
@@ -78,11 +81,11 @@ class MLXDecider:
 
     def dist_many_cached(self, state, items, chunk=32):
         """Several questions about the same state: the prefix once, then the questions in a batch from the cache.
-        Questions with more than 26 options go through dist (tournament), one at a time."""
-        if any(len(o) > 26 for _, o in items):
-            small = [(q, o) for q, o in items if len(o) <= 26]
+        Questions with more options than one pass reads go through dist (tournament), one at a time."""
+        if any(len(o) > dc.MAX_ONE_PASS for _, o in items):
+            small = [(q, o) for q, o in items if len(o) <= dc.MAX_ONE_PASS]
             done = iter(self.dist_many_cached(state, small, chunk) if small else [])
-            return [self.dist(state, q, o) if len(o) > 26 else next(done) for q, o in items]
+            return [self.dist(state, q, o) if len(o) > dc.MAX_ONE_PASS else next(done) for q, o in items]
         seqs = [self._ids(state, q, o) for q, o in items]
         P = 0
         while all(len(x) > P + 1 for x in seqs) and len({x[P] for x in seqs}) == 1:

serve.py

@@ -1,7 +1,8 @@
 """Typed-decision server compatible with the TypeSafe API (POST /v1/systemone and /v1/evaluate).
 
 Uses the same core as training and evaluation (decision_core + letter_adapter): same prompt, same
-letter readout, same calibration. All questions in a request go in one pass (batch); >26 options via a tournament;
+letter readout, same calibration. All questions in a request go in one pass (batch), each with up to 588 options
+(labels A..Z, AA, AB, ...; start vLLM with serve_vllm.sh, --max-logprobs 600); more options than that via a tournament;
 System One: by default every question is answered in one pass, without generating text. Optional (off by
 default, not used for the reported results): "mode": "verify" per question turns on a short reasoning step before
 the letter (--verify-budget N).
@@ -27,6 +28,9 @@ import time
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from threading import Lock
 
+VERSION = "1.2"
+
+
 class Decider:
     def __init__(self, a):
         from letter_adapter import LetterAdapter
@@ -130,7 +134,7 @@ def make_handler(dec: Decider):
 
         def do_GET(self):
             if self.path in ("/health", "/v1/health"):
-                self._send(200, {"ok": True, "model": dec.name})
+                self._send(200, {"ok": True, "model": dec.name, "version": VERSION})
             else:
                 self._send(404, {"error": "not found"})
 
@@ -211,18 +215,25 @@ def main():
     ap.add_argument("--sym", action="store_true")
     ap.add_argument("--max-tokens", type=int, default=16000)
     ap.add_argument("--verify-budget", type=int, default=0, help="optional: enables 'mode': 'verify' (short reasoning)")
-    ap.add_argument("--vllm-url", default=None, help="production backend: vLLM URL (serve_vllm.sh)")
+    ap.add_argument("--vllm-url", default=None, help="production backend: vLLM URL (serve_vllm.sh); several "
+                    "comma-separated URLs (e.g. one server per GPU) spread the requests, least busy first")
     ap.add_argument("--sglang-url", default=None, help="production backend (parallel): URL of the SGLang server")
+    ap.add_argument("--max-one-pass", type=int, default=None,
+                    help="most options read in one pass (default: the model's decision_config.json, else 588)")
     a = ap.parse_args()
     cfg_path = os.path.join(a.model, "decision_config.json")
+    cfg = {}
     if os.path.exists(cfg_path):  # exported model: the configuration ships with the weights
         cfg = json.load(open(cfg_path))
         os.environ.setdefault("PROMPT_STYLE", cfg.get("prompt_version", "letter-v1-semif").rsplit("-", 1)[-1])
         if a.calib is None and cfg.get("calib"):
             a.calib = os.path.join(a.model, cfg["calib"])
+    import decision_core  # after PROMPT_STYLE is set
+    decision_core.set_max_one_pass(a.max_one_pass or cfg.get("max_one_pass"))
     dec = Decider(a)
     dec.decide("warm-up", {"type": "noul", "instructions": "Is this a warm-up?", "criteria": {"true": "yes", "false": "no"}})
-    print(f"ready at http://{a.host}:{a.port} (calib={bool(a.calib)}, sym={a.sym}, backend={'vllm' if a.vllm_url else 'sglang' if a.sglang_url else 'pytorch'}, verify={a.verify_budget})", flush=True)
+    print(f"serve.py {VERSION} ready at http://{a.host}:{a.port} (calib={bool(a.calib)}, sym={a.sym}, backend={'vllm' if a.vllm_url else 'sglang' if a.sglang_url else 'pytorch'}, verify={a.verify_budget}, one pass up to {decision_core.MAX_ONE_PASS} options)", flush=True)
+    ThreadingHTTPServer.request_queue_size = 1024  # the default (5) drops connections when many clients arrive at once
     ThreadingHTTPServer((a.host, a.port), make_handler(dec)).serve_forever()
 
 

serve_vllm.sh

@@ -10,6 +10,8 @@ from packaging.version import Version
 if Version(vllm.__version__.split("+")[0]) < Version("0.30.0"):
     sys.exit(f"vLLM {vllm.__version__} is too old for this model: please install vllm>=0.30.0")
 PY
+# --max-logprobs 600: every one of up to 588 option labels (A..Z, AA, AB, ...) is read in the same pass.
+# --mamba-cache-mode align: what vLLM 0.30 uses for this hybrid model anyway ("all" falls back to it with a warning).
 exec vllm serve "$1" --host 127.0.0.1 --port "$2" --served-model-name decider --dtype bfloat16 \
-  --gpu-memory-utilization "${3:-0.85}" --max-model-len 16384 --enable-prefix-caching --mamba-cache-mode all \
-  --logprobs-mode processed_logprobs --max-logprobs 32
+  --gpu-memory-utilization "${3:-0.85}" --max-model-len 16384 --enable-prefix-caching --mamba-cache-mode align \
+  --logprobs-mode processed_logprobs --max-logprobs 600