decider LoRA fine-tune

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,83 @@
+{
+  "architectures": [
+    "Qwen3_5ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_output_gate": true,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 248044,
+  "full_attention_interval": 4,
+  "head_dim": 256,
+  "hidden_act": "silu",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 9216,
+  "layer_types": [
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention"
+  ],
+  "linear_conv_kernel_dim": 4,
+  "linear_key_head_dim": 128,
+  "linear_num_key_heads": 16,
+  "linear_num_value_heads": 32,
+  "linear_value_head_dim": 128,
+  "mamba_ssm_dtype": "float32",
+  "max_position_embeddings": 262144,
+  "mlp_only_layers": [],
+  "model_type": "qwen3_5_text",
+  "mtp_num_hidden_layers": 1,
+  "mtp_use_dedicated_embeddings": false,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 4,
+  "pad_token_id": null,
+  "partial_rotary_factor": 0.25,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "mrope_interleaved": true,
+    "mrope_section": [
+      11,
+      11,
+      10
+    ],
+    "partial_rotary_factor": 0.25,
+    "rope_theta": 10000000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.19.0",
+  "use_cache": true,
+  "vocab_size": 248320
+}

decider/batching.py

@@ -0,0 +1,95 @@
+"""Batch planning for the server's queue: which queued rows share one forward.
+
+Pure python, no torch, no engine: `plan_batches` takes the row lengths and two callables that describe the engine's
+bucket grid, and returns the groups to run.  `decider.serve.batcher` calls it once per collection.
+
+Cost model.  A forward of B rows padded to length T costs `overhead + B * T` token-units: `overhead` is the fixed cost of
+one forward (launch, replay, the slot gather and the device-to-host copy) expressed as the number of padded tokens that
+take the same time, and `B * T` is the padded token work.  Two rows of different lengths are therefore worth merging
+into the longer row's bucket exactly when the padding they add is cheaper than a second forward's overhead.
+`DEFAULT_MERGE_OVERHEAD_TOKENS` is measured on decider-2b (docs/SERVING.md, section on the batching policy); the server
+reads `DECIDER_MERGE_OVERHEAD_TOKENS` over it.
+
+The partition is exact, not greedy.  Rows are sorted by padded length descending (stable, so rows of the same bucket keep
+their arrival order) and split into consecutive groups; each group runs at the bucket of its longest member, so a group
+starting at position k costs `overhead + g * T[k]`.  A dynamic program over the sorted sequence takes the cheapest split,
+subject to `g <= min(max_batch, max_rows(T))`, with ties going to the larger group.  The per-bucket grouping the server
+did before is one of the partitions the program may choose (rows of equal length are adjacent in the sorted order), so
+the planned cost is never above it, and with `overhead = 0` the plan is exactly that grouping.
+"""
+
+DEFAULT_MERGE_OVERHEAD_TOKENS = 512
+
+
+def group_cap(T, max_rows, max_batch):
+    """Rows one forward may take at padded length T: the engine's widest captured batch bucket, and the server's cap."""
+    return max(1, min(int(max_batch), int(max_rows(T))))
+
+
+def batch_cost(size, T, overhead=DEFAULT_MERGE_OVERHEAD_TOKENS):
+    """Cost of one forward of `size` rows padded to `T`, in token-units."""
+    return overhead + size * T
+
+
+def plan_cost(groups, overhead=DEFAULT_MERGE_OVERHEAD_TOKENS):
+    """Cost of a plan: the sum over its forwards."""
+    return sum(batch_cost(len(idx), T, overhead) for T, idx in groups)
+
+
+def per_bucket_groups(lengths, pad_len, max_rows, max_batch):
+    """The grouping of 1.1.0: rows of the same padded length, chunked to the cap.  The baseline `plan_batches` improves on."""
+    return _exact_bucket(list(range(len(lengths))), [pad_len(n) for n in lengths], max_rows, max_batch)
+
+
+def _exact_bucket(idx, pads, max_rows, max_batch):
+    buckets = {}
+    for i in idx:
+        buckets.setdefault(pads[i], []).append(i)
+    out = []
+    for T, members in buckets.items():
+        cap = group_cap(T, max_rows, max_batch)
+        for k in range(0, len(members), cap):
+            out.append((T, members[k:k + cap]))
+    return out
+
+
+def plan_batches(lengths, pad_len, max_rows, max_batch, overhead=DEFAULT_MERGE_OVERHEAD_TOKENS, mergeable=None):
+    """Partition queued rows into forwards.
+
+    lengths:    token count of every queued row, in arrival order.
+    pad_len(n): the engine's padded length for a row of n tokens (`EngineV2.pad_len`).
+    max_rows(T):rows the engine runs in one forward at padded length T (`EngineV2.max_rows`).
+    max_batch:  the server's own cap (`DECIDER_MAX_BATCH`).
+    overhead:   fixed cost of a forward in token-units (see the module docstring).
+    mergeable(n): False for a row that must not be padded into another row's bucket.  Rows above the last captured
+                  length bucket run eager at a request-specific shape, so they are grouped by exact length as before.
+
+    -> [(padded length, [row indices]), ...].  Every row appears in exactly one group; a group's padded length is at
+    least every member's own padded length; a group holds at most `min(max_batch, max_rows(T))` rows.
+    """
+    n = len(lengths)
+    if n == 0:
+        return []
+    pads = [pad_len(x) for x in lengths]
+    merge = [True] * n if mergeable is None else [bool(mergeable(x)) for x in lengths]
+    out = _exact_bucket([i for i in range(n) if not merge[i]], pads, max_rows, max_batch)
+
+    order = sorted((i for i in range(n) if merge[i]), key=lambda i: -pads[i])    # stable: equal buckets keep arrival order
+    m = len(order)
+    dp = [0] * (m + 1)                       # dp[k]: cheapest cost of covering order[k:]
+    take = [0] * (m + 1)
+    for k in range(m - 1, -1, -1):
+        T = pads[order[k]]
+        cap = min(group_cap(T, max_rows, max_batch), m - k)
+        best, best_g = None, 1
+        for g in range(1, cap + 1):
+            c = overhead + g * T + dp[k + g]
+            if best is None or c <= best:        # ties go to the larger group: same modelled cost, one fewer launch
+                best, best_g = c, g
+        dp[k], take[k] = best, best_g
+    k = 0
+    while k < m:
+        g = take[k]
+        out.append((pads[order[k]], sorted(order[k:k + g])))                     # inside a group, arrival order
+        k += g
+    return out

decider/calibrate.py

@@ -0,0 +1,222 @@
+"""Fit decider_config.json "temperature_by_type" by NLL, one temperature per answer type (decider.temperature).
+
+    python -m decider.calibrate records.jsonl [--min-rows 50]
+    -> {"temperature_by_type": {"choice": 1.48, "noul": 2.22, "score": 1.38}, "temperature": 1.6, "rows": {...}, "nll": {...}}
+
+A record is one answer with its gold, read at temperature 1:
+    {"type": "choice" | "noul" | "score", "logits": [one value per option], "gold": option index}
+    {"type": "score", "level_logits": [[no, yes], one pair per level], "gold": level index}      Score with isolated levels
+"probs" / "level_probs" (probabilities at temperature 1) may stand in for "logits" / "level_logits": log p is the logit up to a
+constant, which the softmax ignores.  A noul gold is 1 for true, 0 for false.  Every record is checked first (check_record):
+a malformed one (gold out of range or not an integer, wrong shape, NaN, a row without a finite logit, an isolated Score whose
+levels all have P(yes) = 0) raises ValueError naming its position.
+
+An isolated-levels Score answer is fitted through the readout the server uses: every level row is softmax(row / T), and the
+answer is P(yes) of each level divided by their sum (decider.systemone.combine_isolated).  So the "score" temperature is fitted on
+Score answers whichever readout the model serves them with; fit it on records collected with the same isolated_levels setting.
+
+`collect(decider, examples)` produces records from a decider.infer.Decider and labelled /v1/systemone-shaped examples.
+"temperature" in the output is one temperature fitted on all records together, for comparison; a type with fewer than
+--min-rows records is left out of the map (it then uses "temperature" of the config).
+"""
+import json, math, sys
+import numpy as np
+
+from decider.temperature import TYPES
+
+GRID = np.exp(np.linspace(math.log(0.05), math.log(20.0), 801))       # 0.05 .. 20, about 0.75 % apart
+
+
+def _log(x):
+    x = np.asarray(x, dtype=np.float64)
+    with np.errstate(divide="ignore"):
+        return np.where(x > 0, np.log(np.clip(x, 1e-300, None)), -np.inf)
+
+
+def _logits(rec, key):
+    if key in rec:
+        return np.asarray(rec[key], dtype=np.float64)
+    alt = {"logits": "probs", "level_logits": "level_probs"}[key]
+    return _log(rec[alt])
+
+
+def _lse(z, axis=-1):
+    m = np.max(z, axis=axis, keepdims=True)
+    m = np.where(np.isfinite(m), m, 0.0)
+    with np.errstate(divide="ignore"):                              # an all -inf row gives -inf, handled by the callers
+        return (m + np.log(np.sum(np.exp(z - m), axis=axis, keepdims=True))).squeeze(axis)
+
+
+def nll(rec, T):
+    """Negative log-likelihood of one record's gold at temperature T."""
+    return float(_Batch([rec]).nll(T))
+
+
+def _isolated(rec):
+    return "level_logits" in rec or "level_probs" in rec
+
+
+def check_record(rec, i=None):
+    """Raise ValueError unless `rec` is a well-formed record (module docstring): an integral gold within the options or levels,
+    a 1-D row of at least two options or an [levels, 2] array of at least two levels, logits that are not NaN or +inf,
+    probabilities in [0, 1] with positive mass, and isolated levels only on a Score record."""
+    where = f"record {i}" if i is not None else "record"
+    if not isinstance(rec, dict):
+        raise ValueError(f"{where}: not a JSON object")
+    if rec.get("type") not in TYPES:
+        raise ValueError(f"{where}: type {rec.get('type')!r}, expected one of {', '.join(TYPES)}")
+    iso = _isolated(rec)
+    keys = ("level_logits", "level_probs") if iso else ("logits", "probs")
+    if sum(k in rec for k in keys) != 1 or (iso and ("logits" in rec or "probs" in rec)):
+        raise ValueError(f"{where}: give exactly one of logits, probs, level_logits, level_probs")
+    if iso and rec["type"] != "score":
+        raise ValueError(f"{where}: isolated levels (level_logits / level_probs) belong to a score record, not {rec['type']!r}")
+    key = keys[0] if keys[0] in rec else keys[1]
+    try:
+        a = np.asarray(rec[key], dtype=np.float64)
+    except (TypeError, ValueError):
+        raise ValueError(f"{where}: {key} is not a numeric array") from None
+    if iso and (a.ndim != 2 or a.shape[1] != 2 or a.shape[0] < 2):
+        raise ValueError(f"{where}: {key} must be one [no, yes] pair per level, at least two levels; got shape {a.shape}")
+    if not iso and (a.ndim != 1 or a.shape[0] < 2):
+        raise ValueError(f"{where}: {key} must be one value per option, at least two options; got shape {a.shape}")
+    if key.endswith("probs"):
+        if not np.all(np.isfinite(a)) or np.any(a < 0) or np.any(a > 1) or np.any(a.sum(-1) <= 0):
+            raise ValueError(f"{where}: {key} must be probabilities in [0, 1] with positive mass per row")
+    elif np.any(np.isnan(a)) or np.any(a == np.inf) or not np.all(np.any(np.isfinite(a), axis=-1)):
+        raise ValueError(f"{where}: {key} contains NaN or +inf, or a row without a finite value")
+    if rec["type"] == "noul" and a.shape[0] != 2:
+        raise ValueError(f"{where}: a noul record has exactly two options (false, true); got {a.shape[0]}")
+    if iso and not np.any(a[:, 1] > 0 if key == "level_probs" else np.isfinite(a[:, 1])):
+        raise ValueError(f"{where}: no level has a yes probability above 0, so the served Score answer has no distribution")
+    g = rec.get("gold")
+    if isinstance(g, bool) or not isinstance(g, (int, np.integer)) or not 0 <= g < a.shape[0]:
+        raise ValueError(f"{where}: gold must be an integer index in 0..{a.shape[0] - 1}, got {g!r}")
+    return rec
+
+
+class _Batch:
+    """Records packed into padded arrays, so one temperature is scored over all of them at once."""
+    def __init__(self, records):
+        records = [check_record(r, i) for i, r in enumerate(records)]
+        lists = [r for r in records if not _isolated(r)]
+        isos = [r for r in records if _isolated(r)]
+        self.n = len(lists) + len(isos)
+        self.L = self.Lg = self.I = self.Im = self.Ig = None
+        if lists:
+            zs = [_logits(r, "logits") for r in lists]
+            K = max(len(z) for z in zs)
+            self.L = np.full((len(zs), K), -np.inf)
+            for i, z in enumerate(zs):
+                self.L[i, :len(z)] = z
+            self.Lg = np.array([int(r["gold"]) for r in lists])
+        if isos:
+            zs = [_logits(r, "level_logits").reshape(-1, 2) for r in isos]
+            n = max(len(z) for z in zs)
+            self.I = np.zeros((len(zs), n, 2)); self.Im = np.zeros((len(zs), n), dtype=bool)
+            for i, z in enumerate(zs):
+                self.I[i, :len(z)] = z; self.Im[i, :len(z)] = True
+            self.Ig = np.array([int(r["gold"]) for r in isos])
+
+    def nll(self, T):
+        """Sum of the records' NLL at temperature T."""
+        tot = 0.0
+        if self.L is not None:
+            z = self.L / T
+            zg = z[np.arange(len(z)), self.Lg]
+            tot += float(np.sum(np.where(np.isfinite(zg), _lse(z) - zg, 690.0)))
+        if self.I is not None:                          # in log space: P(yes) of very confident rows underflows otherwise
+            z = self.I / T
+            with np.errstate(invalid="ignore"):
+                lp = np.where(self.Im, z[..., 1] - _lse(z), -np.inf)          # log P(yes) per level, -inf on padding
+            norm = _lse(lp)                                                  # log of the summed P(yes)
+            lg = lp[np.arange(len(z)), self.Ig]
+            with np.errstate(invalid="ignore"):
+                per = np.where(np.isfinite(lg), norm - lg, 690.0)               # check_record guarantees a finite norm
+            tot += float(np.sum(per))
+        return tot
+
+
+def mean_nll(records, T):
+    b = records if isinstance(records, _Batch) else _Batch(records)
+    return b.nll(T) / b.n
+
+
+def fit(records, grid=GRID):
+    """The temperature with the lowest mean NLL on the grid, refined by a golden-section search between its neighbours."""
+    records = records if isinstance(records, _Batch) else _Batch(list(records))
+    if not records.n:
+        raise ValueError("no records to fit")
+    vals = [mean_nll(records, T) for T in grid]
+    i = int(np.argmin(vals))
+    lo, hi = math.log(grid[max(i - 1, 0)]), math.log(grid[min(i + 1, len(grid) - 1)])
+    r = (math.sqrt(5) - 1) / 2
+    a, b = hi - r * (hi - lo), lo + r * (hi - lo)
+    fa, fb = mean_nll(records, math.exp(a)), mean_nll(records, math.exp(b))
+    for _ in range(40):
+        if fa < fb:
+            hi, b, fb = b, a, fa; a = hi - r * (hi - lo); fa = mean_nll(records, math.exp(a))
+        else:
+            lo, a, fa = a, b, fb; b = lo + r * (hi - lo); fb = mean_nll(records, math.exp(b))
+    T = math.exp((lo + hi) / 2)
+    return T if mean_nll(records, T) <= vals[i] else float(grid[i])
+
+
+def fit_by_type(records, min_rows=50):
+    """-> {"temperature_by_type": {type: T}, "temperature": pooled T, "rows": {type: n}, "nll": {type: {"T=1": .., "fitted": ..}}}"""
+    records = [check_record(r, i) for i, r in enumerate(records)]
+    groups = {t: [] for t in TYPES}
+    for r in records:
+        groups[r["type"]].append(r)
+    out = {"temperature_by_type": {}, "temperature": round(fit(records), 3) if records else None,
+           "rows": {t: len(g) for t, g in groups.items()}, "nll": {}}
+    for t, g in groups.items():
+        if len(g) < max(1, min_rows):
+            continue
+        b = _Batch(g); T = fit(b)
+        out["temperature_by_type"][t] = round(T, 3)
+        out["nll"][t] = {"T=1": round(mean_nll(b, 1.0), 4), "fitted": round(mean_nll(b, T), 4)}
+    return out
+
+
+def collect(decider, examples, independent=True):
+    """Records for fit_by_type from a decider.infer.Decider, read at temperature 1 on the uncached state-first path.
+    examples: iterable of (state, questions, golds) with questions as /v1/systemone takes them and golds {question id: gold},
+    the gold being a Choice option name, a Noul true/false, or a Score level index.  Questions without a gold are skipped.
+    Score questions are read with the Decider's isolated_levels setting, as system_one serves them."""
+    recs = []
+    for state, questions, golds in examples:
+        rqs, index, items = decider._system_one_items(state, questions, independent, layout="state_first")
+        rows = decider._system_one_probs(items, "state_first", temperature=1.0)
+        for k, kind, s, n in index:
+            if k not in golds:
+                continue
+            rq, g = rqs[k], golds[k]
+            if rq["type"] == "noul" and not (isinstance(g, bool) or g in (0, 1)):
+                raise ValueError(f"gold of noul question {k!r} must be true or false, got {g!r}")
+            if rq["type"] == "score" and (isinstance(g, bool) or not isinstance(g, int) or not 0 <= g < len(rq["names"])):
+                raise ValueError(f"gold of score question {k!r} must be a level index in 0..{len(rq['names']) - 1}, got {g!r}")
+            key = bool(g) if rq["type"] == "noul" else g
+            if key not in rq["names"]:
+                raise ValueError(f"gold of choice question {k!r} is not one of its options: {g!r}")
+            gold = rq["names"].index(key)
+            if kind == "iso":
+                recs.append({"type": rq["type"], "level_probs": [rows[s + j][:2] for j in range(n)], "gold": gold})
+            else:
+                recs.append({"type": rq["type"], "probs": rows[s][:len(rq["options"])], "gold": gold})
+    return recs
+
+
+def main(argv=None):
+    import argparse
+    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    ap.add_argument("records", help="JSON lines, one record per line (see the module docstring)")
+    ap.add_argument("--min-rows", type=int, default=50, help="fit a type only when it has at least this many records")
+    a = ap.parse_args(argv)
+    with open(a.records) as f:
+        recs = [json.loads(line) for line in f if line.strip()]
+    print(json.dumps(fit_by_type(recs, a.min_rows), indent=1))
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

decider/engine.py

@@ -0,0 +1,216 @@
+"""Low-latency inference engine: shape-bucketed CUDA graphs over the one-pass decision model.
+
+Right padding + causal layers => pad positions never influence earlier slots, so no attention
+mask is needed and every (B, T) bucket can be captured once and replayed.  The graph outputs
+option-letter logits for all positions [B, T, K]; slots are gathered outside.
+"""
+import time, torch, torch._dynamo, torch.nn.functional as F
+from decider.model import DecisionModel, collate
+from decider.prompt import build, MAX_OPTIONS
+from decider.temperature import scaled_softmax, slot_temperatures
+
+T_BUCKETS = [64, 128, 192, 256, 320, 384, 512, 640, 768, 1024, 1280, 1536, 2048]
+B_BUCKETS = [1, 2, 4, 8, 16, 32, 64]
+GRAPH_MAX_T = 2048          # longer inputs (up to the 32k request budget) run eagerly: compute dominates there, and one graph
+LONG_STEP = 1024            # per (B, T) shape would cost a compile + capture for every new length
+
+
+def _bucket(x, buckets):
+    for b in buckets:
+        if x <= b:
+            return b
+    return None
+
+
+def fused_causal_conv1d_fn(hidden_states, weight, bias=None, activation=None, **kwargs):
+    """Depthwise causal conv (kernel k) as k shifted multiply-adds: fuses under torch.compile,
+    unlike the cuDNN grouped conv fallback (which was ~11% of batched GPU time)."""
+    B, C, T = hidden_states.shape; k = weight.shape[-1]
+    x = F.pad(hidden_states.to(weight.dtype), (k - 1, 0))
+    out = x[:, :, k - 1:k - 1 + T] * weight[:, k - 1][None, :, None]
+    for j in range(k - 1):
+        out = out + x[:, :, j:j + T] * weight[:, j][None, :, None]
+    if bias is not None:
+        out = out + bias[None, :, None]
+    if activation == "silu":
+        out = F.silu(out)
+    elif activation is not None:
+        from transformers.activations import ACT2FN
+        out = ACT2FN[activation](out)
+    return out.to(hidden_states.dtype)
+
+
+def patch_conv():
+    from transformers.models.qwen3_5 import modeling_qwen3_5 as mq
+    mq.causal_conv1d_fn = fused_causal_conv1d_fn
+
+
+def read_slots(out, rows, slots, nopts, temperature, n_per_item):
+    """One gather + one softmax + one device-to-host copy for the whole batch (was: three small kernels and a sync per item).
+    out [B, T, K] logits; rows/slots/nopts: flat python lists, one entry per question; n_per_item: questions per item.
+    temperature: a number for every question, or a flat list with one temperature per question (decider.temperature)."""
+    dev = out.device; idx = torch.tensor([rows, slots, nopts], dtype=torch.long).to(dev, non_blocking=True)
+    lg = out[idx[0], idx[1]]                                                                  # [N, K]
+    lg = lg.masked_fill(torch.arange(lg.shape[1], device=dev)[None, :] >= idx[2][:, None], float("-inf"))
+    p = scaled_softmax(lg, temperature).cpu()
+    return list(torch.split(p, n_per_item))
+
+
+def fill_ids(items_ids, B, T, pad):
+    import numpy as np
+    a = np.full((B, T), pad, dtype=np.int64)
+    for b, x in enumerate(items_ids): a[b, :len(x)] = x
+    return torch.from_numpy(a)
+
+
+def set_attention_backend_policy():
+    """Turn off the cuDNN scaled-dot-product-attention backend. On Blackwell with torch 2.14 / CUDA 13 it returns wrong,
+    finite output for masked rectangular attention, which is what the shared-state path (`Engine.score_shared`) and the schema
+    cache run when a suffix is scored against a cached prefix; the math and memory-efficient backends are correct. Measured on
+    a Decision Index row: the cached path answered a wrong option at p=0.93 where the full forward and the corrected cached path
+    both give option_4 at p=0.95 (decider2/SERVING_V2_REVIEW.md in the research notes). Must run before torch.compile and CUDA
+    graph capture: captured graphs keep the backend they were captured with."""
+    if hasattr(torch.backends.cuda, "enable_cudnn_sdp"):
+        torch.backends.cuda.enable_cudnn_sdp(False)
+
+
+class Engine:
+    """compile: torch.compile the forward (needs use_cache=False; ~1.4x batched, fuses elementwise work).
+    fp8: e4m3 weights + per-token activation scaling on the big linears (Hopper tensor cores).
+    conv_patch: fusable depthwise causal conv instead of the cuDNN fallback."""
+    def __init__(self, path, device="cuda", dtype=torch.bfloat16, use_graphs=True, max_ctx_tokens=1536,
+                 compile=True, fp8=False, conv_patch=True):
+        set_attention_backend_policy()
+        if conv_patch:
+            if str(device).startswith("mps"):
+                from decider.mps_ops import patch_mps
+                patch_mps()
+            else:
+                patch_conv()
+        self.m = DecisionModel(path, dtype=dtype, grad_ckpt=False).to(device).eval()
+        use_graphs = use_graphs and torch.device(device).type == "cuda"
+        self.tok = self.m.tok; self.dev = device; self.use_graphs = use_graphs; self.max_ctx = max_ctx_tokens
+        self.core, self.W = self.m.lm.model, self.m.lm.lm_head.weight[self.m.letters].detach().clone()
+        self.cfg = dict(compile=compile, fp8=fp8, conv_patch=conv_patch, graphs=use_graphs)
+        if fp8:
+            from decider.fp8 import convert_to_fp8
+            self.cfg["fp8_layers"] = convert_to_fp8(self.core)
+        if compile:
+            torch._dynamo.config.cache_size_limit = 128
+            self._fwd_impl = torch.compile(self._fwd_eager, dynamic=False)
+        else:
+            self._fwd_impl = self._fwd_eager
+        self.graphs = {}                       # (B, T) -> (static_ids, static_out, graph)
+        self.pool = torch.cuda.graph_pool_handle() if (use_graphs and str(device).startswith("cuda")) else None
+        self.stats = dict(graph_captures=0, forwards=0)
+
+    def _fwd_eager(self, ids):
+        h = self.core(input_ids=ids, use_cache=False).last_hidden_state
+        return F.linear(h, self.W).float()                          # [B, T, K]
+
+    @torch.no_grad()
+    def _fwd(self, ids):
+        return self._fwd_impl(ids)
+
+    def _capture(self, B, T):
+        s_ids = torch.full((B, T), self.tok.pad_token_id, dtype=torch.long, device=self.dev)
+        st = torch.cuda.Stream(); st.wait_stream(torch.cuda.current_stream())
+        with torch.cuda.stream(st):
+            for _ in range(3): self._fwd(s_ids)                     # warm-up: compile / triton autotune
+        torch.cuda.current_stream().wait_stream(st)
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g, pool=self.pool):
+            s_out = self._fwd(s_ids)
+        self.stats["graph_captures"] += 1
+        return s_ids, s_out, g
+
+    @torch.no_grad()
+    def logits_all(self, ids):
+        """ids: [B, T] long on device (already right-padded to a bucket). Returns [B, T, K] float."""
+        B, T = ids.shape; self.stats["forwards"] += 1
+        if T > GRAPH_MAX_T:
+            self.stats["long_forwards"] = self.stats.get("long_forwards", 0) + 1
+            return self._fwd_eager(ids)
+        if not self.use_graphs:
+            return self._fwd(ids)
+        key = (B, T)
+        if key not in self.graphs:
+            self.graphs[key] = self._capture(B, T)
+        s_ids, s_out, g = self.graphs[key]
+        s_ids.copy_(ids); g.replay()
+        return s_out
+
+    @torch.no_grad()
+    def score_items(self, items, temperature=1.0):
+        """items: list of dicts from prompt.build. Returns list of [n_q, MAX_OPTIONS] prob tensors (cpu).
+        temperature: a number, or one entry per item (a number or one number per slot; decider.temperature.for_items)."""
+        Tmax = max(len(it["ids"]) for it in items)
+        T = _bucket(Tmax, T_BUCKETS) or -(-Tmax // LONG_STEP) * LONG_STEP
+        B = (_bucket(len(items), B_BUCKETS) or len(items)) if T <= GRAPH_MAX_T else len(items)
+        ids = fill_ids([it["ids"] for it in items], B, T, self.tok.pad_token_id)
+        out = self.logits_all(ids.to(self.dev, non_blocking=True))
+        return read_slots(out, [b for b, it in enumerate(items) for _ in it["slots"]], [s for it in items for s in it["slots"]],
+                          [n for it in items for n in it["nopts"]], slot_temperatures(temperature, items), [len(it["slots"]) for it in items])
+
+    @torch.no_grad()
+    def score_shared(self, items, temperature=1.0, min_prefix=192):
+        """Rows that start with the same tokens (one state, one question per row): run the shared prefix once, fork its
+        cache (attention KV + delta-net conv/recurrent states), and run only the question suffixes.
+        Same answers as score_items up to kernel round-off; cost ~ state + sum(questions) instead of n * state.
+        The fork is made in chunks that fit `DECIDER_SHARED_FORK_GB`, so the peak memory does not grow with the question
+        count; the implementation is decider.shared_prefix, shared with EngineV2."""
+        from decider import shared_prefix                      # imported here: decider.shared_prefix imports this module
+        out = shared_prefix.score_shared(self, items, temperature, min_prefix)
+        if out is None:
+            return self.score_items(items, temperature)
+        self.stats["shared_prefix_calls"] = self.stats.get("shared_prefix_calls", 0) + 1
+        return out
+
+    def warmup(self, shapes=((1, 128), (1, 256), (1, 384), (1, 512), (8, 256), (8, 512), (32, 256), (32, 512))):
+        t = time.time()
+        for B, T in shapes:
+            self.logits_all(torch.full((B, T), self.tok.pad_token_id, dtype=torch.long, device=self.dev))
+        torch.cuda.synchronize(); return time.time() - t
+
+
+if __name__ == "__main__":
+    import sys, random, numpy as np
+    from decider import data as D
+    from decider.infer import Decider
+    path = sys.argv[1] if len(sys.argv) > 1 else "runs/r3_v2/model"
+    cfg = dict(compile="nocompile" not in sys.argv[2:], fp8="fp8" in sys.argv[2:], conv_patch="noconv" not in sys.argv[2:])
+    _, evals = D.load_cache("data/tasks.pkl")
+    eng = Engine(path, **cfg); print("engine cfg", eng.cfg)
+    rng = random.Random(0)
+    exs = evals["support_tickets"][:64] + evals["clinc_oos"][:64] + evals["race"][:32]
+    from decider.prompt import chat_for_model
+    chat = chat_for_model(path, eng.tok)
+    items = [build(e, eng.tok, rng, max_ctx_tokens=1536, chat=chat) for e in exs]
+    # correctness vs eager masked forward (DecisionModel.slot_logits)
+    ref = []
+    with torch.no_grad():
+        for i in range(0, len(items), 16):
+            b = collate(items[i:i + 16], eng.tok.pad_token_id)
+            lg = eng.m.slot_logits(b["input_ids"].cuda(), b["attention_mask"].cuda(), b["slot_idx"].cuda(), b["slot_batch"].cuda(), b["nopts"].cuda())
+            ref.append(torch.softmax(lg, -1).cpu())
+    ref = torch.cat(ref)
+    got = torch.cat(eng.score_items(items))
+    print(f"max |p_graph - p_eager| = {(ref - got).abs().max():.4f} over {len(ref)} questions; argmax agreement {(ref.argmax(1) == got.argmax(1)).float().mean():.4f}")
+    print(f"warmup capture of 8 buckets: {eng.warmup():.1f}s; captures so far {eng.stats['graph_captures']}")
+    # latency: single real requests
+    for name, pool in [("support_tickets", exs[:64]), ("clinc_oos", exs[64:128]), ("race", exs[128:])]:
+        its = [build(e, eng.tok, rng, chat=chat) for e in pool]
+        ts = []
+        for it in its[:40]:
+            torch.cuda.synchronize(); t = time.time(); eng.score_items([it]); torch.cuda.synchronize(); ts.append(time.time() - t)
+        ts = np.array(ts[5:]) * 1000
+        print(f"single request {name:16s}: p50 {np.median(ts):5.1f} ms  p90 {np.percentile(ts, 90):5.1f} ms  (avg {np.mean([len(i['ids']) for i in its]):.0f} tok, {len(its[0]['slots'])} q)")
+        for bs in (8, 32):
+            ts = []
+            for i in range(0, min(len(its), bs * 6), bs):
+                chunk = its[i:i + bs]
+                if len(chunk) < bs: break
+                torch.cuda.synchronize(); t = time.time(); eng.score_items(chunk); torch.cuda.synchronize(); ts.append(time.time() - t)
+            ts = np.array(ts[1:]) * 1000
+            print(f"   batch {bs:2d}: p50 {np.median(ts):6.1f} ms -> {bs/np.median(ts)*1000:6.0f} ctx/s, {bs*len(its[0]['slots'])/np.median(ts)*1000:6.0f} decisions/s")
+    print("stats", eng.stats, "graphs", len(eng.graphs), f"mem {torch.cuda.memory_reserved()/1e9:.1f} GB")

decider/engine_v2.py

@@ -0,0 +1,213 @@
+"""Serving engine: every CUDA graph is keyed on (batch bucket, padded length bucket) only.
+
+Difference from decider.engine.Engine (the library engine behind decider.infer.Decider):
+  * the full bucket grid is captured at start-up by `warmup()`, then `seal()` forbids any further capture, so no request
+    can ever pay a graph capture or a torch.compile;
+  * the length ladder reaches 8192 instead of 2048, so long rows replay a graph instead of running eager;
+  * a batch larger than the widest captured batch bucket for its length is split into captured chunks rather than
+    capturing a new shape; rows longer than the last bucket run eager in chunks of at most `token_budget` padded tokens;
+  * torch.compile and FP8 are off by default.
+
+Numerics are the same as Engine: right padding + causal layers means padded positions never influence earlier slots, so
+the answer for a row does not depend on which (B, T) bucket it was padded into, up to kernel reduction order.
+
+The attention backend policy of decider.engine (cuDNN SDPA off) is applied in __init__, before compile and capture.  It
+is what makes `score_shared` (a suffix scored against a cached prefix) return the same answers as the full forward; see
+docs/CHANGELOG.md 1.0.2 and tests/test_engine_v2_cuda.py.
+"""
+import time, torch, torch.nn.functional as F
+from decider import shared_prefix
+from decider.engine import read_slots, fill_ids, patch_conv, set_attention_backend_policy
+from decider.model import DecisionModel
+from decider.temperature import item_slice, slot_temperatures
+
+T_BUCKETS = [64, 128, 192, 256, 320, 384, 512, 640, 768, 1024, 1280, 1536, 2048, 3072, 4096, 6144, 8192]
+B_BUCKETS = [1, 2, 4, 8, 16, 32]
+TOKEN_BUDGET = 32768        # capture (B, T) only when B * T fits this; B = 1 is always captured
+LONG_STEP = 1024            # rows longer than the last bucket: pad to a multiple of this and run eager
+
+
+class EngineV2:
+    """compile / fp8 default to False.  `model` lets a test inject a stand-in DecisionModel instead of loading one."""
+
+    def __init__(self, path=None, device="cuda", dtype=torch.bfloat16, use_graphs=None, compile=False, fp8=False,
+                 conv_patch=None, t_buckets=None, b_buckets=None, token_budget=TOKEN_BUDGET, max_ctx_tokens=32768,
+                 model=None):
+        set_attention_backend_policy()              # before compile and capture: captured graphs keep their backend
+        if conv_patch is None:
+            conv_patch = bool(compile)          # the unrolled depthwise conv only pays off inside a compiled region
+        if str(device).startswith("mps"):
+            from decider.mps_ops import patch_mps       # the Apple Silicon path of decider.infer.Decider
+            patch_mps()
+        elif conv_patch:
+            patch_conv()
+        self.m = model if model is not None else DecisionModel(path, dtype=dtype, grad_ckpt=False).to(device).eval()
+        self.tok = self.m.tok; self.dev = device; self.max_ctx = max_ctx_tokens
+        self.core = self.m.lm.model
+        self.W = self.m.lm.lm_head.weight[self.m.letters].detach().clone()
+        self.t_buckets = sorted(t_buckets or T_BUCKETS)
+        self.b_buckets = sorted(b_buckets or B_BUCKETS)
+        self.token_budget = int(token_budget)
+        self.use_graphs = (str(device).startswith("cuda") if use_graphs is None else bool(use_graphs))
+        self.cfg = dict(compile=compile, fp8=fp8, conv_patch=conv_patch, graphs=self.use_graphs,
+                        t_buckets=self.t_buckets, b_buckets=self.b_buckets, token_budget=self.token_budget)
+        if fp8:
+            from decider.fp8 import convert_to_fp8
+            self.cfg["fp8_layers"] = convert_to_fp8(self.core)
+        if compile:
+            from torch import _dynamo                    # not `import torch._dynamo`: that rebinds `torch` locally
+            n = len(self.graph_shapes())
+            # one specialisation per captured shape, times the frames inside the model.  Without the accumulated
+            # limits dynamo stops compiling part way through warm-up and silently leaves the rest eager, which with
+            # conv_patch on is slower than not compiling at all.
+            _dynamo.config.cache_size_limit = max(256, n + 16)
+            _dynamo.config.accumulated_cache_size_limit = max(1 << 16, 64 * n)
+            for k in ("accumulated_recompile_limit", "recompile_limit"):
+                if hasattr(_dynamo.config, k):
+                    setattr(_dynamo.config, k, max(1 << 16, 64 * n))
+            self._fwd_impl = torch.compile(self._fwd_eager, dynamic=False)
+        else:
+            self._fwd_impl = self._fwd_eager
+        self.graphs = {}                 # (B, T) -> (static ids, static out, graph)
+        self.b_for_t = {}                # T -> sorted captured batch buckets
+        self.pool = torch.cuda.graph_pool_handle() if (self.use_graphs and str(device).startswith("cuda")) else None
+        self.sealed = False
+        self.stats = dict(graph_captures=0, forwards=0, replays=0, eager_forwards=0, eager_rows=0,
+                          shared_calls=0, unbucketed_requests=0)
+
+    # ---- bucket arithmetic -------------------------------------------------
+    def graph_shapes(self):
+        """The full grid captured at start-up: every (B, T) whose padded token count fits the budget, plus all of B = 1."""
+        return [(B, T) for T in self.t_buckets for B in self.b_buckets if B == 1 or B * T <= self.token_budget]
+
+    def t_bucket(self, n):
+        """Padded length bucket for a row of n tokens, or None when it is longer than the last bucket."""
+        for t in self.t_buckets:
+            if n <= t:
+                return t
+        return None
+
+    def pad_len(self, n):
+        return self.t_bucket(n) or -(-n // LONG_STEP) * LONG_STEP
+
+    def max_rows(self, T):
+        """Rows per forward at padded length T: the widest captured batch bucket, or, when nothing is captured at T, as
+        many rows as fit the token budget (at least 1)."""
+        av = self.b_for_t.get(T)
+        return av[-1] if av else max(1, self.token_budget // max(T, 1))
+
+    def b_plan(self, n, T):
+        """Batch sizes covering n rows at length T, each one a captured bucket.  One chunk when a bucket fits."""
+        av = self.b_for_t.get(T)
+        if not av:
+            return [n]
+        fit = next((b for b in av if b >= n), None)
+        if fit is not None:
+            return [fit]
+        out = []; big = av[-1]
+        while n > big:
+            out.append(big); n -= big
+        out.append(next(b for b in av if b >= n))
+        return out
+
+    # ---- forward -----------------------------------------------------------
+    def _fwd_eager(self, ids):
+        h = self.core(input_ids=ids, use_cache=False).last_hidden_state
+        return F.linear(h, self.W).float()                          # [B, T, K]
+
+    @torch.no_grad()
+    def _fwd(self, ids):
+        return self._fwd_impl(ids)
+
+    def _capture(self, B, T):
+        s_ids = torch.full((B, T), self.tok.pad_token_id, dtype=torch.long, device=self.dev)
+        st = torch.cuda.Stream(); st.wait_stream(torch.cuda.current_stream())
+        with torch.cuda.stream(st):
+            for _ in range(3): self._fwd(s_ids)
+        torch.cuda.current_stream().wait_stream(st)
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g, pool=self.pool):
+            s_out = self._fwd(s_ids)
+        self.stats["graph_captures"] += 1
+        return s_ids, s_out, g
+
+    @torch.no_grad()
+    def logits_all(self, ids):
+        """ids: [B, T] long on device, already right-padded to a captured shape where one exists."""
+        B, T = ids.shape; self.stats["forwards"] += 1
+        if self.use_graphs:
+            g = self.graphs.get((B, T))
+            if g is None and not self.sealed:            # only reachable before seal(); a sealed engine never captures
+                g = self.graphs[(B, T)] = self._capture(B, T)
+                self.b_for_t.setdefault(T, [])
+                if B not in self.b_for_t[T]: self.b_for_t[T] = sorted(self.b_for_t[T] + [B])
+            if g is not None:
+                s_ids, s_out, gr = g
+                s_ids.copy_(ids); gr.replay(); self.stats["replays"] += 1
+                return s_out
+        self.stats["eager_forwards"] += 1; self.stats["eager_rows"] += B
+        return self._fwd_eager(ids)                      # never the compiled callable: a new shape must not compile
+
+    # ---- scoring -----------------------------------------------------------
+    @torch.no_grad()
+    def score_items(self, items, temperature=1.0):
+        """items: dicts from prompt.build / build_rows.  -> one [n_q, MAX_OPTIONS] cpu probability tensor per item.
+        temperature: a number, or one entry per item (a number or one number per slot; decider.temperature.for_items)."""
+        if not items:
+            return []
+        slot_temperatures(temperature, items)                      # a length mismatch fails before any forward
+        Tmax = max(len(it["ids"]) for it in items)
+        T = self.t_bucket(Tmax)
+        if T is None:
+            self.stats["unbucketed_requests"] += 1
+            T = -(-Tmax // LONG_STEP) * LONG_STEP
+            per = self.max_rows(T); plan = [per] * (len(items) // per) + ([len(items) % per] if len(items) % per else [])
+        else:
+            plan = self.b_plan(len(items), T)
+        out = []; i = 0
+        for B in plan:
+            chunk = items[i:i + B]; i += len(chunk)
+            ids = fill_ids([it["ids"] for it in chunk], B, T, self.tok.pad_token_id)
+            lg = self.logits_all(ids.to(self.dev, non_blocking=True))
+            out += read_slots(lg, [b for b, it in enumerate(chunk) for _ in it["slots"]],
+                              [s for it in chunk for s in it["slots"]], [n for it in chunk for n in it["nopts"]],
+                              slot_temperatures(item_slice(temperature, i - len(chunk), i), chunk), [len(it["slots"]) for it in chunk])
+        return out
+
+    @torch.no_grad()
+    def score_shared(self, items, temperature=1.0, min_prefix=192, budget_bytes=None, rows_per_fork=None):
+        """Rows that start with the same tokens (one state, one question per row): run the shared prefix once, fork its
+        cache in chunks that fit a byte budget, run only the question suffixes.  The algorithm of Engine.score_shared,
+        shared with it in decider.shared_prefix: it is per request, never per schema, so it adds no state that outlives
+        the request and no shape that depends on the question set.  The prefix and suffix forwards are eager
+        (request-specific shapes).  Correct only with the cuDNN SDPA backend off, which __init__ arranges."""
+        out = shared_prefix.score_shared(self, items, temperature, min_prefix, budget_bytes, rows_per_fork)
+        if out is None:
+            return self.score_items(items, temperature)
+        self.stats["shared_calls"] += 1
+        return out
+
+    # ---- start-up ----------------------------------------------------------
+    def warmup(self, shapes=None, log=None):
+        """Capture the whole grid.  Call seal() afterwards: from then on an unknown shape runs eager, never captures."""
+        t = time.time(); shapes = list(shapes if shapes is not None else self.graph_shapes())
+        for j, (B, T) in enumerate(shapes):
+            self.logits_all(torch.full((B, T), self.tok.pad_token_id, dtype=torch.long, device=self.dev))
+            if log and (j + 1) % 10 == 0:
+                log(f"[engine_v2] {j + 1}/{len(shapes)} graphs, {time.time() - t:.0f}s")
+        if self.use_graphs:
+            torch.cuda.synchronize()
+        return time.time() - t
+
+    def seal(self):
+        self.sealed = True
+        self.b_for_t = {}
+        for B, T in self.graphs:
+            self.b_for_t.setdefault(T, []).append(B)
+        for T in self.b_for_t:
+            self.b_for_t[T].sort()
+        return self
+
+    def describe(self):
+        return dict(self.cfg, graphs=len(self.graphs), sealed=self.sealed,
+                    grid={str(T): self.b_for_t.get(T, []) for T in self.t_buckets})

decider/fp8.py

@@ -0,0 +1,54 @@
+"""FP8 (e4m3) linear layers for Hopper via torch._scaled_mm.
+Weights: per-output-channel scales, quantised once.  Activations: per-token dynamic scales.
+Under torch.compile the quantisation ops fuse into the surrounding elementwise work."""
+import torch, torch.nn as nn
+
+E4M3_MAX = 448.0
+
+
+def _quant_rowwise(x):
+    s = x.abs().amax(dim=-1, keepdim=True).float().clamp(min=1e-12) / E4M3_MAX
+    return (x.float() / s).clamp(-E4M3_MAX, E4M3_MAX).to(torch.float8_e4m3fn), s
+
+
+class FP8Linear(nn.Module):
+    def __init__(self, lin: nn.Linear):
+        super().__init__()
+        wq, sw = _quant_rowwise(lin.weight.detach())          # [N,K] fp8, [N,1]
+        self.register_buffer("wq", wq.contiguous())               # [N,K]; passed as wq.t() -> [K,N] column-major, as _scaled_mm wants
+        self.register_buffer("sw_t", sw.t().contiguous())        # [1,N]
+        self.bias = None if lin.bias is None else nn.Parameter(lin.bias.detach().clone(), requires_grad=False)
+        self.in_features, self.out_features = lin.in_features, lin.out_features
+        self.out_dtype = lin.weight.dtype
+
+    def forward(self, x):
+        shp = x.shape[:-1]
+        x2 = x.reshape(-1, self.in_features)
+        xq, sx = _quant_rowwise(x2)
+        y = torch._scaled_mm(xq, self.wq.t(), scale_a=sx, scale_b=self.sw_t, bias=self.bias, out_dtype=self.out_dtype)
+        return y.reshape(*shp, self.out_features)
+
+
+def convert_to_fp8(model, skip=("lm_head",), min_dim=1024):
+    """Replace nn.Linear (with in/out >= min_dim) by FP8Linear in place. Returns count."""
+    n = 0
+    for name, mod in list(model.named_modules()):
+        for cname, child in list(mod.named_children()):
+            full = f"{name}.{cname}" if name else cname
+            if isinstance(child, nn.Linear) and not any(s in full for s in skip) and min(child.in_features, child.out_features) >= min_dim:
+                setattr(mod, cname, FP8Linear(child)); n += 1
+    return n
+
+
+if __name__ == "__main__":
+    import time
+    lin = nn.Linear(2048, 6144, bias=False).cuda().to(torch.bfloat16)
+    f8 = FP8Linear(lin)
+    x = torch.randn(8192, 2048, device="cuda", dtype=torch.bfloat16)
+    ref = lin(x); got = f8(x)
+    print("rel err", ((ref.float() - got.float()).abs().mean() / ref.float().abs().mean()).item())
+    for f, name in [(lin, "bf16 linear"), (f8, "fp8 linear (eager)"), (torch.compile(f8), "fp8 linear (compiled)")]:
+        for _ in range(3): f(x)
+        torch.cuda.synchronize(); t = time.time()
+        for _ in range(20): f(x)
+        torch.cuda.synchronize(); print(f"{name:24s} {(time.time()-t)/20*1000:.3f} ms")

decider/infer.py

@@ -0,0 +1,357 @@
+"""Usable inference API: typed decisions with probabilities, all from one forward pass.
+
+    from decider.infer import Decider
+    d = Decider("runs/r2_full/model")
+    out = d.decide("My card was charged twice for the same purchase.",
+                   [{"question": "Which department should handle this?", "options": ["billing", "technical", "sales"]},
+                    {"question": "How urgent is this?", "options": ["low", "medium", "high"]}])
+    # -> [{'choice': 'billing', 'confidence': 0.97, 'probs': {...}}, {...}]
+"""
+import logging
+
+import torch
+from decider.model import DecisionModel, collate
+from decider.prompt import build, MAX_OPTIONS, resolve_layout, chat_template
+from decider import temperature as TT
+from dataclasses import dataclass
+
+
+@dataclass
+class Q:
+    text: str; options: list; gold: int = 0
+
+
+@dataclass
+class Example:
+    context: str; qs: list; task: str = "infer"; image: bytes = None
+
+
+logger = logging.getLogger(__name__)
+NEUTRAL_NONE = "not listed here"
+
+
+def neutralize_options(options):
+    """The training augmentation used the literal 'none of the above', and the model learned that exact string as an
+    abstain signal (it abstains even on clear cases when the string is offered). Any option that reads like it is
+    rewritten to a neutral phrasing for the model and mapped back in the output."""
+    out, back = [], {}
+    for o in options:
+        key = o.strip().lower()
+        if key.startswith("none of the above") or key in ("none of the above", "none", "n/a", "none of these"):
+            out.append(NEUTRAL_NONE); back[NEUTRAL_NONE] = o
+        else:
+            out.append(o)
+    return out, back
+
+
+class _NoShuffle:                              # keep option order as given
+    def shuffle(self, x): pass
+    def sample(self, xs, k): return xs[:k]
+
+
+class CompiledSchema:
+    def __init__(self, d, rqs, h, index):
+        from decider.systemone import row_types
+        self.d, self.rqs, self.h, self.index = d, rqs, h, index
+        self.types = row_types(rqs, index)                         # answer type of every schema row (decider.temperature)
+
+    def batch(self, states, max_state_tokens=32768):
+        from decider.systemone import render_state, assemble
+        T = TT.for_types(self.d.T_schema, self.d.T_schema_by_type, self.types)
+        probs = self.d._se.score(self.h, [render_state(s) for s in states], temperature=T, max_ctx_tokens=max_state_tokens)
+        return [{"model": self.d.name, "answers": assemble(self.rqs, self.index, [p.tolist() for p in pr])} for pr in probs]
+
+    def __call__(self, state, max_state_tokens=32768):
+        return self.batch([state], max_state_tokens)[0]
+
+
+class Decider:
+    """One-pass decisions with automatic CUDA, MPS, or CPU device selection.
+
+    CUDA uses shape-bucketed graphs by default. MPS defaults to float16 and uses
+    the optional MPS patch; CPU defaults to bfloat16. Set ``use_graphs=False``
+    for eager execution or debugging.
+    """
+    def __init__(self, path, device=None, dtype=None, temperature=None, abstain_below=0.0, use_graphs=None, temperature_by_type=None):
+        """The prompt layout comes from decider_config.json: "layout": "chat" (chat-trained checkpoints) wraps every prompt in the
+        tokenizer's chat template (decider.prompt.build_chat); no "layout" key is the plain layout of every earlier model.
+        An unknown layout raises ValueError before the weights are loaded.
+
+        Temperatures (decider.temperature): decider_config.json "temperature" and the optional "temperature_by_type"
+        {"choice": T, "noul": T, "score": T}.  temperature= overrides "temperature" and switches the config's by-type map off;
+        temperature_by_type= sets the map explicitly.  An invalid temperature or map key raises ValueError before the weights
+        are loaded."""
+        if device is None:
+            device = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
+        if dtype is None:
+            dtype = torch.float16 if str(device).startswith("mps") else torch.bfloat16
+        logger.info("Decider device=%s dtype=%s", device, dtype)
+        import json, os
+        cfg = {}
+        try:                                          # model folder may carry decider_config.json (temperature, flags)
+            from huggingface_hub import hf_hub_download
+            cfg_path = os.path.join(path, "decider_config.json") if os.path.isdir(path) else hf_hub_download(path, "decider_config.json")
+            cfg = json.load(open(cfg_path))
+        except Exception:
+            pass
+        self.layout = resolve_layout(cfg)
+        (temperature, self.T_by_type), (T_schema, self.T_schema_by_type) = TT.from_config(cfg, temperature, temperature_by_type)
+        self.neutralize_none = bool(cfg.get("neutralize_none", True))   # v4 and earlier learned the literal string as an abstain signal
+        if use_graphs is None:
+            use_graphs = str(device).startswith("cuda")
+        if use_graphs:
+            from decider.engine import Engine
+            self.eng = Engine(path, device=device, dtype=dtype); self.m = self.eng.m
+        else:
+            if str(device).startswith("mps"):
+                from decider.mps_ops import patch_mps
+                patch_mps()
+            self.eng = None; self.m = DecisionModel(path, dtype=dtype, grad_ckpt=False).to(device).eval()
+        self.chat = chat_template(self.m.tok) if self.layout == "chat" else None      # None: plain layout, prompts as in 1.1.x
+        self.dev = device; self.T = temperature; self.abstain_below = abstain_below
+        self.name = "decider-" + str(cfg.get("version", "dev"))
+        self.schema_first = bool(cfg.get("schema_first", False)) and self.eng is not None      # default layout. Questions-first (the cacheable one) costs accuracy
+        self.T_schema = T_schema                                              # (about 1.5 points on fixed label sets, more elsewhere): opt in with schema()
+        self.isolated_levels = bool(cfg.get("isolated_levels", False))      # Score levels judged one per row (v8+)
+        self._se = None; self._schemas = {}
+
+    def _decide_items(self, requests, max_ctx_tokens=1536):
+        """The prompt rows of decide_batch: -> (requests with neutralized options, one item per request)."""
+        exs = []
+        if self.neutralize_none:
+            requests = [(context, [dict(q, options=neutralize_options(q["options"])[0], _back=neutralize_options(q["options"])[1]) for q in qs]) for context, qs in requests]
+        for context, qs in requests:
+            for q in qs:
+                assert 2 <= len(q["options"]) <= MAX_OPTIONS, f"2..{MAX_OPTIONS} options required"
+            exs.append(Example(context, [Q(q["question"], list(q["options"]), 0) for q in qs], "infer"))
+        items = [build(e, self.m.tok, _NoShuffle(), max_options=MAX_OPTIONS, max_ctx_tokens=max_ctx_tokens, chat=self.chat) for e in exs]
+        for it, (_, qs) in zip(items, requests):                     # answer type per slot: a plain question is "choice"
+            it["types"] = [q.get("_type", "choice") for q in qs]
+        return requests, items
+
+    @torch.no_grad()
+    def decide_batch(self, requests, max_ctx_tokens=1536):
+        """requests: list of (context:str, questions:list[dict(question, options)]). One forward pass for everything."""
+        requests, items = self._decide_items(requests, max_ctx_tokens)
+        T = TT.for_items(self.T, self.T_by_type, items)                  # the scalar self.T when there is no by-type map
+        if self.eng is not None:
+            probs = torch.cat(self.eng.score_items(items, temperature=T))
+        else:
+            b = collate(items, self.m.tok.pad_token_id)
+            logits = self.m.slot_logits(b["input_ids"].to(self.dev), b["attention_mask"].to(self.dev), b["slot_idx"].to(self.dev),
+                                        b["slot_batch"].to(self.dev), b["nopts"].to(self.dev))
+            probs = TT.scaled_softmax(logits, TT.slot_temperatures(T, items)).cpu()
+        out, k = [], 0
+        for context, qs in requests:
+            res = []
+            for q in qs:
+                p = probs[k, :len(q["options"])].tolist(); k += 1
+                j = max(range(len(p)), key=p.__getitem__); back = q.get("_back", {})
+                names = [back.get(o, o) for o in q["options"]]
+                res.append(dict(choice=names[j] if p[j] >= self.abstain_below else None, confidence=p[j],
+                                probs={o: pi for o, pi in zip(names, p)}, probs_list=p))
+            out.append(res)
+        return out
+
+    def decide(self, context, questions, **kw):
+        return self.decide_batch([(context, questions)], **kw)[0]
+
+    # ---- Jev-shaped interface (decider.systemone): state + {id: Choice | Score | Noul with criteria}
+    # ---- schema cache (v7+): the questions are run once, requests only run the state (decider.schema_engine)
+    def schema(self, questions, independent=True, isolated=None, compile=False):
+        """Compile a fixed set of Jev-shaped questions: schema(state) -> answers; schema.batch([state, ...]) -> [answers]."""
+        import json
+        from decider.schema_engine import SchemaEngine
+        from decider.systemone import render_question
+        isolated = self.isolated_levels if isolated is None else isolated
+        key = (json.dumps(questions, sort_keys=True, ensure_ascii=False), independent, isolated)
+        if key not in self._schemas:
+            if self._se is None: self._se = SchemaEngine(self.eng, chat=self.chat)
+            if len(self._schemas) >= 64:                                  # drop the oldest schema and its graphs
+                old = next(iter(self._schemas)); hid = self._schemas.pop(old)[1].id
+                for k in [k for k in self._se.graphs if k[0] == hid]: del self._se.graphs[k]
+            from decider.systemone import plan_rows
+            rqs = {k: render_question(v) for k, v in questions.items()}
+            rows, index = plan_rows(rqs, isolated and independent)
+            h = self._se.prepare(rows, independent=independent, compile=compile)      # compile=True: ~25 s per (batch, length) shape, 1.6x faster after
+            self._schemas[key] = (rqs, h, index)
+        return CompiledSchema(self, *self._schemas[key])
+
+    def system_one(self, state, questions, independent=True, max_state_tokens=32768, max_fwd_tokens=65536, layout=None, isolated=None):
+        layout = layout or ("schema_first" if self.schema_first else "state_first")
+        isolated = (self.isolated_levels if isolated is None else isolated) and independent
+        if layout == "schema_first" and self.eng is not None:
+            return self.schema(questions, independent, isolated)(state, max_state_tokens)
+        """independent=True scores every question in its own row (state + that question only), so adding, removing or
+        reordering questions cannot change any other answer; the state is run once and its cache forked to every
+        question (Engine.score_shared).  independent=False packs all questions behind one copy of the state in one row
+        (later questions can then see earlier question texts)."""
+        from decider.systemone import unique_tokens, assemble
+        rqs, index, items = self._system_one_items(state, questions, independent, max_state_tokens, layout, isolated)
+        flatp = self._system_one_probs(items, layout, max_fwd_tokens, TT.for_items(self.T, self.T_by_type, items))
+        return {"model": self.name, "answers": assemble(rqs, index, flatp),
+                "usage": {"input_tokens": unique_tokens(items), "output_tokens": 0}}
+
+    @torch.no_grad()
+    def _system_one_probs(self, items, layout, max_fwd_tokens=65536, temperature=None):
+        """Score system_one's uncached rows.  temperature: as Engine.score_items takes it (a number, or one entry per item).
+        -> one probability list per question row, in row order."""
+        if self.eng is not None and len(items) > 1 and layout == "state_first":
+            probs = self.eng.score_shared(items, temperature=temperature)
+        else:
+            probs = []; per = max(1, max_fwd_tokens // max(len(it["ids"]) for it in items))
+            for i in range(0, len(items), per):
+                T = TT.item_slice(temperature, i, i + per)
+                if self.eng is not None:
+                    probs += self.eng.score_items(items[i:i + per], temperature=T)
+                else:
+                    bt = collate(items[i:i + per], self.m.tok.pad_token_id)
+                    lg = self.m.slot_logits(*[bt[k].to(self.dev) for k in ("input_ids", "attention_mask", "slot_idx", "slot_batch", "nopts")])
+                    pr = TT.scaled_softmax(lg, TT.slot_temperatures(T, items[i:i + per])).cpu(); c = 0
+                    for it in items[i:i + per]:
+                        probs.append(pr[c:c + len(it["slots"])]); c += len(it["slots"])
+        return [p.tolist() for ps in probs for p in ps]
+
+    def _system_one_items(self, state, questions, independent=True, max_state_tokens=32768, layout=None, isolated=None):
+        """The prompt rows of system_one's uncached path: -> (rendered questions, answer index, items)."""
+        from decider.systemone import render_state, render_question, plan_rows, row_types
+        layout = layout or ("schema_first" if self.schema_first else "state_first")
+        isolated = (self.isolated_levels if isolated is None else isolated) and independent
+        ctx = render_state(state); rqs = {k: render_question(v) for k, v in questions.items()}
+        opts = (lambda r: neutralize_options(r["options"])[0]) if self.neutralize_none else (lambda r: list(r["options"]))
+        flat, index = plan_rows(rqs, isolated)
+        rows = [[r] for r in flat] if independent else [flat]
+        items = [build(Example(ctx, [Q(r["question"], opts(r), 0) for r in row]), self.m.tok, _NoShuffle(), max_options=MAX_OPTIONS,
+                       max_ctx_tokens=max_state_tokens, layout=layout, chat=self.chat) for row in rows]
+        types = row_types(rqs, index)                                 # answer type per row (decider.temperature)
+        for it, ts in zip(items, [[t] for t in types] if independent else [types]):
+            it["types"] = ts
+        return rqs, index, items
+
+    # ---- typed schema interface: {question: {"type": "bool"} | {"type": "choice", "options": [...]}
+    #                                         | {"type": "scale", "legend": {"0": "none", "1": "low", ...}}}
+    SCHEMA_FORM = ('schema: a map {question: field}, each field one of {"type": "choice", "options": [option strings]}, '
+                   '{"type": "bool"} or {"type": "scale", "legend": [descriptions] or {level number: description}}')
+
+    @staticmethod
+    def _check_schema(schema):
+        """Raise ValueError naming the expected form when `schema` does not have it (the HTTP server turns this into a 422).
+        Every schema that 1.1.2 answered stays accepted, except options or a legend given as a bare string (1.1.2 split it into
+        characters); an empty schema is answered with {}."""
+        import json, math
+        form = Decider.SCHEMA_FORM
+        if schema is None:
+            raise ValueError("schema is required; " + form)
+        if not isinstance(schema, dict):
+            raise ValueError(f"schema is a {type(schema).__name__}; " + form)
+        for qtext, spec in schema.items():
+            where = f"schema[{qtext!r}]"
+            if not isinstance(spec, dict):
+                raise ValueError(f"{where} is a {type(spec).__name__}, not a field object; " + form)
+            t = spec.get("type", "choice")
+            if t == "choice":
+                opts = spec.get("options")
+                if not isinstance(opts, (list, tuple, dict)) or not opts or not all(isinstance(o, str) for o in opts):
+                    raise ValueError(f'{where}: "options" must be a non-empty list of strings; ' + form)
+            elif t == "scale":
+                leg = spec.get("legend")
+                if not isinstance(leg, (list, tuple, dict)) or not leg:
+                    raise ValueError(f'{where}: "legend" must be a non-empty list or {{level number: description}} map; ' + form)
+                try:                                           # the legend is echoed in the answer, which must serialise
+                    json.dumps(leg, allow_nan=False)
+                except (TypeError, ValueError):
+                    raise ValueError(f'{where}: the "legend" contains a value that is not finite JSON (NaN or Infinity); ' + form)
+                if isinstance(leg, dict):
+                    try:
+                        ok = all(math.isfinite(float(k)) for k in leg)
+                    except (TypeError, ValueError, OverflowError):
+                        ok = False
+                    if not ok:
+                        raise ValueError(f'{where}: the keys of a "legend" map must be finite numbers, e.g. {{"0": "none", "1": "low"}}; ' + form)
+            elif t != "bool":
+                raise ValueError(f"{where}: unknown field type {t!r}; " + form)
+
+    @staticmethod
+    def _schema_to_questions(schema):
+        Decider._check_schema(schema)
+        qs = []
+        for qtext, spec in schema.items():
+            t = spec.get("type", "choice")
+            if t == "bool":
+                qs.append(dict(question=qtext, options=["no", "yes"], _type="noul"))
+            elif t == "choice":
+                qs.append(dict(question=qtext, options=list(spec["options"]), _type="choice"))
+            elif t == "scale":
+                leg = spec["legend"]
+                keys = sorted(leg, key=lambda k: float(k)) if isinstance(leg, dict) else list(range(len(leg)))
+                labels = [f"{k}: {leg[k]}" if isinstance(leg, dict) else f"{i}: {leg[i]}" for i, k in enumerate(keys)]
+                qs.append(dict(question=qtext, options=labels, _keys=keys, _legend=leg, _type="score"))
+            else:
+                raise ValueError(f"unknown field type {t}")
+        return qs
+
+    def decide_json_batch(self, requests, **kw):
+        """requests: list of (context, schema). Returns one dict per context keyed by question."""
+        qss = [self._schema_to_questions(schema) for _, schema in requests]
+        raw = self.decide_batch([(ctx, qs) for (ctx, _), qs in zip(requests, qss)], **kw)
+        out = []
+        for (ctx, schema), qs, res in zip(requests, qss, raw):
+            o = {}
+            for (qtext, spec), q, r in zip(schema.items(), qs, res):
+                t = spec.get("type", "choice")
+                if t == "bool":
+                    o[qtext] = {"noul": round(r["probs"]["yes"], 4), "type": "noul"}
+                elif t == "choice":
+                    o[qtext] = {"choice": r["choice"], "confidence": round(r["confidence"], 4), "type": "choice",
+                                "probabilities": {k: round(v, 4) for k, v in r["probs"].items()}}
+                else:
+                    p = [r["probs"][lab] for lab in q["options"]]
+                    keys = q["_keys"]; n = len(p)
+                    score = sum(float(k) * pi for k, pi in zip(keys, p))          # expected level on the legend scale
+                    j = max(range(n), key=p.__getitem__)
+                    o[qtext] = {"score": round(score, 2), "confidence": round(p[j], 4), "type": "scale", "legend": q["_legend"],
+                                "probabilities": {str(keys[i]): round(pi, 4) for i, pi in enumerate(p)}}
+            out.append(o)
+        return out
+
+    def decide_json(self, context, schema, **kw):
+        return self.decide_json_batch([(context, schema)], **kw)[0]
+
+
+if __name__ == "__main__":
+    import sys, json, time
+    d = Decider(sys.argv[1] if len(sys.argv) > 1 else "runs/r1_200k/model")
+    demo = [
+        ("My card was charged twice for the same purchase and I want the extra charge refunded.",
+         [{"question": "Which department should handle this?", "options": ["billing", "technical support", "sales"]},
+          {"question": "What is the customer's sentiment?", "options": ["angry", "neutral", "happy"]},
+          {"question": "Does this need a refund action?", "options": ["no", "yes"]}]),
+        ("hey can u turn the lights off in the kitchen",
+         [{"question": "What is the intent?", "options": ["smart home control", "set alarm", "play music", "none of the above"]},
+          {"question": "Is this request toxic?", "options": ["no", "yes"]}]),
+        ("The quarterly report shows revenue fell 12% while costs rose sharply.",
+         [{"question": "What is the financial sentiment?", "options": ["bearish", "neutral", "bullish"]}]),
+    ]
+    t = time.time(); res = d.decide_batch(demo); dt = time.time() - t
+    for (ctx, qs), r in zip(demo, res):
+        print("\n>>", ctx)
+        for q, a in zip(qs, r):
+            print(f"   {q['question']:45s} -> {a['choice']!s:22s} p={a['confidence']:.2f}  " + " ".join(f"{o}:{p:.2f}" for o, p in a['probs'].items()))
+    print(f"\n{sum(len(q) for _, q in demo)} decisions in {dt*1000:.0f} ms (one forward pass)")
+    schema = {
+        "Revenue currently impacted?": {"type": "bool"},
+        "What business impact?": {"type": "choice", "options": ["none", "degraded", "outage"]},
+        "Integration issue present?": {"type": "bool"},
+        "Account health status?": {"type": "choice", "options": ["healthy", "watch", "at risk"]},
+        "Which incident scope?": {"type": "choice", "options": ["single_account", "multi_account", "platform_wide"]},
+        "Security concern present?": {"type": "bool"},
+        "Duplicate charge reported?": {"type": "bool"},
+        "Churn likelihood level?": {"type": "scale", "legend": {"0": "none", "1": "low", "2": "medium", "3": "high"}},
+        "Human attention needed?": {"type": "bool"},
+        "Immediate feature request?": {"type": "bool"},
+    }
+    ctx = ("Hi, since this morning our Stripe webhook integration stopped firing and our checkout is down for all customers. "
+           "We are losing orders every minute and our partner launch is on Thursday. Also I think we got billed twice last week. "
+           "If this is not fixed today we will have to look at other providers.")
+    t = time.time(); js = d.decide_json(ctx, schema); dt = time.time() - t
+    print(f"\n>> {ctx[:80]}...\n" + json.dumps(js, indent=1)[:3000]); print(f"{len(schema)} typed fields in {dt*1000:.0f} ms (one forward pass)")

decider/metrics.py

@@ -0,0 +1,37 @@
+import numpy as np
+
+
+def ece(conf, correct, bins=15):
+    conf = np.asarray(conf); correct = np.asarray(correct, dtype=float)
+    edges = np.linspace(0, 1, bins + 1); e = 0.0
+    for lo, hi in zip(edges[:-1], edges[1:]):
+        m = (conf > lo) & (conf <= hi)
+        if m.any():
+            e += m.mean() * abs(conf[m].mean() - correct[m].mean())
+    return float(e)
+
+
+def aurc(conf, correct):
+    """Area under risk-coverage curve (lower is better)."""
+    order = np.argsort(-np.asarray(conf)); c = np.asarray(correct, dtype=float)[order]
+    risk = np.cumsum(1 - c) / np.arange(1, len(c) + 1)
+    return float(risk.mean())
+
+
+def sel_acc(conf, correct, coverage):
+    order = np.argsort(-np.asarray(conf)); c = np.asarray(correct, dtype=float)[order]
+    n = max(1, int(round(coverage * len(c))))
+    return float(c[:n].mean())
+
+
+def summarize(probs, golds, nopts):
+    """probs [N,K] (masked entries 0), golds [N], nopts [N]."""
+    probs = np.asarray(probs); golds = np.asarray(golds); nopts = np.asarray(nopts)
+    pred = probs.argmax(1); conf = probs.max(1); correct = (pred == golds)
+    p_gold = probs[np.arange(len(golds)), golds]
+    nll = -np.log(np.clip(p_gold, 1e-12, 1)).mean()
+    onehot = np.zeros_like(probs); onehot[np.arange(len(golds)), golds] = 1
+    brier = ((probs - onehot) ** 2).sum(1).mean()
+    return dict(n=int(len(golds)), acc=float(correct.mean()), nll=float(nll), brier=float(brier), ece=ece(conf, correct),
+                aurc=aurc(conf, correct), acc_at_80=sel_acc(conf, correct, 0.8), acc_at_50=sel_acc(conf, correct, 0.5),
+                chance=float((1.0 / nopts).mean()), mean_conf=float(conf.mean()))

decider/model.py

@@ -0,0 +1,49 @@
+"""Backbone -> slot hidden states -> restricted logits over option letters."""
+import torch, torch.nn as nn, torch.nn.functional as F
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from decider.prompt import letter_ids, MAX_OPTIONS
+
+
+class DecisionModel(nn.Module):
+    def __init__(self, name, dtype=torch.bfloat16, grad_ckpt=True):
+        super().__init__()
+        if hasattr(torch.backends.cuda, "enable_cudnn_sdp"):   # as decider.engine.set_attention_backend_policy: the cuDNN SDPA
+            torch.backends.cuda.enable_cudnn_sdp(False)         # backend is wrong for masked attention on Blackwell (torch 2.14)
+        self.tok = AutoTokenizer.from_pretrained(name)
+        self.lm = AutoModelForCausalLM.from_pretrained(name, dtype=dtype)
+        if grad_ckpt:
+            self.lm.gradient_checkpointing_enable()
+        self.register_buffer("letters", torch.tensor(letter_ids(self.tok)), persistent=False)
+
+    def slot_logits(self, input_ids, attention_mask, slot_idx, slot_batch, nopts):
+        """input_ids [B,T]; slot_idx/slot_batch [N] flat slot positions; nopts [N].
+        Returns [N, MAX_OPTIONS] logits with invalid options masked to -inf."""
+        h = self.lm.model(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
+        hs = h[slot_batch, slot_idx]                                   # [N,H]
+        W = self.lm.lm_head.weight[self.letters]                       # [K,H]
+        logits = F.linear(hs, W).float()                               # [N,K]
+        ar = torch.arange(MAX_OPTIONS, device=logits.device)[None, :]
+        logits = logits.masked_fill(ar >= nopts[:, None], float("-inf"))
+        return logits
+
+    def forward(self, batch):
+        return self.slot_logits(batch["input_ids"], batch["attention_mask"], batch["slot_idx"], batch["slot_batch"], batch["nopts"])
+
+
+def collate(items, pad_id):
+    """items: list of dicts from prompt.build (+ 'task', 'ex_id'). Right-pad."""
+    T = max(len(it["ids"]) for it in items)
+    T = ((T + 63) // 64) * 64          # few distinct shapes -> fewer kernel (re)compiles
+    B = len(items)
+    input_ids = torch.full((B, T), pad_id, dtype=torch.long)
+    attn = torch.zeros((B, T), dtype=torch.long)
+    slot_idx, slot_batch, golds, nopts, tasks, qidx = [], [], [], [], [], []
+    for b, it in enumerate(items):
+        n = len(it["ids"])
+        input_ids[b, :n] = torch.tensor(it["ids"])
+        attn[b, :n] = 1
+        for k, s in enumerate(it["slots"]):
+            slot_idx.append(s); slot_batch.append(b); golds.append(it["golds"][k]); nopts.append(it["nopts"][k])
+            tasks.append(it.get("task", "")); qidx.append(k)
+    return dict(input_ids=input_ids, attention_mask=attn, slot_idx=torch.tensor(slot_idx), slot_batch=torch.tensor(slot_batch),
+                golds=torch.tensor(golds), nopts=torch.tensor(nopts), tasks=tasks, qidx=qidx)

decider/mps_moe.py

@@ -0,0 +1,102 @@
+"""Two MPS slow paths in transformers' Qwen3.5-MoE reference code, replaced in-process (decider-35b-a3b on Apple Silicon).
+
+Contributed by @nassersala in issue #6 under the project's Apache-2.0 licence; adapted for the package (out-of-range expert
+ids are dropped as `histc` drops them, and the replacements are installed from `decider.mps_ops.patch_mps`).
+
+On an M-series Mac (torch 2.14, transformers 5.17) `torch.histc` takes about 45 ms a call on MPS (once per MoE layer, to
+count tokens per expert) and `torch.linalg.solve_triangular` about 17 ms (twice per linear-attention layer): most of a
+2.8-4 s decision. With both replaced the 35B takes 0.23-0.5 s a decision for typical inputs (reported in issue #6).
+The `histc` replacement is an exact count. The block inverse is within 1e-6 of the MPS solver on real systems; on six items
+the probabilities moved at most 0.033, less than switching to the exact CPU solver does (0.042), and no argmax changed.
+
+The replacements are visible only inside those two transformers modules (install() gives each its own view of `torch`), and
+there they act only on MPS tensors of the matching call shape; everything else goes to torch.
+"""
+import torch
+
+_histc, _solve = torch.histc, torch.linalg.solve_triangular
+
+
+def count_ids(x, bins, min=0):
+    """histc for integer-valued ids binned one per bin: a count of ids min..min+bins-1; other values are dropped as histc
+    drops them (transformers passes expert-parallel sentinels >= num_experts and relies on that)."""
+    idx = x.long() - int(min)
+    keep = (idx >= 0) & (idx < bins)
+    out = torch.zeros(bins, device=x.device, dtype=x.dtype)
+    return out.scatter_add_(0, idx.clamp(0, bins - 1), keep.to(x.dtype))
+
+
+def histc(input, bins=100, min=0, max=0, *, out=None):
+    # expert ids 0..n-1 into n bins over [0, n-1]: bin k holds id k exactly, so this is a count (histc: ~45 ms on MPS)
+    if input.device.type == "mps" and input.dim() == 1 and out is None and min == 0 and max > 0 and bins == max - min + 1:
+        return count_ids(input, bins, min)
+    return _histc(input, bins=bins, min=min, max=max) if out is None else _histc(input, bins=bins, min=min, max=max, out=out)
+
+
+def unit_lower_inverse(A):
+    """Inverse of unit lower-triangular A (last dim a power of two) by block doubling:
+    [[A11, 0], [A21, A22]]^-1 = [[X11, 0], [-X22 A21 X11, X22]]."""
+    n = A.shape[-1]
+    inv = torch.ones(*A.shape[:-2], n, 1, 1, device=A.device, dtype=A.dtype)  # n diagonal 1x1 blocks
+    b = 1
+    while b < n:
+        nb = n // (2 * b)
+        blocks = A.reshape(*A.shape[:-2], nb, 2 * b, nb, 2 * b).diagonal(dim1=-4, dim2=-2).movedim(-1, -3)
+        x11, x22, a21 = inv[..., 0::2, :, :], inv[..., 1::2, :, :], blocks[..., b:, :b]
+        new = torch.zeros(*A.shape[:-2], nb, 2 * b, 2 * b, device=A.device, dtype=A.dtype)
+        new[..., :b, :b], new[..., b:, b:], new[..., b:, :b] = x11, x22, -(x22 @ a21 @ x11)
+        inv, b = new, 2 * b
+    return inv[..., 0, :, :]
+
+
+def solve_unit_lower(A, B):
+    """solve_triangular(A, B, upper=False, unitriangular=True): only the strict lower triangle of A is read."""
+    n = A.shape[-1]
+    return unit_lower_inverse(A.tril(-1) + torch.eye(n, device=A.device, dtype=A.dtype)) @ B
+
+
+def solve_triangular(A, B, *, upper, left=True, unitriangular=False, out=None):
+    # unit lower-triangular: invert by block doubling, all batched matmuls (the MPS solver: ~17 ms per call)
+    n = A.shape[-1]
+    if A.device.type == "mps" and not upper and left and unitriangular and out is None and n > 0 and n & (n - 1) == 0:
+        return solve_unit_lower(A, B)
+    return _solve(A, B, upper=upper, left=left, unitriangular=unitriangular, out=out)
+
+
+class _Proxy:
+    """A module's view of `torch` (or `torch.linalg`) with some attributes replaced; everything else is the real one."""
+    def __init__(self, real, **over):
+        self._real, self._over = real, over
+
+    def __getattr__(self, name):
+        return self._over[name] if name in self._over else getattr(self._real, name)
+
+
+_installed = {}
+
+
+def install():
+    """Replace the two operations inside the two transformers modules that call them, and nowhere else: the grouped-MoE
+    expert count in `transformers.integrations.moe` and the gated delta rule in `modeling_qwen3_5_moe` see a `torch` whose
+    `histc` / `linalg.solve_triangular` are the replacements; every other caller keeps torch's.  -> names patched."""
+    import importlib
+    targets = {"transformers.integrations.moe": dict(histc=histc),
+               "transformers.models.qwen3_5_moe.modeling_qwen3_5_moe": dict(linalg=_Proxy(torch.linalg, solve_triangular=solve_triangular))}
+    for name, over in targets.items():
+        if name in _installed:
+            continue
+        try:
+            mod = importlib.import_module(name)
+        except ImportError:
+            continue
+        if getattr(mod, "torch", None) is not torch:
+            continue                                  # not the layout this was written for: leave it alone
+        _installed[name] = mod
+        mod.torch = _Proxy(torch, **over)
+    return sorted(_installed)
+
+
+def uninstall():
+    for name, mod in list(_installed.items()):
+        mod.torch = torch
+        del _installed[name]

decider/mps_ops.py

@@ -0,0 +1,268 @@
+"""MPS kernels for Qwen3.5 gated-delta attention.
+
+The optimization is measured against Transformers' PyTorch reference fallback for
+this kernel; it is not an end-to-end model speedup claim.
+
+The gated-delta and L2-normalization code follows Transformers 5.17's
+``modeling_qwen3_5.py`` under the Apache License 2.0. The MPS inversion and
+backend dispatch are Decider additions.
+"""
+import functools
+import inspect
+import logging
+import warnings
+
+import torch, torch.nn.functional as F
+from decider.engine import fused_causal_conv1d_fn
+
+
+# Adapted from transformers 5.17.0's modeling_qwen3_5.py (Apache-2.0).
+# The original license applies to the gated-delta and normalization portions;
+# the MPS inversion and dispatch below are Decider additions.
+# Native Metal Shading Language (MSL) JIT kernel via MLX/Metal
+_metal_invert_kernel = None
+_metal_failure_reported = False
+_compat_warning_reported = False
+try:
+    import mlx.core as mx
+    import mlx.core.fast as fast
+
+    _METAL_INVERT_SRC = """
+    uint col = thread_position_in_threadgroup.x;
+    uint mat_idx = threadgroup_position_in_grid.x;
+    threadgroup float s_inv[4096];
+
+    for (uint r = 0; r < 64; ++r) {
+        s_inv[r * 64 + col] = (r == col) ? 1.0f : 0.0f;
+    }
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    uint mat_offset = mat_idx * 4096;
+    for (uint r = 1; r < 64; ++r) {
+        float sum = 0.0f;
+        if (col < r) {
+            for (uint k = col; k < r; ++k) {
+                sum += L[mat_offset + r * 64 + k] * s_inv[k * 64 + col];
+            }
+        }
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+        if (col < r) {
+            s_inv[r * 64 + col] = -sum;
+        }
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+    }
+    for (uint r = 0; r < 64; ++r) {
+        out_inv[mat_offset + r * 64 + col] = s_inv[r * 64 + col];
+    }
+    """
+    _metal_invert_kernel = fast.metal_kernel(
+        name="invert_unitriangular_64",
+        input_names=["L"],
+        output_names=["out_inv"],
+        source=_METAL_INVERT_SRC,
+    )
+except (ImportError, OSError, RuntimeError, AttributeError):
+    logging.getLogger(__name__).debug("MLX Metal kernel unavailable; using PyTorch MPS fallback", exc_info=True)
+    _metal_invert_kernel = None
+
+
+def l2norm(x: torch.Tensor, dim: int = -1, eps: float = 1e-6) -> torch.Tensor:
+    return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps)
+
+
+def fast_invert_unitriangular_64(L: torch.Tensor) -> torch.Tensor:
+    """Invert batches of 64x64 unit lower-triangular matrices."""
+    global _metal_invert_kernel, _metal_failure_reported
+    if _metal_invert_kernel is not None and L.is_mps and not L.requires_grad:
+        assert L.dtype == torch.float32, "Metal inversion requires float32 input"
+        try:
+            orig_shape = L.shape
+            L_flat = L.reshape(-1, 64, 64).contiguous()
+            num_m = L_flat.shape[0]
+            torch.mps.synchronize()
+            L_mx = mx.from_dlpack(torch.to_dlpack(L_flat))
+            mx.eval(L_mx)
+            out_mx = _metal_invert_kernel(
+                inputs=[L_mx],
+                grid=(num_m * 64, 1, 1),
+                threadgroup=(64, 1, 1),
+                output_shapes=[(num_m, 64, 64)],
+                output_dtypes=[mx.float32],
+            )[0]
+            mx.eval(out_mx)
+            torch.mps.synchronize()
+            return torch.from_dlpack(out_mx).to(L.device).reshape(orig_shape)
+        except Exception:
+            # Optional acceleration must not turn a recoverable backend issue into
+            # a failed request; disable the failing kernel for this process.
+            _metal_invert_kernel = None
+            if not _metal_failure_reported:
+                logging.getLogger(__name__).warning("MLX Metal inversion failed; using the PyTorch MPS fallback", exc_info=True)
+                _metal_failure_reported = True
+
+    N = 64
+    inv = torch.eye(N, device=L.device, dtype=L.dtype).expand_as(L).clone()
+
+    # b = 1: 32 blocks of 2x2. For [[1, 0], [l, 1]], inverse is [[1, 0], [-l, 1]]
+    idx_row = torch.arange(1, 64, 2, device=L.device)
+    idx_col = torch.arange(0, 64, 2, device=L.device)
+    inv[..., idx_row, idx_col] = -L[..., idx_row, idx_col]
+
+    # b = 2: 16 blocks of 4x4
+    for j in range(16):
+        r1, r2, r3 = j * 4, j * 4 + 2, j * 4 + 4
+        inv[..., r2:r3, r1:r2] = -inv[..., r2:r3, r2:r3] @ L[..., r2:r3, r1:r2] @ inv[..., r1:r2, r1:r2]
+
+    # b = 4: 8 blocks of 8x8
+    for j in range(8):
+        r1, r2, r3 = j * 8, j * 8 + 4, j * 8 + 8
+        inv[..., r2:r3, r1:r2] = -inv[..., r2:r3, r2:r3] @ L[..., r2:r3, r1:r2] @ inv[..., r1:r2, r1:r2]
+
+    # b = 8: 4 blocks of 16x16
+    for j in range(4):
+        r1, r2, r3 = j * 16, j * 16 + 8, j * 16 + 16
+        inv[..., r2:r3, r1:r2] = -inv[..., r2:r3, r2:r3] @ L[..., r2:r3, r1:r2] @ inv[..., r1:r2, r1:r2]
+
+    # b = 16: 2 blocks of 32x32
+    for j in range(2):
+        r1, r2, r3 = j * 32, j * 32 + 16, j * 32 + 32
+        inv[..., r2:r3, r1:r2] = -inv[..., r2:r3, r2:r3] @ L[..., r2:r3, r1:r2] @ inv[..., r1:r2, r1:r2]
+
+    # b = 32: 1 block of 64x64
+    inv[..., 32:64, 0:32] = -inv[..., 32:64, 32:64] @ L[..., 32:64, 0:32] @ inv[..., 0:32, 0:32]
+    return inv
+
+
+def mps_chunk_gated_delta_rule(
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    g: torch.Tensor,
+    beta: torch.Tensor,
+    chunk_size: int = 64,
+    initial_state: torch.Tensor | None = None,
+    output_final_state: bool = False,
+    use_qk_l2norm_in_kernel: bool = False,
+    **kwargs,
+) -> tuple[torch.Tensor, torch.Tensor | None]:
+    """Optimized chunk-gated delta rule for Apple Silicon (MPS)."""
+    assert chunk_size == 64, f"MPS optimized delta rule requires chunk_size=64, got {chunk_size}"
+    initial_dtype = query.dtype
+    batch_size, sequence_length, _, k_head_dim = key.shape
+    num_v_heads, v_head_dim = value.shape[-2:]
+    recurrent_state_shape = (batch_size, num_v_heads, k_head_dim, v_head_dim)
+    padded_output_shape = (batch_size, num_v_heads, -1, v_head_dim)
+    decay = g
+
+    query, key, value, beta, decay = [
+        x.transpose(1, 2).to(torch.float32, memory_format=torch.contiguous_format)
+        for x in (query, key, value, beta, decay)
+    ]
+    if use_qk_l2norm_in_kernel:
+        query = l2norm(query, dim=-1, eps=1e-6)
+        key = l2norm(key, dim=-1, eps=1e-6)
+    scaling = query.shape[-1] ** -0.5
+    query = query * scaling
+
+    pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size
+    query = F.pad(query, (0, 0, 0, pad_size))
+    key = F.pad(key, (0, 0, 0, pad_size))
+    value = F.pad(value, (0, 0, 0, pad_size))
+    beta = F.pad(beta, (0, pad_size))
+    decay = F.pad(decay, (0, pad_size))
+
+    v_beta = value * beta.unsqueeze(-1)
+    k_beta = key * beta.unsqueeze(-1)
+
+    query, key, k_beta, v_beta = [
+        x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1])
+        for x in (query, key, k_beta, v_beta)
+    ]
+    decay = decay.reshape(decay.shape[0], decay.shape[1], -1, chunk_size)
+
+    strictly_upper_mask = torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device).triu(1)
+    cum_decay = decay.cumsum(dim=3)
+    pairwise_decay = (cum_decay.unsqueeze(4) - cum_decay.unsqueeze(3)).masked_fill(strictly_upper_mask, float("-inf")).exp()
+
+    ut_system = (k_beta @ key.transpose(-1, -2)) * pairwise_decay
+    intra_chunk_attn = (query @ key.transpose(-1, -2)) * pairwise_decay
+    decayed_k_beta = k_beta * cum_decay.exp().unsqueeze(-1)
+
+    # Fast block divide-and-conquer unitriangular inverse on MPS
+    L = ut_system.tril(-1)
+    inv = fast_invert_unitriangular_64(L)
+    new_values = inv @ v_beta
+    k_cumdecay = inv @ decayed_k_beta
+
+    if initial_state is None:
+        last_recurrent_state = torch.zeros(recurrent_state_shape, dtype=new_values.dtype, device=new_values.device)
+    else:
+        last_recurrent_state = initial_state.to(new_values)
+    core_attn_out = torch.zeros_like(new_values)
+
+    query = query * cum_decay.exp().unsqueeze(-1)
+    key = key * (cum_decay[..., -1:] - cum_decay).exp().unsqueeze(-1)
+    chunk_decay = cum_decay[..., -1].exp()[..., None, None]
+
+    num_chunks = query.shape[2]
+    qk = torch.cat([query, k_cumdecay], dim=3)
+    kt = key.transpose(-1, -2)
+
+    for i in range(num_chunks):
+        qk_state = qk[:, :, i] @ last_recurrent_state
+        inter_chunk_attn = qk_state[:, :, :chunk_size]
+        v_new = new_values[:, :, i] - qk_state[:, :, chunk_size:]
+        core_attn_out[:, :, i] = inter_chunk_attn + intra_chunk_attn[:, :, i] @ v_new
+        last_recurrent_state = last_recurrent_state * chunk_decay[:, :, i] + kt[:, :, i] @ v_new
+
+    last_recurrent_state = None if not output_final_state else last_recurrent_state
+    core_attn_out = core_attn_out.reshape(padded_output_shape)[:, :, :sequence_length]
+    core_attn_out = core_attn_out.transpose(1, 2).to(initial_dtype, memory_format=torch.contiguous_format)
+    return core_attn_out, last_recurrent_state
+
+
+def patch_mps():
+    """Patch Qwen3.5 operations on MPS tensors while preserving other backends."""
+    global _compat_warning_reported
+    if not torch.backends.mps.is_available():
+        return False
+    import os
+    if os.environ.get("DECIDER_MPS_MOE_PATCH", "1") != "0":     # histc + unit-triangular solve replacements (issue #6, @nassersala)
+        from decider import mps_moe
+        mps_moe.install()
+    try:
+        import transformers
+        import transformers.models.qwen3_5.modeling_qwen3_5 as mq
+        version = tuple(int(part) for part in transformers.__version__.split(".")[:2])
+        if not all(callable(getattr(mq, name, None)) for name in ("torch_chunk_gated_delta_rule", "causal_conv1d_fn")):
+            raise AttributeError("unsupported Transformers Qwen3.5 implementation")
+        params = inspect.signature(mq.torch_chunk_gated_delta_rule).parameters
+        required = {"query", "key", "value", "g", "beta", "chunk_size"}
+        if version < (5, 17) or not required.issubset(params):
+            if not _compat_warning_reported:
+                warnings.warn(f"MPS patch requires Transformers >=5.17 with the Qwen3.5 gated-delta signature; got {transformers.__version__}", RuntimeWarning, stacklevel=2)
+                _compat_warning_reported = True
+            return False
+        if getattr(mq.torch_chunk_gated_delta_rule, "_decider_mps_patch", False):
+            return True
+
+        original_delta, original_conv = mq.torch_chunk_gated_delta_rule, mq.causal_conv1d_fn
+        delta_params = tuple(inspect.signature(original_delta).parameters)
+        query_index, chunk_index = delta_params.index("query"), delta_params.index("chunk_size")
+
+        @functools.wraps(original_delta)
+        def delta(*args, **kwargs):
+            query = args[query_index] if len(args) > query_index else kwargs["query"]
+            chunk_size = args[chunk_index] if len(args) > chunk_index else kwargs.get("chunk_size", 64)
+            return mps_chunk_gated_delta_rule(*args, **kwargs) if query.is_mps and chunk_size == 64 else original_delta(*args, **kwargs)
+
+        @functools.wraps(original_conv)
+        def conv(hidden_states, *args, **kwargs):
+            return fused_causal_conv1d_fn(hidden_states, *args, **kwargs) if hidden_states.is_mps else original_conv(hidden_states, *args, **kwargs)
+
+        delta._decider_mps_patch = True
+        mq.torch_chunk_gated_delta_rule, mq.causal_conv1d_fn = delta, conv
+        return True
+    except (ImportError, AttributeError, TypeError, ValueError):
+        logging.getLogger(__name__).debug("MPS patch unavailable; using Transformers fallback", exc_info=True)
+        return False

decider/prompt.py

@@ -0,0 +1,287 @@
+"""Prompt construction.  One context, N typed questions, N answer slots.
+
+All N decisions are read from a single forward pass: the logits at each
+"Answer k: (" slot are restricted to the option-letter tokens.  No answer
+letters are ever inserted, so slot k sees the context and all questions but
+no earlier answers (the decisions are conditionally independent given input).
+"""
+import random
+
+LETTERS = "ABCDEFGHIJ"
+NARROW = len(LETTERS)      # <= NARROW options: the original "(A) .. (J)" rendering, tokenized as a string (unchanged since v1)
+MAX_OPTIONS = 255          # width of the label head. > NARROW options: "wide" rendering, one label token per option:
+                           # A..Z then the first 229 two-letter upper-case strings that are single tokens (AA, AB, ...)
+ABSTAIN_PREFIXES = ("none of the above", "none of these", "not listed", "no suitable", "does not apply", "cannot tell")
+ABSTAIN_EXACT = ("other", "unsure", "something else", "neither of these", "other / not covered")
+
+
+def is_abstain_option(o):
+    o = o.strip().lower()
+    return o.startswith(ABSTAIN_PREFIXES) or o in ABSTAIN_EXACT
+
+
+_LABELS = {}
+_OPT_CACHE = {}
+
+
+def _enc_opt(tok, text):
+    """Token ids of ") <option text>" (cached: fixed label sets repeat the same strings millions of times)."""
+    key = (id(tok), text)
+    v = _OPT_CACHE.get(key)
+    if v is None:
+        v = tok.encode(f") {text}", add_special_tokens=False)
+        if len(_OPT_CACHE) < 2_000_000:
+            _OPT_CACHE[key] = v
+    return v
+
+
+def label_table(tok):
+    """(label strings, label token ids), MAX_OPTIONS entries; the first NARROW are A..J so narrow questions are unchanged."""
+    key = id(tok)
+    if key not in _LABELS:
+        import string
+        U = string.ascii_uppercase
+        names = list(U) + [a + b for a in U for b in U]
+        out = []
+        for n in names:
+            t = tok.encode(n, add_special_tokens=False)
+            if len(t) == 1:
+                out.append((n, t[0]))
+            if len(out) == MAX_OPTIONS:
+                break
+        assert len(out) == MAX_OPTIONS and len({i for _, i in out}) == MAX_OPTIONS
+        _LABELS[key] = ([n for n, _ in out], [i for _, i in out],
+                        tok.encode("\n(", add_special_tokens=False))
+    return _LABELS[key]
+
+
+def _select(q, rng, max_options):
+    opts = list(range(len(q.options)))
+    if len(opts) > max_options:
+        # always keep the gold and any abstain-style option (its mere presence must not carry information)
+        forced = {q.gold} | {i for i, o in enumerate(q.options) if is_abstain_option(o)}
+        others = [i for i in opts if i not in forced]
+        opts = rng.sample(others, max_options - len(forced)) + list(forced)
+    rng.shuffle(opts)
+    return opts
+
+
+def _options_ids(tok, q, opts):
+    if len(opts) <= NARROW:
+        return tok.encode("".join(f"\n({LETTERS[j]}) {q.options[oi]}" for j, oi in enumerate(opts)), add_special_tokens=False)
+    _, lab_ids, open_ids = label_table(tok); out = []
+    for j, oi in enumerate(opts):
+        out += open_ids + [lab_ids[j]] + _enc_opt(tok, q.options[oi])
+    return out
+
+
+def build_schema_first(example, tok, rng=None, max_options=NARROW, max_ctx_tokens=1536, chat=None):
+    """Schema-first layout: all question/option blocks, then the context, then one answer slot per question.
+
+        Question 1: ...\nOptions:\n(A) ...          <- prefix: depends only on the questions, so its cache (attention KV and
+        \n\nQuestion 2: ...                            delta-net states) is computed once per schema and reused for every state
+        \n\nContext:\n<state>\n\nAnswer 1: (\nAnswer 2: (
+
+    The three parts are tokenized separately, so `ids[:prefix_len]` is identical for every state.
+    chat: a ChatTemplate for a chat-layout model (see `build_chat`); None renders the plain layout above."""
+    rng = rng or random
+    perms = [_select(q, rng, max_options) for q in example.qs]
+    pre = schema_prefix_ids(tok, example.qs, perms, chat=chat)
+    suf, slots = schema_suffix_ids(tok, example.context, len(example.qs), max_ctx_tokens, chat=chat)
+    return dict(ids=pre + suf, slots=[len(pre) + s for s in slots], golds=[p.index(q.gold) if q.gold in p else -1 for p, q in zip(perms, example.qs)],
+                nopts=[len(p) for p in perms], perms=perms, prefix_len=len(pre))
+
+
+def schema_prefix_ids(tok, qs, perms=None, chat=None):
+    """Token ids of the question/option blocks (the cacheable part of the schema-first layout).
+    chat: a ChatTemplate; the prefix then starts with the template head (the user turn opens before the first question)."""
+    multi = len(qs) > 1; pre = [] if chat is None else list(chat.head)
+    for k, q in enumerate(qs):
+        opts = perms[k] if perms is not None else list(range(len(q.options)))
+        pre += tok.encode(f"{chr(10) * 2 if k else ''}Question{' ' + str(k + 1) if multi else ''}: {q.text}\nOptions:", add_special_tokens=False) + _options_ids(tok, q, opts)
+    return pre
+
+
+def schema_suffix_ids(tok, context, n_q, max_ctx_tokens=1536, chat=None):
+    """Token ids after the schema prefix: the context and one answer slot per question. Returns (ids, slot positions in ids).
+    chat: a ChatTemplate; the context then ends the user turn, and the template tail and the answer pieces follow."""
+    ids = tok.encode("\n\nContext:\n", add_special_tokens=False) + tok.encode(context, add_special_tokens=False)[:max_ctx_tokens]; slots = []
+    if chat is not None:
+        ids += chat.tail
+        for k in range(n_q):
+            ids += chat.answer_ids(tok, k, n_q > 1); slots.append(len(ids) - 1)
+        return ids, slots
+    for k in range(n_q):
+        ids += tok.encode(f"{chr(10) * 2 if k == 0 else chr(10)}Answer{' ' + str(k + 1) if n_q > 1 else ''}: (", add_special_tokens=False); slots.append(len(ids) - 1)
+    return ids, slots
+
+
+def build(example, tok, rng=None, max_options=NARROW, max_ctx_tokens=1536, layout="state_first", chat=None):
+    """Returns dict(ids=list[int], slots=list[int], golds=list[int], nopts=list[int], perms=list[list[int]]).
+    chat: a ChatTemplate (chat_template(tok)) for a model trained in the chat layout (decider_config.json "layout": "chat");
+    None, the default, is the plain layout every earlier model uses, unchanged."""
+    if layout == "schema_first":
+        return build_schema_first(example, tok, rng, max_options, max_ctx_tokens, chat=chat)
+    if chat is not None:
+        return build_chat(example, tok, chat, rng, max_options, max_ctx_tokens)
+    rng = rng or random
+    ctx_ids = tok.encode("Context:\n" + example.context, add_special_tokens=False)[:max_ctx_tokens]
+    ids = list(ctx_ids)
+    slots, golds, nopts, perms = [], [], [], []
+    multi = len(example.qs) > 1
+    for k, q in enumerate(example.qs):
+        opts = list(range(len(q.options)))
+        if len(opts) > max_options:
+            # always keep the gold and any abstain-style option (its mere presence must not carry information)
+            forced = {q.gold} | {i for i, o in enumerate(q.options) if is_abstain_option(o)}
+            others = [i for i in opts if i not in forced]
+            keep = rng.sample(others, max_options - len(forced)) + list(forced)
+            opts = keep
+        rng.shuffle(opts)
+        head = f"\n\nQuestion{' ' + str(k + 1) if multi else ''}: {q.text}\nOptions:"
+        tail = f"\nAnswer{' ' + str(k + 1) if multi else ''}: ("
+        if len(opts) <= NARROW:
+            lines = [head] + [f"\n({LETTERS[j]}) {q.options[oi]}" for j, oi in enumerate(opts)] + [tail]
+            piece = tok.encode("".join(lines), add_special_tokens=False)
+        else:                                   # wide: "\n(" + <label token> + ") text", built from ids so every label is one token
+            _, lab_ids, open_ids = label_table(tok)
+            piece = tok.encode(head, add_special_tokens=False)
+            for j, oi in enumerate(opts):
+                piece += open_ids + [lab_ids[j]] + _enc_opt(tok, q.options[oi])
+            piece += tok.encode(tail, add_special_tokens=False)
+        ids.extend(piece)
+        slots.append(len(ids) - 1)          # position of " (" token
+        golds.append(opts.index(q.gold) if q.gold in opts else -1)
+        nopts.append(len(opts))
+        perms.append(opts)
+    return dict(ids=ids, slots=slots, golds=golds, nopts=nopts, perms=perms)
+
+
+# ---- chat layout (1.2.0) ----------------------------------------------------------------------------------------------
+# A model whose decider_config.json has "layout": "chat" (or "chat_template": true) was trained with every prompt wrapped
+# in its tokenizer's chat template: one user turn holding the plain content, thinking switched off, and the answer pieces in
+# the assistant turn.  For Qwen3.5, state-first:
+#
+#     <|im_start|>user\n  Context:\n<state>  \n\nQuestion: <q>\nOptions:\n(A) ..\n(B) ..
+#     <|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n  Answer: (
+#
+# With several questions in one row all question blocks come first and the answer pieces ("Answer 1: (", "\nAnswer 2: (",
+# ...) follow the template tail.  Schema-first: the user turn holds the question blocks and then "\n\nContext:\n<state>".
+# Every part is tokenized on its own (head, context, each question header, each option block, tail, each answer piece);
+# the letter is read at the final " (" token as in the plain layout.
+LAYOUTS = ("plain", "chat")
+
+
+def resolve_layout(cfg):
+    """The prompt layout a decider_config.json names: "plain" (no "layout" key and no "chat_template": true, which is every
+    released model) or "chat".  Raises ValueError for any other value, so a model trained in a layout
+    this version does not know is refused instead of being read in the wrong one."""
+    cfg = cfg or {}
+    layout = cfg.get("layout")
+    if layout is None:
+        layout = "chat" if cfg.get("chat_template") is True else "plain"
+    if layout not in LAYOUTS:
+        raise ValueError(f"decider_config.json names the prompt layout {layout!r}; this version of decider-ai knows "
+                         f"{', '.join(repr(x) for x in LAYOUTS)}. Upgrade decider-ai, or check the model's config.")
+    if layout == "plain" and cfg.get("chat_template") is True:
+        raise ValueError('decider_config.json says "layout": "plain" and "chat_template": true; these contradict each other')
+    return layout
+
+
+class ChatTemplate:
+    """Token ids of the tokenizer's chat template around one user turn, and the answer pieces of the assistant turn.
+
+    head/tail come from tok.apply_chat_template([user turn], add_generation_prompt=True) with thinking disabled
+    (enable_thinking=False where the template knows the switch; a thinking block the template leaves open is closed here),
+    with no system prompt.  This is the computation of the chat-layout research server the chat models were trained against."""
+    SENTINEL = "@@DECIDER_USER_CONTENT@@"
+
+    def __init__(self, tok, answer="Answer"):
+        if not getattr(tok, "chat_template", None):
+            raise ValueError('the model is configured for the chat layout ("layout": "chat") but its tokenizer has no chat template')
+        msgs = [{"role": "user", "content": self.SENTINEL}]
+        kw = {"enable_thinking": False} if "enable_thinking" in tok.chat_template else {}
+        s = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, **kw)
+        head, tail = s.split(self.SENTINEL)
+        for opened, closed in (("<think>\n", "</think>\n\n"), ("<think>", "</think>\n\n"), ("<|channel>thought\n", "<channel|>")):
+            if tail.endswith(opened):
+                tail += closed
+        self.head_text, self.tail_text = head, tail
+        self.head = tok.encode(head, add_special_tokens=False)
+        self.tail = tok.encode(tail, add_special_tokens=False)
+        self.answer = answer
+        names, ids, _ = label_table(tok)
+        pre = tok.encode(f"{answer}: (", add_special_tokens=False)
+        for n, i in list(zip(names, ids))[:NARROW]:
+            if tok.encode(f"{answer}: ({n}", add_special_tokens=False) != pre + [i]:
+                raise ValueError(f"chat layout: the label {n!r} does not tokenize as one token after {answer!r}: (")
+        if tok.encode(tail + f"{answer}: (", add_special_tokens=False) != self.tail + pre:
+            raise ValueError("chat layout: the chat template tail merges with the answer prefix")
+
+    def answer_ids(self, tok, k, multi):
+        """Token ids of answer piece k: "Answer: (" (or "Answer k: (" with several questions), later pieces after a newline."""
+        return tok.encode(f"{'' if k == 0 else chr(10)}{self.answer}{' ' + str(k + 1) if multi else ''}: (", add_special_tokens=False)
+
+
+_CHAT = {}
+
+
+def chat_template(tok):
+    """The ChatTemplate of a tokenizer, cached per tokenizer object."""
+    key = id(tok)
+    if key not in _CHAT:
+        _CHAT[key] = (tok, ChatTemplate(tok))           # keep tok alive so its id is not reused
+    return _CHAT[key][1]
+
+
+def chat_for(tok, cfg):
+    """The ChatTemplate to pass as `chat=` for a model with this decider_config.json, or None for a plain-layout model."""
+    return chat_template(tok) if resolve_layout(cfg) == "chat" else None
+
+
+def load_decider_config(path):
+    """decider_config.json of a model folder or a Hub repository id; {} when the model has none (a raw base model)."""
+    import json, os
+    if os.path.isdir(path):
+        f = os.path.join(path, "decider_config.json")
+        return json.load(open(f)) if os.path.exists(f) else {}
+    try:
+        from huggingface_hub import hf_hub_download
+        return json.load(open(hf_hub_download(path, "decider_config.json")))
+    except Exception:
+        return {}
+
+
+def chat_for_model(path, tok):
+    """chat_for(tok, the model's decider_config.json): what every script that builds prompts for `path` passes as `chat=`."""
+    return chat_for(tok, load_decider_config(path))
+
+
+def build_chat(example, tok, chat, rng=None, max_options=NARROW, max_ctx_tokens=1536):
+    """State-first chat layout (see the comment above LAYOUTS).  Same return fields as `build`.  max_ctx_tokens caps the
+    tokens of "Context:\\n" + state, as in the plain layout; the template head and tail are not counted."""
+    rng = rng or random
+    qs = example.qs; multi = len(qs) > 1
+    ids = list(chat.head) + tok.encode("Context:\n" + example.context, add_special_tokens=False)[:max_ctx_tokens]
+    golds, nopts, perms = [], [], []
+    for k, q in enumerate(qs):
+        opts = _select(q, rng, max_options)
+        ids += tok.encode(f"\n\nQuestion{' ' + str(k + 1) if multi else ''}: {q.text}\nOptions:", add_special_tokens=False) + _options_ids(tok, q, opts)
+        golds.append(opts.index(q.gold) if q.gold in opts else -1); nopts.append(len(opts)); perms.append(opts)
+    ids += chat.tail
+    slots = []
+    for k in range(len(qs)):
+        ids += chat.answer_ids(tok, k, multi); slots.append(len(ids) - 1)
+    return dict(ids=ids, slots=slots, golds=golds, nopts=nopts, perms=perms)
+
+
+def letter_ids(tok):
+    ids = label_table(tok)[1]
+    for j, L in enumerate(LETTERS):
+        assert tok.encode(L, add_special_tokens=False) == [ids[j]], L
+    return ids
+
+
+def render(example, tok, **kw):
+    b = build(example, tok, **kw)
+    return tok.decode(b["ids"])

decider/prompt_fast.py

@@ -0,0 +1,90 @@
+"""Row construction for the systemone path that tokenizes the state once per request instead of once per question.
+
+`prompt.build` encodes `"Context:\\n" + state` for every row it is called with, so an independent request with n
+questions tokenizes the whole state n times.  The question block is encoded as its own string there and simply appended,
+so the ids are exactly `ctx_ids + question_piece`; this module reuses that fact and shares `ctx_ids` across the rows.
+
+`build_rows` returns the same item dicts as
+    [prompt.build(Example(ctx, [Q(text, options, 0) for text, options in row]), tok, <no-shuffle rng>,
+                  max_options=MAX_OPTIONS, max_ctx_tokens=max_ctx_tokens) for row in rows]
+(ids, slots, golds, nopts, perms) and is checked against it in tests/test_prompt_fast.py.
+"""
+from decider.prompt import LETTERS, NARROW, MAX_OPTIONS, label_table, _enc_opt, _options_ids
+
+
+def context_ids(tok, context, max_ctx_tokens=32768):
+    return tok.encode("Context:\n" + context, add_special_tokens=False)[:max_ctx_tokens]
+
+
+def question_piece(tok, text, options, k=0, multi=False):
+    """Token ids of one `\\n\\nQuestion...: <text>\\nOptions:...\\nAnswer...: (` block, independent of the state."""
+    head = f"\n\nQuestion{' ' + str(k + 1) if multi else ''}: {text}\nOptions:"
+    tail = f"\nAnswer{' ' + str(k + 1) if multi else ''}: ("
+    if len(options) <= NARROW:
+        return tok.encode(head + "".join(f"\n({LETTERS[j]}) {o}" for j, o in enumerate(options)) + tail,
+                          add_special_tokens=False)
+    _, lab_ids, open_ids = label_table(tok)
+    piece = tok.encode(head, add_special_tokens=False)
+    for j, o in enumerate(options):
+        piece += open_ids + [lab_ids[j]] + _enc_opt(tok, o)
+    return piece + tok.encode(tail, add_special_tokens=False)
+
+
+class _Opts:
+    def __init__(self, options): self.options = options
+
+
+def _chat_row(tok, chat, ctx, row):
+    """One chat-layout row after the shared ids `ctx` (template head + context): every question block, the template tail,
+    then the answer pieces.  Equal to prompt.build_chat for the same row (tests/test_layout.py)."""
+    multi = len(row) > 1
+    ids = list(ctx)
+    for k, (text, options) in enumerate(row):
+        ids += tok.encode(f"\n\nQuestion{' ' + str(k + 1) if multi else ''}: {text}\nOptions:", add_special_tokens=False)
+        ids += _options_ids(tok, _Opts(options), list(range(len(options))))
+    ids += chat.tail
+    slots = []
+    for k in range(len(row)):
+        ids += chat.answer_ids(tok, k, multi); slots.append(len(ids) - 1)
+    return ids, slots
+
+
+def build_rows(tok, context, rows, max_ctx_tokens=32768, chat=None):
+    """rows: list of rows, each a list of (question text, options).  -> (items, len(ctx_ids)).
+
+    Option order is kept as given (no shuffling, no subsetting): systemone.render_question already caps a choice at
+    MAX_OPTIONS options, so prompt.build's sampling branch is unreachable here.
+    chat: a ChatTemplate (prompt.chat_template) for a chat-layout model; the shared ids are then the template head plus the
+    context, and each row is prompt.build_chat's rendering."""
+    ctx = context_ids(tok, context, max_ctx_tokens)
+    if chat is not None:
+        ctx = list(chat.head) + ctx
+    items = []
+    for row in rows:
+        multi = len(row) > 1
+        ids = list(ctx); slots = []; nopts = []
+        for k, (text, options) in enumerate(row):
+            if not 2 <= len(options) <= MAX_OPTIONS:
+                raise ValueError(f"2..{MAX_OPTIONS} options required")
+            if chat is None:
+                ids.extend(question_piece(tok, text, options, k, multi))
+                slots.append(len(ids) - 1)
+            nopts.append(len(options))
+        if chat is not None:
+            ids, slots = _chat_row(tok, chat, ctx, row)
+        items.append(dict(ids=ids, slots=slots, golds=[0] * len(row), nopts=nopts,
+                          perms=[list(range(len(o))) for _, o in row]))
+    return items, len(ctx)
+
+
+def unique_tokens(items, ctx_len):
+    """systemone.unique_tokens(items) without re-walking the shared context (all rows start with the same ctx_len ids).
+    No rows (a request with no questions) counts 0, as systemone.unique_tokens does."""
+    if not items:
+        return 0
+    sufs = [it["ids"][ctx_len:] for it in items]
+    if len(sufs) < 2:
+        return ctx_len + sum(len(s) for s in sufs)
+    lcp = 0; short = min(len(s) for s in sufs); s0 = sufs[0]
+    while lcp < short and all(s[lcp] == s0[lcp] for s in sufs): lcp += 1
+    return ctx_len + lcp + sum(len(s) - lcp for s in sufs)

decider/schema_engine.py

@@ -0,0 +1,150 @@
+"""Schema cache: compute a question schema once, then score states against it.
+
+In production the questions are fixed and only the state changes.  With the schema-first prompt layout
+(prompt.build_schema_first) the question/option blocks are a prefix that does not depend on the state, so their
+cache - attention K/V for the 6 full-attention layers, conv + recurrent state for the 18 delta-net layers - is computed
+once (`prepare`).  A request then runs only "Context: <state>" plus one answer slot per question, as a CUDA graph per
+(batch, length) bucket.  The prefix cache is read-only during a request (nothing is written back), so one copy serves
+every batch and every graph.
+
+    se = SchemaEngine(engine); h = se.prepare([{"question": ..., "options": [...]}, ...])
+    probs = se.score(h, ["state 1", "state 2", ...])          # list of [n_questions, MAX_OPTIONS] tensors
+"""
+import time, types, torch, torch.nn.functional as F
+from decider.prompt import schema_prefix_ids, schema_suffix_ids, MAX_OPTIONS
+from decider.engine import read_slots, fill_ids
+
+TS_BUCKETS = [32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024]
+B_BUCKETS = [1, 2, 4, 8, 16, 32, 64]
+
+
+class _Q:
+    def __init__(self, text, options): self.text, self.options = text, options
+
+
+class PrefixCache:
+    """Duck-typed transformers Cache over fixed, read-only prefixes, for one suffix forward pass.
+    A handle holds P prefixes (P = 1: all questions packed in one prefix; P = n_questions: one prefix per question, so every
+    question is scored independently).  A batch of R states has R * P rows; row r * P + p continues prefix p."""
+    def __init__(self, h, R):
+        rep = (lambda t: t.expand(R, *t.shape[1:])) if h.P == 1 else (lambda t: t.repeat(R, *([1] * (t.dim() - 1))))
+        self.tp = h.tpmax; self.k = {i: rep(k) for i, k in h.k.items()}; self.v = {i: rep(v) for i, v in h.v.items()}
+        self.conv = {i: rep(c).contiguous() for i, c in h.conv.items()}
+        self.layers = {i: types.SimpleNamespace(record_past=False, recurrent_states={0: rep(r).contiguous()}) for i, r in h.rec.items()}
+
+    def has_previous_state(self, layer_idx=None, state_idx=None): return True
+    def get_seq_length(self, *a, **k): return self.tp
+    def update(self, key, value, layer_idx, *a, **k): return torch.cat([self.k[layer_idx], key], 2), torch.cat([self.v[layer_idx], value], 2)
+    def update_conv_state(self, x, layer_idx, **k): return torch.cat([self.conv[layer_idx].to(x.dtype), x], -1)
+    def update_recurrent_state(self, s, layer_idx, **k): return s
+
+
+class SchemaEngine:
+    def __init__(self, engine, use_graphs=True, chat=None):
+        """chat: the ChatTemplate of a chat-layout model (decider.prompt.chat_template); the prefix then starts with the template
+        head and the suffix ends with the template tail and the answer pieces (decider.prompt.build_schema_first with chat)."""
+        self.chat = chat
+        self.e = engine; self.core = engine.core; self.W = engine.W; self.tok = engine.tok; self.dev = engine.dev
+        self.use_graphs = use_graphs and engine.use_graphs; self.graphs = {}; self.stats = dict(prepared=0, captures=0, replays=0, eager=0)
+        self.compile = bool(engine.cfg.get("compile")); self._compiled = {}
+        if self.compile:                        # every compiled schema graph specialises the model frames again (its cache tensors are constants)
+            import torch._dynamo
+            torch._dynamo.config.cache_size_limit = 4096; torch._dynamo.config.accumulated_cache_size_limit = 1 << 16
+
+    @torch.no_grad()
+    def prepare(self, questions, independent=False, compile=False):
+        """questions: [{"question": str, "options": [str]}] in the order answers are wanted.  Runs the prefix(es) once.
+        independent=False: one prefix holding every question (cheapest: a request costs state + n slots).
+        independent=True:  one prefix per question, one row per question (a request costs n * (state + 1 slot); no question
+                           can influence another)."""
+        qs = [_Q(q["question"], list(q["options"])) for q in questions]
+        groups = [[q] for q in qs] if independent else [qs]; pres = [schema_prefix_ids(self.tok, g, chat=self.chat) for g in groups]
+        h = types.SimpleNamespace(P=len(groups), nq=len(qs), slots_per_row=1 if independent else len(qs), nopts=[len(q.options) for q in qs], tps=[len(p) for p in pres],
+                                  tpmax=max(len(p) for p in pres), k={}, v={}, conv={}, rec={}, id=self.stats["prepared"],
+                                  compile=bool(compile and self.compile))
+        parts = []
+        for pre in pres:
+            out = self.core(input_ids=torch.tensor(pre, device=self.dev)[None], use_cache=True).past_key_values; d = dict(k={}, v={}, conv={}, rec={})
+            for i, layer in enumerate(out.layers):
+                if getattr(layer, "recurrent_states", None) is not None and layer.recurrent_states.get(0) is not None:
+                    d["conv"][i] = layer.conv_states[0]; d["rec"][i] = layer.recurrent_states[0]
+                else:                                                   # right-pad every prefix's K/V to the longest; the mask hides the padding
+                    pad = (0, 0, 0, h.tpmax - len(pre)); d["k"][i] = F.pad(layer.keys, pad); d["v"][i] = F.pad(layer.values, pad)
+            parts.append(d)
+        for name in ("k", "v", "conv", "rec"):
+            getattr(h, name).update({i: torch.cat([d[name][i] for d in parts], 0).clone() for i in parts[0][name]})
+        self.stats["prepared"] += 1
+        return h
+
+    def _fwd(self, ids, cache, mask, pos):
+        hs = self.core(input_ids=ids, past_key_values=cache, attention_mask={"full_attention": mask, "linear_attention": None}, position_ids=pos, use_cache=True).last_hidden_state
+        return F.linear(hs, self.W).float()
+
+    def _static(self, h, R, Ts):
+        """R request slots -> R * P rows.  Mask: a row sees its own prefix (not the padding up to tpmax) and the causal suffix."""
+        ar = torch.arange(Ts, device=self.dev); tps = torch.tensor(h.tps, device=self.dev).repeat(R)                                # [R*P]
+        pre = (torch.arange(h.tpmax, device=self.dev)[None, :] < tps[:, None])[:, None, None, :].expand(-1, 1, Ts, -1)             # [B,1,Ts,tpmax]
+        mask = torch.cat([pre, (ar[:, None] >= ar[None, :])[None, None].expand(len(tps), 1, -1, -1)], 3).contiguous()
+        return PrefixCache(h, R), mask, (tps[:, None] + ar[None, :]).contiguous()
+
+    def _capture(self, h, R, Ts):
+        B = R * h.P
+        ids = torch.full((B, Ts), self.tok.pad_token_id, dtype=torch.long, device=self.dev); cache, mask, pos = self._static(h, R, Ts)
+        fwd = self._fwd
+        if h.compile:                                       # one compiled function per graph (20-30 s each: only for preloaded schemas): the cache tensors are constants of that graph
+            fwd = torch.compile(lambda i: self._fwd(i, cache, mask, pos), dynamic=False)
+            call = lambda: fwd(ids)
+        else:
+            call = lambda: fwd(ids, cache, mask, pos)
+        st = torch.cuda.Stream(); st.wait_stream(torch.cuda.current_stream())
+        with torch.cuda.stream(st):
+            for _ in range(3): call()
+        torch.cuda.current_stream().wait_stream(st)
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g, pool=self.e.pool):
+            out = call()
+        self.stats["captures"] += 1
+        return ids, out, g, (cache, mask, pos)
+
+    def warmup(self, h, batch_sizes=(1, 8, 32), state_tokens=(64, 128, 256)):
+        """Capture (and, for a compiled schema, compile) the graphs for these request-batch sizes and suffix lengths ahead of traffic."""
+        t = time.time()
+        for R in batch_sizes:
+            for Ts in state_tokens:
+                Ts = next((x for x in TS_BUCKETS if x >= Ts), TS_BUCKETS[-1])
+                if (h.id, R, Ts) not in self.graphs: self.graphs[(h.id, R, Ts)] = self._capture(h, R, Ts)
+        torch.cuda.synchronize(); return time.time() - t
+
+    def tokenize(self, h, context, max_ctx_tokens=1536):
+        """CPU part of a request (do it outside any GPU lock): -> (suffix ids, slot positions)."""
+        return schema_suffix_ids(self.tok, context, h.slots_per_row, max_ctx_tokens, chat=self.chat)
+
+    @staticmethod
+    def bucket(n_tokens):
+        return next((t for t in TS_BUCKETS if t >= n_tokens), -(-n_tokens // 256) * 256)
+
+    def score(self, h, contexts, temperature=1.0, max_ctx_tokens=1536):
+        """-> one [n_questions, MAX_OPTIONS] probability tensor per context.  temperature: as in score_rows."""
+        return self.score_rows(h, [self.tokenize(h, c, max_ctx_tokens) for c in contexts], temperature)
+
+    @torch.no_grad()
+    def score_rows(self, h, rows, temperature=1.0):
+        """rows: [(suffix ids, slots)] from tokenize().  temperature: a number, or a list with one temperature per schema row
+        (h.nq values, in the order of prepare's questions), applied to every request."""
+        if isinstance(temperature, (list, tuple)) and len(temperature) != h.nq:
+            raise ValueError(f"temperature: {len(temperature)} values for a schema with {h.nq} rows")
+        Tmax = max(len(r[0]) for r in rows); Ts = next((t for t in TS_BUCKETS if t >= Tmax), None); n = len(rows)
+        R = next((b for b in B_BUCKETS if b >= n), n) if Ts else n; Ts = Ts or -(-Tmax // 256) * 256
+        ids = fill_ids([x for x, _ in rows for _ in range(h.P)], R * h.P, Ts, self.tok.pad_token_id).to(self.dev, non_blocking=True)
+        if self.use_graphs and Ts <= TS_BUCKETS[-1]:
+            key = (h.id, R, Ts)
+            if key not in self.graphs: self.graphs[key] = self._capture(h, R, Ts)
+            s_ids, s_out, g, _ = self.graphs[key]; s_ids.copy_(ids); g.replay(); out = s_out; self.stats["replays"] += 1
+        else:
+            out = self._fwd(ids, *self._static(h, R, Ts)); self.stats["eager"] += 1
+        if h.P == 1:                                           # packed: n slots in one row per request
+            rws = [r for r in range(n) for _ in range(h.nq)]; sls = [x for _, sl in rows for x in sl]
+        else:                                                  # independent: one slot in each of the request's P rows
+            rws = [r * h.P + p for r in range(n) for p in range(h.P)]; sls = [sl[0] for _, sl in rows for _ in range(h.P)]
+        temps = list(temperature) * n if isinstance(temperature, (list, tuple)) else temperature
+        return read_slots(out, rws, sls, h.nopts * n, temps, [h.nq] * n)

decider/serve.py

@@ -0,0 +1,562 @@
+"""HTTP server.
+  POST /decide        {"context": str, "schema": {...}}                     -> typed JSON decisions (all questions packed in one row)
+                      schema: {question: {"type": "choice", "options": [...]} | {"type": "bool"} | {"type": "scale", "legend": [...]}};
+                      a missing or malformed schema is a 422 naming this form (decider.infer.Decider._check_schema)
+  POST /v1/systemone  {"state": str|object|array, "questions": {id: {...}}} -> the TypeSafe/Jev wire format (decider.systemone):
+                      Choice (up to 255 described options), Score, Noul; every question is scored in its own row, so answers
+                      are independent of each other ("independent": false packs them behind one copy of the state instead).
+  GET  /v1/models  /health  /stats
+
+   uvicorn decider.serve:app --host 0.0.0.0 --port 8000        (env: DECIDER_MODEL and the variables below)
+
+Execution (1.1.1; docs/SERVING.md has the design and the measurements):
+  * decider.engine_v2.EngineV2: CUDA graphs keyed on (batch bucket, padded length bucket) only.  The whole grid is captured
+    during start-up and the engine is then sealed, so no request pays a graph capture or a torch.compile.  Rows longer than the
+    last length bucket (8,192 tokens) run eager, in chunks bounded by DECIDER_GRAPH_TOKEN_BUDGET padded tokens.
+  * one GPU thread runs every forward; tokenisation runs on a CPU pool; rows waiting at the same moment are partitioned by
+    decider.batching.plan_batches, which pads a shorter row into a longer row's bucket when that costs less than a second
+    forward (cost model: DECIDER_MERGE_OVERHEAD_TOKENS + rows * padded length).  When a collection held rows from more than
+    one live request the next one waits up to DECIDER_BATCH_ADAPTIVE_WAIT_MS for more rows, unless its own first row took
+    more than 50 ms to arrive (a quiet queue drops the window again); after a single-request collection it does not wait.
+  * an independent request with more than one question over a state of at least DECIDER_SHARED_MIN_TOKENS tokens runs the state
+    once and forks its cache per question (EngineV2.score_shared, decider.shared_prefix).  The fork is made in chunks that fit
+    DECIDER_SHARED_FORK_GB, so the peak memory does not grow with the question count.  The cuDNN SDPA backend is off
+    (decider.engine.set_attention_backend_policy), which is what makes that path agree with the full forward.
+  * the schema cache (questions-first layout, prefix cached per question set) is honoured exactly as before: on when
+    decider_config.json has "schema_first": true, or with DECIDER_SCHEMA_CACHE=1 on a model trained for that layout.  It is
+    the one path that does GPU work after start-up that is not a graph replay: the first request with a new schema runs its
+    prefix and captures one graph per (schema, batch bucket, state bucket) as in 1.0.x (/stats -> schema_cache.captures).
+    Its requests are bounded and admitted like the others, on prefix plus suffix tokens, before any GPU preparation.
+  * prompt layout (1.2.0): a model whose decider_config.json has "layout": "chat" (a chat-trained checkpoint) is read in the chat layout
+    (decider.prompt.build_chat: the tokenizer's chat template around one user turn, thinking off, "Answer: (" in the assistant
+    turn) on every route, including the shared-prefix fork and the schema cache.  Every other model is read in the plain
+    layout, with the same token ids as 1.1.x.  A config naming an unknown layout stops start-up with a ValueError.
+  * requests are bounded before they reach the GPU: HTTP 413 when a row, the request or the expanded question count exceeds the
+    limits below, HTTP 503 when the outstanding work exceeds DECIDER_MAX_QUEUE_ROWS.
+
+Variables (default):  DECIDER_DEVICE (auto: cuda, else mps, else cpu)  DECIDER_COMPILE (0)  DECIDER_FP8 (0)  DECIDER_SHARED (1)  DECIDER_SHARED_MIN_TOKENS (768)
+  DECIDER_SHARED_FORK_GB (8)  DECIDER_MERGE_OVERHEAD_TOKENS (512)  DECIDER_BATCH_ADAPTIVE_WAIT_MS (2)
+  DECIDER_MAX_BATCH (32)  DECIDER_BATCH_WAIT_MS (0; DECIDER_MAX_WAIT_MS is an alias)  DECIDER_MAX_STATE_TOKENS (32768)
+  DECIDER_T_BUCKETS  DECIDER_B_BUCKETS  DECIDER_GRAPH_TOKEN_BUDGET (32768)  DECIDER_WARMUP (1)  DECIDER_TOKENIZE_THREADS (8)
+  DECIDER_MAX_ROWS (1024)  DECIDER_MAX_ROW_TOKENS (DECIDER_MAX_STATE_TOKENS + 4096)  DECIDER_MAX_REQUEST_TOKENS (1048576)
+  DECIDER_MAX_QUEUE_ROWS (4096)  DECIDER_TEMPERATURE  DECIDER_SCHEMA_CACHE (0)  DECIDER_SCHEMA_MIN_SEEN (2)  DECIDER_SCHEMAS
+
+Temperatures (1.4.0, decider.temperature): decider_config.json "temperature", and optionally "temperature_by_type"
+{"choice": T, "noul": T, "score": T} (a /decide "bool" field is "noul", a "scale" field is "score"; a missing type uses
+"temperature").  Every row carries the answer type of each of its slots, so a batch that mixes requests and types still
+applies each answer's own temperature.  DECIDER_TEMPERATURE replaces "temperature" and switches the by-type map off.
+/health and the ready line report the temperature every answer type gets.
+"""
+import asyncio, json, os, time
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from decider import systemone as S1
+from decider import temperature as TT
+from decider.batching import DEFAULT_MERGE_OVERHEAD_TOKENS, plan_batches
+from decider.prompt import build, MAX_OPTIONS, resolve_layout, chat_template
+from decider.prompt_fast import build_rows, unique_tokens
+
+
+def _env_int(name, default):
+    return int(os.environ.get(name, default))
+
+
+MODEL = os.environ.get("DECIDER_MODEL", "runs/r3_v2/model")
+MAX_BATCH = _env_int("DECIDER_MAX_BATCH", 32)
+BATCH_WAIT_MS = float(os.environ.get("DECIDER_BATCH_WAIT_MS", os.environ.get("DECIDER_MAX_WAIT_MS", "0")))
+ADAPTIVE_WAIT_MS = float(os.environ.get("DECIDER_BATCH_ADAPTIVE_WAIT_MS", "2"))   # window after a collection that held >1 request; 0 disables
+ADAPTIVE_IDLE_RESET_MS = 50.0                                               # an idler queue than this drops the adaptive window again
+MERGE_OVERHEAD_TOKENS = _env_int("DECIDER_MERGE_OVERHEAD_TOKENS", DEFAULT_MERGE_OVERHEAD_TOKENS)
+MAX_STATE_TOKENS = _env_int("DECIDER_MAX_STATE_TOKENS", 32768)
+SHARED = os.environ.get("DECIDER_SHARED", "1") == "1"
+SHARED_MIN_TOKENS = _env_int("DECIDER_SHARED_MIN_TOKENS", 768)
+DEVICE = os.environ.get("DECIDER_DEVICE", "auto")                          # auto | cuda[:i] | mps | cpu
+COMPILE = os.environ.get("DECIDER_COMPILE", "0") == "1"
+FP8 = os.environ.get("DECIDER_FP8", "0") == "1"
+WARMUP = os.environ.get("DECIDER_WARMUP", "1") == "1"
+TOKENIZE_THREADS = _env_int("DECIDER_TOKENIZE_THREADS", 8)
+GRAPH_TOKEN_BUDGET = _env_int("DECIDER_GRAPH_TOKEN_BUDGET", 32768)
+# request bounds, checked after tokenisation and before anything is queued
+MAX_ROWS = _env_int("DECIDER_MAX_ROWS", 1024)                               # scoring rows per request (questions, expanded score levels)
+MAX_ROW_TOKENS = _env_int("DECIDER_MAX_ROW_TOKENS", MAX_STATE_TOKENS + 4096)  # tokens in one row: truncated state + question block
+MAX_REQUEST_TOKENS = _env_int("DECIDER_MAX_REQUEST_TOKENS", 1 << 20)       # sum of row lengths of one request
+MAX_QUEUE_ROWS = _env_int("DECIDER_MAX_QUEUE_ROWS", 4096)                   # rows admitted and not yet scored, over all requests
+DECIDE_MAX_CTX_TOKENS = 1536                                                # /decide context cap, unchanged from 1.0.x
+
+MODEL_NAME = "decider"; TEMP = 1.0; TEMP_SCHEMA = 1.0; TEMP_BY_TYPE = {}; TEMP_SCHEMA_BY_TYPE = {}; RELEASE_DATE = "2026-09-17"; ISOLATED = False; NEUTRALIZE_NONE = True
+SCHEMA_FIRST = False; LAYOUT = "plain"; CHAT = None; se = None; squeue = None; schemas = {}; seen = {}
+eng = None; queue = None; gpu = None; cpu = None; batcher_task = None; schema_task = None
+outstanding = 0; REQ_SEQ = 0
+stats = dict(requests=0, batches=0, decisions=0, rows=0, shared_prefix_requests=0, errors=0, rejected_too_large=0,
+             rejected_overloaded=0, batch_hist={}, bucket_hist={})
+
+
+def _ints(name):
+    v = os.environ.get(name)
+    return [int(x) for x in v.split(",")] if v else None
+
+
+def load_config(path):
+    """decider_config.json from a model folder or a Hub repository (the way decider.infer.Decider resolves it)."""
+    try:
+        if os.path.isdir(path):
+            return json.load(open(os.path.join(path, "decider_config.json")))
+        from huggingface_hub import hf_hub_download
+        return json.load(open(hf_hub_download(path, "decider_config.json")))
+    except Exception:
+        return {}
+
+
+# ---- request preparation (CPU) -------------------------------------------
+class _NoShuffle:
+    def shuffle(self, x): pass
+    def sample(self, xs, k): return xs[:k]
+
+
+def prepare(tok, state, questions, independent, isolated=False, max_state_tokens=32768, chat=None):
+    """Render, plan the rows, tokenize the state once.  -> (rqs, index, items, ctx_len).  The rows are those
+    `prompt.build` produces for the same request (tests/test_prompt_fast.py, tests/test_serve_prepare.py); chat: the
+    ChatTemplate of a chat-layout model, None for the plain layout."""
+    ctx = S1.render_state(state)
+    rqs = {k: S1.render_question(v) for k, v in questions.items()}
+    flat, index = S1.plan_rows(rqs, isolated and independent)
+    pairs = [(r["question"], list(r["options"])) for r in flat]
+    rows = [[p] for p in pairs] if independent else [pairs]
+    items, ctx_len = build_rows(tok, ctx, rows, max_ctx_tokens=max_state_tokens, chat=chat)
+    types = S1.row_types(rqs, index)                                 # answer type per slot (decider.temperature)
+    for it, ts in zip(items, [[t] for t in types] if independent else [types]):
+        it["types"] = ts
+    return rqs, index, items, ctx_len
+
+
+def _prepare_s1(state, questions, independent):
+    return prepare(eng.tok, state, questions, independent, ISOLATED, MAX_STATE_TOKENS, chat=CHAT)
+
+
+def _prepare_decide(context, schema):
+    from decider.infer import Decider, Example, Q, neutralize_options
+    qs = Decider._schema_to_questions(schema)
+    for q in qs:
+        if NEUTRALIZE_NONE:
+            q["options"], q["_back"] = neutralize_options(q["options"])
+    ex = Example(context, [Q(q["question"], list(q["options"]), 0) for q in qs])
+    it = build(ex, eng.tok, _NoShuffle(), max_options=MAX_OPTIONS, max_ctx_tokens=DECIDE_MAX_CTX_TOKENS, chat=CHAT)
+    it["types"] = [q["_type"] for q in qs]                            # bool -> noul, scale -> score (decider.temperature)
+    return qs, it
+
+
+def _format_decide(schema, qs, probs):
+    o = {}
+    for (qtext, spec), q, p in zip(schema.items(), qs, probs):
+        p = p[:len(q["options"])].tolist(); t = spec.get("type", "choice"); j = max(range(len(p)), key=p.__getitem__)
+        back = q.get("_back", {}); names = [back.get(x, x) for x in q["options"]]
+        if t == "bool":
+            o[qtext] = {"noul": round(p[1], 4), "type": "noul"}
+        elif t == "choice":
+            o[qtext] = {"choice": names[j], "confidence": round(p[j], 4), "type": "choice",
+                        "probabilities": {k: round(v, 4) for k, v in zip(names, p)}}
+        else:
+            keys = q["_keys"]; score = sum(float(k) * pi for k, pi in zip(keys, p))
+            o[qtext] = {"score": round(score, 2), "confidence": round(p[j], 4), "type": "scale", "legend": q["_legend"],
+                        "probabilities": {str(keys[i]): round(pi, 4) for i, pi in enumerate(p)}}
+    return o
+
+
+# ---- limits ---------------------------------------------------------------
+def check_size(row_lengths, max_rows=None, max_row_tokens=None, max_request_tokens=None):
+    """Raise HTTPException(413) when a request exceeds the row count, per-row token or total token limit."""
+    max_rows = MAX_ROWS if max_rows is None else max_rows
+    max_row_tokens = MAX_ROW_TOKENS if max_row_tokens is None else max_row_tokens
+    max_request_tokens = MAX_REQUEST_TOKENS if max_request_tokens is None else max_request_tokens
+    n, total, longest = len(row_lengths), sum(row_lengths), max(row_lengths, default=0)
+    if n > max_rows:
+        msg = f"too many questions: the request expands to {n} scoring rows, the limit is {max_rows} (DECIDER_MAX_ROWS)"
+    elif longest > max_row_tokens:
+        msg = f"too many tokens: one row has {longest} tokens, the limit is {max_row_tokens} per row (DECIDER_MAX_ROW_TOKENS)"
+    elif total > max_request_tokens:
+        msg = f"too many tokens: the request has {total} tokens over {n} rows, the limit is {max_request_tokens} (DECIDER_MAX_REQUEST_TOKENS)"
+    else:
+        return
+    stats["rejected_too_large"] += 1
+    raise HTTPException(413, msg)
+
+
+def _admit(n):
+    """Reserve n rows of outstanding work or raise HTTPException(503)."""
+    global outstanding
+    if outstanding + n > MAX_QUEUE_ROWS:
+        stats["rejected_overloaded"] += 1
+        raise HTTPException(503, f"server busy: {outstanding} rows queued, the limit is {MAX_QUEUE_ROWS} (DECIDER_MAX_QUEUE_ROWS); retry later")
+    outstanding += n
+
+
+def _release(n):
+    global outstanding
+    outstanding -= n
+
+
+# ---- GPU work ----------------------------------------------------------------
+def _score_items(items):
+    return eng.score_items(items, temperature=TT.for_items(TEMP, TEMP_BY_TYPE, items))      # the scalar TEMP without a by-type map
+
+
+def _score_shared(items):
+    return eng.score_shared(items, temperature=TT.for_items(TEMP, TEMP_BY_TYPE, items))
+
+
+async def _collect(q, wait_ms=None, adaptive_ms=0.0, idle_reset_ms=None):
+    """Take what is already queued and go.
+
+    `wait_ms` (DECIDER_BATCH_WAIT_MS) is the unconditional collection window.  `adaptive_ms` is the extra window the
+    batcher asks for after a collection that held more than one live request; it is dropped again when the first row of
+    this collection took longer than `idle_reset_ms` to arrive, so an isolated request after a quiet period never waits.
+    """
+    wait = BATCH_WAIT_MS if wait_ms is None else wait_ms
+    reset = ADAPTIVE_IDLE_RESET_MS if idle_reset_ms is None else idle_reset_ms
+    t0 = time.monotonic()
+    batch = [await q.get()]
+    if adaptive_ms > 0 and (time.monotonic() - t0) * 1000 <= reset:
+        wait = max(wait, adaptive_ms)
+    deadline = time.monotonic() + wait / 1000
+    while len(batch) < MAX_BATCH:
+        try:
+            batch.append(q.get_nowait())
+        except asyncio.QueueEmpty:
+            timeout = deadline - time.monotonic()
+            if timeout <= 0: break
+            try: batch.append(await asyncio.wait_for(q.get(), timeout))
+            except asyncio.TimeoutError: break
+    return batch
+
+
+def _bucketed(n):
+    """False for a row longer than the engine's last captured length bucket.  Those run eager at a request-specific shape,
+    so they are grouped by exact padded length as in 1.1.0 instead of being padded into another row's bucket."""
+    t_bucket = getattr(eng, "t_bucket", None)
+    return True if t_bucket is None else t_bucket(n) is not None
+
+
+def adaptive_ms(batch):
+    """The extra collection window the next collection may use: DECIDER_BATCH_ADAPTIVE_WAIT_MS when this collection held
+    live rows from more than one request, 0 otherwise.  Rows whose future is already done (a cancelled or failed request)
+    are not counted: they are not evidence that requests are overlapping."""
+    if ADAPTIVE_WAIT_MS <= 0:
+        return 0.0
+    return ADAPTIVE_WAIT_MS if len({rid for fut, _, rid in batch if not fut.done()}) > 1 else 0.0
+
+
+async def batcher():
+    """One forward per planned group.  Rows queued at the same moment are partitioned by decider.batching.plan_batches:
+    a shorter row is padded into a longer row's bucket when that costs less than running a second forward."""
+    loop = asyncio.get_running_loop()
+    extra = 0.0                                      # the adaptive window the previous collection earned
+    while True:
+        batch = await _collect(queue, BATCH_WAIT_MS, extra)
+        extra = adaptive_ms(batch)
+        groups = plan_batches([len(it["ids"]) for _, it, _ in batch], eng.pad_len, eng.max_rows, MAX_BATCH,
+                              MERGE_OVERHEAD_TOKENS, _bucketed)
+        for T, idx in groups:
+            part = [batch[i] for i in idx]
+            stats["bucket_hist"][T] = stats["bucket_hist"].get(T, 0) + len(part)
+            try:
+                probs = await loop.run_in_executor(gpu, _score_items, [it for _, it, _ in part])
+                for (fut, _, _), p in zip(part, probs):
+                    if not fut.done(): fut.set_result(p)
+            except Exception as e:
+                for fut, _, _ in part:
+                    if not fut.done(): fut.set_exception(e)
+            stats["batches"] += 1
+            stats["batch_hist"][len(part)] = stats["batch_hist"].get(len(part), 0) + 1
+
+
+# ---- schema cache (questions-first layout; opt-in) --------------------------------
+def _schema_key(questions, independent):
+    return (json.dumps(questions, sort_keys=True, ensure_ascii=False), independent)
+
+
+class _SQ:
+    def __init__(self, text, options): self.text, self.options = text, options
+
+
+def _plan_schema(questions, independent, state):
+    """CPU part of a schema-cache request: prefix token lengths (from the cached handle when the schema is known, otherwise
+    tokenised here, without touching the GPU) and the suffix row (context plus answer slots).  -> (tps, (suffix ids, slots)).
+    Raises ValueError for an invalid question."""
+    from decider.prompt import schema_prefix_ids, schema_suffix_ids
+    rqs = {k: S1.render_question(v) for k, v in questions.items()}; rows, _ = S1.plan_rows(rqs, ISOLATED and independent)
+    cached = schemas.get(_schema_key(questions, independent))
+    if cached is not None:
+        tps = list(cached[1].tps)
+    else:
+        qs = [_SQ(x["question"], list(x["options"])) for x in rows]
+        tps = [len(schema_prefix_ids(eng.tok, g, chat=CHAT)) for g in ([[q] for q in qs] if independent else [qs])]
+    row = schema_suffix_ids(eng.tok, S1.render_state(state), 1 if independent else len(rows), MAX_STATE_TOKENS, chat=CHAT)
+    return tps, row
+
+
+def _schema_handle(questions, independent, compile=False):
+    """Compile (or look up) the question schema: its prefix is run once, requests then only run the state.  GPU thread."""
+    key = _schema_key(questions, independent)
+    if key not in schemas:
+        rqs = {k: S1.render_question(v) for k, v in questions.items()}; rows, index = S1.plan_rows(rqs, ISOLATED and independent)
+        if len(schemas) >= 128:
+            old = next(iter(schemas)); hid = schemas.pop(old)[1].id
+            for k in [k for k in se.graphs if k[0] == hid]: del se.graphs[k]
+        h = se.prepare(rows, independent=independent, compile=compile)
+        h.types = S1.row_types(rqs, index)                             # answer type per schema row (decider.temperature)
+        schemas[key] = (rqs, h, index)
+    return schemas[key]
+
+
+def _worth_caching(questions, independent):
+    """A schema gets a cached prefix and CUDA graphs from its second request on."""
+    key = _schema_key(questions, independent)
+    if key in schemas: return True
+    if len(seen) > 50000: seen.clear()
+    seen[key] = seen.get(key, 0) + 1
+    return seen[key] >= _env_int("DECIDER_SCHEMA_MIN_SEEN", 2)
+
+
+def _score_schema(h, rows):
+    return se.score_rows(h, rows, temperature=TT.for_types(TEMP_SCHEMA, TEMP_SCHEMA_BY_TYPE, h.types))
+
+
+async def schema_batcher():
+    loop = asyncio.get_running_loop()
+    while True:
+        batch = await _collect(squeue)
+        groups = {}
+        for fut, h, row in batch: groups.setdefault((h.id, se.bucket(len(row[0]))), (h, []))[1].append((fut, row))
+        for h, items in groups.values():
+            step = max(1, MAX_BATCH // h.P)
+            for i in range(0, len(items), step):
+                chunk = items[i:i + step]
+                try:
+                    probs = await loop.run_in_executor(gpu, _score_schema, h, [c for _, c in chunk])
+                    for (fut, _), p in zip(chunk, probs):
+                        if not fut.done(): fut.set_result(p)
+                except Exception as e:
+                    for fut, _ in chunk:
+                        if not fut.done(): fut.set_exception(e)
+            stats["schema_batches"] = stats.get("schema_batches", 0) + 1
+
+
+# ---- start-up / shutdown -------------------------------------------------------
+def apply_config(cfg):
+    global MODEL_NAME, TEMP, TEMP_SCHEMA, TEMP_BY_TYPE, TEMP_SCHEMA_BY_TYPE, RELEASE_DATE, ISOLATED, NEUTRALIZE_NONE, SCHEMA_FIRST, LAYOUT
+    LAYOUT = resolve_layout(cfg)                    # ValueError for an unknown layout, before the engine is built
+    NEUTRALIZE_NONE = bool(cfg.get("neutralize_none", True))
+    MODEL_NAME = "decider-" + str(cfg.get("version", "dev"))
+    # ValueError for a temperature that is not finite and > 0 or an unknown by-type key, before the engine is built.
+    # DECIDER_TEMPERATURE replaces "temperature" and switches "temperature_by_type" off (decider.temperature).
+    (TEMP, TEMP_BY_TYPE), (TEMP_SCHEMA, TEMP_SCHEMA_BY_TYPE) = TT.from_config(cfg, os.environ.get("DECIDER_TEMPERATURE"))
+    RELEASE_DATE = str(cfg.get("release_date", RELEASE_DATE))
+    ISOLATED = bool(cfg.get("isolated_levels", False))
+    trained = bool(cfg.get("schema_first", False) or cfg.get("schema_first_trained", False))
+    SCHEMA_FIRST = trained and (bool(cfg.get("schema_first", False)) or os.environ.get("DECIDER_SCHEMA_CACHE", "0") == "1")
+
+
+def start_workers(loop=None):
+    """Queues, executors and batcher tasks.  Needs `eng` set; a test can set a stand-in engine and call this directly."""
+    global queue, gpu, cpu, batcher_task, squeue, schema_task
+    loop = loop or asyncio.get_running_loop()
+    if gpu is None: gpu = ThreadPoolExecutor(max_workers=1, thread_name_prefix="gpu")
+    if cpu is None: cpu = ThreadPoolExecutor(max_workers=TOKENIZE_THREADS, thread_name_prefix="tok")
+    queue = asyncio.Queue(); batcher_task = loop.create_task(batcher())
+    if SCHEMA_FIRST:
+        squeue = asyncio.Queue(); schema_task = loop.create_task(schema_batcher())
+
+
+def resolve_device(requested=None):
+    """The device and dtype the server runs on.  `auto` picks as decider.infer.Decider does: CUDA, else MPS, else CPU; float16
+    on MPS, bfloat16 elsewhere.  An explicit device that is not available, or FP8 / torch.compile off CUDA, is a start-up error
+    that says so, instead of the torch assertion a CUDA call raises on a build without CUDA."""
+    import torch
+    req = (DEVICE if requested is None else requested).strip().lower()
+    if req in ("", "auto"):
+        dev = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
+    else:
+        dev = req
+        kind = dev.split(":")[0]
+        if kind == "cuda" and not torch.cuda.is_available():
+            raise RuntimeError(f"DECIDER_DEVICE={req}, but torch.cuda.is_available() is False on this machine. "
+                               "Set DECIDER_DEVICE=mps or cpu (or leave it at auto).")
+        if kind == "mps" and not torch.backends.mps.is_available():
+            raise RuntimeError(f"DECIDER_DEVICE={req}, but torch.backends.mps.is_available() is False on this machine.")
+        if kind not in ("cuda", "mps", "cpu"):
+            raise RuntimeError(f"DECIDER_DEVICE={req}: expected auto, cuda, cuda:<index>, mps or cpu.")
+    if not dev.startswith("cuda") and (FP8 or COMPILE):
+        raise RuntimeError(f"DECIDER_FP8 and DECIDER_COMPILE need CUDA; the server is starting on {dev}. Unset them.")
+    return dev, (torch.float16 if dev.startswith("mps") else torch.bfloat16)
+
+
+async def _start():
+    global eng, se, gpu, CHAT
+    from decider.engine_v2 import EngineV2
+    apply_config(load_config(MODEL))
+    gpu = ThreadPoolExecutor(max_workers=1, thread_name_prefix="gpu")
+    loop = asyncio.get_running_loop()
+    dev, dtype = resolve_device()
+    eng = await loop.run_in_executor(gpu, lambda: EngineV2(
+        MODEL, device=dev, dtype=dtype, compile=COMPILE, fp8=FP8, max_ctx_tokens=MAX_STATE_TOKENS, t_buckets=_ints("DECIDER_T_BUCKETS"),
+        b_buckets=_ints("DECIDER_B_BUCKETS"), token_budget=GRAPH_TOKEN_BUDGET))
+    CHAT = chat_template(eng.tok) if LAYOUT == "chat" else None
+    print("[serve] engine", dict({k: v for k, v in eng.cfg.items() if k not in ("t_buckets", "b_buckets")}, device=dev, layout=LAYOUT), flush=True)
+    if SCHEMA_FIRST:
+        from decider.schema_engine import SchemaEngine
+        se = SchemaEngine(eng, chat=CHAT); print("[serve] schema cache on", flush=True)
+        pre = os.environ.get("DECIDER_SCHEMAS")      # JSON: [{"questions": {...}, "independent": true, "batch_sizes": [1, 8, 32], "state_tokens": [64, 256]}]
+        for spec in (json.load(open(pre)) if pre else []):
+            _, h, _ = await loop.run_in_executor(gpu, _schema_handle, spec["questions"], spec.get("independent", True), COMPILE)
+            t = await loop.run_in_executor(gpu, se.warmup, h, spec.get("batch_sizes", (1, 8, 32)), spec.get("state_tokens", (64, 128, 256)))
+            print(f"[serve] preloaded schema with {h.nq} rows, prefix {sum(h.tps)} tokens, graphs ready in {t:.0f}s", flush=True)
+    if WARMUP and eng.use_graphs:                   # off CUDA there are no graphs to capture; every request runs eager
+        t = await loop.run_in_executor(gpu, lambda: eng.warmup(log=lambda s: print(s, flush=True)))
+        print(f"[serve] captured {len(eng.graphs)} graphs in {t:.0f}s", flush=True)
+    eng.seal()
+    print("[serve] ready", json.dumps(dict(model=MODEL_NAME, layout=LAYOUT, temperature=TEMP, temperature_by_type=TT.effective(TEMP, TEMP_BY_TYPE),
+                                          **({"temperature_schema_first_by_type": TT.effective(TEMP_SCHEMA, TEMP_SCHEMA_BY_TYPE)} if SCHEMA_FIRST else {}),
+                                          isolated_levels=ISOLATED, schema_first=SCHEMA_FIRST,
+                                          shared=SHARED, graphs=len(eng.graphs), limits=dict(
+                                              max_rows=MAX_ROWS, max_row_tokens=MAX_ROW_TOKENS, max_request_tokens=MAX_REQUEST_TOKENS,
+                                              max_queue_rows=MAX_QUEUE_ROWS))), flush=True)
+    start_workers(loop)
+
+
+def _stop():
+    global gpu, cpu, batcher_task, schema_task
+    for t in (batcher_task, schema_task):
+        if t is not None: t.cancel()
+    for ex in (gpu, cpu):
+        if ex is not None: ex.shutdown(wait=False, cancel_futures=True)
+    gpu = cpu = batcher_task = schema_task = None
+
+
+@asynccontextmanager
+async def lifespan(app):
+    await _start()
+    yield
+    _stop()
+
+
+app = FastAPI(title="decider", lifespan=lifespan)
+
+
+# ---- routes -------------------------------------------------------------------
+class Req(BaseModel):
+    context: str
+    schema_: object = None                  # any JSON value: Decider._check_schema gives the 422 (pydantic's echoes NaN/Infinity and cannot be serialised)
+    model_config = {"populate_by_name": True}
+    def __init__(self, **kw):
+        if "schema" in kw: kw["schema_"] = kw.pop("schema")
+        super().__init__(**kw)
+
+
+class S1Req(BaseModel):
+    state: object
+    questions: dict
+    model: str | None = None
+    independent: bool = True
+    layout: str | None = None            # "state_first" forces the uncached layout on a schema-first model
+
+
+def _alive():
+    return batcher_task is not None and not batcher_task.done() and (schema_task is None or not schema_task.done())
+
+
+async def _queued(items):
+    """Queue one request's rows.  Every row carries the request's id, which is what the batcher's adaptive wait reads."""
+    global REQ_SEQ
+    loop = asyncio.get_running_loop(); futs = []
+    REQ_SEQ += 1; rid = REQ_SEQ
+    for it in items:
+        f = loop.create_future(); futs.append(f); queue.put_nowait((f, it, rid))
+    return await asyncio.gather(*futs)
+
+
+@app.post("/decide")
+async def decide(r: Req):
+    loop = asyncio.get_running_loop()
+    try:
+        qs, it = await loop.run_in_executor(cpu, _prepare_decide, r.context, r.schema_)
+    except (ValueError, KeyError) as e:
+        stats["errors"] += 1
+        raise HTTPException(422, str(e))
+    if not qs:                                       # empty schema: nothing to score (the 1.1.2 answer, without a forward)
+        stats["requests"] += 1; return {}
+    check_size([len(it["ids"])])
+    _admit(1)
+    try:
+        probs = (await _queued([it]))[0]
+    except Exception:
+        stats["errors"] += 1; raise
+    finally:
+        _release(1)
+    stats["requests"] += 1; stats["decisions"] += len(qs); stats["rows"] += 1
+    return _format_decide(r.schema_, qs, probs)
+
+
+@app.post("/v1/systemone")
+async def systemone(r: S1Req):
+    loop = asyncio.get_running_loop()
+    if SCHEMA_FIRST and r.questions and r.layout != "state_first" and _worth_caching(r.questions, r.independent):
+        try:                                       # CPU: rows, prefix lengths, suffix ids -> the complete cost before any GPU work
+            tps, row = await loop.run_in_executor(cpu, _plan_schema, r.questions, r.independent, r.state)
+        except ValueError as e:
+            stats["errors"] += 1
+            raise HTTPException(422, str(e))
+        check_size([tp + len(row[0]) for tp in tps])
+        _admit(len(tps))
+        try:
+            rqs, h, index = await loop.run_in_executor(gpu, _schema_handle, r.questions, r.independent)
+            fut = loop.create_future(); await squeue.put((fut, h, row)); p = await fut
+        except Exception:
+            stats["errors"] += 1; raise
+        finally:
+            _release(len(tps))
+        stats["requests"] += 1; stats["decisions"] += len(rqs); stats["schema_requests"] = stats.get("schema_requests", 0) + 1
+        return {"model": MODEL_NAME, "answers": S1.assemble(rqs, index, [pk.tolist() for pk in p]),
+                "usage": {"input_tokens": len(row[0]) * h.P, "cached_tokens": sum(h.tps), "output_tokens": 0}}
+    try:
+        rqs, index, items, ctx_len = await loop.run_in_executor(cpu, _prepare_s1, r.state, r.questions, r.independent)
+    except ValueError as e:
+        stats["errors"] += 1
+        raise HTTPException(422, str(e))
+    check_size([len(it["ids"]) for it in items])
+    _admit(len(items))
+    try:
+        if SHARED and len(items) > 1 and min(len(it["ids"]) for it in items) >= SHARED_MIN_TOKENS:
+            res = await loop.run_in_executor(gpu, _score_shared, items)      # long state: run it once, fork the cache per question
+            stats["shared_prefix_requests"] += 1
+        else:
+            res = await _queued(items)
+    except Exception:
+        stats["errors"] += 1; raise
+    finally:
+        _release(len(items))
+    probs = [p for ps in res for p in ps]                                   # one prob row per question, request order
+    stats["requests"] += 1; stats["decisions"] += len(rqs); stats["rows"] += len(items)
+    return {"model": MODEL_NAME, "answers": S1.assemble(rqs, index, [p.tolist() for p in probs]),
+            "usage": {"input_tokens": unique_tokens(items, ctx_len), "output_tokens": 0}}
+
+
+@app.get("/v1/models")
+async def models():
+    return {"models": [{"name": MODEL_NAME, "description": "decider: one-pass typed decisions with calibrated probabilities", "release_date": RELEASE_DATE}]}
+
+
+@app.get("/health")
+async def health():
+    return {"ok": eng is not None and bool(getattr(eng, "sealed", True)) and _alive(), "model": MODEL,
+            "device": str(getattr(eng, "dev", "")) if eng is not None else None, "layout": LAYOUT,
+            "temperature": TEMP, "temperature_by_type": TT.effective(TEMP, TEMP_BY_TYPE),
+            **({"temperature_schema_first_by_type": TT.effective(TEMP_SCHEMA, TEMP_SCHEMA_BY_TYPE)} if SCHEMA_FIRST else {})}
+
+
+@app.get("/stats")
+async def get_stats():
+    return dict(stats, outstanding_rows=outstanding, engine=eng.stats if eng else None, graphs=len(eng.graphs) if eng else 0,
+                sealed=bool(eng and getattr(eng, "sealed", False)), schema_cache=dict(se.stats, schemas=len(schemas)) if se else None,
+                limits=dict(max_rows=MAX_ROWS, max_row_tokens=MAX_ROW_TOKENS, max_request_tokens=MAX_REQUEST_TOKENS, max_queue_rows=MAX_QUEUE_ROWS))

decider/shared_prefix.py

@@ -0,0 +1,200 @@
+"""Shared-prefix scoring with a bounded memory fork, used by both engines.
+
+Rows that start with the same tokens (one state, one question per row) run the shared prefix once and then only the
+question suffixes, against a copy of the prefix cache.  Until 1.1.0 the copy was made with
+`cache.reorder_cache(zeros(n))`, which expands the prefix to all n rows at once: a 31k-token state with 32 questions
+costs n times the prefix cache, which is 133 GB on a 31B model and is wasteful on the 2B.  Here the prefix cache is
+forked in chunks of m rows, m chosen so that one fork fits a byte budget, and the rows are scored chunk by chunk.
+
+The budget is `DECIDER_SHARED_FORK_GB` (8 GB), capped at half of the memory currently free on the device (device-free
+plus the caching allocator's reserved-but-unused blocks); `m = clamp(budget // prefix_bytes, 1, n)`.  Chunking changes
+which rows share a forward and how far each chunk's suffixes are padded, so answers can move by the usual bf16
+reduction-order amount; they do not depend on it mathematically (right padding, causal layers).
+
+Cache layout.  transformers 5.17 keeps one layer object per layer in `cache.layers`.  An attention layer carries
+`keys` and `values` tensors `[batch, heads, seq, head_dim]`; a sparse-attention layer adds `indexer_keys`
+`[batch, seq, dim]`; a linear-attention layer carries `conv_states` and `recurrent_states` dicts of tensors whose first
+dimension is the batch.  Qwen3.5 gives a `DynamicCache` whose 24 layers are a mix of the two objects.  `cache_state_tensors`
+enumerates whatever of `STATE_NAMES` is present rather than assuming a layout, and `fork_cache` builds a new cache
+object from them without touching the original (the linear-attention layers update their states with `copy_()`, so a
+fork that shared storage with the prefix would corrupt it for the next chunk).
+
+A layout we do not know is not chunked.  If a layer holds any other tensor attribute, or a dict of tensors, outside
+`STATE_NAMES`, we cannot tell whether it carries a batch dimension, so `score_shared` takes one fork of all n rows --
+the 1.1.0 behaviour, correct but unbounded -- and counts it in `engine.stats["shared_unchunked_layout"]`.
+
+The prefix and suffix forwards are eager, at request-specific shapes, and are correct only with the cuDNN SDPA backend
+off (`decider.engine.set_attention_backend_policy`, applied in both engines' `__init__`).
+"""
+import copy
+import os
+
+import torch
+import torch.nn.functional as F
+
+from decider.engine import fill_ids, read_slots
+from decider.temperature import item_slice, slot_temperatures
+
+DEFAULT_FORK_GB = 8.0
+STATE_NAMES = ("keys", "values", "indexer_keys", "conv_states", "recurrent_states")
+
+
+def common_prefix_len(ids):
+    """Length of the longest common prefix of the rows, capped one token below the shortest row."""
+    lcp = 0
+    short = min(len(x) for x in ids) - 1
+    while lcp < short and all(x[lcp] == ids[0][lcp] for x in ids):
+        lcp += 1
+    return lcp
+
+
+def _get(container, key):
+    return container[key] if isinstance(container, dict) else getattr(container, key)
+
+
+def _set(container, key, value):
+    if isinstance(container, dict):
+        container[key] = value
+    else:
+        setattr(container, key, value)
+
+
+def cache_state_tensors(cache):
+    """Every tensor in the cache whose first dimension is the batch, as (container, key) pairs.
+
+    Covers `STATE_NAMES`: the attention layers' `keys`/`values`, a sparse-attention layer's `indexer_keys` and the
+    linear-attention layers' `conv_states`/`recurrent_states`, whether those are dicts of tensors (transformers 5.17) or
+    single tensors, and ignores anything not present."""
+    out = []
+    for layer in getattr(cache, "layers", None) or []:
+        for attr in STATE_NAMES:
+            v = getattr(layer, attr, None)
+            if isinstance(v, torch.Tensor):
+                if v.numel():
+                    out.append((layer, attr))
+            elif isinstance(v, dict):
+                for k, t in v.items():
+                    if isinstance(t, torch.Tensor) and t.numel():
+                        out.append((v, k))
+    return out
+
+
+def cache_row_bytes(cache):
+    """Bytes one row of this cache holds, summed over the layers and over `STATE_NAMES`."""
+    return sum(_get(c, k).nbytes // max(_get(c, k).shape[0], 1) for c, k in cache_state_tensors(cache))
+
+
+def unknown_state_names(cache):
+    """Attribute names the cache's layers hold that carry tensors and are not in `STATE_NAMES`.
+
+    A tensor outside the enumerated set may or may not have a batch dimension, and `fork_cache` would leave it at one
+    row.  When this is not empty the caller must not chunk."""
+    out = set()
+    for layer in getattr(cache, "layers", None) or []:
+        for name, v in vars(layer).items():
+            if name in STATE_NAMES:
+                continue
+            if isinstance(v, torch.Tensor) or (isinstance(v, dict) and any(isinstance(t, torch.Tensor) for t in v.values())):
+                out.add(name)
+    return out
+
+
+def fork_budget_bytes(device=None, gb=None):
+    """DECIDER_SHARED_FORK_GB, capped at half of the memory free on the device right now.
+
+    The cap applies only when the CUDA memory queries succeed: on CPU, on MPS and when the driver does not know the
+    device string, the configured budget is kept as it is."""
+    gb = float(os.environ.get("DECIDER_SHARED_FORK_GB", DEFAULT_FORK_GB)) if gb is None else float(gb)
+    budget = int(gb * (1 << 30))
+    try:
+        free, _ = torch.cuda.mem_get_info(device)
+        free += torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)
+        budget = min(budget, free // 2)
+    except Exception:                       # no CUDA device, or a device string the driver does not know
+        pass
+    return max(int(budget), 1)
+
+
+def chunk_rows(prefix_bytes, n, budget_bytes=None, device=None):
+    """Rows per fork: as many copies of the prefix cache as the budget holds, at least 1 and at most n.  One row is the
+    minimum even when a single copy is over the budget: the budget bounds the fork, it cannot make it free."""
+    if budget_bytes is None:
+        budget_bytes = fork_budget_bytes(device)
+    if prefix_bytes <= 0:
+        return max(1, int(n))
+    return max(1, min(int(n), int(budget_bytes // prefix_bytes)))
+
+
+def fork_cache(cache, m, row=0):
+    """A new cache holding `m` copies of `cache`'s row `row`.  The original is not read from again and not modified."""
+    fork = copy.copy(cache)
+    layers = getattr(cache, "layers", None)
+    if layers is not None:
+        new = []
+        for layer in layers:
+            nl = copy.copy(layer)
+            for k, v in list(vars(nl).items()):          # the per-state dicts are mutated by the forward: give the fork its own
+                if isinstance(v, dict):
+                    setattr(nl, k, dict(v))
+            new.append(nl)
+        fork.layers = new
+    idx = {}
+    for container, key in cache_state_tensors(fork):
+        t = _get(container, key)
+        i = idx.get(t.device)
+        if i is None:
+            i = idx[t.device] = torch.full((m,), row, dtype=torch.long, device=t.device)
+        _set(container, key, t.index_select(0, i))       # a fresh contiguous tensor: the fork never shares storage
+    return fork
+
+
+def _count(engine, key):
+    stats = getattr(engine, "stats", None)
+    if isinstance(stats, dict):
+        stats[key] = stats.get(key, 0) + 1
+
+
+@torch.no_grad()
+def score_shared(engine, items, temperature=1.0, min_prefix=192, budget_bytes=None, rows_per_fork=None):
+    """Score `items` through the shared prefix.  -> one probability tensor per item, in item order, or None when the
+    request does not qualify (fewer than two rows, or a common prefix below `min_prefix`) and the caller should use
+    `score_items`.
+
+    `temperature`: a number, or one entry per item (decider.temperature.slot_temperatures).
+    `rows_per_fork` forces the chunk size; it exists for the tests that compare chunked against unchunked answers."""
+    ids = [it["ids"] for it in items]
+    n = len(ids)
+    if n < 2:
+        return None
+    lcp = common_prefix_len(ids)
+    if lcp < min_prefix:
+        return None
+    slot_temperatures(temperature, items)                  # a length mismatch fails before any forward
+    core, W, dev, pad = engine.core, engine.W, engine.dev, engine.tok.pad_token_id
+    pre = torch.tensor(ids[0][:lcp], device=dev)[None]
+    cache = core(input_ids=pre, use_cache=True).past_key_values
+    unknown = unknown_state_names(cache)
+    if unknown:                                  # a state we cannot fork row by row: one fork of everything, as in 1.1.0
+        _count(engine, "shared_unchunked_layout")
+        m = n
+    elif rows_per_fork:
+        m = int(rows_per_fork)
+    else:
+        m = chunk_rows(cache_row_bytes(cache), n, budget_bytes, dev)
+    m = max(1, min(m, n))
+    out = []
+    for i in range(0, n, m):
+        part = items[i:i + m]
+        b = len(part)
+        fork = fork_cache(cache, b)
+        Ts = max(len(it["ids"]) for it in part) - lcp
+        suf = fill_ids([it["ids"][lcp:] for it in part], b, Ts, pad)
+        h = core(input_ids=suf.to(dev), past_key_values=fork, use_cache=True).last_hidden_state
+        rows = [j for j, it in enumerate(part) for _ in it["slots"]]
+        sl = [s - lcp for it in part for s in it["slots"]]
+        idx = torch.tensor([rows, sl], device=dev)
+        out += read_slots(F.linear(h[idx[0], idx[1]], W).float()[:, None, :], list(range(len(rows))), [0] * len(rows),
+                          [k for it in part for k in it["nopts"]],
+                          slot_temperatures(item_slice(temperature, i, i + b), part), [len(it["slots"]) for it in part])
+        del fork, h, suf                                  # drop this chunk's fork before the next one is built
+    return out

decider/systemone.py

@@ -0,0 +1,199 @@
+"""Jev-shaped requests on top of the decider prompt format (same wire format as TypeSafe's POST /v1/systemone).
+
+    state      str | dict | list            JSON state is serialised compactly; questions may name a part by path (`ticket.messages[0].text`)
+    questions  {id: {"type": "choice", "instructions": ..., "criteria": {name: description | {...} | [...] | None}}      up to 255 options
+                     {"type": "score",  "instructions": ..., "criteria": [level 0 description, level 1 description, ...]}  2..10 levels
+                     {"type": "noul",   "instructions": ... (optional), "criteria": {"true": ..., "false": ...} (optional)}}
+    a noul question needs instructions or at least one true/false description.
+    ids are never shown to the model.  `instructions` and every description may be a string or any JSON value.
+"""
+import json, math
+
+MAX_CHOICE, MAX_LEVELS = 255, 10
+
+
+def _txt(v):
+    return v if isinstance(v, str) else json.dumps(v, ensure_ascii=False)
+
+
+ANNOTATE_MIN = 8
+
+
+def annotate_indices(x, min_len=ANNOTATE_MIN):
+    """Write each element's position into long arrays ({"_index": i, ...}).  A path such as `records[47].text` otherwise makes
+    the model count 47 elements; with the index written down it is a lookup (json_k64 probe: 0.49 -> 0.57 accuracy)."""
+    if isinstance(x, list):
+        if len(x) >= min_len:
+            return [({"_index": i, **annotate_indices(v, min_len)} if isinstance(v, dict) else {"_index": i, "value": annotate_indices(v, min_len)}) for i, v in enumerate(x)]
+        return [annotate_indices(v, min_len) for v in x]
+    if isinstance(x, dict):
+        return {k: annotate_indices(v, min_len) for k, v in x.items()}
+    return x
+
+
+def render_state(state, index_arrays=True):
+    if isinstance(state, str):
+        return state
+    return json.dumps(annotate_indices(state) if index_arrays else state, ensure_ascii=False)
+
+
+def render_question(spec):
+    """-> dict(question=str, options=[str], type=..., names=[...])  (names: what the answer reports for each option)"""
+    t = spec.get("type", "choice"); crit = spec.get("criteria", spec.get("options"))
+    raw = spec.get("instructions", spec.get("question", ""))
+    if t in ("noul", "bool") and raw in (None, ""):          # the criteria carry the question (NOUL_WITHOUT_INSTRUCTIONS)
+        ins = NOUL_WITHOUT_INSTRUCTIONS
+        described = isinstance(crit, dict) and any(d not in (None, "") for d in (crit.get("true", crit.get(True)), crit.get("false", crit.get(False))))
+        if not described and (crit is None or isinstance(crit, dict)):     # a non-map is rejected below with the criteria message
+            raise ValueError("noul question without instructions: criteria must describe true or false")
+    else:
+        ins = _txt(raw)
+    if not ins:
+        raise ValueError("question without instructions")
+    if t == "choice":
+        if isinstance(crit, (list, tuple)):
+            crit = {str(c): None for c in crit}
+        if not isinstance(crit, dict) or not 2 <= len(crit) <= MAX_CHOICE:
+            raise ValueError(f"choice criteria: a map of 2..{MAX_CHOICE} options")
+        names = list(crit); opts = [n if crit[n] in (None, "") else f"{n}: {_txt(crit[n])}" for n in names]
+    elif t == "score":
+        if isinstance(crit, dict):                                   # legend form {"0": "...", "1": "..."}
+            crit = [crit[k] for k in sorted(crit, key=float)]
+        if not isinstance(crit, (list, tuple)) or not 2 <= len(crit) <= MAX_LEVELS:
+            raise ValueError(f"score criteria: an ordered list of 2..{MAX_LEVELS} level descriptions")
+        names = list(range(len(crit))); opts = [f"{i}: {_txt(c)}" for i, c in enumerate(crit)]
+    elif t in ("noul", "bool"):
+        if crit is not None and not isinstance(crit, dict):
+            raise ValueError("noul criteria: a map of optional true/false descriptions")
+        names = [False, True]; c = crit if crit is not None else {}
+        f, tr = c.get("false", c.get(False)), c.get("true", c.get(True))
+        opts = ["no" if f in (None, "") else f"no: {_txt(f)}", "yes" if tr in (None, "") else f"yes: {_txt(tr)}"]
+    else:
+        raise ValueError(f"unknown question type {t!r}")
+    return dict(question=ins, options=opts, type="noul" if t == "bool" else t, names=names, legend=[_txt(c) for c in crit] if t == "score" else None,
+                isolated=bool(spec.get("isolated", True)))
+
+
+# A noul question may omit `instructions` (TypeSafe's OpenAPI file marks it optional).  The question id is never shown to the
+# model, so the question text is this fixed sentence and the true/false descriptions, rendered as the options "no: ..." and
+# "yes: ...", say what is being asked.  A request that gives instructions is rendered exactly as before.
+NOUL_WITHOUT_INSTRUCTIONS = "Which answer fits the context?"
+
+
+# ---- isolated levels: every Score level is judged in its own row, without its number or its neighbours
+ISOLATED = "{q}\nProposed answer: {level}\nDoes the proposed answer fit?"
+_NUM = None
+
+
+def strip_level_number(text):
+    """"2: somewhat" -> "somewhat" (dataset legends carry the number; an isolated level must not)."""
+    import re
+    return re.sub(r"^\s*-?\d+\s*:\s*", "", text)
+
+
+def isolated_rows(question, levels):
+    """-> one yes/no question per level: [(question text, ["no", "yes"])]."""
+    return [(ISOLATED.format(q=question, level=strip_level_number(l)), ["no", "yes"]) for l in levels]
+
+
+def combine_isolated(p_yes):
+    """Per-level P(fits), each computed without reference to any other level -> a distribution over levels.
+    Also returns the unnormalised mass: near 1 when exactly one level fits, low when none does, high when several do."""
+    tot = sum(p_yes) or 1e-9
+    return [x / tot for x in p_yes], tot
+
+
+def plan_rows(rqs, isolated=True):
+    """One scoring row per question; a Score question with isolated levels becomes one yes/no row per level.
+    -> (rows [{"question", "options"}], index [(id, "iso" | "list", first row, n rows)])"""
+    rows, index = [], []
+    for k, r in rqs.items():
+        if isolated and r["type"] == "score" and r.get("isolated", True):
+            rws = isolated_rows(r["question"], r["legend"]); index.append((k, "iso", len(rows), len(rws))); rows += [dict(question=t, options=o) for t, o in rws]
+        else:
+            index.append((k, "list", len(rows), 1)); rows.append(dict(question=r["question"], options=r["options"]))
+    return rows, index
+
+
+def row_types(rqs, index):
+    """The answer type ("choice", "noul" or "score") of every plan_rows row, in row order.  An isolated Score question's
+    yes/no level rows carry "score": together they are one Score answer (decider.temperature)."""
+    types = [None] * sum(n for _, _, _, n in index)
+    for k, _, s, n in index:
+        types[s:s + n] = [rqs[k]["type"]] * n
+    return types
+
+
+def assemble(rqs, index, probs):
+    """probs: one probability list per row (plan_rows order) -> {id: answer}."""
+    out = {}
+    for k, kind, s, n in index:
+        if kind == "iso":
+            fit = [float(probs[s + j][1]) for j in range(n)]; p, mass = combine_isolated(fit); a = format_answer(rqs[k], p)
+            a["level_fit"] = {str(j): round(x, 4) for j, x in enumerate(fit)}; a["fit_mass"] = round(mass, 4); out[k] = a
+        else:
+            out[k] = format_answer(rqs[k], probs[s])
+    return out
+
+
+def certainty(p):
+    """1 - normalised entropy: 1 when all mass is on one option, 0 when the distribution is flat."""
+    h = -sum(x * math.log(x) for x in p if x > 0)
+    return max(0.0, 1.0 - h / math.log(len(p))) if len(p) > 1 else 1.0
+
+
+def _clip01(x):
+    return min(1.0, max(0.0, x))
+
+
+def _normalised(p):
+    """As the adapter's _normalize: a distribution with zero total counts as uniform."""
+    tot = sum(p)
+    return [1.0 / len(p)] * len(p) if tot == 0 else [x / tot for x in p]
+
+
+def choice_confidence(p):
+    """TypeSafe's Choice confidence: the largest probability rescaled so that a uniform distribution gives 0 and all mass on one
+    option gives 1, (n * p_max - 1) / (n - 1); 1 for a single option."""
+    n = len(p); p = _normalised(p)
+    return 1.0 if n <= 1 else _clip01((n * max(p) - 1) / (n - 1))
+
+
+def score_confidence(p):
+    """TypeSafe's Score confidence (system-one-adapter-python, confidence_metrics.score_confidence): 1 minus the expected distance
+    from the most likely level, divided by D = mean over levels i of |i - (n - 1)/2| (the mean distance of the levels from the
+    middle of the scale), floored at 0; 1 for a single level.
+    With two levels it equals choice_confidence."""
+    n = len(p); p = _normalised(p)
+    if n <= 1:
+        return 1.0
+    k = max(range(n), key=p.__getitem__)
+    spread = sum(x * abs(i - k) for i, x in enumerate(p))
+    uniform = sum(abs(i - (n - 1) / 2) for i in range(n)) / n
+    return _clip01(1.0 - spread / uniform)
+
+
+def format_answer(rq, p, nd=4):
+    """rq: render_question output; p: probabilities in option order.
+    `confidence` is TypeSafe's (choice_confidence / score_confidence); `x_p_max` is the largest probability, which was
+    `confidence` before 1.3.0."""
+    p = [float(x) for x in p[:len(rq["options"])]]; s = sum(p) or 1.0; p = [x / s for x in p]
+    j = max(range(len(p)), key=p.__getitem__)
+    if rq["type"] == "noul":
+        return {"type": "noul", "noul": round(p[1], nd)}
+    if rq["type"] == "choice":
+        return {"type": "choice", "choice": rq["names"][j], "confidence": round(choice_confidence(p), nd), "x_p_max": round(p[j], nd),
+                "certainty": round(certainty(p), nd), "probabilities": {n: round(x, nd) for n, x in zip(rq["names"], p)}}
+    return {"type": "score", "score": round(sum(i * x for i, x in enumerate(p)), 2), "confidence": round(score_confidence(p), nd), "x_p_max": round(p[j], nd),
+            "certainty": round(certainty(p), nd),
+            "legend": {str(i): d for i, d in enumerate(rq["legend"])}, "probabilities": {str(i): round(x, nd) for i, x in enumerate(p)}}
+
+
+def unique_tokens(items):
+    """Input tokens of a request whose rows share a prefix (the state): the prefix counts once."""
+    ids = [it["ids"] for it in items]
+    if len(ids) < 2:
+        return sum(len(x) for x in ids)
+    lcp = 0; short = min(len(x) for x in ids)
+    while lcp < short and all(x[lcp] == ids[0][lcp] for x in ids): lcp += 1
+    return lcp + sum(len(x) - lcp for x in ids)

decider/temperature.py

@@ -0,0 +1,144 @@
+"""Answer temperatures from decider_config.json (1.4.0).
+
+Every answer is softmax(logits / T) over its option letters.  decider_config.json sets T:
+
+    "temperature": 1.3                                   one value for every answer (the only form before 1.4.0)
+    "temperature_by_type": {"choice": 1.48, "noul": 2.22, "score": 1.38}
+                                                         optional; one value per answer type, a missing type uses "temperature"
+    "temperature_schema_first": 1.18                     optional; the schema cache (questions-first layout), as before
+    "temperature_schema_first_by_type": {...}            optional; per answer type on the schema cache
+
+The answer types are the /v1/systemone question types.  A /decide field maps onto them: "choice" -> "choice", "bool" -> "noul",
+"scale" -> "score".  Plain `Decider.decide` questions (a question and its options, no type) are "choice".  A Score question
+read with isolated levels (one yes/no row per level) uses the "score" temperature on every one of its level rows: the rows form
+one Score answer, and a temperature fitted on Score answers is fitted through that same readout (decider.calibrate).
+
+On the state-first layout the temperature of an answer of type t is temperature_by_type[t], else temperature.  On the schema
+cache it is temperature_schema_first_by_type[t], else temperature_schema_first, else (no schema-first value at all) the
+state-first temperature of t.  An explicit override (Decider(temperature=...), DECIDER_TEMPERATURE) replaces the state-first
+"temperature" and switches temperature_by_type off, so the override is the one temperature of every state-first answer, as it
+was before 1.4.0.
+
+A config without a by-type map gives every path the single number it gave in 1.3.0, and that number reaches the engines as the
+same Python float, so the probabilities are bit-identical to 1.3.0.
+"""
+import math
+
+TYPES = ("choice", "noul", "score")
+FIELD_TYPES = {"choice": "choice", "bool": "noul", "scale": "score"}        # /decide field type -> answer type
+_KEYS_TEXT = ('the keys are "choice", "noul" and "score" (a /decide "bool" field is "noul", a "scale" field is "score"; '
+              'a /v1/systemone "bool" question is "noul")')
+
+
+def positive(value, where):
+    """A temperature: a number (or a numeric string, as float() reads it, which 1.3.0 accepted for "temperature") that is finite
+    and > 0.  Raises ValueError naming `where`."""
+    if isinstance(value, bool):
+        raise ValueError(f"{where} must be a finite number > 0, got {value!r}")
+    try:
+        v = float(value)
+    except (TypeError, ValueError):
+        raise ValueError(f"{where} must be a finite number > 0, got {value!r}") from None
+    if not math.isfinite(v) or v <= 0:
+        raise ValueError(f"{where} must be a finite number > 0, got {value!r}")
+    return v
+
+
+def by_type(m, where):
+    """Validate a {answer type: temperature} map.  None -> {}.  Unknown keys, non-numbers and values that are not finite and > 0
+    raise ValueError."""
+    if m is None:
+        return {}
+    if not isinstance(m, dict):
+        raise ValueError(f"{where} must be a map {{answer type: temperature}}, got {type(m).__name__}; " + _KEYS_TEXT)
+    out = {}
+    for k, v in m.items():
+        if k not in TYPES:
+            raise ValueError(f"{where} has the unknown key {k!r}; " + _KEYS_TEXT)
+        if isinstance(v, bool) or not isinstance(v, (int, float)):
+            raise ValueError(f"{where}[{k!r}] must be a finite number > 0, got {v!r}")
+        out[k] = positive(v, f"{where}[{k!r}]")
+    return out
+
+
+def from_config(cfg, temperature=None, temperature_by_type=None):
+    """-> ((T, by_type) for the state-first layout, (T, by_type) for the schema cache).
+
+    temperature / temperature_by_type: explicit overrides (Decider arguments, DECIDER_TEMPERATURE).  An explicit temperature
+    without an explicit map switches the config's map off (see the module docstring)."""
+    cfg = cfg or {}
+    where = "decider_config.json"
+    if temperature is not None:
+        T = positive(temperature, "temperature")
+        m = by_type(temperature_by_type, "temperature_by_type") if temperature_by_type is not None else {}
+    else:
+        T = positive(cfg.get("temperature", 1.0), f'{where} "temperature"')
+        m = (by_type(temperature_by_type, "temperature_by_type") if temperature_by_type is not None
+             else by_type(cfg.get("temperature_by_type"), f'{where} "temperature_by_type"'))
+    ms = by_type(cfg.get("temperature_schema_first_by_type"), f'{where} "temperature_schema_first_by_type"')
+    if "temperature_schema_first" in cfg:
+        schema = (positive(cfg["temperature_schema_first"], f'{where} "temperature_schema_first"'), ms)
+    else:
+        schema = (T, {**m, **ms})
+    return (T, m), schema
+
+
+def effective(T, m):
+    """{answer type: the temperature it gets} for reporting (/health, the ready line)."""
+    return {t: m.get(t, T) for t in TYPES}
+
+
+def for_types(T, m, types):
+    """One temperature per slot, or the scalar T itself when there is no map (the 1.3.0 call)."""
+    if not m:
+        return T
+    return [m.get(t, T) for t in types]
+
+
+def item_types(it):
+    """The answer type of every slot of a prompt item ("types", set where the item is built); None for an item without it."""
+    ts = it.get("types")
+    return list(ts) if ts is not None else [None] * len(it["slots"])
+
+
+def for_items(T, m, items):
+    """The `temperature` argument of Engine.score_items / score_shared: the scalar T when there is no map (the 1.3.0 call),
+    else one list of per-slot temperatures per item."""
+    if not m:
+        return T
+    return [[m.get(t, T) for t in item_types(it)] for it in items]
+
+
+def slot_temperatures(temperature, items):
+    """Engine side.  temperature: a scalar (returned as it is), or one entry per item, each a scalar or one value per slot of
+    that item.  -> the scalar, or a flat list with one temperature per slot in item order."""
+    if not isinstance(temperature, (list, tuple)):
+        return temperature
+    if len(temperature) != len(items):
+        raise ValueError(f"temperature: {len(temperature)} entries for {len(items)} items")
+    flat = []
+    for t, it in zip(temperature, items):
+        n = len(it["slots"])
+        if isinstance(t, (list, tuple)):
+            if len(t) != n:
+                raise ValueError(f"temperature: {len(t)} values for an item with {n} slots")
+            flat += list(t)
+        else:
+            flat += [t] * n
+    return flat
+
+
+def item_slice(temperature, lo, hi):
+    """The per-item temperature entries of items[lo:hi] (a scalar is shared by every item)."""
+    return temperature[lo:hi] if isinstance(temperature, (list, tuple)) else temperature
+
+
+def scaled_softmax(lg, temperature):
+    """softmax(lg / T) over the last axis.  A scalar T is the 1.3.0 expression unchanged; a list gives one T per row of lg."""
+    import torch
+    if isinstance(temperature, (list, tuple)):
+        if len(temperature) != lg.shape[0]:
+            raise ValueError(f"temperature: {len(temperature)} values for {lg.shape[0]} slots")
+        t = torch.tensor(temperature, dtype=lg.dtype).to(lg.device, non_blocking=True)[:, None]
+        return torch.softmax(lg / t, -1)
+    return torch.softmax(lg / temperature, -1)

decider_config.json

@@ -0,0 +1,18 @@
+{
+ "temperature": 1.097,
+ "temperature_by_type": {
+  "noul": 1.097
+ },
+ "neutralize_none": false,
+ "version": "4b-v2.1-tr-lora-tr-lora",
+ "base": "hayriyigit/decider-4b-tr@df9aacb9f4c9e4e64178176502f51ae4db271f26 + LoRA (merged)",
+ "layout": "plain",
+ "max_options": 255,
+ "max_state_tokens": 32768,
+ "schema_first": false,
+ "schema_first_trained": false,
+ "isolated_levels": true,
+ "release_date": "2026-09-24",
+ "requires": "decider-ai>=1.4.0 for temperature_by_type; older versions serve every answer at temperature",
+ "stage": "LoRA rank 64 (alpha 128) on q_proj, k_proj, v_proj, o_proj, in_proj_qkv, in_proj_z, out_proj, gate_proj, up_proj, down_proj; LR 0.0001, 1 epochs; cross-entropy on Turkish decision rows, KL toward the base model on English replay rows; temperatures fitted with decider.calibrate.fit_by_type on held-out Turkish rows"
+}

eval_report.json

@@ -0,0 +1,331 @@
+{
+ "model": "decider-4b-v2.1-tr-lora-tr-lora",
+ "path": "/workspace/laya/answerability_run/model",
+ "revision": "eb5fbdfc9448473ec25e399882912863afbdb70e",
+ "fp8_layers": 0,
+ "limit": 0,
+ "skipped": {},
+ "seconds": 502,
+ "report": {
+  "all": {
+   "n": 16243,
+   "accuracy": 0.8931,
+   "brier": 0.1653,
+   "nll": 0.2898,
+   "ece": 0.0426
+  },
+  "source/hotpot_tr": {
+   "n": 1500,
+   "accuracy": 0.98,
+   "brier": 0.032,
+   "nll": 0.0605,
+   "ece": 0.0063
+  },
+  "source/hotpot_tr/noul": {
+   "n": 1500,
+   "accuracy": 0.98,
+   "brier": 0.032,
+   "nll": 0.0605,
+   "ece": 0.0063
+  },
+  "source/musique_tr": {
+   "n": 500,
+   "accuracy": 0.898,
+   "brier": 0.1518,
+   "nll": 0.2529,
+   "ece": 0.0296
+  },
+  "source/musique_tr/noul": {
+   "n": 500,
+   "accuracy": 0.898,
+   "brier": 0.1518,
+   "nll": 0.2529,
+   "ece": 0.0296
+  },
+  "source/squad_tr": {
+   "n": 11873,
+   "accuracy": 0.8614,
+   "brier": 0.2146,
+   "nll": 0.3759,
+   "ece": 0.0572
+  },
+  "source/squad_tr/noul": {
+   "n": 11873,
+   "accuracy": 0.8614,
+   "brier": 0.2146,
+   "nll": 0.3759,
+   "ece": 0.0572
+  },
+  "source/synth_tr": {
+   "n": 1170,
+   "accuracy": 0.9923,
+   "brier": 0.0113,
+   "nll": 0.023,
+   "ece": 0.0038
+  },
+  "source/synth_tr/noul": {
+   "n": 1170,
+   "accuracy": 0.9923,
+   "brier": 0.0113,
+   "nll": 0.023,
+   "ece": 0.0038
+  },
+  "source/wiki2_tr": {
+   "n": 1200,
+   "accuracy": 1.0,
+   "brier": 0.0001,
+   "nll": 0.0006,
+   "ece": 0.0006
+  },
+  "source/wiki2_tr/noul": {
+   "n": 1200,
+   "accuracy": 1.0,
+   "brier": 0.0001,
+   "nll": 0.0006,
+   "ece": 0.0006
+  },
+  "type/noul": {
+   "n": 16243,
+   "accuracy": 0.8931,
+   "brier": 0.1653,
+   "nll": 0.2898,
+   "ece": 0.0426
+  },
+  "workflow/hotpot_tr/all_hops": {
+   "n": 500,
+   "accuracy": 0.97,
+   "brier": 0.0527,
+   "nll": 0.1011,
+   "ece": 0.0126
+  },
+  "workflow/hotpot_tr/distractors_only": {
+   "n": 500,
+   "accuracy": 0.996,
+   "brier": 0.0059,
+   "nll": 0.01,
+   "ece": 0.0065
+  },
+  "workflow/hotpot_tr/missing_hop": {
+   "n": 500,
+   "accuracy": 0.974,
+   "brier": 0.0374,
+   "nll": 0.0704,
+   "ece": 0.0127
+  },
+  "workflow/musique_tr/all_hops": {
+   "n": 250,
+   "accuracy": 0.824,
+   "brier": 0.2596,
+   "nll": 0.4257,
+   "ece": 0.0587
+  },
+  "workflow/musique_tr/missing_hop": {
+   "n": 250,
+   "accuracy": 0.972,
+   "brier": 0.044,
+   "nll": 0.0801,
+   "ece": 0.0199
+  },
+  "workflow/squad_tr/answerability": {
+   "n": 11873,
+   "accuracy": 0.8614,
+   "brier": 0.2146,
+   "nll": 0.3759,
+   "ece": 0.0572
+  },
+  "workflow/synth_tr/synth_answerable": {
+   "n": 352,
+   "accuracy": 0.983,
+   "brier": 0.0226,
+   "nll": 0.0439,
+   "ece": 0.0118
+  },
+  "workflow/synth_tr/synth_eksik_oznitelik": {
+   "n": 127,
+   "accuracy": 0.9921,
+   "brier": 0.0208,
+   "nll": 0.0531,
+   "ece": 0.014
+  },
+  "workflow/synth_tr/synth_kismi": {
+   "n": 232,
+   "accuracy": 0.9957,
+   "brier": 0.0055,
+   "nll": 0.0096,
+   "ece": 0.0073
+  },
+  "workflow/synth_tr/synth_kurum": {
+   "n": 113,
+   "accuracy": 1.0,
+   "brier": 0.0,
+   "nll": 0.0005,
+   "ece": 0.0005
+  },
+  "workflow/synth_tr/synth_kısmi": {
+   "n": 2,
+   "accuracy": 1.0,
+   "brier": 0.0,
+   "nll": 0.0001,
+   "ece": 0.0001
+  },
+  "workflow/synth_tr/synth_varlik": {
+   "n": 115,
+   "accuracy": 0.9913,
+   "brier": 0.0115,
+   "nll": 0.016,
+   "ece": 0.0056
+  },
+  "workflow/synth_tr/synth_varlık": {
+   "n": 3,
+   "accuracy": 1.0,
+   "brier": 0.0,
+   "nll": 0.0002,
+   "ece": 0.0002
+  },
+  "workflow/synth_tr/synth_yer_facet": {
+   "n": 113,
+   "accuracy": 1.0,
+   "brier": 0.0002,
+   "nll": 0.0025,
+   "ece": 0.0025
+  },
+  "workflow/synth_tr/synth_zaman": {
+   "n": 113,
+   "accuracy": 1.0,
+   "brier": 0.0001,
+   "nll": 0.0023,
+   "ece": 0.0023
+  },
+  "workflow/wiki2_tr/other_entity_same_attribute": {
+   "n": 600,
+   "accuracy": 1.0,
+   "brier": 0.0,
+   "nll": 0.0002,
+   "ece": 0.0002
+  },
+  "workflow/wiki2_tr/same_entity": {
+   "n": 600,
+   "accuracy": 1.0,
+   "brier": 0.0003,
+   "nll": 0.001,
+   "ece": 0.0009
+  },
+  "hotpot_tr/answerability": {
+   "n": 1500,
+   "acc_answerable": 0.97,
+   "acc_unanswerable": 0.984,
+   "auroc": 0.997
+  },
+  "hotpot_tr/all_hops/answerability": {
+   "n": 500,
+   "acc_answerable": 0.97,
+   "acc_unanswerable": null
+  },
+  "hotpot_tr/distractors_only/answerability": {
+   "n": 500,
+   "acc_answerable": null,
+   "acc_unanswerable": 0.996
+  },
+  "hotpot_tr/missing_hop/answerability": {
+   "n": 500,
+   "acc_answerable": null,
+   "acc_unanswerable": 0.972
+  },
+  "musique_tr/answerability": {
+   "n": 500,
+   "acc_answerable": 0.832,
+   "acc_unanswerable": 0.972,
+   "auroc": 0.9783
+  },
+  "musique_tr/all_hops/answerability": {
+   "n": 250,
+   "acc_answerable": 0.832,
+   "acc_unanswerable": null
+  },
+  "musique_tr/missing_hop/answerability": {
+   "n": 250,
+   "acc_answerable": null,
+   "acc_unanswerable": 0.972
+  },
+  "squad_tr/answerability": {
+   "n": 11873,
+   "acc_answerable": 0.9239,
+   "acc_unanswerable": 0.7978,
+   "auroc": 0.9346
+  },
+  "squad_tr/answerability/answerability": {
+   "n": 11873,
+   "acc_answerable": 0.9239,
+   "acc_unanswerable": 0.7978,
+   "auroc": 0.9346
+  },
+  "synth_tr/answerability": {
+   "n": 1170,
+   "acc_answerable": 0.983,
+   "acc_unanswerable": 0.9963,
+   "auroc": 0.9996
+  },
+  "synth_tr/synth_answerable/answerability": {
+   "n": 352,
+   "acc_answerable": 0.983,
+   "acc_unanswerable": null
+  },
+  "synth_tr/synth_eksik_oznitelik/answerability": {
+   "n": 127,
+   "acc_answerable": null,
+   "acc_unanswerable": 0.9921
+  },
+  "synth_tr/synth_kismi/answerability": {
+   "n": 232,
+   "acc_answerable": null,
+   "acc_unanswerable": 0.9957
+  },
+  "synth_tr/synth_kurum/answerability": {
+   "n": 113,
+   "acc_answerable": null,
+   "acc_unanswerable": 1.0
+  },
+  "synth_tr/synth_kısmi/answerability": {
+   "n": 2,
+   "acc_answerable": null,
+   "acc_unanswerable": 1.0
+  },
+  "synth_tr/synth_varlik/answerability": {
+   "n": 115,
+   "acc_answerable": null,
+   "acc_unanswerable": 0.9913
+  },
+  "synth_tr/synth_varlık/answerability": {
+   "n": 3,
+   "acc_answerable": null,
+   "acc_unanswerable": 1.0
+  },
+  "synth_tr/synth_yer_facet/answerability": {
+   "n": 113,
+   "acc_answerable": null,
+   "acc_unanswerable": 1.0
+  },
+  "synth_tr/synth_zaman/answerability": {
+   "n": 113,
+   "acc_answerable": null,
+   "acc_unanswerable": 1.0
+  },
+  "wiki2_tr/answerability": {
+   "n": 1200,
+   "acc_answerable": 1.0,
+   "acc_unanswerable": 1.0,
+   "auroc": 1.0
+  },
+  "wiki2_tr/other_entity_same_attribute/answerability": {
+   "n": 600,
+   "acc_answerable": null,
+   "acc_unanswerable": 1.0
+  },
+  "wiki2_tr/same_entity/answerability": {
+   "n": 600,
+   "acc_answerable": 1.0,
+   "acc_unanswerable": null
+  }
+ },
+ "train_key": "dc5d693207f19271"
+}

export.json

@@ -0,0 +1,20 @@
+{
+ "train_key": "dc5d693207f19271",
+ "calibration": {
+  "temperature_by_type": {
+   "noul": 1.097
+  },
+  "temperature": 1.097,
+  "rows": {
+   "choice": 0,
+   "noul": 1500,
+   "score": 0
+  },
+  "nll": {
+   "noul": {
+    "T=1": 0.056,
+    "fitted": 0.0556
+   }
+  }
+ }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 248044,
+  "transformers_version": "5.17.0",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2d6fc070f1b6a8f967388e035f1051c5789e2e1c45e871da81fa2aac20ef614
+size 8411558400

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
+size 19989325

tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}