Preserve Boolean criteria and add pinned accuracy reproduction

README.md

@@ -128,12 +128,48 @@ Julia 1 and Dumont support the same named-question interface: `predict(state=...
 | --- | --- | --- |
 | `choice` | Mapping of 2–20 IDs to nonempty descriptions | `choice`: winning ID |
 | `score` | Ordered list of 2–20 rubric descriptions | `score`: expected zero-based rubric index |
-| `noul` | Omit criteria; false/true order is automatic | `noul`: probability of true |
+| `noul` | Optional mapping of `false` and `true` to descriptions; omitted criteria use literal false/true | `noul`: probability of true |
 
 Each named answer includes `type` and `probabilities`, keyed by caller IDs for choices, zero-based strings for scores, or `false`/`true` for Boolean decisions. Choice and score also include `max_probability`. These are full softmax probabilities, without display rounding; they are not guaranteed certainty. Questions are independently scored in a batch.
 
 Julia 1's native 2–20 option limit still applies. The runtime defaults to the checkpoint’s **8,192-token** combined state/question/options limit. The example uses a 512-token question-and-options budget; each option has a 48-token limit. Strict encoding rejects overflow. Historical accuracy benchmarks above used 1,024 tokens. An [8,192-token CPU smoke test](metrics/context-8k-smoke.json) passed; 8k task accuracy is not established.
 
+### Reproduce typed-decision accuracy
+
+```sh
+python -m pip install -e '.[benchmark]'
+python scripts/reproduce_typed.py --output benchmark/typed-cpu --literal-noul-ablation
+```
+
+Run from this model repository with the real weights downloaded. Use a fresh output
+folder. The script verifies the published weight hash, downloads the pinned test
+Parquet and checks its SHA-256, then writes predictions and CPU results for all
+400 cases / 2,000 questions. `--dataset PATH` reuses a local copy. The optional
+ablation also evaluates Boolean questions with literal false/true descriptions.
+
+Dataset revision: `c76749ec58bd8c3d2ea706b31c333a9059c38f90`.
+Test Parquet SHA-256: `4f294f218ea1da27f3efef936359389c62ea4d3973a41457732990f1d31b647c`.
+
+Preserve the dataset's `criteria` descriptions, including Boolean descriptions in
+false/true order. Only questions without Boolean criteria use literal `false` and
+`true`. Accuracy uses the highest-probability option for all three types; do not
+round the score API's expected index. No threshold is fitted on the test data.
+
+400 of the 600 Boolean questions have descriptive criteria. Earlier versions of
+the named-question API incorrectly discarded those descriptions; the low-level
+benchmark retained them. The API now preserves supplied descriptions, while
+questions without criteria keep their existing literal false/true behavior.
+
+The historical report used CUDA BF16; this harness runs CPU FP32 and records its
+own results. It does not regenerate the unrelated classification or MASSIVE metrics.
+
+The September 26 CPU FP32 reproduction (torch 2.14.0, transformers 5.0.0,
+marker-only head disabled) gives **426/600 choice, 542/800 score, 483/600 noul**.
+The historical CUDA BF16 totals above remain separate; the smaller CPU/GPU
+prediction differences have not been isolated to a single cause. Replacing only
+the Boolean descriptions with literal false/true reproduces **391/600 noul** on
+the same CPU run. See [the paired CPU results](metrics/typed-cpu-20260926.json).
+
 ### Existing list API
 
 The existing interface remains supported:

julia/typed.py

@@ -18,7 +18,14 @@ def predict_typed(engine, state, questions):
         elif kind=='score':
             if not isinstance(criteria,list):raise ValueError('Score requires an ordered rubric')
             labels=criteria;keys=[str(i) for i in range(len(labels))]
-        elif kind=='noul':keys=['false','true'];labels=['false','true']
+        elif kind=='noul':
+            keys=['false','true']
+            if criteria is None:
+                labels=list(keys)
+            else:
+                if not isinstance(criteria,dict) or set(criteria)!=set(keys):
+                    raise ValueError('Noul criteria must map false and true to descriptions')
+                labels=[criteria[key] for key in keys]
         else:raise ValueError('Unsupported question type')
         row=dict(state=state,question=q.get('instructions'),type=kind,options=labels)
         validate_row(row,len(rows)+1)

metrics/typed-cpu-20260926.json

@@ -0,0 +1,47 @@
+{
+  "weights_sha256": "df853bf7fe424420011f3d0c47a05d7341aa9eefa7fb9f203ea4aada4ad95b72",
+  "dataset_revision": "c76749ec58bd8c3d2ea706b31c333a9059c38f90",
+  "dataset_sha256": "4f294f218ea1da27f3efef936359389c62ea4d3973a41457732990f1d31b647c",
+  "torch": "2.14.0+cpu",
+  "transformers": "5.0.0",
+  "device": "cpu",
+  "threads": 4,
+  "marker_only_head": false,
+  "encoding": {
+    "max_length": 1024,
+    "head_length": 512,
+    "strict": true
+  },
+  "original_criteria": {
+    "by_type": {
+      "choice": {
+        "count": 600,
+        "correct": 426,
+        "accuracy": 0.71
+      },
+      "noul": {
+        "count": 600,
+        "correct": 483,
+        "accuracy": 0.805
+      },
+      "score": {
+        "count": 800,
+        "correct": 542,
+        "accuracy": 0.6775
+      }
+    },
+    "elapsed_seconds": 601.7335131389846
+  },
+  "literal_noul_ablation": {
+    "by_type": {
+      "noul": {
+        "count": 600,
+        "correct": 391,
+        "accuracy": 0.6516666666666666
+      }
+    },
+    "elapsed_seconds": 170.3590434489888
+  },
+  "executed_probe_sha256": "ac274e1eff64d96d377a53ecbca2c9676b9197c78abb84744fef84c65f0d6ad1",
+  "note": "Same verified weights, pinned data and CPU FP32 settings for both passes; only noul option descriptions change. Historical CUDA BF16 totals remain separate."
+}

pyproject.toml

@@ -13,6 +13,7 @@ include = ["julia*"]
 exclude = ["julia.router.tests*"]
 
 [project.optional-dependencies]
+benchmark = ["pyarrow>=15"]
 cuda = ["bitsandbytes>=0.48,<0.50", "accelerate>=1.10,<2"]
 
 [tool.setuptools.package-data]

scripts/reproduce_typed.py

@@ -0,0 +1,111 @@
+"""Reproduce Julia-1 typed decisions; optionally ablate Boolean descriptions.
+Requires torch, transformers 5.0.x, safetensors, numpy and pyarrow.
+"""
+import argparse
+import collections
+import hashlib
+import json
+from pathlib import Path
+import sys
+import time
+import urllib.request
+import shutil
+
+REVISION = 'c76749ec58bd8c3d2ea706b31c333a9059c38f90'
+DATA_SHA256 = '4f294f218ea1da27f3efef936359389c62ea4d3973a41457732990f1d31b647c'
+WEIGHTS_SHA256 = 'df853bf7fe424420011f3d0c47a05d7341aa9eefa7fb9f203ea4aada4ad95b72'
+
+def digest(path):
+    with Path(path).open('rb') as stream:
+        return hashlib.file_digest(stream, 'sha256').hexdigest()
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('--checkpoint', type=Path, default=Path(__file__).resolve().parents[1])
+    ap.add_argument('--dataset', type=Path, help='Existing pinned Parquet; otherwise download and verify it')
+    ap.add_argument('--output', type=Path, required=True)
+    ap.add_argument('--threads', type=int, default=4)
+    ap.add_argument('--literal-noul-ablation', action='store_true')
+    args = ap.parse_args()
+    checkpoint = args.checkpoint.resolve()
+    if digest(checkpoint / 'model.safetensors') != WEIGHTS_SHA256:
+        raise ValueError('Weights do not match accuracy-20260924.json')
+    args.output.mkdir(parents=True, exist_ok=False)
+    if args.dataset is None:
+        args.dataset = args.output / 'test-00000-of-00001.parquet'
+        url = (f'https://huggingface.co/datasets/LocalLLaMA/typed-decisions/resolve/{REVISION}'
+               '/all/test-00000-of-00001.parquet')
+        temporary = args.dataset.with_suffix('.partial')
+        with urllib.request.urlopen(url, timeout=180) as source, temporary.open('wb') as target:
+            shutil.copyfileobj(source, target)
+        temporary.replace(args.dataset)
+    if digest(args.dataset) != DATA_SHA256:
+        raise ValueError('Dataset does not match the pinned published evaluation')
+    sys.path.insert(0, str(checkpoint))
+    import pyarrow.parquet as pq
+    import torch
+    import transformers
+    from julia.router.engine import FastEngine
+    torch.set_num_threads(args.threads)
+    cases = pq.read_table(args.dataset).to_pylist()
+    rows, metadata = [], []
+    for case in cases:
+        state = json.loads(case['state'])
+        gold = json.loads(case['gold'])
+        for question_id, question in json.loads(case['questions']).items():
+            kind = question['type']
+            criteria = question.get('criteria')
+            if criteria is None and kind == 'noul':
+                criteria = {'false': 'false', 'true': 'true'}
+            if isinstance(criteria, list):
+                criteria = {str(i): value for i, value in enumerate(criteria)}
+            if not isinstance(criteria, dict):
+                raise ValueError('Missing option descriptions')
+            keys = ['false', 'true'] if kind == 'noul' else list(criteria)
+            rows.append(dict(state=state, question=question['instructions'],
+                             type=kind, options=[criteria[key] for key in keys]))
+            metadata.append(dict(id=case['id']+':'+question_id, type=kind,
+                                 keys=keys, gold=str(gold[question_id]['label'])))
+    counts = collections.Counter(row['type'] for row in rows)
+    assert len(cases) == 400 and counts == {'choice': 600, 'score': 800, 'noul': 600}, counts
+    engine = FastEngine(checkpoint, device='cpu', transformer_backend='torch',
+                        strict_encoding=True, max_length=1024, head_length=512,
+                        batch_size=16, marker_only_head=False)
+    result = dict(weights_sha256=WEIGHTS_SHA256, dataset_revision=REVISION,
+                  dataset_sha256=DATA_SHA256, torch=torch.__version__,
+                  transformers=transformers.__version__, device='cpu',
+                  threads=args.threads, marker_only_head=False,
+                  encoding=dict(max_length=1024, head_length=512, strict=True),
+                  runtime_sha256={str(p.relative_to(checkpoint)): digest(p)
+                                  for p in sorted((checkpoint/'julia').rglob('*.py'))})
+    def evaluate(name, input_rows, labels):
+        started = time.monotonic()
+        stats = collections.defaultdict(lambda: dict(count=0, correct=0))
+        with (args.output / (name+'-predictions.jsonl')).open('w') as stream:
+            # One example per forward matches the original benchmark protocol.
+            for index, (row, label) in enumerate(zip(input_rows, labels), 1):
+                answer = engine.predict([row])[0]
+                prediction = label['keys'][answer['index']]
+                correct = prediction == label['gold']
+                stats[label['type']]['count'] += 1
+                stats[label['type']]['correct'] += int(correct)
+                stream.write(json.dumps(dict(**label, prediction=prediction, correct=correct,
+                                             options=row['options'], probabilities=answer['probabilities']))+'\n')
+                if index % 200 == 0:
+                    print(json.dumps(dict(mode=name, done=index, total=len(input_rows))), flush=True)
+        for item in stats.values():
+            item['accuracy'] = item['correct']/item['count']
+        summary = dict(by_type=dict(stats), elapsed_seconds=time.monotonic()-started)
+        print(json.dumps(dict(mode=name, **summary)), flush=True)
+        return summary
+    result['original_criteria'] = evaluate('original-criteria', rows, metadata)
+    if args.literal_noul_ablation:
+        selected = [(dict(row, options=['false', 'true']), label)
+                    for row, label in zip(rows, metadata) if row['type']=='noul']
+        result['literal_noul_ablation'] = evaluate('literal-noul',
+                                                 [x[0] for x in selected], [x[1] for x in selected])
+    (args.output/'results.json').write_text(json.dumps(result, indent=2)+'\n')
+    print(json.dumps(result, indent=2), flush=True)
+
+if __name__ == '__main__':
+    main()

tests/test_typed_api.py

@@ -25,3 +25,24 @@ class TypedAPITests(unittest.TestCase):
         self.engine.logits.return_value=[[1.,0.]]
         q={'q':dict(type='choice',instructions='q',criteria={'a':'A','b':'B'})}
         self.assertEqual(self.engine.predict('s',q),self.engine.predict(state='s',questions=q))
+
+    def test_noul_preserves_descriptions_in_false_true_order(self):
+        self.engine.logits.return_value=[[0.,2.]]
+        questions={'review':dict(type='noul',instructions='Needs human review?',
+            criteria={'true':'A human should inspect this run.',
+                      'false':'No human attention is warranted.'})}
+        answer=self.engine.predict(state='context',questions=questions)['answers']['review']
+        self.assertEqual(self.engine.logits.call_args.args[0][0]['options'],
+                         ['No human attention is warranted.','A human should inspect this run.'])
+        self.assertEqual(list(answer['probabilities']),['false','true'])
+        self.assertGreater(answer['noul'],.5)
+
+    def test_noul_rejects_invalid_criteria_before_inference(self):
+        self.engine.logits.return_value=[[0.,2.]]
+        for criteria in ({}, {'true':'yes'}, {'false':'no','true':'yes','other':'maybe'},
+                         ['no','yes'], {'false':'','true':'yes'}):
+            with self.subTest(criteria=criteria):
+                questions={'q':dict(type='noul',instructions='q',criteria=criteria)}
+                with self.assertRaises(ValueError):
+                    self.engine.predict(state='context',questions=questions)
+        self.engine.logits.assert_not_called()