Release Jev-Style v2 with calibrated decision inference and evaluation records

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Jev-Style-v2-Q8_0-Calibrated.gguf filter=lfs diff=lfs merge=lfs -text

Jev-Style-v2-Q8_0-Calibrated.calibration.json

@@ -0,0 +1,49 @@
+{
+  "temperature": 1.0,
+  "fitted_temperature_folded": 1.0403540135054734,
+  "temperature_folded": true,
+  "folded_tensor": "output_norm.weight",
+  "source_calibration": {
+    "temperature": 1.0403540135054734,
+    "calibration_n": 3100,
+    "objective": "sample_mean_soft_cross_entropy",
+    "bounds": [
+      0.05,
+      20
+    ],
+    "nll_before": 0.5142612871187658,
+    "nll_after": 0.5139652439657095,
+    "backend": "gguf",
+    "model": "results/Jev-Style-v2-Q8_0.gguf",
+    "source_sha256": "2fde7f45dce3440abfde145bb30ad61e2643b1f853866b5760b235685328dc1c"
+  },
+  "input_sha256": "f659d164d0fed4e645f711cbf56c177ee63c1856e6875f48a27ebac2cfb11175",
+  "output_sha256": "5c2aa0d35b24a27f03228b2c62ebaaebd9b5b785844634d4217278d822751494",
+  "validation_required": false,
+  "validation": {
+    "n": 500,
+    "argmax_agreement": 0.992,
+    "cuda_same_subset": {
+      "accuracy": 0.7910177949703642,
+      "macro_f1": 0.7760857891780776,
+      "nll": 0.6000106706289864,
+      "brier": 0.317876961372947,
+      "ece": 0.16696329399611096
+    },
+    "q8_same_subset": {
+      "accuracy": 0.7868855635654054,
+      "macro_f1": 0.773615445189482,
+      "nll": 0.6015417570028033,
+      "brier": 0.318930434743987,
+      "ece": 0.16630794459650552
+    },
+    "accuracy_difference": -0.004132231404958775,
+    "nll_difference": 0.0015310863738169367,
+    "temperature_folded": true,
+    "passed": true,
+    "practical_deployment_gate_passed": true,
+    "initial_strict_accuracy_target_met_on_subset": false,
+    "initial_accuracy_loss_target": 0.003,
+    "note": "500-example subset check: 99.2% agreement. Observed task-macro loss is 0.413 pp, so the initial 0.3 pp accuracy goal is not met on this subset. Prefer BF16/MLX when accuracy is the priority; this is not a bound on population degradation."
+  }
+}

Jev-Style-v2-Q8_0-Calibrated.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c2aa0d35b24a27f03228b2c62ebaaebd9b5b785844634d4217278d822751494
+size 2012004256

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Alibaba Cloud
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,118 @@
+---
+license: apache-2.0
+language:
+- en
+library_name: gguf
+pipeline_tag: text-generation
+base_model: chaoliangUNSW/Jev-Style-Qwen3.5-2B-Decision-v2
+base_model_relation: quantized
+tags:
+- decision-model
+- classification
+- calibration
+- qwen3.5
+- jev-style
+- single-prefill
+---
+# Jev-Style-Qwen3.5-2B-Decision v2 · GGUF Q8_0
+
+A compact decision model for classification, routing and typed choices. Give it a state, a question and a list of options; receive a selected option and calibrated probabilities in one prefill.
+
+[HF BF16](https://huggingface.co/chaoliangUNSW/Jev-Style-Qwen3.5-2B-Decision-v2) | [GGUF Q8_0](https://huggingface.co/chaoliangUNSW/Jev-Style-Qwen3.5-2B-Decision-v2-GGUF) | [MLX BF16](https://huggingface.co/chaoliangUNSW/Jev-Style-Qwen3.5-2B-Decision-v2-MLX-bf16)
+
+## This release: calibrated Q8_0 GGUF
+
+- **2.01 GB** model file, approximately **46.5% smaller** than the BF16 GGUF export.
+- **99.2% choice agreement** with merged CUDA BF16 on the frozen 500-decision deployment subset.
+- Temperature fitted on 3,100 calibration records and already folded into the final normalization tensor.
+- Ready for exact-option inference through llama.cpp, with a Python client and native C++ logit reader included.
+
+The deployment subset's real-label task-macro accuracy is **78.69%** for Q8 and **79.10%** for CUDA BF16 on those same cases. The main evaluation table below describes the CUDA reference; the 500-case deployment subset is a separate measurement.
+
+### Download and run
+
+```bash
+python -m pip install -U huggingface_hub
+hf download chaoliangUNSW/Jev-Style-Qwen3.5-2B-Decision-v2-GGUF --local-dir jev-v2-gguf
+cd jev-v2-gguf
+llama-server -m Jev-Style-v2-Q8_0-Calibrated.gguf -c 2048 -ngl 99 --port 8080
+```
+
+In another terminal, from the same directory:
+
+```bash
+python jev_decision_client.py --url http://127.0.0.1:8080 \
+  --state "The film was excellent." \
+  --question "What is the sentiment of this review?" \
+  --options negative positive
+```
+
+Use a llama.cpp build with Qwen3.5 support. Conversion and native evaluation used commit `b29c606e28a01b1bc8c1351026a0fa6e616bf6c4`. The client uses the native `/completion` endpoint, requests complete declared-option log-probabilities and increases the candidate count as needed. The supplied `gguf_logits.cpp` reads all declared-option logits directly through the C API.
+
+**Runtime calibration temperature is 1.0** for this file: its fitted temperature has already been incorporated. The accompanying calibration JSON records the exact settings and checksum. Serve the raw decision prompt shown below, with the full declared option list.
+
+## Highlights
+
+- **81.27% macro accuracy** for the released merged BF16 model across 11 real-label task groups (3,277 decisions).
+- **9 of 12 task-group accuracy point estimates ahead of English Laya** in the fixed CUDA reference comparison.
+- **+4.53 percentage points over Jev-Style v1** and **+6.12 points over English Laya** in reference macro accuracy on the same evaluation panel.
+- **18.4% lower NLL and 20.0% lower Brier score** than English Laya in the reference comparison.
+- **6.0% option-permutation flip rate**, compared with 9.25% for v1 and 12.0% for English Laya, on 400 Choice/Bool decisions.
+- **One H100 80GB, 36.9 minutes of main training**, with a 2B-class text backbone and rank-32 LoRA.
+
+The comparison uses the frozen English task panel and the CUDA reference structure. Deployment variants are measured separately below. The 9/12 count describes task-level point estimates.
+
+## Reference evaluation
+
+Real-label results are macro-averaged with equal task weights. All three models use the same calibration records and global temperature-fitting objective.
+
+| Metric | Jev-Style v1 | English Laya | Jev-Style v2 reference |
+|---|---:|---:|---:|
+| Accuracy ↑ | 76.68% | 75.09% | **81.20%** |
+| Macro-F1 ↑ | 75.42% | 73.45% | **79.78%** |
+| NLL ↓ | 0.5752 | 0.6318 | **0.5154** |
+| Brier ↓ | 0.3290 | 0.3482 | **0.2787** |
+
+Accuracy improvements have paired 95% intervals of **+3.58 to +5.52 points vs v1** and **+4.64 to +7.52 points vs English Laya** within this frozen task panel.
+
+The panel covers sentiment, news, natural-language inference, question answering, emotion and email classification. The twelfth task group contains 2,000 teacher-reference typed decisions from 400 states and is reported separately from the real-label macro. Per-task results, all probability metrics, robustness measurements and baseline sensitivity results are supplied in the evaluation files.
+
+## Decision interface
+
+Provide an English state, a question, and **2–26 unique options**, within a **1,024-token prompt**. A single prefill produces one logit per declared option. Apply the supplied calibration once and normalize over those options to obtain the decision probabilities.
+
+```text
+You are a decision function. Read the state, then answer the question by choosing exactly one option.
+
+[State]
+The film was excellent.
+
+[Question]
+What is the sentiment of this review?
+
+[Options]
+A. negative
+B. positive
+
+Answer:
+```
+
+The supplied clients implement this exact prompt and read the next-position ` A` through ` Z` token scores. Use this decision interface for Choice, Bool and ordered Score tasks; `decide_bool` returns the probability of yes, and `decide_score` also returns the expected zero-based level. For ordered scores, supply options from lowest to highest.
+
+## Training
+
+Continued from the uncalibrated Jev-Style v1 text backbone derived from [Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base). Training used a BF16 backbone, FP32 rank-32 LoRA (alpha 32), 186 adapted modules and 33,638,400 trainable parameters. Effective batch size was 64 with a 1,024-token budget.
+
+The 60,000-record training pool combines original-task replay with emotion, email, typed workflow decisions, label transformations and programmatic threshold rules. A two-stage schedule increases hard-example sampling while retaining approximately 50% original-task replay. Main training completed 1,000 optimizer updates and processed 11,605,632 tokens in 36.9 minutes on one H100 80GB.
+
+Development (2,050 records), calibration (3,100 records) and final evaluation (5,277 decisions) were handled separately. Checkpoint selection used development results; calibration used the calibration split. This release records one training seed. The evaluation JSON files document the dataset, rendering and deployment protocols.
+
+## License and attribution
+
+Apache-2.0. See [LICENSE](LICENSE). This release builds on Qwen3.5-2B-Base and Jev-Style v1. Training data retain their original source licenses; source and split details are recorded in the data manifest. The release contains model artifacts and aggregate evaluation records.
+
+## Contact
+
+I welcome internship, employment, and research collaboration opportunities. Please contact me at [**yanhcaoliang369@gmail.com**](mailto:yanhcaoliang369@gmail.com).
+
+欢迎提供实习、工作及科研合作机会，请邮件联系：[yanhcaoliang369@gmail.com](mailto:yanhcaoliang369@gmail.com)。

SHA256SUMS.json

@@ -0,0 +1,58 @@
+{
+  "Jev-Style-v2-Q8_0-Calibrated.calibration.json": {
+    "bytes": 1807,
+    "sha256": "7f3ae96ebf0271483365b8941124f3be9521e9e3ed4c3d62240e51c9b416ebb4"
+  },
+  "Jev-Style-v2-Q8_0-Calibrated.gguf": {
+    "bytes": 2012004256,
+    "sha256": "5c2aa0d35b24a27f03228b2c62ebaaebd9b5b785844634d4217278d822751494"
+  },
+  "LICENSE": {
+    "bytes": 11343,
+    "sha256": "50cbab8a892c5f2993b8c7351a99182507472def3b1374558308605d99b86b32"
+  },
+  "README.md": {
+    "bytes": 7065,
+    "sha256": "09d5aa25952f46b1a2f65702fc94898af22fe4a9734a541ca8ac45b479a906cd"
+  },
+  "evaluation/baseline_sensitivity.json": {
+    "bytes": 2015,
+    "sha256": "aadc4566710a397044edbfb9c353ee2e0baf14a6deb5e30f4234eabd66294026"
+  },
+  "evaluation/data_manifest.json": {
+    "bytes": 10680,
+    "sha256": "8d03d4bba9400ffe062b248733213a6e67a3d6c187340e9aa796c319f2feef23"
+  },
+  "evaluation/deployment.json": {
+    "bytes": 963,
+    "sha256": "a49d48a3c5cd3173dfc6badb11d33a3e22120e9a559b8e4ccbb2bd91653bc757"
+  },
+  "evaluation/http_smoke.json": {
+    "bytes": 542,
+    "sha256": "b33c0ea9f7a6cf49bba573b98dcf966c1dbf033ef21e6b9a92c8bc217a46a9f3"
+  },
+  "evaluation/laya_source.json": {
+    "bytes": 97,
+    "sha256": "b12ecce8dca6afad9e43c31c2385be255514d78856009ef9a6222bc955ca0757"
+  },
+  "evaluation/laya_typed_source.json": {
+    "bytes": 113,
+    "sha256": "c2c2bc4931bb489636c5471f25f2fa31c3baca347c0465f119b8039aac1d3619"
+  },
+  "evaluation/reference_comparison.json": {
+    "bytes": 27042,
+    "sha256": "36f56a490fc67e774cbb25b38d4dc9b5e3a42e1f65fcd748af092e127a98eff8"
+  },
+  "gguf_logits.cpp": {
+    "bytes": 3001,
+    "sha256": "a29f2dbd608f6448a6d440995baed130c8ae35c1a2bed0e653f5da0d32b6b067"
+  },
+  "jev_decision_client.py": {
+    "bytes": 4113,
+    "sha256": "7d02a340b80326ceefc6d20378bdf7c05cb5087b8ee243394eebd4c43eb47b36"
+  },
+  "requirements.txt": {
+    "bytes": 64,
+    "sha256": "7265d12f38635b9f592b0e2d62b1cc0a9788ebaa2f37d84eea2212ae16952d7c"
+  }
+}

evaluation/baseline_sensitivity.json

@@ -0,0 +1,54 @@
+{
+  "protocol": "Post-run MPS sensitivity audit, separate from the original same-CUDA main comparison",
+  "selection": {
+    "selected_rendering": "semantic",
+    "dev_metrics": {
+      "neutral": {
+        "accuracy": 0.7714285714285715,
+        "macro_f1": 0.7459788544523509,
+        "nll": 0.6996133521392657,
+        "brier": 0.3261039060421592,
+        "ece": 0.12216485276160671
+      },
+      "semantic": {
+        "accuracy": 0.7838095238095237,
+        "macro_f1": 0.7599396892684371,
+        "nll": 0.718921979422934,
+        "brier": 0.3180294403591556,
+        "ece": 0.12064957876186524
+      }
+    },
+    "criterion": "highest real-label dev macro accuracy, then lower dev NLL",
+    "device": "mps",
+    "scope": "Post-run sensitivity audit of Laya choice-key formatting, not a new training experiment. No test-based rendering selection."
+  },
+  "english_semantic_real_macro": {
+    "accuracy": 0.7484148342632099,
+    "macro_f1": 0.7329973270680202,
+    "nll": 0.6309314045108828,
+    "brier": 0.3475500737329656,
+    "ece": 0.12349987305363984
+  },
+  "v2_point_wins_vs_english_semantic": 10,
+  "typed_native_teacher_metrics": {
+    "accuracy": 0.7475,
+    "macro_f1": 0.6203131476624142,
+    "nll": 0.8965023905846662,
+    "brier": 0.06935947732109757,
+    "ece": 0.17598656338286872
+  },
+  "v2_typed_teacher_metrics": {
+    "accuracy": 0.7345,
+    "macro_f1": 0.5843823268841445,
+    "nll": 0.9071323454613555,
+    "brier": 0.07585474596137866,
+    "ece": 0.13429560744677668
+  },
+  "native_typed_accuracy_gap_pp": -1.3000000000000012,
+  "caveats": [
+    "Development rendering selection used real-label macro accuracy, not test results.",
+    "This is a device/rendering sensitivity check; do not merge it into the original same-GPU experiment.",
+    "The typed checkpoint has upstream training exposure to the public train split from which these calibration examples were drawn.",
+    "Teacher agreement is not real-world correctness; probability matching is described by soft NLL/Brier."
+  ]
+}

evaluation/data_manifest.json

@@ -0,0 +1,317 @@
+{
+  "version": "h100-v2-20260923",
+  "seed": 20260923,
+  "max_length": 1024,
+  "sources": [
+    {
+      "url": "https://huggingface.co/datasets/SetFit/enron_spam/resolve/1916f66c89d52221ae33eb57d44498b4f3a5df22/train.jsonl",
+      "path": "source_project/h100_v2/data/sources/enron/train.jsonl",
+      "sha256": "b846cf15bcad7676c37ea3535622e39e6bff5c9c004f634513e4f9c2951e5f02",
+      "bytes": 101069043
+    },
+    {
+      "url": "https://huggingface.co/datasets/SetFit/enron_spam/resolve/1916f66c89d52221ae33eb57d44498b4f3a5df22/test.jsonl",
+      "path": "source_project/h100_v2/data/sources/enron/test.jsonl",
+      "sha256": "f07a4cca7bfc1d845d4ff59650e69a0ad31d9b5c0409ddad088219f7fc32bb52",
+      "bytes": 6273613
+    },
+    {
+      "url": "https://huggingface.co/datasets/LocalLLaMA/typed-decisions/resolve/c76749ec58bd8c3d2ea706b31c333a9059c38f90/all/train-00000-of-00001.parquet",
+      "path": "source_project/h100_v2/data/sources/typed/train-00000-of-00001.parquet",
+      "sha256": "46a58d63edfd86e23229c78afe8b72307bb4ca9fb0e8df180cabb3c67ec9dcd5",
+      "bytes": 598824
+    },
+    {
+      "url": "https://huggingface.co/datasets/LocalLLaMA/typed-decisions/resolve/c76749ec58bd8c3d2ea706b31c333a9059c38f90/all/test-00000-of-00001.parquet",
+      "path": "source_project/h100_v2/data/sources/typed/test-00000-of-00001.parquet",
+      "sha256": "4f294f218ea1da27f3efef936359389c62ea4d3973a41457732990f1d31b647c",
+      "bytes": 222140
+    },
+    {
+      "url": "https://huggingface.co/datasets/stanfordnlp/imdb/resolve/e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/test-00000-of-00001.parquet",
+      "path": "source_project/h100_v2/data/sources/imdb/test-00000-of-00001.parquet",
+      "sha256": "b52e26e2f872d282ffac460bf9770b25ac6f102cda0e6ca7158df98c94e8b3da",
+      "bytes": 20470363
+    },
+    {
+      "url": "https://huggingface.co/datasets/facebook/anli/resolve/8e4813d81f46d313dac7892e1c28076917cfcdf9/plain_text/test_r1-00000-of-00001.parquet",
+      "path": "source_project/h100_v2/data/sources/anli/test_r1-00000-of-00001.parquet",
+      "sha256": "c4a3d304c4671941d6bad5a07632a79713c5a1be485ccf75b81b6df93f61045e",
+      "bytes": 353376
+    },
+    {
+      "url": "https://huggingface.co/datasets/facebook/anli/resolve/8e4813d81f46d313dac7892e1c28076917cfcdf9/plain_text/test_r2-00000-of-00001.parquet",
+      "path": "source_project/h100_v2/data/sources/anli/test_r2-00000-of-00001.parquet",
+      "sha256": "df5daccdd5623cfcaa34be0100721783485f4181a42796b1d0ac0cd7601e7acb",
+      "bytes": 361549
+    },
+    {
+      "url": "https://huggingface.co/datasets/facebook/anli/resolve/8e4813d81f46d313dac7892e1c28076917cfcdf9/plain_text/test_r3-00000-of-00001.parquet",
+      "path": "source_project/h100_v2/data/sources/anli/test_r3-00000-of-00001.parquet",
+      "sha256": "3232c4217979da00b2cd6ed97d099a8a8edf04530193ea52e3c8d69190de92a2",
+      "bytes": 434550
+    },
+    {
+      "url": "https://raw.githubusercontent.com/tommccoy1/hans/7299f6f657089ce06a0f98e7e81f8d0f5b7741ce/heuristics_evaluation_set.txt",
+      "path": "source_project/h100_v2/data/sources/hans/evaluation.tsv",
+      "sha256": "c55b62feef9913070e88f38938dc2492018c945ac81f70139346472494124e79",
+      "bytes": 15462062
+    },
+    {
+      "path": "source_project/data/calib.jsonl",
+      "sha256": "42097c0b0583e7ae6976d8b882374e9f5d9c0ad517d96866cf2f18fc39b21a6a",
+      "purpose": "exclude_v1_calibration_and_ood_from_training"
+    },
+    {
+      "path": "source_project/runs/b2_x4/holdout.jsonl",
+      "sha256": "e6e2e5dcc126565ffc88dbf589123522bf3c6b98568d750f00717ee29262db5f",
+      "purpose": "exclude_v1_calibration_and_ood_from_training"
+    },
+    {
+      "path": "source_project/data/eval_ood.jsonl",
+      "sha256": "0d8e6c82aa7f5701264e1af8423762f1e0bc93593ed47341a1fe0b5de8eba637",
+      "purpose": "exclude_v1_calibration_and_ood_from_training"
+    },
+    {
+      "path": "source_cache/huggingface/datasets/nyu-mll___glue/sst2/0.0.0/bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c/glue-train.arrow",
+      "sha256": "47387d613631ae60c690f2dd67b8f80d0c39e36d9080ca438eb0f18e1ca4cb4a",
+      "task": "sst2"
+    },
+    {
+      "path": "source_cache/huggingface/datasets/nyu-mll___glue/mnli/0.0.0/bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c/glue-train.arrow",
+      "sha256": "c53e4467acd3becf55c0f898391de3b453df8f0f4c98cb5e8cc8c4c706ecebca",
+      "task": "mnli"
+    },
+    {
+      "path": "source_cache/huggingface/datasets/fancyzhx___ag_news/default/0.0.0/eb185aade064a813bc0b7f42de02595523103ca4/ag_news-train.arrow",
+      "sha256": "b3eec28c0aed7d6616f047cf5148d94aba4f475f3b905b3dc9d331fa3435d16d",
+      "task": "ag_news"
+    },
+    {
+      "path": "source_cache/huggingface/datasets/google___boolq/default/0.0.0/35b264d03638db9f4ce671b711558bf7ff0f80d5/boolq-train.arrow",
+      "sha256": "297dd7c22cfaf5bbdde0467a2ec2eecab261118e8ee34f375a3326fc4ea6486e",
+      "task": "boolq"
+    },
+    {
+      "path": "source_cache/huggingface/datasets/SetFit___sst5/default/0.0.0/e51bdcd8cd3a30da231967c1a249ba59361279a3/sst5-train.arrow",
+      "sha256": "c3221ca614196a083a72efecd569148782022d434999bbd4a1da73d52e2a64b3",
+      "task": "sst5"
+    },
+    {
+      "path": "source_cache/huggingface/datasets/dair-ai___emotion/split/0.0.0/cab853a1dbdf4c42c2b3ef2173804746df8825fe/emotion-train.arrow",
+      "sha256": "d362b3fb7dabba0613533182e3e0dce2340972e5f90d9ee02783de28bdf8e095"
+    },
+    {
+      "path": "source_cache/huggingface/datasets/dair-ai___emotion/split/0.0.0/cab853a1dbdf4c42c2b3ef2173804746df8825fe/emotion-test.arrow",
+      "sha256": "3b039c3c9f5b0dd907e302b9e2e0c71278d66865883a3de4509841154aeaa115"
+    }
+  ],
+  "group_overlap_counts": {
+    "train__dev": 0,
+    "train__cal": 0,
+    "train__test": 0,
+    "train__counterfactual_test": 0,
+    "dev__cal": 0,
+    "dev__test": 0,
+    "dev__counterfactual_test": 0,
+    "cal__test": 0,
+    "cal__counterfactual_test": 0,
+    "test__counterfactual_test": 0
+  },
+  "sst_subphrases_excluded": 7213,
+  "length_audit": {
+    "legacy_test": {
+      "read": 1777,
+      "eligible": 1777,
+      "too_long": 0
+    },
+    "sst2": {
+      "read": 60000,
+      "eligible": 52933,
+      "too_long": 0
+    },
+    "mnli": {
+      "read": 60000,
+      "eligible": 59997,
+      "too_long": 0
+    },
+    "ag_news": {
+      "read": 60000,
+      "eligible": 59953,
+      "too_long": 0
+    },
+    "boolq": {
+      "read": 9427,
+      "eligible": 9427,
+      "too_long": 0
+    },
+    "sst5": {
+      "read": 8544,
+      "eligible": 8533,
+      "too_long": 0
+    },
+    "emotion_train": {
+      "read": 16000,
+      "eligible": 15969,
+      "too_long": 0
+    },
+    "emotion_test": {
+      "read": 2000,
+      "eligible": 2000,
+      "too_long": 0
+    },
+    "enron_train": {
+      "read": 31665,
+      "eligible": 26225,
+      "too_long": 2532
+    },
+    "enron_test": {
+      "read": 2000,
+      "eligible": 1830,
+      "too_long": 152
+    },
+    "imdb": {
+      "read": 25000,
+      "eligible": 24250,
+      "too_long": 551
+    },
+    "hans": {
+      "read": 30000,
+      "eligible": 30000,
+      "too_long": 0
+    },
+    "anli": {
+      "read": 3200,
+      "eligible": 3200,
+      "too_long": 0
+    },
+    "final_test": {
+      "read": 5277,
+      "eligible": 5277,
+      "too_long": 0
+    },
+    "typed_train": {
+      "read": 4000,
+      "eligible": 4000,
+      "too_long": 0
+    },
+    "typed_dev": {
+      "read": 1000,
+      "eligible": 1000,
+      "too_long": 0
+    },
+    "typed_cal": {
+      "read": 1000,
+      "eligible": 1000,
+      "too_long": 0
+    },
+    "selected_core": {
+      "read": 30000,
+      "eligible": 30000,
+      "too_long": 0
+    },
+    "selected_new": {
+      "read": 18000,
+      "eligible": 18000,
+      "too_long": 0
+    },
+    "selected_challenge": {
+      "read": 12000,
+      "eligible": 12000,
+      "too_long": 0
+    }
+  },
+  "files": {
+    "train": {
+      "path": "train.jsonl",
+      "n": 60000,
+      "tasks": {
+        "enron_spam": 11812,
+        "sst2": 3000,
+        "emotion": 6000,
+        "boolq": 6000,
+        "ag_news": 6597,
+        "ag_news_verification": 1109,
+        "mnli": 13591,
+        "mnli_verification": 2330,
+        "sst5": 3000,
+        "policy_counterfactual": 2000,
+        "typed_decisions": 4000,
+        "sst2_verification": 561
+      },
+      "sha256": "07e3395d464794f32fdf4790815af9ee76a8d4eeb28f73313cd4427e0434fdef"
+    },
+    "dev": {
+      "path": "dev.jsonl",
+      "n": 2050,
+      "tasks": {
+        "typed_decisions": 1000,
+        "enron_spam": 150,
+        "sst2": 150,
+        "sst5": 150,
+        "emotion": 150,
+        "boolq": 150,
+        "mnli": 150,
+        "ag_news": 150
+      },
+      "sha256": "d46758f4b71eeb4774dc881136c56aa6a3f42ec715e9e5d40040bccbc13904e6"
+    },
+    "cal": {
+      "path": "cal.jsonl",
+      "n": 3100,
+      "tasks": {
+        "sst5": 300,
+        "emotion": 300,
+        "typed_decisions": 1000,
+        "sst2": 300,
+        "enron_spam": 300,
+        "mnli": 300,
+        "ag_news": 300,
+        "boolq": 300
+      },
+      "sha256": "2fde7f45dce3440abfde145bb30ad61e2643b1f853866b5760b235685328dc1c"
+    },
+    "test": {
+      "path": "test.jsonl",
+      "n": 5277,
+      "tasks": {
+        "ag_news": 300,
+        "imdb": 300,
+        "anli": 300,
+        "typed_decisions": 2000,
+        "mnli": 300,
+        "emotion": 300,
+        "rte": 277,
+        "sst5": 300,
+        "hans": 300,
+        "boolq": 300,
+        "enron_spam": 300,
+        "sst2": 300
+      },
+      "sha256": "d5f92e48cf742aecb3c8e40714cb51ddd72fefd176a95dc8b7209523e2daa2bb"
+    },
+    "counterfactual_test": {
+      "path": "counterfactual_test.jsonl",
+      "n": 400,
+      "tasks": {
+        "policy_counterfactual": 400
+      },
+      "sha256": "a4b4a435a7a91f49a57bcaf4cb758f56fb530eed0aeac3166cd0220dbd4401df"
+    }
+  },
+  "caveats": [
+    "This is a new CUDA experiment; prior MLX v2 artifacts remain unchanged.",
+    "Legacy five-task and RTE tests are historical regression sets, not unseen benchmark discovery.",
+    "OOD is relative to this fine-tuning; base pretraining contamination is unknown.",
+    "All source text must fit 1024 tokens; full-length rejection counts are reported, so this is a length-filtered suite.",
+    "Challenge gold rows are lexically selected candidates, not yet mined v1 mistakes.",
+    "Synthetic counterfactual results describe this generator, not general real-world reasoning.",
+    "Typed-decisions targets are teacher distributions and are reported separately.",
+    "Training and calibration source licenses remain upstream; this bundle is for the user's experiment, not a dataset republication."
+  ],
+  "train_pools": {
+    "challenge": 12000,
+    "core": 30000,
+    "new": 18000
+  }
+}

evaluation/deployment.json

@@ -0,0 +1,26 @@
+{
+  "n": 500,
+  "argmax_agreement": 0.992,
+  "cuda_same_subset": {
+    "accuracy": 0.7910177949703642,
+    "macro_f1": 0.7760857891780776,
+    "nll": 0.6000106706289864,
+    "brier": 0.317876961372947,
+    "ece": 0.16696329399611096
+  },
+  "q8_same_subset": {
+    "accuracy": 0.7868855635654054,
+    "macro_f1": 0.773615445189482,
+    "nll": 0.6015417570028033,
+    "brier": 0.318930434743987,
+    "ece": 0.16630794459650552
+  },
+  "accuracy_difference": -0.004132231404958775,
+  "nll_difference": 0.0015310863738169367,
+  "temperature_folded": true,
+  "passed": true,
+  "practical_deployment_gate_passed": true,
+  "initial_strict_accuracy_target_met_on_subset": false,
+  "initial_accuracy_loss_target": 0.003,
+  "note": "500-example subset check: 99.2% agreement. Observed task-macro loss is 0.413 pp, so the initial 0.3 pp accuracy goal is not met on this subset. Prefer BF16/MLX when accuracy is the priority; this is not a bound on population degradation."
+}

evaluation/http_smoke.json

@@ -0,0 +1,24 @@
+{
+  "passed": true,
+  "cases": [
+    {
+      "options": 2,
+      "choice": "positive",
+      "candidate_count": 64,
+      "max_probability_difference_vs_native": 7.650678379711451e-08
+    },
+    {
+      "options": 4,
+      "choice": "science and technology",
+      "candidate_count": 64,
+      "max_probability_difference_vs_native": 3.4161285047654318e-06
+    },
+    {
+      "options": 5,
+      "choice": "neutral",
+      "candidate_count": 64,
+      "max_probability_difference_vs_native": 6.717687878710077e-05
+    }
+  ],
+  "server": ""
+}

evaluation/laya_source.json

@@ -0,0 +1,4 @@
+{
+  "repo": "convaiinnovations/laya",
+  "revision": "5e7b2b1b8ca2ecdd3f2322d94069c9b6ce7e844b"
+}

evaluation/laya_typed_source.json

@@ -0,0 +1,4 @@
+{
+  "repo": "convaiinnovations/laya-typed-decisions",
+  "revision": "dd079950600224fb459af2a0cb1d74e1e57ee9cf"
+}

evaluation/reference_comparison.json

@@ -0,0 +1,888 @@
+{
+  "selection": {
+    "passed_id_guard": true,
+    "selected": "cloud_run/runs/h100/checkpoints/step-001000",
+    "best_dev_nll": 0.3434923910639061,
+    "step": 1000
+  },
+  "english_laya_task_point_estimate_wins": 9,
+  "summaries": {
+    "v1": {
+      "temperature": 1.5364534560220011,
+      "by_task": {
+        "ag_news": {
+          "n": 300,
+          "accuracy": 0.8766666666666667,
+          "macro_f1": 0.8785087042568024,
+          "nll": 0.3288212186160579,
+          "brier": 0.17065989210780397,
+          "ece": 0.03637787729240994,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "anli": {
+          "n": 300,
+          "accuracy": 0.48,
+          "macro_f1": 0.48082817201988187,
+          "nll": 1.345594522107734,
+          "brier": 0.7635947518392052,
+          "ece": 0.26481392729155384,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "boolq": {
+          "n": 300,
+          "accuracy": 0.8266666666666667,
+          "macro_f1": 0.8199362851470521,
+          "nll": 0.386092244929183,
+          "brier": 0.24103588842113643,
+          "ece": 0.03688531973385248,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "emotion": {
+          "n": 300,
+          "accuracy": 0.5833333333333334,
+          "macro_f1": 0.5269104214377847,
+          "nll": 1.10472924334084,
+          "brier": 0.5309111092881537,
+          "ece": 0.08188870752844603,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "enron_spam": {
+          "n": 300,
+          "accuracy": 0.7733333333333333,
+          "macro_f1": 0.7663123167155426,
+          "nll": 0.4716031847178109,
+          "brier": 0.304713806192544,
+          "ece": 0.04452538080141978,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "hans": {
+          "n": 300,
+          "accuracy": 0.68,
+          "macro_f1": 0.6483516483516483,
+          "nll": 0.7800930922318069,
+          "brier": 0.5081508480232925,
+          "ece": 0.23496644756044477,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "imdb": {
+          "n": 300,
+          "accuracy": 0.9666666666666667,
+          "macro_f1": 0.9665939156385016,
+          "nll": 0.11574640839950448,
+          "brier": 0.05897405464269712,
+          "ece": 0.0223053165075559,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "mnli": {
+          "n": 300,
+          "accuracy": 0.8666666666666667,
+          "macro_f1": 0.8665719621181807,
+          "nll": 0.3487014769045736,
+          "brier": 0.19113681357849138,
+          "ece": 0.04962698032414739,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "rte": {
+          "n": 277,
+          "accuracy": 0.8447653429602888,
+          "macro_f1": 0.8414593565733605,
+          "nll": 0.3921244048911151,
+          "brier": 0.2421531176029872,
+          "ece": 0.051523040766813605,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst2": {
+          "n": 300,
+          "accuracy": 0.9266666666666666,
+          "macro_f1": 0.926614481409002,
+          "nll": 0.17247762464555924,
+          "brier": 0.0977477259340223,
+          "ece": 0.031843922574865374,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst5": {
+          "n": 300,
+          "accuracy": 0.61,
+          "macro_f1": 0.574389202812619,
+          "nll": 0.8806898747967381,
+          "brier": 0.5095396501659217,
+          "ece": 0.08166816518918596,
+          "reference": "label",
+          "normalized_score_mae": 0.11861205756797243
+        },
+        "typed_decisions": {
+          "n": 2000,
+          "accuracy": 0.5335,
+          "macro_f1": 0.3923581429900535,
+          "nll": 1.125339108915838,
+          "brier": 0.2201981319388216,
+          "ece": 0.09412416488373214,
+          "reference": "teacher",
+          "normalized_score_mae": 0.14582647383250627
+        }
+      },
+      "real_label_macro": {
+        "accuracy": 0.7667968493600263,
+        "macro_f1": 0.7542251333163978,
+        "nll": 0.575152117780084,
+        "brier": 0.3289652416178414,
+        "ece": 0.08512955323369954
+      },
+      "teacher_macro": {
+        "accuracy": 0.5335,
+        "macro_f1": 0.3923581429900535,
+        "nll": 1.125339108915838,
+        "brier": 0.2201981319388216,
+        "ece": 0.09412416488373214
+      },
+      "core_macro": {
+        "accuracy": 0.8213333333333332,
+        "macro_f1": 0.8132041271487311,
+        "nll": 0.42335648797842235,
+        "brier": 0.24202399404147518,
+        "ece": 0.04728045302289223
+      },
+      "permutation": {
+        "n": 400,
+        "semantic_flip_rate": 0.0925,
+        "permuted_accuracy": 0.6675
+      },
+      "counterfactual": {
+        "pairs": 200,
+        "both_correct": 0.63,
+        "scope": "programmatic threshold-policy shift only"
+      }
+    },
+    "laya": {
+      "temperature": 1.6244894678214752,
+      "by_task": {
+        "ag_news": {
+          "n": 300,
+          "accuracy": 0.89,
+          "macro_f1": 0.8916595697302985,
+          "nll": 0.3890859987330656,
+          "brier": 0.17812422915561885,
+          "ece": 0.1436283133976796,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "anli": {
+          "n": 300,
+          "accuracy": 0.49666666666666665,
+          "macro_f1": 0.4950624077400388,
+          "nll": 1.0684693571593755,
+          "brier": 0.6556895709821131,
+          "ece": 0.1762710459363415,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "boolq": {
+          "n": 300,
+          "accuracy": 0.7566666666666667,
+          "macro_f1": 0.7354273633343401,
+          "nll": 0.5134514580247954,
+          "brier": 0.33961628293233154,
+          "ece": 0.026724078995199588,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "emotion": {
+          "n": 300,
+          "accuracy": 0.6033333333333334,
+          "macro_f1": 0.5030792685810473,
+          "nll": 1.2404968190139007,
+          "brier": 0.5863981791717614,
+          "ece": 0.18257059239020804,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "enron_spam": {
+          "n": 300,
+          "accuracy": 0.9633333333333334,
+          "macro_f1": 0.9633133594957255,
+          "nll": 0.21805558070253217,
+          "brier": 0.09912737327536653,
+          "ece": 0.13892227969938,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "hans": {
+          "n": 300,
+          "accuracy": 0.75,
+          "macro_f1": 0.7351663743688133,
+          "nll": 0.6434251988011115,
+          "brier": 0.4040020709975337,
+          "ece": 0.14393662951910027,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "imdb": {
+          "n": 300,
+          "accuracy": 0.9366666666666666,
+          "macro_f1": 0.9366321664017077,
+          "nll": 0.2330622005969465,
+          "brier": 0.1196734142724245,
+          "ece": 0.10107641838144815,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "mnli": {
+          "n": 300,
+          "accuracy": 0.85,
+          "macro_f1": 0.8472960359831189,
+          "nll": 0.41495303657797333,
+          "brier": 0.21352458299435156,
+          "ece": 0.09690900164370211,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "rte": {
+          "n": 277,
+          "accuracy": 0.779783393501805,
+          "macro_f1": 0.7696858174879029,
+          "nll": 0.4566424939641287,
+          "brier": 0.29922837708221717,
+          "ece": 0.05653944816228468,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst2": {
+          "n": 300,
+          "accuracy": 0.9166666666666666,
+          "macro_f1": 0.9166435120866907,
+          "nll": 0.3031605758271119,
+          "brier": 0.16675708608896114,
+          "ece": 0.11399632140488353,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst5": {
+          "n": 300,
+          "accuracy": 0.31666666666666665,
+          "macro_f1": 0.2850066699248626,
+          "nll": 1.469504647138718,
+          "brier": 0.7679831033340562,
+          "ece": 0.17926135006709534,
+          "reference": "label",
+          "normalized_score_mae": 0.21794930561640727
+        },
+        "typed_decisions": {
+          "n": 2000,
+          "accuracy": 0.3755,
+          "macro_f1": 0.2166732389396975,
+          "nll": 1.2182328820568178,
+          "brier": 0.2535778087531912,
+          "ece": 0.07021472076361834,
+          "reference": "teacher",
+          "normalized_score_mae": 0.22102582796001108
+        }
+      },
+      "real_label_macro": {
+        "accuracy": 0.750889399409255,
+        "macro_f1": 0.7344520495576862,
+        "nll": 0.6318461242308782,
+        "brier": 0.3481931154806123,
+        "ece": 0.12362140723612026
+      },
+      "teacher_macro": {
+        "accuracy": 0.3755,
+        "macro_f1": 0.2166732389396975,
+        "nll": 1.2182328820568178,
+        "brier": 0.2535778087531912,
+        "ece": 0.07021472076361834
+      },
+      "core_macro": {
+        "accuracy": 0.7460000000000001,
+        "macro_f1": 0.7352066302118622,
+        "nll": 0.6180311432603329,
+        "brier": 0.3332010569010639,
+        "ece": 0.11210381310171202
+      },
+      "permutation": {
+        "n": 400,
+        "semantic_flip_rate": 0.12,
+        "permuted_accuracy": 0.67
+      },
+      "counterfactual": {
+        "pairs": 200,
+        "both_correct": 0.075,
+        "scope": "programmatic threshold-policy shift only"
+      }
+    },
+    "laya_typed": {
+      "temperature": 0.7527652652933652,
+      "by_task": {
+        "typed_decisions": {
+          "n": 2000,
+          "accuracy": 0.746,
+          "macro_f1": 0.6362003375359054,
+          "nll": 0.9000079785440322,
+          "brier": 0.07263492026918578,
+          "ece": 0.13953533609326807,
+          "reference": "teacher",
+          "normalized_score_mae": 0.07586843216276863
+        }
+      },
+      "real_label_macro": {},
+      "teacher_macro": {
+        "accuracy": 0.746,
+        "macro_f1": 0.6362003375359054,
+        "nll": 0.9000079785440322,
+        "brier": 0.07263492026918578,
+        "ece": 0.13953533609326807
+      },
+      "core_macro": {}
+    },
+    "v2": {
+      "temperature": 1.0423505400296817,
+      "by_task": {
+        "ag_news": {
+          "n": 300,
+          "accuracy": 0.88,
+          "macro_f1": 0.8820997305693538,
+          "nll": 0.31346308023310226,
+          "brier": 0.1660708606601504,
+          "ece": 0.036905068677083444,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "anli": {
+          "n": 300,
+          "accuracy": 0.4866666666666667,
+          "macro_f1": 0.48858002939264306,
+          "nll": 1.5326536998028522,
+          "brier": 0.7840283119508541,
+          "ece": 0.29666781411882187,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "boolq": {
+          "n": 300,
+          "accuracy": 0.8166666666666667,
+          "macro_f1": 0.8098004633952347,
+          "nll": 0.39781728129187827,
+          "brier": 0.2371245824960182,
+          "ece": 0.06916750434052144,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "emotion": {
+          "n": 300,
+          "accuracy": 0.8533333333333334,
+          "macro_f1": 0.7543092974717506,
+          "nll": 0.39460058817048616,
+          "brier": 0.21137011176529708,
+          "ece": 0.06414777184386307,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "enron_spam": {
+          "n": 300,
+          "accuracy": 0.9766666666666667,
+          "macro_f1": 0.9766352540637968,
+          "nll": 0.0405273058138434,
+          "brier": 0.02532319226345078,
+          "ece": 0.013309812113907561,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "hans": {
+          "n": 300,
+          "accuracy": 0.68,
+          "macro_f1": 0.6483516483516483,
+          "nll": 1.086154851651223,
+          "brier": 0.5556954554093221,
+          "ece": 0.27724140990986645,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "imdb": {
+          "n": 300,
+          "accuracy": 0.9633333333333334,
+          "macro_f1": 0.9632643519497723,
+          "nll": 0.10724673447705721,
+          "brier": 0.0607675831159509,
+          "ece": 0.021863761868199296,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "mnli": {
+          "n": 300,
+          "accuracy": 0.88,
+          "macro_f1": 0.881280297400005,
+          "nll": 0.3067923222260661,
+          "brier": 0.17883574492282395,
+          "ece": 0.04206252397025968,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "rte": {
+          "n": 277,
+          "accuracy": 0.8592057761732852,
+          "macro_f1": 0.8552128985565519,
+          "nll": 0.4517337258263279,
+          "brier": 0.23701608121320944,
+          "ece": 0.0908439947482558,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst2": {
+          "n": 300,
+          "accuracy": 0.93,
+          "macro_f1": 0.9299057621913907,
+          "nll": 0.1858724715082013,
+          "brier": 0.10517183633091974,
+          "ece": 0.04078288098575952,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst5": {
+          "n": 300,
+          "accuracy": 0.6066666666666667,
+          "macro_f1": 0.5860584803091198,
+          "nll": 0.8520690951275738,
+          "brier": 0.5043373916500422,
+          "ece": 0.04808122820563478,
+          "reference": "label",
+          "normalized_score_mae": 0.11676450596913004
+        },
+        "typed_decisions": {
+          "n": 2000,
+          "accuracy": 0.7345,
+          "macro_f1": 0.5843823268841445,
+          "nll": 0.9071323454613555,
+          "brier": 0.07585474596137866,
+          "ece": 0.13429560744677668,
+          "reference": "teacher",
+          "normalized_score_mae": 0.08219189555212383
+        }
+      },
+      "real_label_macro": {
+        "accuracy": 0.8120490099551472,
+        "macro_f1": 0.797772564877388,
+        "nll": 0.5153573778298738,
+        "brier": 0.27870374107073076,
+        "ece": 0.09100670643474301
+      },
+      "teacher_macro": {
+        "accuracy": 0.7345,
+        "macro_f1": 0.5843823268841445,
+        "nll": 0.9071323454613555,
+        "brier": 0.07585474596137866,
+        "ece": 0.13429560744677668
+      },
+      "core_macro": {
+        "accuracy": 0.8226666666666667,
+        "macro_f1": 0.8178289467730208,
+        "nll": 0.41120285007736435,
+        "brier": 0.2383080832119909,
+        "ece": 0.04739984123585177
+      },
+      "permutation": {
+        "n": 400,
+        "semantic_flip_rate": 0.06,
+        "permuted_accuracy": 0.8
+      },
+      "counterfactual": {
+        "pairs": 200,
+        "both_correct": 0.715,
+        "scope": "programmatic threshold-policy shift only"
+      }
+    }
+  },
+  "paired": {
+    "v2_minus_v1": {
+      "accuracy_macro_difference": 0.04525216059512088,
+      "ci95": [
+        0.035818489806266444,
+        0.055249750194765204
+      ],
+      "tasks": [
+        "ag_news",
+        "imdb",
+        "anli",
+        "mnli",
+        "emotion",
+        "rte",
+        "sst5",
+        "hans",
+        "boolq",
+        "enron_spam",
+        "sst2"
+      ],
+      "bootstrap_repeats": 2000,
+      "note": "paired cluster bootstrap within fixed tasks; does not measure training-seed uncertainty or arbitrary-task generalization"
+    },
+    "v2_minus_laya": {
+      "accuracy_macro_difference": 0.06115961054589213,
+      "ci95": [
+        0.046367898711433334,
+        0.07522936785812957
+      ],
+      "tasks": [
+        "ag_news",
+        "imdb",
+        "anli",
+        "mnli",
+        "emotion",
+        "rte",
+        "sst5",
+        "hans",
+        "boolq",
+        "enron_spam",
+        "sst2"
+      ],
+      "bootstrap_repeats": 2000,
+      "note": "paired cluster bootstrap within fixed tasks; does not measure training-seed uncertainty or arbitrary-task generalization"
+    },
+    "ood_v2_minus_v1": {
+      "accuracy_macro_difference": 0.004443441636582431,
+      "ci95": [
+        -0.010560018050541518,
+        0.018542644404332124
+      ],
+      "tasks": [
+        "imdb",
+        "anli",
+        "rte",
+        "hans"
+      ],
+      "bootstrap_repeats": 2000,
+      "note": "paired cluster bootstrap within fixed tasks; does not measure training-seed uncertainty or arbitrary-task generalization"
+    },
+    "untruncated_v2_minus_laya": {
+      "accuracy_macro_difference": 0.059998031088527796,
+      "ci95": [
+        0.04506266490813522,
+        0.07413016918623276
+      ],
+      "tasks": [
+        "ag_news",
+        "imdb",
+        "anli",
+        "mnli",
+        "emotion",
+        "rte",
+        "sst5",
+        "hans",
+        "boolq",
+        "enron_spam",
+        "sst2"
+      ],
+      "bootstrap_repeats": 2000,
+      "note": "paired cluster bootstrap within fixed tasks; does not measure training-seed uncertainty or arbitrary-task generalization"
+    }
+  },
+  "calibration_controls": {
+    "v1_original_temperature_on_cuda": {
+      "temperature": 1.3202217760439754,
+      "by_task": {
+        "ag_news": {
+          "n": 300,
+          "accuracy": 0.8766666666666667,
+          "macro_f1": 0.8785087042568024,
+          "nll": 0.3281366243432546,
+          "brier": 0.1726640358398538,
+          "ece": 0.039331689468595894,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "anli": {
+          "n": 300,
+          "accuracy": 0.48,
+          "macro_f1": 0.48082817201988187,
+          "nll": 1.4662612150178047,
+          "brier": 0.7922295711067328,
+          "ece": 0.2898982693205049,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "boolq": {
+          "n": 300,
+          "accuracy": 0.8266666666666667,
+          "macro_f1": 0.8199362851470521,
+          "nll": 0.38989404321203175,
+          "brier": 0.24272104113468998,
+          "ece": 0.05031788332225141,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "emotion": {
+          "n": 300,
+          "accuracy": 0.5833333333333334,
+          "macro_f1": 0.5269104214377847,
+          "nll": 1.1076530050710907,
+          "brier": 0.5309906441117376,
+          "ece": 0.07362572075843835,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "enron_spam": {
+          "n": 300,
+          "accuracy": 0.7733333333333333,
+          "macro_f1": 0.7663123167155426,
+          "nll": 0.48267648677303093,
+          "brier": 0.3087812059355877,
+          "ece": 0.08231533185271653,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "hans": {
+          "n": 300,
+          "accuracy": 0.68,
+          "macro_f1": 0.6483516483516483,
+          "nll": 0.8677628723642957,
+          "brier": 0.5334069806799419,
+          "ece": 0.2588078500628962,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "imdb": {
+          "n": 300,
+          "accuracy": 0.9666666666666667,
+          "macro_f1": 0.9665939156385016,
+          "nll": 0.11163361134820626,
+          "brier": 0.05894937304526226,
+          "ece": 0.014493677943709362,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "mnli": {
+          "n": 300,
+          "accuracy": 0.8666666666666667,
+          "macro_f1": 0.8665719621181807,
+          "nll": 0.3366354718105848,
+          "brier": 0.19027442302778316,
+          "ece": 0.029256404603792807,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "rte": {
+          "n": 277,
+          "accuracy": 0.8447653429602888,
+          "macro_f1": 0.8414593565733605,
+          "nll": 0.41086459374614426,
+          "brier": 0.2482190559064413,
+          "ece": 0.07849875720418192,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst2": {
+          "n": 300,
+          "accuracy": 0.9266666666666666,
+          "macro_f1": 0.926614481409002,
+          "nll": 0.17368569141206347,
+          "brier": 0.09951329459784344,
+          "ece": 0.025793675811563426,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst5": {
+          "n": 300,
+          "accuracy": 0.61,
+          "macro_f1": 0.574389202812619,
+          "nll": 0.857938284145913,
+          "brier": 0.5043493515158285,
+          "ece": 0.038674490185143656,
+          "reference": "label",
+          "normalized_score_mae": 0.1170761133878696
+        },
+        "typed_decisions": {
+          "n": 2000,
+          "accuracy": 0.5335,
+          "macro_f1": 0.3923581429900535,
+          "nll": 1.150007781593213,
+          "brier": 0.2343584931681744,
+          "ece": 0.09236133305092582,
+          "reference": "teacher",
+          "normalized_score_mae": 0.1431887051207915
+        }
+      },
+      "real_label_macro": {
+        "accuracy": 0.7667968493600263,
+        "macro_f1": 0.7542251333163978,
+        "nll": 0.5939219908404018,
+        "brier": 0.3347362706274275,
+        "ece": 0.08918306823034496
+      },
+      "teacher_macro": {
+        "accuracy": 0.5335,
+        "macro_f1": 0.3923581429900535,
+        "nll": 1.150007781593213,
+        "brier": 0.2343584931681744,
+        "ece": 0.09236133305092582
+      },
+      "core_macro": {
+        "accuracy": 0.8213333333333332,
+        "macro_f1": 0.8132041271487311,
+        "nll": 0.41725802298476955,
+        "brier": 0.24190442922319977,
+        "ece": 0.03667482867826944
+      }
+    },
+    "laya_as_shipped_sdk": {
+      "temperature": 1.0,
+      "by_task": {
+        "ag_news": {
+          "n": 300,
+          "accuracy": 0.89,
+          "macro_f1": 0.8916595697302985,
+          "nll": 0.29744314000921096,
+          "brier": 0.15355589074399054,
+          "ece": 0.02859855627603831,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "anli": {
+          "n": 300,
+          "accuracy": 0.49666666666666665,
+          "macro_f1": 0.4950624077400388,
+          "nll": 1.2987562065605402,
+          "brier": 0.7558626718222238,
+          "ece": 0.3062399090728766,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "boolq": {
+          "n": 300,
+          "accuracy": 0.7566666666666667,
+          "macro_f1": 0.7354273633343401,
+          "nll": 0.5367802184868166,
+          "brier": 0.3478546186892215,
+          "ece": 0.07923804256532056,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "emotion": {
+          "n": 300,
+          "accuracy": 0.6033333333333334,
+          "macro_f1": 0.5030792685810473,
+          "nll": 1.6927465184074462,
+          "brier": 0.6582166983170381,
+          "ece": 0.286246200733411,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "enron_spam": {
+          "n": 300,
+          "accuracy": 0.9633333333333334,
+          "macro_f1": 0.9633133594957255,
+          "nll": 0.13450873780547198,
+          "brier": 0.06280437074987427,
+          "ece": 0.07201542413641238,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "hans": {
+          "n": 300,
+          "accuracy": 0.75,
+          "macro_f1": 0.7351663743688133,
+          "nll": 0.9031531229475278,
+          "brier": 0.4485972065279667,
+          "ece": 0.20662327977696504,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "imdb": {
+          "n": 300,
+          "accuracy": 0.9366666666666666,
+          "macro_f1": 0.9366321664017077,
+          "nll": 0.1794530845307123,
+          "brier": 0.10226999518680399,
+          "ece": 0.04792647661314227,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "mnli": {
+          "n": 300,
+          "accuracy": 0.85,
+          "macro_f1": 0.8472960359831189,
+          "nll": 0.3746265039588178,
+          "brier": 0.2065960080844294,
+          "ece": 0.042795663681334174,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "rte": {
+          "n": 277,
+          "accuracy": 0.779783393501805,
+          "macro_f1": 0.7696858174879029,
+          "nll": 0.5010537889652635,
+          "brier": 0.3191090893275295,
+          "ece": 0.11343595147528629,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst2": {
+          "n": 300,
+          "accuracy": 0.9166666666666666,
+          "macro_f1": 0.9166435120866907,
+          "nll": 0.25252279399989175,
+          "brier": 0.1443710213782037,
+          "ece": 0.046774318938379265,
+          "reference": "label",
+          "normalized_score_mae": null
+        },
+        "sst5": {
+          "n": 300,
+          "accuracy": 0.31666666666666665,
+          "macro_f1": 0.2850066699248626,
+          "nll": 1.7214212379962721,
+          "brier": 0.8533385970923033,
+          "ece": 0.29467911014112425,
+          "reference": "label",
+          "normalized_score_mae": 0.22147871144215314
+        },
+        "typed_decisions": {
+          "n": 2000,
+          "accuracy": 0.3755,
+          "macro_f1": 0.2166732389396975,
+          "nll": 1.2881079328908613,
+          "brier": 0.29687600807669234,
+          "ece": 0.14465272165417856,
+          "reference": "teacher",
+          "normalized_score_mae": 0.2245792671818757
+        }
+      },
+      "real_label_macro": {
+        "accuracy": 0.750889399409255,
+        "macro_f1": 0.7344520495576862,
+        "nll": 0.7174968503334519,
+        "brier": 0.36841601526541684,
+        "ece": 0.1385975394009355
+      },
+      "teacher_macro": {
+        "accuracy": 0.3755,
+        "macro_f1": 0.2166732389396975,
+        "nll": 1.2881079328908613,
+        "brier": 0.29687600807669234,
+        "ece": 0.14465272165417856
+      },
+      "core_macro": {
+        "accuracy": 0.7460000000000001,
+        "macro_f1": 0.7352066302118622,
+        "nll": 0.6365587788902018,
+        "brier": 0.3411432271976297,
+        "ece": 0.09841713832043932
+      }
+    }
+  },
+  "limitations": [
+    "Single training seed.",
+    "Historical v1 core and RTE regression examples.",
+    "Length-filtered inputs.",
+    "Typed reference is teacher agreement, reported separately.",
+    "Synthetic counterfactual family is narrow.",
+    "Point-estimate task wins are not all statistically significant wins."
+  ]
+}

gguf_logits.cpp

@@ -0,0 +1,64 @@
+// Exact next-position option logits. JSONL in/out; no sampling or text decoding.
+// Build against the same llama.cpp version used to convert the GGUF.
+#include "llama.h"
+#include "ggml-backend.h"
+#include "json.hpp"
+#include <cmath>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+using json = nlohmann::json;
+
+int main(int argc, char **argv) {
+    if (argc != 2) { std::cerr << "usage: gguf_logits model.gguf\n"; return 2; }
+    ggml_backend_load_all();
+    llama_backend_init();
+    auto mp = llama_model_default_params(); mp.n_gpu_layers = 99;
+    auto *model = llama_model_load_from_file(argv[1], mp);
+    if (!model) return 3;
+    auto cp = llama_context_default_params();
+    cp.n_ctx = 2048; cp.n_batch = 1024; cp.n_ubatch = 1024; cp.n_seq_max = 1;
+    cp.n_threads = 8; cp.n_threads_batch = 8;
+    auto *ctx = llama_init_from_model(model, cp);
+    if (!ctx) { llama_model_free(model); return 4; }
+    auto *vocab = llama_model_get_vocab(model);
+    std::vector<llama_token> labels;
+    for (char c = 'A'; c <= 'Z'; ++c) {
+        std::string label = std::string(" ") + c;
+        llama_token token[4];
+        int n = llama_tokenize(vocab, label.data(), label.size(), token, 4, false, false);
+        if (n != 1) { std::cerr << "option is not a single token\n"; return 5; }
+        labels.push_back(token[0]);
+    }
+    std::string line;
+    while (std::getline(std::cin, line)) {
+        try {
+            if (line.size() > 200000) throw std::runtime_error("request too large");
+            auto in = json::parse(line);
+            std::string prompt = in.at("prompt").get<std::string>();
+            int k = in.at("n_options").get<int>();
+            if (k < 2 || k > 26) throw std::runtime_error("need 2-26 options");
+            std::vector<llama_token> tokens(1025);
+            int n = llama_tokenize(vocab, prompt.data(), prompt.size(), tokens.data(), tokens.size(), false, true);
+            if (n <= 0 || n > 1024) throw std::runtime_error("prompt exceeds the validated context budget");
+            tokens.resize(n);
+            llama_memory_clear(llama_get_memory(ctx), true);
+            auto batch = llama_batch_get_one(tokens.data(), n);
+            if (llama_decode(ctx, batch) != 0) throw std::runtime_error("prefill failed");
+            llama_synchronize(ctx);
+            float *all = llama_get_logits_ith(ctx, -1);
+            if (!all) throw std::runtime_error("missing logits");
+            std::vector<float> logits;
+            for (int i = 0; i < k; ++i) {
+                if (!std::isfinite(all[labels[i]])) throw std::runtime_error("nonfinite option logit");
+                logits.push_back(all[labels[i]]);
+            }
+            std::cout << json({{"logits", logits}, {"prompt_tokens", n}}).dump() << std::endl;
+        } catch (const std::exception &e) {
+            std::cout << json({{"error", e.what()}}).dump() << std::endl;
+        }
+    }
+    llama_free(ctx); llama_model_free(model); llama_backend_free();
+    return 0;
+}

jev_decision_client.py

@@ -0,0 +1,76 @@
+"""Exact declared-option probabilities through llama.cpp's /completion endpoint.
+
+No artificial probability floor. Candidate count is increased if an option is
+missing; if complete finite log-probabilities cannot be obtained, fail explicitly.
+This client requires llama.cpp's native endpoint, not generic chat logprobs APIs.
+"""
+import argparse
+import json
+import math
+import urllib.request
+
+LETTERS="ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+HEADER="You are a decision function. Read the state, then answer the question by choosing exactly one option."
+
+
+class Client:
+    def __init__(self,url="http://127.0.0.1:8080",temperature=1.0):
+        if not math.isfinite(temperature) or temperature<=0:raise ValueError("invalid calibration temperature")
+        self.url=url.rstrip("/");self.temperature=temperature;self.label_ids={}
+
+    def post(self,path,payload):
+        request=urllib.request.Request(self.url+path,json.dumps(payload).encode(),{"Content-Type":"application/json"})
+        with urllib.request.urlopen(request,timeout=120) as response:return json.load(response)
+
+    def labels(self,n):
+        for letter in LETTERS[:n]:
+            if letter not in self.label_ids:
+                ids=self.post("/tokenize",{"content":" "+letter,"add_special":False})["tokens"]
+                if len(ids)!=1 or not isinstance(ids[0],int):raise ValueError("option label must be a single token")
+                self.label_ids[letter]=ids[0]
+        return [self.label_ids[c] for c in LETTERS[:n]]
+
+    def prompt_logits(self,prompt,n):
+        wanted=self.labels(n)
+        for count in [64,256,1024,4096]:
+            result=self.post("/completion",{"prompt":prompt,"n_predict":1,"temperature":-1.0,
+                "n_probs":count,"post_sampling_probs":False,"cache_prompt":False,
+                "repeat_penalty":1.0,"presence_penalty":0.,"frequency_penalty":0.})
+            positions=result.get("completion_probabilities",result.get("probs"))
+            if not positions:raise RuntimeError("server did not return token probabilities")
+            candidates=positions[0].get("top_logprobs")
+            if candidates is None:raise RuntimeError("server must return raw finite log-probabilities")
+            by_id={c["id"]:c["logprob"] for c in candidates}
+            if all(i in by_id and by_id[i] is not None and math.isfinite(by_id[i]) for i in wanted):
+                return [by_id[i] for i in wanted],count
+        raise RuntimeError("Some declared option probabilities are missing/nonfinite. Use native exact-logit inference; no values have been guessed.")
+
+    def decide(self,state,question,options):
+        if not 2<=len(options)<=26 or len(set(options))!=len(options):raise ValueError("need 2–26 unique options")
+        lines="\n".join(f"{LETTERS[i]}. {o}" for i,o in enumerate(options))
+        prompt=f"{HEADER}\n\n[State]\n{state}\n\n[Question]\n{question}\n\n[Options]\n{lines}\n\nAnswer:"
+        logits,count=self.prompt_logits(prompt,len(options))
+        z=[x/self.temperature for x in logits];m=max(z)
+        p=[math.exp(x-m) for x in z];total=sum(p);p=[x/total for x in p]
+        return {"choice":options[max(range(len(p)),key=p.__getitem__)],"probabilities":dict(zip(options,p)),"candidate_count":count}
+
+    def decide_bool(self,state,proposition):
+        return self.decide(state,proposition,["yes","no"])["probabilities"]["yes"]
+
+    def decide_score(self,state,question,levels):
+        result=self.decide(state,question,levels)
+        result["expected_level"]=sum(i*result["probabilities"][level] for i,level in enumerate(levels))
+        return result
+
+
+def main():
+    p=argparse.ArgumentParser();p.add_argument("--url",default="http://127.0.0.1:8080")
+    p.add_argument("--calibration");p.add_argument("--state",required=True);p.add_argument("--question",required=True)
+    p.add_argument("--options",nargs="+",required=True);a=p.parse_args()
+    t=1.0
+    if a.calibration:
+        with open(a.calibration) as f:t=json.load(f)["temperature"]
+    print(json.dumps(Client(a.url,t).decide(a.state,a.question,a.options),ensure_ascii=False,indent=2))
+
+
+if __name__=="__main__":main()

requirements.txt

@@ -0,0 +1 @@
+# The HTTP decision client uses Python's standard library only.