diff --git a/.gitattributes b/.gitattributes index e02cc0fd205ad99a2897c1d1fc744fa02cf08426..d5f3fa5c0c85bcd687438d45b22cbfbe6fbb80da 100644 --- a/.gitattributes +++ b/.gitattributes @@ -44,3 +44,12 @@ assets/decision-quality.png filter=lfs diff=lfs merge=lfs -text assets/readout.pdf filter=lfs diff=lfs merge=lfs -text assets/readout.png filter=lfs diff=lfs merge=lfs -text tokenizer.json filter=lfs diff=lfs merge=lfs -text +assets/decision-expanded-old_core-600px.png filter=lfs diff=lfs merge=lfs -text +assets/decision-expanded-old_core.png filter=lfs diff=lfs merge=lfs -text +assets/decision-expanded-overview.png filter=lfs diff=lfs merge=lfs -text +assets/decision-expanded-ranking.png filter=lfs diff=lfs merge=lfs -text +assets/decision-expanded-v3_core-600px.png filter=lfs diff=lfs merge=lfs -text +assets/decision-expanded-v3_core.png filter=lfs diff=lfs merge=lfs -text +assets/decision-expanded-v4.png filter=lfs diff=lfs merge=lfs -text +assets/decision-expanded-v5.png filter=lfs diff=lfs merge=lfs -text +assets/decision-question-scaling.png filter=lfs diff=lfs merge=lfs -text diff --git a/ATTRIBUTIONS.md b/ATTRIBUTIONS.md index 04c64ff35e515f5604cf31d6f66a22e8fd55aa4e..2410992a66f3020e95629019fa4d3aac4a1ffe3f 100644 --- a/ATTRIBUTIONS.md +++ b/ATTRIBUTIONS.md @@ -23,3 +23,13 @@ MultiNLI is by Adina Williams, Nikita Nangia and Samuel R. Bowman, *A Broad-Cove - [MultiNLI paper](https://aclanthology.org/N18-1101/). MASSIVE and SLURP remain excluded from custom training, checkpoint selection and calibration. MASSIVE is used only for evaluation. Official Jev outputs are never used as training labels. The initial-release training histories above remain historical; the subsequent selected model and evaluation provenance identify the actual update. + +## Natural-language decision adaptation + +This update adds 8,000 human-annotated training examples to 16,000 retained decision examples: 4,000 Cosmos QA reading questions, 2,000 SQuAD 2.0 answerability judgments, and 2,000 SNLI inference pairs. The selected natural-data checkpoints complete one pass of this 24,000-example mixture. Human source labels are preserved; SQuAD answerability uses its supplied impossible/answerable annotation, and each source is converted to the model's decision interface. Source-parent groups and near duplicates are separated between custom training, selection and calibration. No official Jev output supplies a training label. + +- **Cosmos QA**, by Lifu Huang, Ronan Le Bras, Chandra Bhagavatula and Yejin Choi. Data from the [author repository at the pinned revision](https://github.com/wilburOne/cosmosqa/tree/b6eb99cca4e2a51dd28a9a6f562534872d851639). The [official AllenAI dataset card](https://huggingface.co/datasets/allenai/cosmos_qa/blob/28d9d5e2aae025e73e11177891a88dba51190013/README.md) records the author-confirmed [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) license. +- **SQuAD 2.0**, by Pranav Rajpurkar, Robin Jia and Percy Liang. The official [SQuAD project](https://rajpurkar.github.io/SQuAD-explorer/) distributes the dataset under [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/). The adaptation here is answerability classification, not the official span-extraction benchmark. +- **SNLI 1.0**, by Samuel R. Bowman, Gabor Angeli, Christopher Potts and Christopher D. Manning. The official [Stanford Natural Language Inference project](https://nlp.stanford.edu/projects/snli/) and release README identify [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/) for the corpus. Original entailment, neutral and contradiction labels supply the three decision alternatives. + +These dataset licenses govern their respective source material; they are not replaced by the model package's Apache 2.0 license. Source corpus text, transformed training records and individual evaluation predictions are not redistributed in this model package. New evaluation panels, including supplied-fact QASC questions, are described separately in EVALUATION.md; evaluation data are not used for checkpoint selection or temperature fitting. As with other public datasets, exclusion from this custom training does not establish absence from upstream pretraining. diff --git a/EVALUATION.md b/EVALUATION.md index 7bbf3d8b06732f75bd7af2e87c3a487275a1faa3..6b1a0f062f6285fc44bbd083d43bc4c261e949aa 100644 --- a/EVALUATION.md +++ b/EVALUATION.md @@ -1,93 +1,92 @@ -# Nox: measured quality update - -The primary score averages 20 task-family accuracies equally: one half the original 880-question panel and one half the independently constructed 880-question confirmation panel. The original panel was already observed during development. The added panel was excluded from custom training, selection and calibration; candidate weights, prompt and temperature were fixed before its inference. Once unsealed for the first candidate, it remains an observed regression panel. A later release does not make it fresh again. - -| Model | Overall accuracy ↑ | Choice ↑ | Noul ↑ | Score ↑ | -| --- | ---: | ---: | ---: | ---: | -| Jev · 1.13.0 | **72.74** | **71.77** | **80.36** | **68.06** | -| Nox · 4B · v1.1 | 67.05 | 70.55 | 60.27 | 55.56 | -| Qwen3.5 · 4B · untuned | 56.61 | 60.06 | 53.57 | 52.08 | -| Sol · 2B · v1.0 | 55.58 | 61.93 | 50.89 | 32.64 | -| Decider · 2B | 55.30 | 57.26 | 58.93 | 50.00 | -| Qwen3.5 · 2B · untuned | 48.06 | 50.36 | 45.09 | 47.22 | -| Laya · EN/ML | 47.38 | 53.02 | 52.23 | 19.44 | - -All values are percentages. Choice, Noul and Score columns pool correct/total questions of that type across both panels; their mean is not the overall family-macro score. Score accuracy tests the highest-probability rubric level; expected-value error is reported separately. Missing or invalid predictions count as incorrect. Confidence intervals use 10,000 paired cluster-bootstrap resamples. The two language views of each MASSIVE source utterance share draws within menu strata; synthetic counterfactual blocks remain intact. The original and confirmation panels are resampled independently, then combined with equal weight. - -## Change from the released weights - -| Panel | Accuracy change (points) | Paired 95% interval | -| --- | ---: | ---: | -| Old | +3.53 | [+0.37, +6.59] | -| Fresh | +5.12 | [+2.21, +8.08] | -| Joint | +4.33 | [+2.17, +6.46] | - -Release eligibility follows the prospectively fixed positive joint point improvement plus complete, finite, valid candidate execution. It does not require every family to improve or the confidence interval to exclude zero. An eligible update is not evidence of universal superiority. - -## Native API coverage - -Each separate native panel has 68 requests and 220 requested answers, including repeated packing and isolation probes. These answers are descriptive diagnostics, not 220 independent questions and not part of the primary score. Unsupported requests are reported with the full denominator rather than relabeled as wrong semantic predictions. - -| Model | Original: valid / 220 | Original: correct / 220 | New: valid / 220 | New: correct / 220 | -| --- | ---: | ---: | ---: | ---: | -| Nox · 4B | 220 | 200 | 220 | 138 | -| Sol · 2B | 220 | 116 | 220 | 116 | -| Jev · 1.13.0 | 220 | 203 | 220 | 168 | -| Laya · EN/ML | 208 | 88 | 208 | 98 | -| Decider · 2B | 220 | 111 | 220 | 96 | -| Qwen3.5 · 2B · untuned | 196 | 86 | 196 | 71 | -| Qwen3.5 · 4B · untuned | 196 | 112 | 196 | 104 | - -## Scope and interpretation - -All comparators receive the same frozen requests through documented native or fixed adaptation interfaces. The untuned baselines are post-trained Qwen3.5 parents before decision adaptation, evaluated with a fixed LM-head decision adapter. Laya uses fixed English/multilingual routing. Jev is the official API; server-side truncation and internal model architecture cannot be independently inspected. - -| Model | Original reported truncations | New reported truncations | -| --- | ---: | ---: | -| Nox · 4B | 0 | 0 | -| Sol · 2B | 0 | 0 | -| Jev · 1.13.0 | 0 | 0 | -| Laya · EN/ML | 112 | 40 | -| Decider · 2B | 0 | 0 | -| Qwen3.5 · 2B · untuned | 0 | 0 | -| Qwen3.5 · 4B · untuned | 0 | 0 | - -Zero reported truncations for a remote service is not proof of complete server-side input processing. The Decision candidate additionally passes explicit complete-token and overflow checks. - -The confirmation panel includes unseen MASSIVE English/Chinese intents and eight independently generated decision families: temporal exclusion, constraint assignment, transaction recovery, conflicting-rule reasoning, multiset reconciliation, record identity, capacity constraints and service-loss scoring. Training, selection and calibration exclude MASSIVE and SLURP. Exact overlap was zero; three moderate lexical near-pairs with training were disclosed and retained. Gold labels were not supplied by Jev. - -MASSIVE uses the [official 1.1 archive](https://amazon-massive-nlu-dataset.s3.amazonaws.com/amazon-massive-dataset-1.1.tar.gz), CC BY 4.0. Exposure during base-model pretraining is unknown. Synthetic English/Chinese views share symbolic JSON state; they do not establish natural multilingual document comprehension. - -The [complete aggregate measurements](quality-metrics.json) retain every family, raw and calibrated probability diagnostics, native coverage, failure counts, Brier/NLL and ordinal metrics. Proper probability scores are conditional on valid probability coverage; zero probability on the true answer produces explicit infinite NLL rather than clipping. Per-family regressions remain visible in the capability matrix. - -The question-scaling figure retains measurements from v1.0 weights and is labeled accordingly. Architecture and API behavior are unchanged; these measurements are not a timing claim for the updated weights. - -Evaluation policy SHA-256: `359de2b6b81be91a66c6c538b3655bed06f4728357328cd318b1a765e64565cd`. Scoring receipt SHA-256: `927c92121c2df534f2ba12523e466ecebd8bd8936f949c7792de3e747c92a47c`. - -## Human-source confirmation (V4) - -Nox improves from **75.47% to 78.44%** on this separately constructed, human-annotated panel: **+2.97 percentage points**, paired 95% CI **[+0.77, +5.32]**. Reading comprehension remains a clear gap: Decider reaches **92.03%**, the untuned 4B model with its fixed LM-head adapter **87.97%**, and Jev **94.53%**. This supplemental result does not change the fixed V3 publication rule. - -The score weights 160 balanced BoolQ questions at 50% and 160 parallel Belebele questions in each of English and Chinese at 25% each. All **480 requested examples** are included; the weighted score differs from raw correct/480. Bold marks the best result in each score column. - -| Model | Weighted % | BoolQ % (correct) | Belebele EN % (correct) | Belebele ZH % (correct) | Raw correct | -|---|---:|---:|---:|---:|---:| -| Nox (this update) | 78.44 | 87.50 (140/160) | 71.25 (114/160) | 67.50 (108/160) | 362/480 | -| Sol v1.0 | 69.38 | 74.38 (119/160) | 66.25 (106/160) | 62.50 (100/160) | 325/480 | -| Jev 1.13.0 | **94.53** | **92.50 (148/160)** | **96.88 (155/160)** | **96.25 (154/160)** | 457/480 | -| Laya EN/ML routed | 51.25 | 69.38 (111/160) | 38.12 (61/160) | 28.12 (45/160) | 217/480 | -| Decider 2B | 92.03 | 91.88 (147/160) | 93.12 (149/160) | 91.25 (146/160) | 442/480 | -| Qwen3.5 2B + LM-head adapter | 73.75 | 65.00 (104/160) | 83.75 (134/160) | 81.25 (130/160) | 368/480 | -| Qwen3.5 4B + LM-head adapter | 87.97 | 83.75 (134/160) | 93.12 (149/160) | 91.25 (146/160) | 429/480 | - -Nox's absolute weighted 95% CI is **[73.99%, 82.65%]**, compared with **[70.71%, 80.02%]** for Nox v1.0. Its BoolQ false and true recalls are both **87.50%** (70/80 each). Machine-readable results include all model intervals and Noul/Choice NLL and Brier scores. Probabilities and released calibration are unchanged; zero assigned gold probability yields infinite NLL without epsilon clipping. - -All seven models returned 480/480 valid outputs. Complete inputs were verified for all six local models, with zero truncations; retention inside Jev's closed API is unknown. Sol is the immutable v1.0 reference. Qwen baselines use the same starting model revisions with a fixed LM-head readout, not randomly initialized decision heads. These are quality measurements, not latency comparisons. - -Intervals use 10,000 paired passage-cluster bootstrap draws. BoolQ has 160 passage parents; Belebele has 144, with every selected question and both translations from each passage resampled together. All models share the same draws. - -[BoolQ validation](https://huggingface.co/datasets/google/boolq/tree/35b264d03638db9f4ce671b711558bf7ff0f80d5) (CC BY-SA 3.0) and [Belebele test](https://huggingface.co/datasets/facebook/belebele/tree/7899cdfa4e1e0d733fd77c848e2c273cb1d32be2) (CC BY-SA 4.0) were excluded from custom TRAIN/SELECT/CAL. Original human labels and English/Chinese alignment were independently checked; this was not new human adjudication. The lexical audit found no exact or >=0.5 character-5-gram Jaccard overlaps across 51 data/panel files, but cannot establish semantic independence or unknown upstream-pretraining exposure. BoolQ lacks article-title metadata, so grouping is by passage, not article. - -The candidate was frozen before the panel's first global unseal on 2026-09-21. The panel is now observed regression for subsequent unfrozen candidates. It covers evidence yes/no and four-choice comprehension, not ordinal Score or all decision capabilities. Aggregate data and SHA-256 evidence accompany this section. - -[Reading benchmark aggregate measurements](metrics/natural-confirmation.json). +# Evaluation + +Four complete panels, shared requests and recorded native decisions. Failures and rejections remain in the requested denominator. The four-panel mean uses 25% each for general decisions, compositional tasks, natural reading, and reading/inference. Within-panel source/family weights are retained. Intervals use the frozen paired source-cluster procedure; they are reported without introducing another release gate. + +| Model | Version | General decisions | Compositional tasks | Natural reading | Reading and inference | Mean | +|---|---|---:|---:|---:|---:|---:| +| Jev | 1.13.0 | 79.10 | **66.38** | **94.53** | **89.79** | **82.45** | +| Nox | v1.2 · this release | **83.21** | 51.08 | 78.12 | 84.38 | 74.20 | +| Nox | v1.1 · previous | 82.85 | 51.25 | 78.44 | 79.17 | 72.93 | +| Decider | 2B | 64.01 | 46.58 | 92.03 | 84.38 | 71.75 | +| Qwen3.5 | 4B · untuned | 69.89 | 43.33 | 87.97 | 79.79 | 70.25 | +| Qwen3.5 | 2B · untuned | 57.12 | 39.00 | 73.75 | 72.29 | 60.54 | +| Laya | Upstream default | 57.01 | 37.75 | 51.25 | 63.75 | 52.44 | + +Accuracy (%). Mean weights each panel equally; each panel retains its frozen family/source weights. +Bold marks the exact highest score in a column, including exact ties. Rounded visual ties can differ. + +### General decisions + +| Task | Jev · 1.13.0 | Nox · v1.2 · this release | Nox · v1.1 · previous | Decider · 2B | Qwen3.5 · 4B · untuned | Qwen3.5 · 2B · untuned | Laya · Upstream default | +|---|---:|---:|---:|---:|---:|---:|---:| +| ag news | 85.16 | 85.16 | 83.59 | 86.72 | 84.38 | 80.47 | **91.41** | +| boolean constraints | **100.00** | 93.75 | 93.75 | 71.88 | 62.50 | 37.50 | 43.75 | +| dbpedia 14 | 96.43 | 96.43 | 96.43 | **98.21** | 97.32 | 92.86 | 83.93 | +| natural intents | **100.00** | **100.00** | **100.00** | **100.00** | **100.00** | 96.88 | 85.94 | +| option carrier | 30.21 | **100.00** | **100.00** | 8.33 | 72.92 | 9.38 | 95.83 | +| ordinal rubric | **100.00** | 89.06 | 89.06 | 84.38 | 90.62 | 81.25 | 18.75 | +| relational composition | **56.25** | 54.17 | 55.21 | 54.17 | 51.04 | 48.96 | 25.00 | +| scoped evidence | **89.58** | 78.12 | 75.00 | 47.92 | 51.04 | 36.46 | 37.50 | +| state tracking | 33.33 | **35.42** | **35.42** | 29.17 | 28.12 | 25.00 | 23.96 | +| unknown rejection | **100.00** | **100.00** | **100.00** | 59.38 | 60.94 | 62.50 | 64.06 | + +### Compositional tasks + +| Task | Jev · 1.13.0 | Nox · v1.2 · this release | Nox · v1.1 · previous | Decider · 2B | Qwen3.5 · 4B · untuned | Qwen3.5 · 2B · untuned | Laya · Upstream default | +|---|---:|---:|---:|---:|---:|---:|---:| +| canonical record identity | **76.25** | 50.00 | 50.00 | 56.25 | 50.00 | 46.25 | 47.50 | +| capacitated assignment | **68.75** | 41.25 | 43.75 | 51.25 | 50.00 | 50.00 | 63.75 | +| constraint assignment | **55.00** | 41.25 | 42.50 | 23.75 | 23.75 | 21.25 | 22.50 | +| massive en | **91.67** | 90.83 | **91.67** | 90.00 | **91.67** | 70.83 | 74.17 | +| massive zh | **88.33** | 87.50 | **88.33** | **88.33** | 86.67 | 74.17 | 73.33 | +| multiset reconciliation | **58.75** | 28.75 | 31.25 | 23.75 | 27.50 | 27.50 | 26.25 | +| ordinal service loss | **42.50** | 31.25 | 28.75 | 22.50 | 21.25 | 20.00 | 20.00 | +| paraconsistent rule closure | **66.25** | 33.75 | 35.00 | 26.25 | 25.00 | 27.50 | 18.75 | +| temporal exclusion | 41.25 | **45.00** | 41.25 | 42.50 | 31.25 | 28.75 | 12.50 | +| transaction recovery | **75.00** | 61.25 | 60.00 | 41.25 | 26.25 | 23.75 | 18.75 | + +### Natural reading + +| Task | Jev · 1.13.0 | Nox · v1.2 · this release | Nox · v1.1 · previous | Decider · 2B | Qwen3.5 · 4B · untuned | Qwen3.5 · 2B · untuned | Laya · Upstream default | +|---|---:|---:|---:|---:|---:|---:|---:| +| boolq | **92.50** | 85.62 | 87.50 | 91.88 | 83.75 | 65.00 | 69.38 | +| belebele en | **96.88** | 71.88 | 71.25 | 93.12 | 93.12 | 83.75 | 38.12 | +| belebele zh | **96.25** | 69.38 | 67.50 | 91.25 | 91.25 | 81.25 | 28.12 | + +### Reading and inference + +| Task | Jev · 1.13.0 | Nox · v1.2 · this release | Nox · v1.1 · previous | Decider · 2B | Qwen3.5 · 4B · untuned | Qwen3.5 · 2B · untuned | Laya · Upstream default | +|---|---:|---:|---:|---:|---:|---:|---:| +| cosmos qa | **86.67** | 68.33 | 57.50 | 66.67 | 60.83 | 56.67 | 30.00 | +| squad2 answerability | **90.83** | 85.00 | 75.83 | 84.17 | 81.67 | 82.50 | 57.50 | +| snli | 82.50 | 86.67 | 85.83 | **90.83** | 80.00 | 64.17 | 72.50 | +| qasc | **99.17** | 97.50 | 97.50 | 95.83 | 96.67 | 85.83 | 95.00 | + +### Probability quality + +| Model | Version | Weighted Brier ↓ | Valid probability rows | +|---|---|---:|---:| +| Jev | 1.13.0 | **0.2287** | 2,720 / 2,720 | +| Nox | v1.2 · this release | 0.3416 | 2,720 / 2,720 | +| Nox | v1.1 · previous | 0.3630 | 2,720 / 2,720 | +| Decider | 2B | 0.3535 | 2,720 / 2,720 | +| Qwen3.5 | 4B · untuned | 0.3996 | 2,720 / 2,720 | +| Qwen3.5 | 2B · untuned | 0.5130 | 2,720 / 2,720 | +| Laya | Upstream default | 0.5903 | 2,720 / 2,720 | + +Missing Brier means probability coverage was incomplete; no supported-only average is substituted. +The two native contract supplements are reported separately and are outside this quality mean. + +![Detailed General decisions](assets/decision-expanded-old_core.png) + +![Detailed Compositional tasks](assets/decision-expanded-v3_core.png) + +![Detailed Natural reading](assets/decision-expanded-v4.png) + +![Detailed Reading and inference](assets/decision-expanded-v5.png) + +## Scope + +These panels include previously observed regression sets. They are not all pristine holdouts. The untuned Qwen models use their frozen chat/LM-head adapters. Laya uses its unmodified upstream default language router (no explicit language override); Decider retains its native adapter. Documented native truncation is retained. Jev is a closed service snapshot, not a reproducible weight release. Full native-interface supplements are outside the quality mean. + +[Exact aggregate statistics](metrics/expanded-quality.json) · [Probability and adapter coverage](metrics/comparator-coverage.json) · [Measured latency](QUESTION-SCALING.md) · [Material provenance](metrics/materials-provenance.json) diff --git a/NORMALIZATION_RUNTIME.md b/NORMALIZATION_RUNTIME.md new file mode 100644 index 0000000000000000000000000000000000000000..d9a89dd1133c5f07ec94f3a3278018639c3a6c61 --- /dev/null +++ b/NORMALIZATION_RUNTIME.md @@ -0,0 +1,5 @@ +# Validated normalization runtime + +This Nox bundle installs its recorded FLA normalization profile through the default public entrypoint. It requires the pinned ROCm runtime on gfx942 and a fresh process. The profile covers dimension 128, BF16 input/output, FP32 reciprocal norms, and buckets 1–64 for the actual 32 normalized value heads, batch size at most 8 and complete inputs at most 16,384 tokens. Unknown keys fail closed. Weights, tokenizer, prompt, readout and temperature remain unchanged from the source export. + +No private compilation cache is required. Other FLA kernels retain normal runtime behavior. Numerical validation of this newly bound bundle is still required. Earlier latency measurements do not measure this runtime or candidate. diff --git a/QUESTION-SCALING.md b/QUESTION-SCALING.md index 8409dd27fa34cd1dcab1cdb59fac07da68321f39..dbdcdcd2fe6cbcce30fe90f95abdb210cf9bd3aa 100644 --- a/QUESTION-SCALING.md +++ b/QUESTION-SCALING.md @@ -1,55 +1,64 @@ -# Fixed-question latency scaling — v1.0 - -These measurements use the initial Sol/Nox v1.0 weights. They are retained as versioned evidence and do not measure the updated weights. - -A fixed English state (819 characters), identical question text and a four-choice menu are repeated Q times in one API request. Native rendered rows, including state and question, contain 309 tokens for Sol/Nox, 228 for Laya and 250 for Decider. Tokenizers and templates differ. Every model uses the same physical AMD gfx942 GPU and matches its quality/runtime fingerprint. Native question packing is preserved, with no cross-request cache or concurrent-client load. - -Points show the empirical median of 30 timed requests, from three blocks with randomized model order. Each point receives three warmups and ten measured repetitions per block. Rendering, tokenization, transfers, model forward and answer assembly are included; loading, warmup and network are excluded. Lines connect measured points for readability and do not represent measured intermediate values. The horizontal axis is logarithmic in base 2. - -Q1/Q8/Q32 come from the original GPU-quiet round; Q2/Q4/Q16 are an additive round under the same frozen setup. A repeated Q1 bridge is reported separately and never pooled or selected. This is local request scaling, not maximum server throughput. Laya uses its English branch for these English inputs. - -Base-model appendix retains the original measured Q1/Q8/Q32 points only. - -The machine exposes 261824 MiB of device memory. Parameters, native precision, shipped temperature and model/runtime fingerprints are those of the original v1.0 quality benchmark. No throughput saturation claim is made. p95 is an empirical percentile, not a confidence interval. - -Full aggregate evidence: [question-scaling.json](metrics/question-scaling.json). - -| Model | Questions | Tokens / question | p50 (ms) | p95 (ms) | Samples | Round | -|---|---:|---:|---:|---:|---:|---| -| Sol 2B | 1 | 309 | 21.04 | 21.41 | 30 | original | -| Sol 2B | 2 | 309 | 21.48 | 21.81 | 30 | additive | -| Sol 2B | 4 | 309 | 23.47 | 31.89 | 30 | additive | -| Sol 2B | 8 | 309 | 37.29 | 37.51 | 30 | original | -| Sol 2B | 16 | 309 | 73.88 | 74.46 | 30 | additive | -| Sol 2B | 32 | 309 | 147.50 | 147.91 | 30 | original | -| Nox 4B | 1 | 309 | 27.37 | 28.95 | 30 | original | -| Nox 4B | 2 | 309 | 28.17 | 28.54 | 30 | additive | -| Nox 4B | 4 | 309 | 40.89 | 41.04 | 30 | additive | -| Nox 4B | 8 | 309 | 69.25 | 69.84 | 30 | original | -| Nox 4B | 16 | 309 | 137.64 | 137.82 | 30 | additive | -| Nox 4B | 32 | 309 | 275.65 | 278.26 | 30 | original | -| Laya | 1 | 228 | 11.54 | 12.21 | 30 | original | -| Laya | 2 | 228 | 12.60 | 13.02 | 30 | additive | -| Laya | 4 | 228 | 13.37 | 13.68 | 30 | additive | -| Laya | 8 | 228 | 14.56 | 15.02 | 30 | original | -| Laya | 16 | 228 | 22.47 | 22.80 | 30 | additive | -| Laya | 32 | 228 | 37.89 | 38.30 | 30 | original | -| Decider | 1 | 250 | 23.35 | 23.91 | 30 | original | -| Decider | 2 | 250 | 23.52 | 23.89 | 30 | additive | -| Decider | 4 | 250 | 24.05 | 24.31 | 30 | additive | -| Decider | 8 | 250 | 31.59 | 31.77 | 30 | original | -| Decider | 16 | 250 | 53.12 | 53.32 | 30 | additive | -| Decider | 32 | 250 | 96.67 | 97.02 | 30 | original | - -## Nonpooled Q1 bridge - -Flag threshold: absolute median difference greater than max(10% of the original Q1 median, 2 ms). Original Q1 remains the plotted point in every case; no normalization, replacement or pooling. - -| Model | Original p50 (ms) | Bridge p50 (ms) | Difference (ms) | Flag | -|---|---:|---:|---:|---| -| Sol 2B | 21.04 | 20.87 | -0.17 | No | -| Nox 4B | 27.37 | 27.16 | -0.21 | No | -| Laya | 11.54 | 12.14 | +0.60 | No | -| Decider | 23.35 | 23.18 | -0.17 | No | - -All bridge checks are below the predeclared threshold. +# Request latency + +Fresh matched measurements of this candidate and its own published predecessor. + +The same physical AMD gfx942 GPU runs three independently loaded blocks per model in a counterbalanced order. Each of 18 fixed Choice/Noul/Score cases has three warmups and ten measurements per block. The tables pool all 30 measurements per point. Loading and network time are excluded; tokenization and inference are included. No cross-request prefix cache is enabled. These are sequential requests, not concurrent-service throughput. + +![Choice question scaling](assets/decision-question-scaling.png) + +## Choice + +| Model | Q | p50 ms ↓ | p95 ms ↓ | Input tokens | +|---|---:|---:|---:|---:| +| Nox · v1.2 · this release | 1 | 32.108 | 33.047 | 309 | +| Nox · v1.1 · previous | 1 | **28.976** | **30.240** | 309 | +| Nox · v1.2 · this release | 2 | 33.025 | 34.047 | 618 | +| Nox · v1.1 · previous | 2 | **29.337** | **29.953** | 618 | +| Nox · v1.2 · this release | 4 | **41.706** | **42.005** | 1236 | +| Nox · v1.1 · previous | 4 | 42.041 | 42.413 | 1236 | +| Nox · v1.2 · this release | 8 | **69.483** | **70.719** | 2472 | +| Nox · v1.1 · previous | 8 | 70.106 | 70.850 | 2472 | +| Nox · v1.2 · this release | 16 | **141.299** | **141.921** | 4944 | +| Nox · v1.1 · previous | 16 | 141.369 | 142.338 | 4944 | +| Nox · v1.2 · this release | 32 | **282.326** | **284.856** | 9888 | +| Nox · v1.1 · previous | 32 | 283.681 | 286.013 | 9888 | + +## Noul + +| Model | Q | p50 ms ↓ | p95 ms ↓ | Input tokens | +|---|---:|---:|---:|---:| +| Nox · v1.2 · this release | 1 | 32.282 | 32.977 | 262 | +| Nox · v1.1 · previous | 1 | **28.993** | **29.371** | 262 | +| Nox · v1.2 · this release | 2 | 32.573 | 33.978 | 524 | +| Nox · v1.1 · previous | 2 | **29.196** | **30.459** | 524 | +| Nox · v1.2 · this release | 4 | **38.967** | **39.277** | 1048 | +| Nox · v1.1 · previous | 4 | 39.373 | 39.573 | 1048 | +| Nox · v1.2 · this release | 8 | 64.497 | **65.068** | 2096 | +| Nox · v1.1 · previous | 8 | **64.382** | 65.297 | 2096 | +| Nox · v1.2 · this release | 16 | **129.421** | **130.165** | 4192 | +| Nox · v1.1 · previous | 16 | 129.926 | 130.926 | 4192 | +| Nox · v1.2 · this release | 32 | 260.115 | **261.017** | 8384 | +| Nox · v1.1 · previous | 32 | **259.700** | 262.418 | 8384 | + +## Score + +| Model | Q | p50 ms ↓ | p95 ms ↓ | Input tokens | +|---|---:|---:|---:|---:| +| Nox · v1.2 · this release | 1 | 32.601 | 33.154 | 307 | +| Nox · v1.1 · previous | 1 | **28.816** | **29.119** | 307 | +| Nox · v1.2 · this release | 2 | 33.221 | 33.625 | 614 | +| Nox · v1.1 · previous | 2 | **29.339** | **29.777** | 614 | +| Nox · v1.2 · this release | 4 | **41.776** | 42.379 | 1228 | +| Nox · v1.1 · previous | 4 | 41.965 | **42.220** | 1228 | +| Nox · v1.2 · this release | 8 | **69.535** | **70.140** | 2456 | +| Nox · v1.1 · previous | 8 | 69.583 | 70.672 | 2456 | +| Nox · v1.2 · this release | 16 | **140.447** | **141.969** | 4912 | +| Nox · v1.1 · previous | 16 | 141.124 | 142.791 | 4912 | +| Nox · v1.2 · this release | 32 | **281.576** | **284.902** | 9824 | +| Nox · v1.1 · previous | 32 | 283.684 | 286.097 | 9824 | + +Choice six-point geometric-mean latency ratio (candidate / predecessor): **1.0337**. + +The candidate has 3.37% higher measured latency by this summary. This is a descriptive result for these six request sizes, not a claim of universal speedup. + +The runtime/profile and temperature belong to these exact measured bundles. Older release timing is not reused. diff --git a/README.md b/README.md index c450fdb566400b75d96871ce36baea53a73e4f69..8d485977771209af7f0ab93fc0608805ed0a624a 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,6 @@ tags: - custom-code - pytorch - rocm -- choice -- noul -- scoring -datasets: -- PolyAI/banking77 -- clinc/clinc_oos -- nyu-mll/multi_nli --- ![Decision 1.0 — Your move.](assets/decision-family-header.png) @@ -27,69 +20,47 @@ datasets: *Nox, Latin for night.* -**Your move.** - -A capable decoder for decisions defined by you. Give Nox a state, questions and possible answers. It returns choices, yes/no judgments and rubric scores with probability distributions—one forward pass per question. +**Your move.** Give Nox a state, questions and possible answers. It returns decisions and probabilities with labels defined at runtime. **4.208B parameters · 16K complete-question budget · English / Chinese evaluated · Apache 2.0** -[Decision family](https://huggingface.co/collections/llm-semantic-router/decision-10-6ab12177bd0002394d8409f9) · [Meet Sol](https://huggingface.co/llm-semantic-router/Decision-1.0-Sol) - -## Three ways to decide +[Decision family](https://huggingface.co/collections/llm-semantic-router/decision-10-6ab12177bd0002394d8409f9) | Type | Use it for | Output | -| --- | --- | --- | -| **Choice** | Route a request; select an action from 2–255 candidates. | Selected ID + distribution | -| **Noul** | Check a condition against available evidence. | P(true) | +|---|---|---| +| **Choice** | Route a request or choose among 2–255 actions. | Selected ID + distribution | +| **Noul** | Check a condition against supplied evidence. | P(true) | | **Score** | Apply 2–10 ordered rubric descriptions. | Expected index + distribution | -Your question names and candidate IDs are preserved. Labels are defined at runtime. - ## Measured capability -**67.05% overall**, up 4.33 points from Nox v1.0 across 20 equally weighted capabilities. - -| Model | Overall accuracy ↑ | Choice ↑ | Noul ↑ | Score ↑ | -| --- | ---: | ---: | ---: | ---: | -| Jev · 1.13.0 | **72.74** | **71.77** | **80.36** | **68.06** | -| Nox · 4B · v1.1 | 67.05 | 70.55 | 60.27 | 55.56 | -| Qwen3.5 · 4B · untuned | 56.61 | 60.06 | 53.57 | 52.08 | -| Sol · 2B · v1.0 | 55.58 | 61.93 | 50.89 | 32.64 | -| Decider · 2B | 55.30 | 57.26 | 58.93 | 50.00 | -| Qwen3.5 · 2B · untuned | 48.06 | 50.36 | 45.09 | 47.22 | -| Laya · EN/ML | 47.38 | 53.02 | 52.23 | 19.44 | - -Accuracy (%). Overall is the equal-weight mean of **20 task families**, covering **1,760 questions**; type columns pool their questions. Same requests for every model. Laya's native interface reported 152 truncated inputs. [Methods, coverage and uncertainty](EVALUATION.md). - -![Decision quality ranking](assets/decision-quality.png) - -![Capability comparison across twenty families](assets/decision-capabilities.png) +**74.20% four-panel mean**, +1.27 points versus its published predecessor. -### Reading comprehension +| Model | Mean accuracy ↑ | Decisions | Composition | Reading | Inference | +|---|---:|---:|---:|---:|---:| +| Jev · 1.13.0 | **82.45** | 79.10 | **66.38** | **94.53** | **89.79** | +| Nox · v1.2 · this release | 74.20 | **83.21** | 51.08 | 78.12 | 84.38 | +| Nox · v1.1 · previous | 72.93 | 82.85 | 51.25 | 78.44 | 79.17 | +| Decider · 2B | 71.75 | 64.01 | 46.58 | 92.03 | 84.38 | +| Qwen3.5 · 4B · untuned | 70.25 | 69.89 | 43.33 | 87.97 | 79.79 | +| Qwen3.5 · 2B · untuned | 60.54 | 57.12 | 39.00 | 73.75 | 72.29 | +| Laya · Upstream default | 52.44 | 57.01 | 37.75 | 51.25 | 63.75 | -On a separate **480-question human-annotated reading benchmark**, Nox improves **75.47% → 78.44%**. Jev, Decider and the untuned 4B parent remain stronger on this task. +Accuracy (%), **2,720 decisions**. Each panel contributes one quarter; its original family/source weights are retained. Bold marks each column's exact highest score. [Methods, uncertainty and all task results](EVALUATION.md). -| Model | Weighted accuracy (%) ↑ | -| --- | ---: | -| Jev 1.13.0 | **94.53** | -| Decider 2B | 92.03 | -| Qwen3.5 4B + LM-head adapter | 87.97 | -| Nox · 4B · v1.1 | 78.44 | -| Qwen3.5 2B + LM-head adapter | 73.75 | -| Sol · 2B · v1.0 | 69.38 | -| Laya EN/ML routed | 51.25 | +![Expanded decision ranking](assets/decision-expanded-ranking.png) -Weights: 50% BoolQ, 25% Belebele English, 25% Belebele Chinese. [Exact counts, uncertainty and source limitations](EVALUATION.md#human-source-confirmation-v4). +![Four-panel capability overview](assets/decision-expanded-overview.png) ## More questions, measured -![Latency as questions per request increase](assets/decision-question-scaling.png) +![Current release request latency](assets/decision-question-scaling.png) -**Measured with v1.0 weights.** Same state, same question length, four choices; only the number of questions changes. Each model keeps its native interface (Nox: 309 tokens/question). Median of 30 requests per point on one AMD gfx942 GPU; local Python request time includes tokenization and inference, excluding loading and network. [p95, drift checks and full methods](QUESTION-SCALING.md). +Same inputs and physical AMD gfx942 GPU; 30 measured requests per point across three blocks. Python request latency includes tokenization and inference, excluding loading and network. [p95 and all three native types](QUESTION-SCALING.md). ## Try it -Download the fixed release with `hf download llm-semantic-router/Decision-1.0-Nox --revision v1.1 --local-dir decision-model`, then follow the [ROCm setup](RUNTIME.md). Inside that container, with the model mounted at `/model`: +Download `hf download llm-semantic-router/Decision-1.0-Nox --revision v1.2 --local-dir decision-model`, then follow [ROCm setup](RUNTIME.md). In that container, with the model mounted at `/model`: ```python from decision import DecisionModel @@ -99,16 +70,16 @@ model = DecisionModel.from_pretrained("/model", local_files_only=True) print(model.decide(**REQUEST)["answers"]) ``` -[Actual request and measured output](model-card-example.json) · [Install and API guide](USAGE.md) +[Tested request and output](model-card-example.json) · [Install and API guide](USAGE.md) -The complete state, question and candidates must fit 16,384 tokens. Overflow is rejected. Score returns an expected ordinal index. Runtime: AMD gfx942 validated; CPU/MPS unsupported; NVIDIA unqualified. +The complete state, question and candidates must fit 16,384 tokens; overflow is rejected. The bundled normalization profile loads automatically. AMD gfx942 is validated; CPU/MPS are unsupported and NVIDIA is unqualified. Use a fresh Python process when switching profiles. ## Architecture ![Decision decoder architecture](assets/architecture.png) -A causal Qwen3.5 text backbone combines gated linear attention with full attention. A shared candidate head reads candidate endpoints and the final query vector. Questions run independently in batches of eight. +A causal Qwen3.5 text backbone combines gated linear and full attention. A shared candidate head reads candidate endpoints and the final query vector. Each question uses one forward pass; questions run independently in batches of eight. -[Candidate head](assets/readout.png) · [Editable SVG](assets/architecture.svg) · [Vector atlas](assets/architecture-atlas.pdf) · [Inference code](code/decision_model.py) +[Candidate head](assets/readout.png) · [Vector architecture](assets/architecture.svg) · [Inference code](code/decision_model.py) -Adapted from [Qwen3.5-4B](https://huggingface.co/Qwen/Qwen3.5-4B). It evaluates supplied evidence, without live retrieval; confidence is not a correctness guarantee. [Apache 2.0](https://huggingface.co/llm-semantic-router/Decision-1.0-Nox/blob/main/LICENSE) · [Attributions](ATTRIBUTIONS.md). +Adapted from [Qwen3.5-4B](https://huggingface.co/Qwen/Qwen3.5-4B). It evaluates supplied evidence without live retrieval; confidence does not guarantee correctness. [License](LICENSE) · [Attributions](ATTRIBUTIONS.md). diff --git a/RUNTIME.md b/RUNTIME.md index db2642566b6503f2bab14dcae15a95e18f55c7ff..2106cc049a44e2ec2493aa5b84148b32e716492d 100644 --- a/RUNTIME.md +++ b/RUNTIME.md @@ -2,7 +2,7 @@ The package has a public, digest-pinned installation path. `Dockerfile.runtime` starts from `vllm/vllm-openai-rocm@sha256:1fd21abe66455b4df5a2e83629e97cdcc9d58913b16052d8118b92b239792339`, adds two hash-checked FLA wheels, and installs this repository's loading wrapper. It keeps the base image's ROCm PyTorch and Triton builds. The vLLM server is not used by Decision inference. -**Validation boundary:** the public registry manifest, base-image ancestry, package metadata and critical PyTorch binary hashes have been checked. The recipe built successfully and passed CPU imports and real AMD ROCm gfx942 GPU inference for both initial v1.0 bundles. On the packaged three-question Choice/Noul/Score example, its complete responses matched the qualified research runtime exactly; the wrapper matched the direct engine and rejected an oversized complete input. The initial v1.0 build evidence is in `runtime-build-provenance.json`. Updated weights retain the same installation recipe and are separately checked during release; their actual request and output are in `model-card-example.json`. This example establishes a working public installation path; it is not a full rerun of the quality or timing benchmark. Published benchmark results use the qualified runtime in each bundle's `runtime.json`. +**Validation boundary:** the public registry manifest, base-image ancestry, package metadata and critical PyTorch binary hashes have been checked. The recipe built successfully and passed CPU imports and real AMD ROCm gfx942 GPU GPU inference for both released bundles. On the packaged three-question Choice/Noul/Score example, its complete responses matched the qualified research runtime exactly; the wrapper matched the direct engine and rejected an oversized complete input. Evidence is in `runtime-build-provenance.json`. This example establishes a working public installation path; it is not a full rerun of the quality or timing benchmark. Published benchmark results use the qualified runtime in each bundle's `runtime.json`. ## Build and run diff --git a/RUNTIME_BINDING.json b/RUNTIME_BINDING.json new file mode 100644 index 0000000000000000000000000000000000000000..02627d018524d3d6a21799cb8eb3a8489f9ce2af --- /dev/null +++ b/RUNTIME_BINDING.json @@ -0,0 +1,76 @@ +{ + "source_bundle_manifest_sha256": "6d402d6d55e734851cb0d6417d6f3b416b25ae807c3a0d6dcf12e40acb6ac546", + "profile_sha256": "be32858d15233e0a3fbee0e4257fb02be0b3439deee4eb9c3f61151df7b73850", + "profile_validation_receipt_sha256": "7a66474325b5575fdd15a33e5e0340c19e809bdcc22adf4cbb45baa0e1b32693", + "unchanged_files": [ + { + "file": "backbone/config.json", + "bytes": 1978, + "sha256": "ae3a463b32e95b6cc207a7af4f1defb4195f388eb6f9ff19d2690b73d4966953" + }, + { + "file": "backbone/model-00001-of-00003.safetensors", + "bytes": 3991295368, + "sha256": "5f9c4bc396605b551b5983a1699de8e96755fafdb6afdb228a5cc4d9b438a5da" + }, + { + "file": "backbone/model-00002-of-00003.safetensors", + "bytes": 3979828128, + "sha256": "656cc757ba03f87cedc0e4c88237db1ae8215686f9b6477334a34dc0a32136db" + }, + { + "file": "backbone/model-00003-of-00003.safetensors", + "bytes": 440425856, + "sha256": "0a30a07b6efbaa37d016008848c1fa6dadcb46ae451a0e5e4b927ac9b5771804" + }, + { + "file": "backbone/model.safetensors.index.json", + "bytes": 33047, + "sha256": "1602d52e38d81586af85bc4ce29ce082c5fc5877c763b1ebcab7545320016599" + }, + { + "file": "chat_template.jinja", + "bytes": 7756, + "sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715" + }, + { + "file": "code/decision_model.py", + "bytes": 10114, + "sha256": "d3e28489c09f3bd7130e2d43d92e0b5c4a08e25b09b21303904defb0ff1c3646" + }, + { + "file": "code/predict.py", + "bytes": 3164, + "sha256": "02352e8385ab47157b6459910da54d962e5da4bb4940d571faedc86bc5da9aee" + }, + { + "file": "decision_config.json", + "bytes": 753, + "sha256": "443a9b3f191a8387915c606de8303700fc1a069fe7c8ad46a0eba5528a2549a8" + }, + { + "file": "decision_head.safetensors", + "bytes": 10529624, + "sha256": "8b8e342445035e503b4963e16c5f7a4fef95c08368e27eace312145fc3660772" + }, + { + "file": "temperature.json", + "bytes": 6095, + "sha256": "1d652e75f49b5c232d1e742b189c1c826ad52ced1a931a124e29e65e7d7fb569" + }, + { + "file": "tokenizer.json", + "bytes": 19989325, + "sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523" + }, + { + "file": "tokenizer_config.json", + "bytes": 1123, + "sha256": "bee8eba30f0eb4af73c0fe2cd06d0f89b657d7819941c438157ec42f7c80ea87" + } + ], + "training_selection_calibration_unchanged": true, + "recomputed_or_recalibrated": false, + "private_compiled_cache_required": false, + "new_bundle_offline_proof_required": true +} diff --git a/SOURCE_BUNDLE_MANIFEST.json b/SOURCE_BUNDLE_MANIFEST.json new file mode 100644 index 0000000000000000000000000000000000000000..e93c76ac366e0f8c4a748927663c5917adc30475 --- /dev/null +++ b/SOURCE_BUNDLE_MANIFEST.json @@ -0,0 +1,173 @@ +{ + "format": "research-pointer-bundle-v1", + "status": "candidate-export-awaiting-independent-reload-and-quality-gates", + "files": [ + { + "file": "backbone/config.json", + "bytes": 1978, + "sha256": "ae3a463b32e95b6cc207a7af4f1defb4195f388eb6f9ff19d2690b73d4966953" + }, + { + "file": "backbone/model-00001-of-00003.safetensors", + "bytes": 3991295368, + "sha256": "5f9c4bc396605b551b5983a1699de8e96755fafdb6afdb228a5cc4d9b438a5da" + }, + { + "file": "backbone/model-00002-of-00003.safetensors", + "bytes": 3979828128, + "sha256": "656cc757ba03f87cedc0e4c88237db1ae8215686f9b6477334a34dc0a32136db" + }, + { + "file": "backbone/model-00003-of-00003.safetensors", + "bytes": 440425856, + "sha256": "0a30a07b6efbaa37d016008848c1fa6dadcb46ae451a0e5e4b927ac9b5771804" + }, + { + "file": "backbone/model.safetensors.index.json", + "bytes": 33047, + "sha256": "1602d52e38d81586af85bc4ce29ce082c5fc5877c763b1ebcab7545320016599" + }, + { + "file": "chat_template.jinja", + "bytes": 7756, + "sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715" + }, + { + "file": "code/decision_api.py", + "bytes": 6724, + "sha256": "147b2fec32cbbbbb1b92cf2a19bb887f9d945e4974e141ca7ea93b8c542f5b21" + }, + { + "file": "code/decision_model.py", + "bytes": 10114, + "sha256": "d3e28489c09f3bd7130e2d43d92e0b5c4a08e25b09b21303904defb0ff1c3646" + }, + { + "file": "code/predict.py", + "bytes": 3164, + "sha256": "02352e8385ab47157b6459910da54d962e5da4bb4940d571faedc86bc5da9aee" + }, + { + "file": "decision_config.json", + "bytes": 753, + "sha256": "443a9b3f191a8387915c606de8303700fc1a069fe7c8ad46a0eba5528a2549a8" + }, + { + "file": "decision_head.safetensors", + "bytes": 10529624, + "sha256": "8b8e342445035e503b4963e16c5f7a4fef95c08368e27eace312145fc3660772" + }, + { + "file": "runtime.json", + "bytes": 378, + "sha256": "c5d3521358b2817f4e56ea8150c5c612139b4e2b2c5bb09f512d9a7c0b5298b9" + }, + { + "file": "temperature.json", + "bytes": 6095, + "sha256": "1d652e75f49b5c232d1e742b189c1c826ad52ced1a931a124e29e65e7d7fb569" + }, + { + "file": "tokenizer.json", + "bytes": 19989325, + "sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523" + }, + { + "file": "tokenizer_config.json", + "bytes": 1123, + "sha256": "bee8eba30f0eb4af73c0fe2cd06d0f89b657d7819941c438157ec42f7c80ea87" + } + ], + "tensors": [ + { + "file": "backbone/model-00001-of-00003.safetensors", + "elements": 1995638528, + "elements_by_dtype": { + "BF16": 1995638528 + } + }, + { + "file": "backbone/model-00002-of-00003.safetensors", + "elements": 1989900928, + "elements_by_dtype": { + "BF16": 1989900928 + } + }, + { + "file": "backbone/model-00003-of-00003.safetensors", + "elements": 220211840, + "elements_by_dtype": { + "BF16": 220211840 + } + }, + { + "file": "decision_head.safetensors", + "elements": 2632192, + "elements_by_dtype": { + "F32": 2632192 + } + } + ], + "source_checkpoint_files": [ + { + "file": "backbone/config.json", + "bytes": 1977, + "sha256": "a5ed4156fda05f0f9149c66964d6165916754e7355488a8a07d9b0398acdbdb9" + }, + { + "file": "backbone/model-00001-of-00005.safetensors", + "bytes": 3992269864, + "sha256": "8adb3dbf9db30b3ecd1f29ce6a853bc6d7e3f465857471cfe579c570537250f8" + }, + { + "file": "backbone/model-00002-of-00005.safetensors", + "bytes": 3990302320, + "sha256": "8aa80bed34a466a7208ad177d5e5742773e4140c55cba69911fcc805633afa5e" + }, + { + "file": "backbone/model-00003-of-00005.safetensors", + "bytes": 3937871784, + "sha256": "9af681e845c99e0d933dc3fa643271ee9f5bb382f98454ceb41eb1faf0979f87" + }, + { + "file": "backbone/model-00004-of-00005.safetensors", + "bytes": 3926578128, + "sha256": "34de5c39c2338e1c7110beaf2834c856f3830411002893f7946045c0a083cc92" + }, + { + "file": "backbone/model-00005-of-00005.safetensors", + "bytes": 976029360, + "sha256": "ce20083dd8deb786b14fe1126a12cdbf678a8a3c25367a6676c0ca55d9ae7980" + }, + { + "file": "decision_config.json", + "bytes": 1110, + "sha256": "a12bd49b459771f22d22f29c59fe3cb6779c52a5d39b40cdbe5e247c5181b3fc" + }, + { + "file": "decision_head.safetensors", + "bytes": 10529624, + "sha256": "8b8e342445035e503b4963e16c5f7a4fef95c08368e27eace312145fc3660772" + }, + { + "file": "tokenizer.json", + "bytes": 19989325, + "sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523" + }, + { + "file": "tokenizer_config.json", + "bytes": 1123, + "sha256": "bee8eba30f0eb4af73c0fe2cd06d0f89b657d7819941c438157ec42f7c80ea87" + } + ], + "source_model_code_sha256": "d3e28489c09f3bd7130e2d43d92e0b5c4a08e25b09b21303904defb0ff1c3646", + "source_api_code_sha256": "147b2fec32cbbbbb1b92cf2a19bb887f9d945e4974e141ca7ea93b8c542f5b21", + "dev_sha256": "45b4cd46acc4be53b95c7ed8f333b3533972296a4d0557c7b8c48c9cab6ced61", + "production_predictions_sha256": "cf0b1a17008bddc2e3e18289866993770c41d4a5af5f6f4219508bb34fcabefa", + "temperature_sha256": "1d652e75f49b5c232d1e742b189c1c826ad52ced1a931a124e29e65e7d7fb569", + "runtime_sha256": "c5d3521358b2817f4e56ea8150c5c612139b4e2b2c5bb09f512d9a7c0b5298b9", + "production_batch_size": 8, + "input_length_limit": 16384, + "original_checkpoint_name": "winner", + "no_publication_performed": true +} diff --git a/assets/architecture-atlas.pdf b/assets/architecture-atlas.pdf deleted file mode 100644 index 0026e78ffed2d4b4be209ad5646d3166dbdb985e..0000000000000000000000000000000000000000 --- a/assets/architecture-atlas.pdf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d1582ac115aa6a40129b6221e15a197e33d20847e03d69396f02fdab82771376 -size 306242 diff --git a/assets/architecture.pdf b/assets/architecture.pdf deleted file mode 100644 index 3d94346a23104039a8ca5d04c5cd214ba203d0fc..0000000000000000000000000000000000000000 --- a/assets/architecture.pdf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ee87f997a0aa4343c2ec66038b7526361e0a3c0ae02b383f8928aac9dd9bd74c -size 183111 diff --git a/assets/decision-capabilities.pdf b/assets/decision-capabilities.pdf deleted file mode 100644 index 7ef95dd53592ddac69407a4f74d08702931dff7a..0000000000000000000000000000000000000000 --- a/assets/decision-capabilities.pdf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1db9c9fcad30a0721692eb2143149001686f37e1ab9639395aa40886185f0e1f -size 51584 diff --git a/assets/decision-capabilities.png b/assets/decision-capabilities.png deleted file mode 100644 index 86f86ab1291a6d8ac78b3cd5fa21286fb50d0d75..0000000000000000000000000000000000000000 --- a/assets/decision-capabilities.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bf584ab8924fe6f1e4623a5d402cac6fbfde96f12eaefa8c6b650ede9d95f21b -size 451771 diff --git a/assets/decision-capabilities.svg b/assets/decision-capabilities.svg deleted file mode 100644 index e4e29a43b37904d523aaa65fb33b6920704ad1c6..0000000000000000000000000000000000000000 --- a/assets/decision-capabilities.svg +++ /dev/null @@ -1,2533 +0,0 @@ - - - - - - - - 2026-09-22T00:43:41.565381 - image/svg+xml - - - Decision measured update renderer - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Jev - - - 1.13.0 - - - Nox - - - 4B · v1.1 - - - Qwen3.5 - - - 4B · untuned - - - Sol - - - 2B · v1.0 - - - Decider - - - 2B - - - Qwen3.5 - - - 2B · untuned - - - Laya - - - EN/ML - - - News topics - - - 85.2 - - - 83.6 - - - 84.4 - - - 82.8 - - - 86.7 - - - 80.5 - - - 91.4 - - - Boolean rules - - - 100.0 - - - 93.8 - - - 62.5 - - - 42.2 - - - 71.9 - - - 37.5 - - - 43.8 - - - Entity types - - - 96.4 - - - 96.4 - - - 97.3 - - - 93.8 - - - 98.2 - - - 92.9 - - - 83.9 - - - Intent routing - - - 100.0 - - - 100.0 - - - 100.0 - - - 100.0 - - - 100.0 - - - 96.9 - - - 85.9 - - - Cross-option context - - - 30.2 - - - 100.0 - - - 72.9 - - - 100.0 - - - 8.3 - - - 9.4 - - - 95.8 - - - Ordinal scoring - - - 100.0 - - - 89.1 - - - 90.6 - - - 43.8 - - - 84.4 - - - 81.2 - - - 18.8 - - - Relation composition - - - 56.2 - - - 55.2 - - - 51.0 - - - 45.8 - - - 54.2 - - - 49.0 - - - 25.0 - - - Evidence scope - - - 89.6 - - - 75.0 - - - 51.0 - - - 54.2 - - - 47.9 - - - 36.5 - - - 37.5 - - - State updates - - - 33.3 - - - 35.4 - - - 28.1 - - - 18.8 - - - 29.2 - - - 25.0 - - - 24.0 - - - Outside-menu detection - - - 100.0 - - - 100.0 - - - 60.9 - - - 81.2 - - - 59.4 - - - 62.5 - - - 64.1 - - - Record identity - - - 76.2 - - - 50.0 - - - 50.0 - - - 58.8 - - - 56.2 - - - 46.2 - - - 47.5 - - - Capacity constraints - - - 68.8 - - - 43.8 - - - 50.0 - - - 50.0 - - - 51.2 - - - 50.0 - - - 63.7 - - - Constraint assignment - - - 55.0 - - - 42.5 - - - 23.8 - - - 16.2 - - - 23.8 - - - 21.2 - - - 22.5 - - - New intents · EN - - - 91.7 - - - 91.7 - - - 91.7 - - - 90.0 - - - 90.0 - - - 70.8 - - - 74.2 - - - New intents · ZH - - - 88.3 - - - 88.3 - - - 86.7 - - - 89.2 - - - 88.3 - - - 74.2 - - - 73.3 - - - Multiset reconciliation - - - 58.8 - - - 31.2 - - - 27.5 - - - 26.2 - - - 23.8 - - - 27.5 - - - 26.2 - - - Service-loss scoring - - - 42.5 - - - 28.7 - - - 21.2 - - - 23.8 - - - 22.5 - - - 20.0 - - - 20.0 - - - Conflicting evidence - - - 66.2 - - - 35.0 - - - 25.0 - - - 28.7 - - - 26.2 - - - 27.5 - - - 18.8 - - - Temporal exclusion - - - 41.2 - - - 41.2 - - - 31.2 - - - 35.0 - - - 42.5 - - - 28.7 - - - 12.5 - - - Transaction recovery - - - 75.0 - - - 60.0 - - - 26.2 - - - 31.2 - - - 41.2 - - - 23.8 - - - 18.8 - - - MEAN - - - 72.74 - - - 67.05 - - - 56.61 - - - 55.58 - - - 55.30 - - - 48.06 - - - 47.38 - - - - - - - D E C I S I O N 1 . 0 - - - Capability profile - - - 1,760 decisions · all 20 families · accuracy (%) - - - - - - - - - - - diff --git a/assets/decision-expanded-old_core-600px.png b/assets/decision-expanded-old_core-600px.png new file mode 100644 index 0000000000000000000000000000000000000000..7e3d59c6c97fad779ae2883b578a4bfb43e3dd77 --- /dev/null +++ b/assets/decision-expanded-old_core-600px.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:519923f36d62df99c6769731416d70413c9f5ec3991ba43a75de0714767a12cc +size 140155 diff --git a/assets/decision-expanded-old_core.pdf b/assets/decision-expanded-old_core.pdf new file mode 100644 index 0000000000000000000000000000000000000000..36ff27c92f667f8a821dea19ee62a1f22381fd0f Binary files /dev/null and b/assets/decision-expanded-old_core.pdf differ diff --git a/assets/decision-expanded-old_core.png b/assets/decision-expanded-old_core.png new file mode 100644 index 0000000000000000000000000000000000000000..c2987afc600b1f1d5d402a46a7f362d49fad5dac --- /dev/null +++ b/assets/decision-expanded-old_core.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c59e464a07320a13a182ec841fef0a826eed777d03264260d3a3d6bcf99a2819 +size 261350 diff --git a/assets/decision-expanded-old_core.svg b/assets/decision-expanded-old_core.svg new file mode 100644 index 0000000000000000000000000000000000000000..27c9c63c48354d64653349997b2ff3cf53935c3d --- /dev/null +++ b/assets/decision-expanded-old_core.svg @@ -0,0 +1,1434 @@ + + + + + + + + 2026-09-22T05:19:05.190230 + image/svg+xml + + + Decision expanded aggregate renderer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Jev + + + 1.13.0 + + + Nox + + + v1.2 · this + release + + + Nox + + + v1.1 · + previous + + + Decider + + + 2B + + + Qwen3.5 + + + 4B · + untuned + + + Qwen3.5 + + + 2B · + untuned + + + Laya + + + Upstream + default + + + ag news + + + 85.2 + + + 85.2 + + + 83.6 + + + 86.7 + + + 84.4 + + + 80.5 + + + 91.4 + + + boolean constraints + + + 100.0 + + + 93.8 + + + 93.8 + + + 71.9 + + + 62.5 + + + 37.5 + + + 43.8 + + + dbpedia 14 + + + 96.4 + + + 96.4 + + + 96.4 + + + 98.2 + + + 97.3 + + + 92.9 + + + 83.9 + + + natural intents + + + 100.0 + + + 100.0 + + + 100.0 + + + 100.0 + + + 100.0 + + + 96.9 + + + 85.9 + + + option carrier + + + 30.2 + + + 100.0 + + + 100.0 + + + 8.3 + + + 72.9 + + + 9.4 + + + 95.8 + + + ordinal rubric + + + 100.0 + + + 89.1 + + + 89.1 + + + 84.4 + + + 90.6 + + + 81.2 + + + 18.8 + + + relational composition + + + 56.2 + + + 54.2 + + + 55.2 + + + 54.2 + + + 51.0 + + + 49.0 + + + 25.0 + + + scoped evidence + + + 89.6 + + + 78.1 + + + 75.0 + + + 47.9 + + + 51.0 + + + 36.5 + + + 37.5 + + + state tracking + + + 33.3 + + + 35.4 + + + 35.4 + + + 29.2 + + + 28.1 + + + 25.0 + + + 24.0 + + + unknown rejection + + + 100.0 + + + 100.0 + + + 100.0 + + + 59.4 + + + 60.9 + + + 62.5 + + + 64.1 + + + PANEL MEAN + + + 79.1 + + + 83.2 + + + 82.8 + + + 64.0 + + + 69.9 + + + 57.1 + + + 57.0 + + + + + + + D E C I S I O N 1 . 0 + + + General decisions + + + 880 decisions · accuracy (%) + + + + + + + + + + + diff --git a/assets/decision-expanded-overview-600px.png b/assets/decision-expanded-overview-600px.png new file mode 100644 index 0000000000000000000000000000000000000000..8aab2201341a316a1bd8bc1cd169c8550d442329 Binary files /dev/null and b/assets/decision-expanded-overview-600px.png differ diff --git a/assets/decision-expanded-overview.pdf b/assets/decision-expanded-overview.pdf new file mode 100644 index 0000000000000000000000000000000000000000..944776f1923284c45f1887ede44b7f6f7e59ef77 Binary files /dev/null and b/assets/decision-expanded-overview.pdf differ diff --git a/assets/decision-expanded-overview.png b/assets/decision-expanded-overview.png new file mode 100644 index 0000000000000000000000000000000000000000..0af17bbdacd38de51acb504ce2f241408f4d48e3 --- /dev/null +++ b/assets/decision-expanded-overview.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa9972beb8e9081bb5a8c59e487e69f31567470fd4af6b0d6d36bdb82d60cdf1 +size 167930 diff --git a/assets/decision-expanded-overview.svg b/assets/decision-expanded-overview.svg new file mode 100644 index 0000000000000000000000000000000000000000..df15c54a630f18e5d14a1d79506fdbf9de081928 --- /dev/null +++ b/assets/decision-expanded-overview.svg @@ -0,0 +1,714 @@ + + + + + + + + 2026-09-22T05:19:04.764999 + image/svg+xml + + + Decision expanded aggregate renderer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Jev + + + 1.13.0 + + + Nox + + + v1.2 · this + release + + + Nox + + + v1.1 · + previous + + + Decider + + + 2B + + + Qwen3.5 + + + 4B · + untuned + + + Qwen3.5 + + + 2B · + untuned + + + Laya + + + Upstream + default + + + General decisions + + + 79.1 + + + 83.2 + + + 82.8 + + + 64.0 + + + 69.9 + + + 57.1 + + + 57.0 + + + Compositional tasks + + + 66.4 + + + 51.1 + + + 51.2 + + + 46.6 + + + 43.3 + + + 39.0 + + + 37.8 + + + Natural reading + + + 94.5 + + + 78.1 + + + 78.4 + + + 92.0 + + + 88.0 + + + 73.8 + + + 51.2 + + + Reading and inference + + + 89.8 + + + 84.4 + + + 79.2 + + + 84.4 + + + 79.8 + + + 72.3 + + + 63.7 + + + FOUR-PANEL MEAN + + + 82.4 + + + 74.2 + + + 72.9 + + + 71.8 + + + 70.2 + + + 60.5 + + + 52.4 + + + + + + + D E C I S I O N 1 . 0 + + + Capability overview + + + 2,720 decisions · accuracy (%) + + + + + + + + + + + diff --git a/assets/decision-expanded-ranking-600px.png b/assets/decision-expanded-ranking-600px.png new file mode 100644 index 0000000000000000000000000000000000000000..64ae164f9b9f1314caac05b86445dd4768c8c286 Binary files /dev/null and b/assets/decision-expanded-ranking-600px.png differ diff --git a/assets/decision-expanded-ranking.pdf b/assets/decision-expanded-ranking.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c4e3ac0d899d841a520a1ad3ada741933f22a4c0 Binary files /dev/null and b/assets/decision-expanded-ranking.pdf differ diff --git a/assets/decision-expanded-ranking.png b/assets/decision-expanded-ranking.png new file mode 100644 index 0000000000000000000000000000000000000000..9211da8f4b64b9902306b60152d9a2a740015a35 --- /dev/null +++ b/assets/decision-expanded-ranking.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:038b995974df964403278b2d121bf9b888c52387aeed4e1726ddf5cbbb18f0b3 +size 171628 diff --git a/assets/decision-quality.svg b/assets/decision-expanded-ranking.svg similarity index 56% rename from assets/decision-quality.svg rename to assets/decision-expanded-ranking.svg index 0d287a906213b6b7f0472f3663f45b0e601a90d8..89911fb6cac9a858d413c1ed111d197b21c606f9 100644 --- a/assets/decision-quality.svg +++ b/assets/decision-expanded-ranking.svg @@ -1,16 +1,16 @@ - + - 2026-09-22T00:43:40.873616 + 2026-09-22T05:19:04.293029 image/svg+xml - Decision measured update renderer + Decision expanded aggregate renderer @@ -21,360 +21,323 @@ - - + +iVBORw0KGgoAAAANSUhEUgAAAGEAAABhCAYAAADGBs+jAAAvKUlEQVR4nO19d5ycZ3Xu835tetsp2/uuykqrbnXZktxtbMA2hkAAU3Ih5Bcg5IZwe0Iul9zkJpAQmh1sTIIdsI0hSOAmucqSbKutVtKqbN+d3Z3e+3zv/Z33m1nzy38GSxbJvvbYsrQ7+82pz3nOOa+BpbN0ls7SWTpLZ+ksnaWzdJbO0lk6S2fpLJ2ls3SWzlV7GK7yowAwyTKTZFllqmJXGHMoiuwqQ7KrVV3TwCWzJPEoUNB1PaVwPabrega6ntcr1UqqUgHH1X2uOiVIAMyqqlo1zaeZrSvcFtsmWVMHobBem9XUbLeoDosmm60mRTapssQYUCrrLF8q65l8pZwtVDLZQiWu63xa1fnZTDJxNFbIncjn89OFajWv86tPJVeNEkyKojY7ncttDtdNTNVu8Ti0VZ0Be0NnwKb6GxwwqSqKxQKqYChWOCo6wMEgSTKgl6HIDKoEaIoETZGF9acLVYwHE/zsdCKzEM1cKpXKz+jF7L/OxOOnKpVqAVfJYe/0D7dZrH6vx3ObSTP/TkfAvnlNl8fR7DYxEnA0U0Ysr6PR34BYpoSWlmZks1mhiHKF0xuwSrnCGRhK5SLAOSqVMvRKGV6nGYV8AS6LjAabwiyqxOfiBbwxGsuNzMRO6sXiPyUSsScTuVzknZRBXQ7vyHGYzX6Xy3Ofx+362Nblvr6BFrtUKHOMhfOYieWxkMghmi4hky+x3ZtXosnv4T1dLYyiSbVqaKBSpnjPUSiWUCpVkM0VEIunUSqVsG5VN773+PNCMV6nhbX47Ly70Y7egAUWBTg2ltBfOTN3IRRLPJDPZh7J5HPh/zBKMKuq1evx3O1xub+4Z03Tir5GO5tNlHBpIYfRYAILiTxKVfIDtvh0/gYnPvzunbDabCzgcUCSZV6pVsEkWQhckiSk0xlE42nMLUREKGryN+AffviMUALnHIwxZtZktDbYeEfAjlXtDrQ2mHF6MsGfPzU3Eksk/jqVSj6RLRaz7wT4uCJHYoy5nc41Hrf3K1tXBG7Y0u9XJkIZPHUqhAvBFOKZIphEaRkkMBHrKemSEKOJNEwWKxLpLNas7OGcMejVqvha8oxypQKzSQWv6ijkc1A0DeFERvy5qiigZKxzzotlHaPzKYzOJ3F2yoplrS4MdjjY5+4cWPnC8Pz9x8+HfzeRiv/XUDx+jL7+35USNFXRmn2NH2/0u//n3Vs6AvmyjgOnQxiejiORLQuHrCtAJuFLjJQmlMG5LoQcTWRRKhXhdtkhyRLKlJlJubIivEFVZKSSaWgmDQ1uJ85PhCAzBlU1lFCtKU3XDc+Yj+eE112YTWKw0411nW5ly7LA3sdfndyvmCx/GQovfKdUqeT/XSjBbbE0ePyNX9nYG/joroGANhLM4LVLUUyE0ga6YYawZZkehUOWJEgSCZ+snARmeMZCNAkKJ1ySYbGYYSWlMQmcMyHgcqkIWZHBwKEoCnKFcu19dBGuGGTxPFwihejiPXVdx0KigFBijo0vZPiG3gZ8dE+X99CI46vPn1KviUdCf5LM52Z/q5Xgb2jocjs999+ysXVPa4ONHRwO4fhYFNlCVVgzhShZkkgXkKU3FUFRqKJXhdWSAE0MSKbSzNTgQiKVYwG/l5Py6M91gHG9CpNm4ppmAhMpW0elXCIPBOUO8giFYCuFJVKAQl7HUKnqQhH0uhhMsvlYloeTBWzp8yht7p73/fiQ3GuORz8ZSiROX87YZMSAt/mQEANeX7/X1/jI+7Z37LWbFPbLE3N4dSSEbNFQgPwrL1WRa0LShZWSBfudFnT4HfA5VDDGDYHrVUzOhoSyVE2Doqq179VQLpeRz2UBCl/ling/s8LQ5nWwJrcFJgUij9B7U55QZIlRPaHIEnkOV2SZZ4pVvDg8j1+eXEBJ5+xje7s3BgKNj3U0Nm29LIKqncvy3h6nq8vvafjn929t2Zwp6njm1AJOT8VR5YAsy8ZLkqDIMiQG8gZhjWZVRoffjiaXGflSBVPhNGYjGWRyJaiqysEkPnJpmgIVI2EKJSoqqtUyp7xRLZcQjydZvlCC02lHLF3AdCTDQ8k8HBYNvc1uNDjMQlHkFRIT+UoUd6Q0Ug6FuFPjMTx3Ooy5ZAkf29PZ52nwPNLq82/Db0s4clqtfp/X/727tndsjOcqeOlcBJPhTA3xSIbwFYr7FM/JySkvcLT6HShXOWajGRRLZaEU+jMRBhiDz2NHKpXFfCQJvZYvSJmlUomXyhVUymUUCkXhBcHgAutqC1ANwSvVCqo6w1yMQ5ELzGnV0BlwIJEpIJmrCEAgMQpqivBCTiUIGMbmUihXqrh2pQ8f2dXW8dCL/OGCXrk7HIufvqo9waJpVq+/6Wu3b2y5Nl+osEMjYTYVyRrCF6FHFuFAJErCn5zD7zTzZq8TM+E0phYSICuu6jqquoFo6om5p7MNc6EYxieDCMeSKJXK4s8p5qdTGeRyBaEcioW5XB6dzd7F5EuvSqVCyuWxVJ6PzSe5pCi8q9EBk2I8jwhLEjO8QjynhOlwFs+fCWEiWsBHrm3r8Xl8D3pstnZcrUqQJUnyewOf27HMd4/drLBD56N8IpTljElMoRiukNtLMKmKQESUG3uaPcgVKxifiyNfqoqQQnCy5iDCCyhPWMwa/D4P8oUistkchs+N8VKxJPIAfXEuX0C+WBLCL5dKvFzMU2jjrU0NRtEHqrKrNaXpqFQ5FmIZTCyk0OixwW1TwXXKRVRXUFiSBbQlRQSjebw8EmORdBnv296+zuPxf8OqmRy4GpXgdbj29rQ0/Mn6ngbl6GgSF+bSwtVlmfF6Aibj1zlnDgoJjS5MLiR4PJUHhZNqtcJI2GSRIkxxCK9hTML61X2IRKLi55jNGl45OoRytcoKxRLLZPOMEnYhn0c+nxeCJrJvPjjHbrp2I3QumL564Vzziqp4FctVXJqNCSqkPeCseR1EfqDnIEWQ91JRefhSgiAy27225daAN/B5hTT0Np235Y3sVmuTx9vwV7evCziPTaRxejJei/VG8pVlJj4UVcFuu4k7LSrOT4WRzRcFFDXqAUqURo1QP1ynuoHh9uu34vS5MZGMNVXFqeGLRFHwTCbLs7k8J4Gmk0kUiyWRRcwmlefSSb59fT8afe5F2oLes1Y9C7RFIapSqfK5WAaRRBadfjtII/QIRngyjEeSGD8zlcChkQg293nknjbPH7lc7htwtShBkiTJ5fb90Y1rmgaDqTKOj8VQrPIaDDXwuaqqAv17nGZYTCrG5hLUAxDCQA2Ak2ERAVcqV0kZnDESB8fm9SugqYzFkxmYNOrrSEhncjhy7CyymaxIxrlsDsVSiVASLJoChUoPriM6F8R9994kFElhzfAwehIOTWGwaLLIP6QMStTzyQK6m90GF0IhVjFQHL3ou46PxXH4Qgz3bGlx+hu8f+Wy2ZquCiV4nO6NnQHnJ1obrOzkRArhVKGGgBRomiriPxVTbpsJdosJU6GkSKb04cnSzZoi4rbwhtp7cnDGdc68Hic+fM+NODF0QUBJs0khRcBqNuGpg0eJ6GPJRIolEikUCwUoDHDYNJhUiusSCpkEVrQ6cedNW998YOEPHJtWtmP3pj56PrJ9AQQy+RJBWvS0uIWiyHdUVWXCGxiEcb0xGsN4KItb1jeucjk9X3g7wtJv9AaqLKt2u/M/3zgYcA/PZDASTBoUhIB9IlkbFa8iURjC6EwE1QqhFUPcXpcVd92wASb1zRAk0Ey1ClWV8YefuAszwQUQctQ0BSaTBrNZFbF6bi6EMxemkCuWeDQaFfmAvI88z2Y1QZYAmQHpWAj33LAON+xab1i4+Jvh9KU5HB2aoNqFiUqaisGqjnyxjNlwGq0+uzAgcgIKqYpCBiUhGM/j+EQK/U0O1tbo+ni3z7fhHVWC1+3Z2t/qvoUe8txsihFLSUKXagWZgfWBtoALo7NR0H/S7xESoUMh4OjQKAqlSq1mMPA/Cfkz970bVrOGSDQliiubxcJNmsZdTjtsVhUWi4p9zx7mlCNS6RyKRaotOCxmM0xmDU6HTRiCXq2gFF/Ap+7agXvu2LXIS6XyZUQzReEVxs8l1GQk7Ey+jGyhggY7MbMV4YWEmIw6R8b5YBpD02l2x8Ymd9Vk/Zxq8C1XXgmqoih2h/sPd63w2s4G8ywYz3OyfMMTjERMp8ljYVT1VkkBhFSIsKOWJPFDVR2jM9EaW0SyJmGb8Pnfuwerl3ciGkuI8GGxWkDf4rBZYdFUWEya+LVNZezU2TFYrWYSnqACiVV12u2QGSlEg9Ho5NAzUXz0lvX44qffhwaX3TAEkR9qfDhRfBKRgTXWNpWHJjODIqeAJeCrARxyxSqGpohQ1NDVaL894HKtfUeU4LZY+gIu816ykEtzCV4m+YpcYPQB6MGdNhPFWp5M52sUskEl2ywaXHarUUxxgVyEoFqb/fjyn36CDQ50sVg8IZCV1aLBbjPD6bALD6HYbbfZ0NToZf19XfyhHz0Nj7+RaWazAYM5KUNnlDcEKKC8ZNTOQC6Ovav9+Ic/+zj27Fhbe07jWSn0OGwWkavoGQk2zyUKaPLYhDdRjqBap15wjs9n+PBkDHtX+Z0Wu+tj8q/CuiulBLPNcc+W5T43tSNnYznxaUQ1XMt9hN39LgtCiTyqgpgzoCh9XTZfQCpLVL0hBXr8m3ZvZl/78mdYV3sApWKREznncFjhcthht1kp/zBKkjablUmyxK7dvpHvO3AUqUwO3/3nfXzNujWcapJyucJ1nXPNpDGy4jrWV2BYs1QtoN1exZ/93k34m//+Sawb7BPKI6gcTxlNNSNAUd+6glgqxyifGRLWhYfT+1GNcX4uC4dFhctuerfTYm39dWWp/LotSrPFckeLx4ynT4VRKOuGB5BLEwnGdbisqsDexAMR5y8+mEBAXKAiwf1IwKqVPfjY79zG1q/u5bxaRSabY2ZNBbMzUQmrZHmyxKhpT7SD2aRhcGUve/7QCcwtRIVwRifm8S/7D+Gu6zcgEpwS9EUqk+NepxUWlUKkBIlYV8rU0IVVW1WOHSu92Db4UZy4EMRPnj6CI8fPCX6KMpZhMEyQgO0+O5K1jh+gi14TKW4qmsdEJI9tyxoaI7HkrfFs5oErpwSzZWV3o3NFusgxnyiIh6M4r6hGKCJ3dtrMoh4w0I4OndRASVkUUyasWdWLe951HTZvGGCUB8iDCsUiNe951WKBLBUFfCVagxK3qDskhq6OZsxFkjjw8vFaC9Sw0SPHRkSsv2nrCsTmg8wkg1N813Uwi1nh9F7kCopmgkbvr2ow22xQTWZct6EL21a3YXYhjuPnpvDG8BhGLs0gFEkinc1jNpIWRSL15mRJ5lWqQ5iOdL6MsVAW1w94JdVseY9JUR4qUtFxJZRgtdl3D3S4bdPRAiLpgkGAUbwk5KHrsJgUls4WuUHCUReLw+tyoKezGZvWrcDGtcvQ2uKHw2qhBMsVRWGCwaQEShyOIsNSNiOXyyGdJwRDhJ5OeQCZXJE9/KOnao5leBZZOln3Lw++IZRy+65VCE2MUr+Am00m4q9gMqkwmS0kRSgmk+hZy5pJeC4pRFGK6AyYETD5sWeFC4X8SkSSecylKhiajGPfweMo5gpCGRSSqlUJks4RjOXBJQUBt2X9QtjUXKxUpi+7EhRZljWzdU+z147zwyHR66X4LUxOJizCYNVkvhDPwOdx4Y6bt6O7o4m5XTYu4rMkMZvVxGlQi6pnqYaoKMEzZhENG1nOi34xeUC5RNRCBS1NAczMR/C9H/6S4r5B8tRChJHWjemM/Qdeo5DCP3DbFpZamEY2l4MqAyWLBotdgclshlYTPmF/WTMLJfJqyWgwEXBjRRSTMwiNzCCcKPG9y1qx9fdvw1e//xxbiKY45T6hCF0XkWAqmsPyZrtvbMa6Pp3NTl/2xGw1mdxum2mAkFAkVSRKRgBvIRNGLUMjfAwOLMNnP3m3QEj5XB6xaILlsnkakyDczRRZFt+ryBJTVYXXGz1Gg4ejVKmiRNwO5ywQ8PGhkSn+nYf3CbaUGi9C4oyJCpqU+6vn5dfO4Os/eJqrnhZIJotgXykkCuVJMiQSvkwdO6FGI+GShSuKgZgqRSyEkjgfzCLDNBwfnoAcGcUXP7KbU+1C7yV64wByJR3TkSx6G62SWTNv+3Ug0ltWgqwoLV6H2Zct6lRsGSCci8Y5Mwg0hp3b1uPm6zbg2Mkz1OmiKQmuKDKn6pOoDIOC0ESGrlQqnBoygrRgTFhXsVxGhmhp8WEl/vCPnsEjTzwjCik69QEASuR5orFJyLWqlz4SGcjF8Vl8+RuPYShYguz0I5HJI51MoVQoCBqF6glJNYn304lCp0KtXEK1XMDcVBCvnQli+apebFweYFPxIpsYD8KaCuILH94jQAjVPMJgOEcqVxVAxGzSNtlJk2/xvOVvkCW5u81nM8fTRWSLBp9v6JLzFX3t2L1tLZKpFE4NnwdVt7IsiWa+WUBGTVhPtarzVDorpiaciiKSLr1NoZBHIplCJpMVEPDYyQt4/OcvIpnOGRYqnsAg44xDdYaO3s4morT5QiQpYCjpggSUyRVw/z/vx5GV3fjgnbtgdpgQj8agl4tw+TksTpcRDkmB5SIq+QxC00GMBnNYuWkjtqzvYko5Dd3swoWRCRYMhnigRcLnP7gL33rsVaQyBWF00XRBeJjNonRAVW2oVJKXVQmaqvX4XWZpIU7cPWc9nU3obm9Ca5NXDFcdGzovulNOh1XgaZqCqFQqjNhOi9nIARQKbDYrTJqCaqWKRDwhGjSENpKpDI4eP4efPXUIkzOhWiOAnISCtZG4jQhoqMTjtLFPf/BWmC0m9t1HnubD58YEbUKHoDB9LVXVZy5MYtuGlXjXnrUY0GzQF+aQTcShagr0Yh5cr0DmJTR3NIuexcEXTuGEIsFpruLwKydx7bp2+HytmJyYh5xK4DPv3ojnjk3g+PlZpAsVQXUEXOaGabPJm87nL68SICttbquK0fkMqsTzGA1yPhsMCcsjbySLJw+gMCXKfVXlYjaorDPKF3oyjWQqKSpQJsmilxsKx/D6yYt45bVhTAfDhrBFtDYqcUOqhlCNjpvIQ6yqVzE1HcSqgX589r472JNPv8oPvHJS8ECCnaXvozBX1fHK68M4emIEy3rbsHvTMqzu8qLBVEUhHMTsXBgVMNbX7uPLV/dj794NOD00ymNcZrs3dfO+vhbkswVGhlUp5hFNzfMN3R4UC3lcmIkhlavAY9NsnDGit8cumxIMVlFuIoaThnfJSC+MBzE+HUaD24GWgBs+jw1Ol4OoCWI+mULkl6aJGJyjvnCK9je4SN6RWArj0/M4f3EaU8GwKM4Ep1NDPvVdgnqlbcBRJsYdt6zth8Nm4oeOjeCZQyfZ+fEgOtsaBfO6ddMq/O13HkMqk6cMIRRRrycor5y/NI3R8VmosoRGrwNdfjP6G61Yv7qHH5uMwOaOoL27DasGOUILcb5i7WrEo1FMRkq4lNFwJgp+YTaGqXlqy1ZE27ZQATx2TTFxoQRcPk9gTFJU2UGJlSrhmmpEmEhlsgK5hEJhxLMVUZ3abVZOJJpBfMnUkuT5XAG5QgmZbF54AF/kjsTbCwUsMqo17GL8t/F7op7gOvq6WtHZ4kMkkQYNCd9xw2YEWlr5wZdexx237EKD24UvfeV+5ESdYXgUHXJQ8iz6WYVSGflCAWu6O9idt+6EzdfCf/zyw3j66Kvw2E2wyACrlpHbd1aEylKxgGgBgiEg0KCTigWY4ChWdditJlaQZPtlVYLCmKRKko1CCP1goQIxjWg8DCEMl8OJUCKMUlkSgjZCRz2UGKIkXr4eUniNQBNCJ5LNiB5vjkCKPrP4XaOQoz6FIrNXj53hEluJZr8byzub+Ya1A7A67Pjlc6/iwAtH8J7b9+Czn7wLf/3NfxEeRd8nemrc8Ab6i3rd79u7mt1x+14EWlvxze/vY4ePj8BusyCZK3GaviBjIsUTZA44zajm8pxyDk368UrF+CxcF51CzUqfDQbkulwQlVgTziQCyqiQEoziQAiYGE9DEUQRGwqpC1ooQMR1o3Ffn6gwWss6r/eYxV+iDVkrwgQxWUPeta/ZsWkl7r1tO6ci8bWhUfhcdjQ1B+DyehGcC4uJi0g0gZm5ED509w248+bt9Ua/gaZrw8EmTcJ1G/tw7c7N6FnWz984Pc4f+vGzgtYqlCqc6JJcqYJkroxsmYOGmJkkGc3RWj9ENIMWR+9FC5Srvwab+tbCEclL5xVJ1Qh6CLMiCZKABWwkiyM6mXJy1XhASo4itlOirHPHhrXThDtqnsCZGC418o5JM1jKen4QYaoWjog1Xdm7FsH5KBoaXHA47aLjNjo+LSz4+PBFrF3RxU6dviB6E3/6hx/C7HyUHz121hg406vobPXjll2DIpz1r1yOVK6Mr//jT8SySZ0up4Rf5WJCkFhdY/KDUJcABgZKWwQIFOaoBarITKNvvJyeULPXvGoy1atUIRmyenJZKrRq0V1YPn01cTbLe9sFIiG0RBR3zfLF7C6TaCzY8CkaDBjobcHerQMiaS4OiRmlnFDQ6fOTePTnLzH6wER7dHe2Yuf2jVi7qh9D58ZhNWk4d2kaR4+f5fueOcpTqTS+8Kn3CXKRBOdx23H9tpVY0d/JbrtlDzNZbPzP//YHOHb6knhmMhiaBCQam4YO6NdUoRv2YPSi6WnrI/a1di6j9y9WOC+oSF9WT2DCDyrJaqVC3S0x/1wPI/RgqqKiWBJDVsxpt/J33bgd61d3YXl/N2LxFE4Mj+JbD5LFlWteISLXm7ifQ8DcBNHJ9UGhRbxfcx/OMRmMYN1AD7o7W9DZ3gSL2YRIPI3HnnwGLocVwVCMR+NJeGNRTIxP8fbOdqZIMueSjt1bBlhHewu2bV7DfQE/Hnp0P57Yf6gWJlHzYDIm2oujgQQKtQawqMMDWo0wwqtBOhIL4LKZEE3mdF3XE5dVCRXadqlWwkQT2Exvhgh6YKNLpgvIZreZ+Oc+eTfcTgtbsayHf/uhJ8RA1gfuuhHL+zrwX7/yXQFPxaACNwQsYirnmJ6PY2Yhvpj4hSJqIYCgqSQz7Ng0gIDPxdeuWw1fg0tY+N9864diXaq10S0s1+9xIZfPC+WDBXlPux/Lu5vgdduwbfM63tbWhvOj0/ja/U8Y4/PCC4TGRbKl5yIUKCbFa+iMC7bW+Nj0fEai5zTdwa1mFZdmSwQAw5edO2KV6nQyU+LUoqyjFmEdVWOfIJkt4l03bIci6Xjl6GkcO3kWt920E5FYHGdHLrH+7hZ84dMfWLR+ZvB/iwIXWzUivdRzRy0cieQOrB/oZt3tjdi4fhCrl3Wgs70Fj+9/GU/se0n0rPP5Ig0Ji6GD4EIMB14+hhcPvYEd63vR29XMbr5hF+/ubEYymcLf3f+4WMUylFx7BhFyKAkzVCgGSTSpTSGHho/LorgkdRgkovH5rZrErJqMcCKbhV65/EooVMtjC8mCThZF9IRIzmQVOhdjJyv72+H3ODA8IopG/uyLr8FhNaGnuwMT0/MIzgaxd+da9HU1i/xQFbjf+DBvjvAYyjHmkQyFULdt+8aVaG/ywtfgRm9vJ7OYLZiYmMa3H3oSdquZmDiWzBQYIaeJmXk2Nh0SXTmHzcRWrVrGbr95NzasXQmdS3j6xWN45kXqP9SsuwYAFr26Nl1MaIGQH43l0G/VhpR5HSGRF7rsJjhsKqKZwly+WIhfdiVI1croTDhdCjQ4BM4myyX6WlNlduO1G9l7btoKq80Cv9/HnHYL3E4Du99+8y7QbsF8KCp2EqinXOt5wuCEaj3omhDqOI/CAe2p7d2xBoEGO2tu9lNzh1O1TG/0s/3PwyLTRB0RdkWeyeV5i8/J+zsbKW+wgNeJdWtWYM9129DX2yko8yd+8Qr+19/8QFTtRmoSo0eisKQ/r7uF8MKax7psZrGCtYgialCVful1mEUDK5MtjRQK5dxblelb5o7ypdLsbDgZdlq1Dlq4CMUy2LJhhZhyW97XzfcfOIprt64FTc+NnL+E8SkZk9NBsbLk8bgwPRtmyVQW264ZxC8PHMHoRLBmhYbYCQrWrZCsjARz+95rkE6l2eCKLmzdvB4ulwMWixWpVAbjE9N4987lIMR28MQEuhpd6O1oZrFsASaLDRvXrcTObevgcDhFc+hbD/0Uf/fA47VxSzFnRwUnaYGLvQgR9t58nvoCC/VFJufztd0IwzgoK6iKzJq8dh5Ll3m+VDpM0yWXPxwVi/FkJn+G+gA+t03M/Nx7xx7s2rKWL4SiKBeLePDR/bA77Hzlil50tDXC43JifGIKq5Z3C2+Ymwvz5kYv/9B79+D2G7cuEnP0Iqxv5ADRbsd1WwZZsZBnBHU7O1rR292O1pYmwekPnbkoesBnZ1OoyBZcs7ob4VQOkUyRL1/ehztu38N3btvAHQ4np1bp9374C/b1B54Q8FNcikFc2CJlAiaU/+YspgAaZO1ENBLjS6hIeL5ATcYSC01itAccODcVK5eKhWNvVZ6/lhIqul6tVEovTM8n0Oqzio3KeCKJl48MYf9zh3F+dAqxWByPPPEUWlqa0BTwwOdzYT4Uw+qBfsRTGZwYPo9KsYCxyVl85r73YvXKbhDspXhLfYZ6Um7ye0A7BiSI3q5WWCwWkXzp0M888sYQaN6pu6sN8VwZVpcH1+64Btfv3oKbrt/OVy3rht1uQyKRxCtHTvJHfvIMJ8p8kV2tyVyU7LW6uk6XkFYoF9C/rSYZ2TyFIkJNb86zUght8zvR5rURkzpb0XPnr4gS6GTT6eeOXZjPdDW5YTOrOHdpSryIh28OuOH12HH49WFEYgnW0toCr9cjqtGmQIMIA9TjPTY0gvlQHKeGzuH3P3aXMbH3K55Mwhjo7xCb+q0tjejv7UBbWzNMJpNQ1sj5URRyOfR3NmP3tZtFyKGO3p5rN2P9+kH4GjzCW6ib9uqR4xg6O47x6YVaBb/4UwQkpZmcRpeZ5otEt7aO3OhF1HzAbcVCLC0MpU6/1GuXtkYnypwjmcm/nM2UjCWKKzFtkc3nzk/MJ07brPZtH7n3Nub1OKDzqpgDpQ8wF0rg0NFTeO7F1/Ghu2+G3+tGJpMXE26SrOL40AXctHcrVq3oxuFjZ/Ghe27CNRsG8OrrpxdxOL1Po88lpt9amgO8rbVFDIJRzqBLRggqrhtchoYGDwZW9EFRNdG8V2UGTdOojBeNotNnqMPnwIFXTtSe3kCXYty9BomJWb1msBs0KXjo1AR0biwliq/mOqxWK7JzKeEE5VJZhCOyF7/Lir5mJz9xIVRJ59JPlGkF6EopoVypFHLZ3OOvT6S3bl7XyqaCIUSjCUInxqxmLg+n04bTZy7x6p3XwdvgRiSWFjwMhaeLo1P45TMvY+O6AWzZOICDr5zAbXu3ENWwuOhNtIjFpDJNtXBalSK6QavNhZJ1UqXc0uyH2+kQSdfophFU1gXXQz0PUoLbaUMwksGFsdoQRO2qBofdimQ6uziKefC1i8a4vkBmxjQheVLAYxFzTvQzi8VaL7v26m1pQF9bA/YdvnghmUwfwpUeg8zm0j85NnRhjjA5lfR0KQhRBXOhqBgvpyW/S+NTGJ1aoCQNh90swkgPxe9klgobHHzlGF59bVhMTORyWVCP2hhT18WOgZnmSxWFUVyvRytKnprJBK/Xh8aAHw6Ho8a2chEuCtTIJ6IvlQItlszFcviLv/0BymXqtNEx5k6pjSpCjiCFgGyhhEK5aqxp0cQe7SbIEgJeN2KpvIDhpGgyEnoQu0XF8nY35iMpnsmnH8/l8m+ZrqifX3ukO5PNzCaSyZ+NTsx8ZmBZJ8YmZgSr2dnix1QwCimVRt5iwWsnL+LdN10j5nkIZ/d0tQp+KBxNixA2uxBn58dnOSVht8MKp90Kh80Mn8dBiyLIl4rIpDNi2U9YuBhRJ3aTqlpic424LhYCBeoBYtEoZhdiePCRX+DZF4+hWDEYXKPqNoYK6CwOD4j9OFEJG0Qh7UhLkkA9lAuoiKQQVKb7McTenY5lHY0Y7GvC4wdOh6Lx2KN1muWKegLBiHQqdv+REyMxspDmpgB2XLMGrW0tuPfO3VjZ146ArwFDp8/BbLbCZncglUygtdEnxtiT6YzwDJvVLKYy6IPSJPX6VT0ihFCYSaTSSGeymA/HWThiXKFDP0u0WWXaOzaEK/YghAdVEY5G8YsDr+HTX/wa9j131ICj9SExA46KKxlI8lTgGc0eQ7CCya1t/NutGmxWC0JRIxeQemgogV52s4Z1y5qpw8ZH5yKPRaKZ0XdsmTybzZyNxxI/Gjo79un+7mZGVnrw0Em43Q686/rNYo/sjaFLiCYycLucYjq7weMkGoP7vB5GixyaKnExAEwhIp1hFFKIgiA2NBJLQlUlGofkhKpEhforLVCa1qN6gkJUNpfFzOw8vvODn+MXz71mdNNqexAkQtoKiiVz9f41Z5yz5oCHk5BzhZL4TTIEIu1o2b2/3Y8LU2HROqTcUq1yQeSRslb3NmLDsiY8vP/4fC6f+mbVKBreGSVUq1U9Hg1946XDp97b2RZo0nWd9XY180NHh6BKHFs2DLDJYISfPH0eWzeswEI4CofDLpY+iIkl5tFld4jwQzOeLoeVqla6Ug3ZXF4MgfU0tvLmRh+cDgdUukCkLlaDwBECop7BkTeG8fX7n8DY1NxitbtYeXEurmaoNWAER0FdsqlghFGfyrgFxhh5J6jc3exBKJ4RcNoIFsZyIfVNWv0ubF/bibHZuH5hauFbM8HYJfyG5zdeeoun4heT6eTXXj56uloolviynnas7OvAyTOjlBs4bdxQZetyOQ1351UKP2IAmMKHpiqwWWlGyZi6JhsvVSo8V8jz1mYfVg30ieURj9spoKdBeRsNf6ohJqdm8c0Hn8SX/vcDGJuaXww7i/xTTQ9Up9TGX4wsIoo+8XWCsiC9UQnQ7CeP5QjFjN4MQVLqL5Nn0bNuHexkva0eygXHItHId2uF3jt7twU9RDIe+t65i6Z3rVretWtZbyf6utvgcNjw1MHDePdNO8T+QCKZgc1mE/h+/epe7H/2MPIWjReKRWYxm7i3wSW8I51OM09Vh81u4wMrejm9F5GAVgsN8tIqLheL4zReSQXfdx7+VzHGTof2jTnXhfgpT1o0JiYjRJO13g8X6UH0J4T8RPHFdTFPREMDDqtZLDiKuVpacqQQRErQdWxc3Y0da9r5k8+fSc1HQv8lk8nEcLVcMJJKZxKKFv7isy8f//meHRt83gYnBgd6KcbjpSND6O5owYkzo9iybhmqlRKW9bTy0u7NePnwKSiqCS6nFX097aLiLpeKnBTldjtFfUFbOsRPaarB2FKiDs4t4Ec/PYjH970kRloMIq4+XWQkapMq4ebtAzgxMo1puuCqPlpTi1SMEaVY6w1LCloDLrFTcWkqLBQjkrC4gkEXfYW+dh/bua6Dj83G9OMjM9+MxWMv4W06b9vVAIlY7I2FhYU///5jz5SbGn2iuLrrXbuhmhRRFA0NXxCUA8E8m82OrRtWCrgaiSXFapTH40RPVwvWDK7A2sEVgqjzeT1iC9NEFTA4cllDAd/5/s/wT48/JxRgLHobSNXogBr9beNGMTHKX2eK6o6w+MyiwlZkErDYFJ2cSywOJZPwqb1JuaDJ68DeTb28wWnlP33h9IH5hdn/V6bBo6tNCTrXeTgUfPDYieEHnnr+jaqqaujpbMFH7r0N4WgC0VgSEzMhgTL6e1pxZmQMn/v0+8V+w6WJWZbL5WhgmBGKIshKHkDjlITZCaMTCTc2PolvP/RT7H/uaG3XzRCYIH/qKL9m7FR4PX3oHCbmErVcYNDPtQlyoRzivQZ6mkT9MrOQMFqZuk5NIUZKIAjc4LTgxi3LsLa/Cf/45NGh4Nzs72eyubc0a3pFr9oplUrFhYXZ//bzp1/5yc+eeVU0OmhSjrihXD6H5144IoTb1tworLSQzeJLn/sIhkcm+MGX3qAdNJ7J0JikYWRkhZlMDuOTM3j85y/gj7/8APY9e6Q+FGzEdGNshlHCph4GCdcITBLyFaM/XO/ciX/SRVOKhK6WBvR3+DERjCGSMH5muVJlhIhKpTIvEeVhN+GWHSuxc107Hnjy6NR8JPypWDw58XbK7M0ne5uPx+X2Nja33//h9992x1237ZSoWTJ05hKe2PcCPvG778WGwV4Mj4wTsmF7dm3C2HSI/9+/exjNAQ9uu34Lli/rgd1iRigSo4tFxBrsX/3DvyCRzBpj9ERdqHQts4RsqT7mU9/Xqf271q40BsqMC6pomzPgsaHBaUUkkTO2NRlxYVXoNGWh07iLkYR9bitu3rYCu9a243s/eyN4YXLmY+MTkwcux02dl+1yWqvV5mnr6Pn7O2+57n33vf9mORJPsjNnL+GFV4fYlz73u2hp8uPAC4fR2tqI5X2dPJHMsAcf/QWOvDbEqaAjYdPw1t4d67BqYBk+8KkvC6+oDwvTRSDU7Tp5cU4k1npLQNwwttgVo8qa9gY0NHldsFlUhOMZcb2nkUtovsgoxMS8ES2hcI7OZg/2bl6G9csa8eDPXpsen134+PjExPN1D/2tuiHYarPZA42t/2PbNev/4PP/6R4zJVYaACDrpvGXzrZmvH5iWNwATLNJRG1Qo+fRJ57F0WPDIIr8ui2rRYH31b9/VFSyhSINZdWvYTAmIsjKDXJUN+oOi4mGc4XQKa/QbWLGOq8xO0o7y6JzJq56Lr85mCwBa5e3UxKm6+D4t3986OzY1NQnQ+Hw65fzrtrLfk2zqqhqU1PLB9s7Or/y2U/e3dTR6sf5ixM4d3EKjY1+bN80AOo504YObe43NTWK7X2qGSgX5Ao6vv6dH2JuIcbsFo277Ea9IC6dFaOIRgCqoyGCllRcUbLN5un6nfqofQ3ECghLVk9341HypUGFMvweO7YMdgsFBEMJ/bFnTxycnJ7+g1gi/hvxQlfNXdkUIgKBxvVuT+D/3Hr9tj3vuXWnEo7ExXJHOBqHy+1Ef3c7+rtbwSUVC6EQ9j17FAdefgOZbG20naprjWaRq8Ly67QFzQOJkUTRA6B8YaCfGlwVQ4OCAq8Vz4R4KP4b6KcidqVX97Vg27ourOxuxM+fH8oeOnHxu6HIwlczmUz8isgHV/BYzBabz994X1dX1x9/+N5b23dsHmSJeJzmlZjD4RAc0cilSfzlNx7BuQuT9Rskmbg5RpLE6mp9a1IIvfbrxWWS2tg9Cbe+zyYuuhUCN7zGmHWqijmm/k4/W7eijRP8nA8n9ScODJ2YmZv/i1Bo4amK8Sb/Pm+NJ8G6XO7Opua2z/Z1d35w19ZBn9/nZdlERISok8MXMTWfECGF0I2werp6Wew3CzxKSiH7FvcZ1ZsCYm6p3vsVt7VQnDeEXp+gpi9vcFnR3ebHqt4mrO7xg5be9790ZmE6nPzezMzU38cTiegVlwneoSPLMnM4nP0N3sb7GJPu3dDr79i5plWi0EI3a02FMpiLpFg4keX5QlnMfpJXLIaaGtn3qx/DMF5jTN/IFQRlZbgdFjT7XWhrdGN5lx/tjR4arefPHLkYnpyL/CgcDn0zkUqOVilB/Ef837lINKZodwQcductTofzQy1e+6ZNy5sdPW0+VixXQLf70t0SiUxZrNKmc0WanGOFYoW2QhfHY4gDEvvRGjVkDHREo5otXhvamzzwuKzijr2T5+dLJ87PXIrEEo9GI5EfJTPpCaLk30kZvONK+NUHMZs0zWq3LjeZ7De6nI5bAx7H4IrOgLu3zSN7HCa6lp8Vy1xAzWyJ0x2oAuUIBWiamIGi9iiRdzRDWqoAs6EkPz82nx8LxiYSyczzyUxqXzqdej2dTifq18C90+eqUcK/PSaTolqt9marybyKKaZrrBbLoNNm6XM7zH6nzWTzuJ2aw6rKlDMo/pfKOs/mipVoIl1IZ4uxZCY3k8nlzxWLxWG6P7BSLl6IxmLpdyrk/FYq4d8eWZYlVdXMqqo4HFar2+50um02k81isZkK2TwtUhbjyWQimUjFK5VyIp8vZAnhXMH/Ich/3CO9fRf1Lp2ls3SWztJZOktn6SydpbN0ls7SWTpLB1fT+f9WDhPrfeUbcgAAAABJRU5ErkJggg==" id="imagea791c0e1fc" transform="scale(1 -1) translate(0 -29.1)" x="13.4784" y="-12.66" width="29.1" height="29.1"/> - - + - 0 + 0 - + - 25 + 25 - + - 50 + 50 - + - 75 + 75 - + - 100 + 100 - +" clip-path="url(#p9136e24515)" style="fill: #7b9299"/> - +" clip-path="url(#p9136e24515)" style="fill: #a36a40"/> - +" clip-path="url(#p9136e24515)" style="fill: #7b9299"/> - +" clip-path="url(#p9136e24515)" style="fill: #7b9299"/> - +" clip-path="url(#p9136e24515)" style="fill: #7b9299"/> - +" clip-path="url(#p9136e24515)" style="fill: #7b9299"/> - - - - - +" clip-path="url(#p9136e24515)" style="fill: #7b9299"/> - - + - - - - - - + - - + - - - - - - + - - + - - - - - - + - - + - - - - - - + - - + + + + + + + + + - - - - - - + - - + + + + + + - - - - - - + - - + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + @@ -386,105 +349,105 @@ L 504.9216 102.24 Decision quality - 20-family mean (%) · 95% confidence intervals + Four-panel mean (%) · 95% confidence intervals - MODEL + MODEL - MEAN [95% CI] + MEAN [95% CI] - Jev + Jev - 1.13.0 + 1.13.0 - 72.74% + 82.45% - [70.59, 74.80] + [81.05, 83.79] - Nox + Nox - 4B · v1.1 + v1.2 · this release - 67.05% + 74.20% - [64.87, 69.16] + [72.47, 75.86] - Qwen3.5 + Nox - 4B · untuned + v1.1 · previous - 56.61% + 72.93% - [54.66, 58.56] + [71.18, 74.63] - Sol + Decider - 2B · v1.0 + 2B - 55.58% + 71.75% - [53.11, 58.05] + [70.12, 73.37] - Decider + Qwen3.5 - 2B + 4B · untuned - 55.30% + 70.25% - [52.79, 57.84] + [68.73, 71.76] - Qwen3.5 + Qwen3.5 - 2B · untuned + 2B · untuned - 48.06% + 60.54% - [46.07, 50.03] + [58.82, 62.31] - Laya + Laya - EN/ML + Upstream default - 47.38% + 52.44% - [45.16, 49.63] + [50.66, 54.26] - + - - + + diff --git a/assets/decision-expanded-v3_core-600px.png b/assets/decision-expanded-v3_core-600px.png new file mode 100644 index 0000000000000000000000000000000000000000..8e0f44dfb865f49f834e5e4f8287d3dc441dd136 --- /dev/null +++ b/assets/decision-expanded-v3_core-600px.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba4b96800e5f24d1029911d75fae19d7cf9dfd0f2ded82c05fef3b4997259e8a +size 143107 diff --git a/assets/decision-expanded-v3_core.pdf b/assets/decision-expanded-v3_core.pdf new file mode 100644 index 0000000000000000000000000000000000000000..12a4ab84c38106ac0e585da6fe5005f1e51f25d5 Binary files /dev/null and b/assets/decision-expanded-v3_core.pdf differ diff --git a/assets/decision-expanded-v3_core.png b/assets/decision-expanded-v3_core.png new file mode 100644 index 0000000000000000000000000000000000000000..1774078b267c03c165a062d34adcaae4ec430407 --- /dev/null +++ b/assets/decision-expanded-v3_core.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c9b3b703cacfca5eed2099665cfcd4fab210c2b0dc6e7540e8c6a9b84b2096f +size 269511 diff --git a/assets/decision-expanded-v3_core.svg b/assets/decision-expanded-v3_core.svg new file mode 100644 index 0000000000000000000000000000000000000000..2fdd04f60cbb30bf88575a3906c0cc4424a1390e --- /dev/null +++ b/assets/decision-expanded-v3_core.svg @@ -0,0 +1,1434 @@ + + + + + + + + 2026-09-22T05:19:05.656538 + image/svg+xml + + + Decision expanded aggregate renderer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Jev + + + 1.13.0 + + + Nox + + + v1.2 · this + release + + + Nox + + + v1.1 · + previous + + + Decider + + + 2B + + + Qwen3.5 + + + 4B · + untuned + + + Qwen3.5 + + + 2B · + untuned + + + Laya + + + Upstream + default + + + canonical record identity + + + 76.2 + + + 50.0 + + + 50.0 + + + 56.2 + + + 50.0 + + + 46.2 + + + 47.5 + + + capacitated assignment + + + 68.8 + + + 41.2 + + + 43.8 + + + 51.2 + + + 50.0 + + + 50.0 + + + 63.7 + + + constraint assignment + + + 55.0 + + + 41.2 + + + 42.5 + + + 23.8 + + + 23.8 + + + 21.2 + + + 22.5 + + + massive en + + + 91.7 + + + 90.8 + + + 91.7 + + + 90.0 + + + 91.7 + + + 70.8 + + + 74.2 + + + massive zh + + + 88.3 + + + 87.5 + + + 88.3 + + + 88.3 + + + 86.7 + + + 74.2 + + + 73.3 + + + multiset reconciliation + + + 58.8 + + + 28.7 + + + 31.2 + + + 23.8 + + + 27.5 + + + 27.5 + + + 26.2 + + + ordinal service loss + + + 42.5 + + + 31.2 + + + 28.7 + + + 22.5 + + + 21.2 + + + 20.0 + + + 20.0 + + + paraconsistent rule closure + + + 66.2 + + + 33.8 + + + 35.0 + + + 26.2 + + + 25.0 + + + 27.5 + + + 18.8 + + + temporal exclusion + + + 41.2 + + + 45.0 + + + 41.2 + + + 42.5 + + + 31.2 + + + 28.7 + + + 12.5 + + + transaction recovery + + + 75.0 + + + 61.3 + + + 60.0 + + + 41.2 + + + 26.2 + + + 23.8 + + + 18.8 + + + PANEL MEAN + + + 66.4 + + + 51.1 + + + 51.2 + + + 46.6 + + + 43.3 + + + 39.0 + + + 37.8 + + + + + + + D E C I S I O N 1 . 0 + + + Compositional tasks + + + 880 decisions · accuracy (%) + + + + + + + + + + + diff --git a/assets/decision-expanded-v4-600px.png b/assets/decision-expanded-v4-600px.png new file mode 100644 index 0000000000000000000000000000000000000000..0eb7316f6baaf6e7768c80dd5506288fcc3f6407 Binary files /dev/null and b/assets/decision-expanded-v4-600px.png differ diff --git a/assets/decision-expanded-v4.pdf b/assets/decision-expanded-v4.pdf new file mode 100644 index 0000000000000000000000000000000000000000..36f4af7cf7f3b3ba0519d1472747e82f30899514 Binary files /dev/null and b/assets/decision-expanded-v4.pdf differ diff --git a/assets/decision-expanded-v4.png b/assets/decision-expanded-v4.png new file mode 100644 index 0000000000000000000000000000000000000000..9ae674a6bea1407ac5803cb90ee2aab003b9ebe6 --- /dev/null +++ b/assets/decision-expanded-v4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b392a7f29d9cf0d87920658d2d57ddd2d5fc500f9658b916a518a9570d2fd08 +size 135301 diff --git a/assets/decision-expanded-v4.svg b/assets/decision-expanded-v4.svg new file mode 100644 index 0000000000000000000000000000000000000000..688237b65363d2d313b736ca2d1046ae3023d57f --- /dev/null +++ b/assets/decision-expanded-v4.svg @@ -0,0 +1,594 @@ + + + + + + + + 2026-09-22T05:19:06.155380 + image/svg+xml + + + Decision expanded aggregate renderer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Jev + + + 1.13.0 + + + Nox + + + v1.2 · this + release + + + Nox + + + v1.1 · + previous + + + Decider + + + 2B + + + Qwen3.5 + + + 4B · + untuned + + + Qwen3.5 + + + 2B · + untuned + + + Laya + + + Upstream + default + + + boolq + + + 92.5 + + + 85.6 + + + 87.5 + + + 91.9 + + + 83.8 + + + 65.0 + + + 69.4 + + + belebele en + + + 96.9 + + + 71.9 + + + 71.2 + + + 93.1 + + + 93.1 + + + 83.8 + + + 38.1 + + + belebele zh + + + 96.2 + + + 69.4 + + + 67.5 + + + 91.2 + + + 91.2 + + + 81.2 + + + 28.1 + + + PANEL MEAN + + + 94.5 + + + 78.1 + + + 78.4 + + + 92.0 + + + 88.0 + + + 73.8 + + + 51.2 + + + + + + + D E C I S I O N 1 . 0 + + + Natural reading + + + 480 decisions · accuracy (%) + + + + + + + + + + + diff --git a/assets/decision-expanded-v5-600px.png b/assets/decision-expanded-v5-600px.png new file mode 100644 index 0000000000000000000000000000000000000000..29d72db9ad68186937993e1d22c6d3daa209b150 Binary files /dev/null and b/assets/decision-expanded-v5-600px.png differ diff --git a/assets/decision-expanded-v5.pdf b/assets/decision-expanded-v5.pdf new file mode 100644 index 0000000000000000000000000000000000000000..48ac7aa32e3eaf57adc8b8f8fd6ba7130d9b10b6 Binary files /dev/null and b/assets/decision-expanded-v5.pdf differ diff --git a/assets/decision-expanded-v5.png b/assets/decision-expanded-v5.png new file mode 100644 index 0000000000000000000000000000000000000000..e34073bccea1bfb777fa2d5cfc9a72788d2f9c9c --- /dev/null +++ b/assets/decision-expanded-v5.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef547fd323bf044b1f81edf89f07e1e0175f9a52b9742b14eda6890e7fa0f4d +size 161808 diff --git a/assets/decision-expanded-v5.svg b/assets/decision-expanded-v5.svg new file mode 100644 index 0000000000000000000000000000000000000000..ab64b7088351cf6771f32929fef0254721b0d621 --- /dev/null +++ b/assets/decision-expanded-v5.svg @@ -0,0 +1,714 @@ + + + + + + + + 2026-09-22T05:19:06.857390 + image/svg+xml + + + Decision expanded aggregate renderer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Jev + + + 1.13.0 + + + Nox + + + v1.2 · this + release + + + Nox + + + v1.1 · + previous + + + Decider + + + 2B + + + Qwen3.5 + + + 4B · + untuned + + + Qwen3.5 + + + 2B · + untuned + + + Laya + + + Upstream + default + + + cosmos qa + + + 86.7 + + + 68.3 + + + 57.5 + + + 66.7 + + + 60.8 + + + 56.7 + + + 30.0 + + + squad2 answerability + + + 90.8 + + + 85.0 + + + 75.8 + + + 84.2 + + + 81.7 + + + 82.5 + + + 57.5 + + + snli + + + 82.5 + + + 86.7 + + + 85.8 + + + 90.8 + + + 80.0 + + + 64.2 + + + 72.5 + + + qasc + + + 99.2 + + + 97.5 + + + 97.5 + + + 95.8 + + + 96.7 + + + 85.8 + + + 95.0 + + + PANEL MEAN + + + 89.8 + + + 84.4 + + + 79.2 + + + 84.4 + + + 79.8 + + + 72.3 + + + 63.7 + + + + + + + D E C I S I O N 1 . 0 + + + Reading and inference + + + 480 decisions · accuracy (%) + + + + + + + + + + + diff --git a/assets/decision-mark.png b/assets/decision-mark.png deleted file mode 100644 index 4f92ab5102bce4027a512b1da7c4f548819a5acf..0000000000000000000000000000000000000000 --- a/assets/decision-mark.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d83cf7878c3839f3085feb8c02334217ad2fef0a615b805e1726b292552c46e8 -size 1828679 diff --git a/assets/decision-quality.pdf b/assets/decision-quality.pdf deleted file mode 100644 index 8240d79b1516a51a4316c391d843e19d602012c7..0000000000000000000000000000000000000000 Binary files a/assets/decision-quality.pdf and /dev/null differ diff --git a/assets/decision-quality.png b/assets/decision-quality.png deleted file mode 100644 index e63f80251c9210898c9310a9606c4d5f66c547d3..0000000000000000000000000000000000000000 --- a/assets/decision-quality.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:991ec98a7ea95250b9040fbddba09513e1ae740682c94f8d0f72580c9373c07e -size 171749 diff --git a/assets/decision-question-scaling-600px.png b/assets/decision-question-scaling-600px.png new file mode 100644 index 0000000000000000000000000000000000000000..dbf0ae94c99ae7951c5ceef97ce4250207e6c684 Binary files /dev/null and b/assets/decision-question-scaling-600px.png differ diff --git a/assets/decision-question-scaling.pdf b/assets/decision-question-scaling.pdf index 07870fb8f83ef2b001a40974f8432e7e517194f3..ca7513be52376a124f0a4cda8277807c3b03ded7 100644 Binary files a/assets/decision-question-scaling.pdf and b/assets/decision-question-scaling.pdf differ diff --git a/assets/decision-question-scaling.png b/assets/decision-question-scaling.png index 80114de62e050763ecea7c0a84ae9165a89ef3e2..bd66d7e734ccbbbc6808bf89eb1aa8f0aeb26ef7 100644 Binary files a/assets/decision-question-scaling.png and b/assets/decision-question-scaling.png differ diff --git a/assets/decision-question-scaling.svg b/assets/decision-question-scaling.svg index e736e4d7b0fabb3d743980ba885f2dfc35812224..3729cd761d94bfc6f19c07944184438df5f6d351 100644 --- a/assets/decision-question-scaling.svg +++ b/assets/decision-question-scaling.svg @@ -1,16 +1,16 @@ - + - 2026-09-21T22:05:27.515264 + 2026-09-22T05:19:07.257156 image/svg+xml - Decision measured-latency renderer + Decision expanded aggregate renderer @@ -21,25 +21,25 @@ - - + +iVBORw0KGgoAAAANSUhEUgAAAGEAAABhCAYAAADGBs+jAAAvKUlEQVR4nO19d5ycZ3Xu835tetsp2/uuykqrbnXZktxtbMA2hkAAU3Ih5Bcg5IZwe0Iul9zkJpAQmh1sTIIdsI0hSOAmucqSbKutVtKqbN+d3Z3e+3zv/Z33m1nzy38GSxbJvvbYsrQ7+82pz3nOOa+BpbN0ls7SWTpLZ+ksnaWzdJbO0lk6S2fpLJ2ls3SWzlV7GK7yowAwyTKTZFllqmJXGHMoiuwqQ7KrVV3TwCWzJPEoUNB1PaVwPabrega6ntcr1UqqUgHH1X2uOiVIAMyqqlo1zaeZrSvcFtsmWVMHobBem9XUbLeoDosmm60mRTapssQYUCrrLF8q65l8pZwtVDLZQiWu63xa1fnZTDJxNFbIncjn89OFajWv86tPJVeNEkyKojY7ncttDtdNTNVu8Ti0VZ0Be0NnwKb6GxwwqSqKxQKqYChWOCo6wMEgSTKgl6HIDKoEaIoETZGF9acLVYwHE/zsdCKzEM1cKpXKz+jF7L/OxOOnKpVqAVfJYe/0D7dZrH6vx3ObSTP/TkfAvnlNl8fR7DYxEnA0U0Ysr6PR34BYpoSWlmZks1mhiHKF0xuwSrnCGRhK5SLAOSqVMvRKGV6nGYV8AS6LjAabwiyqxOfiBbwxGsuNzMRO6sXiPyUSsScTuVzknZRBXQ7vyHGYzX6Xy3Ofx+362Nblvr6BFrtUKHOMhfOYieWxkMghmi4hky+x3ZtXosnv4T1dLYyiSbVqaKBSpnjPUSiWUCpVkM0VEIunUSqVsG5VN773+PNCMV6nhbX47Ly70Y7egAUWBTg2ltBfOTN3IRRLPJDPZh7J5HPh/zBKMKuq1evx3O1xub+4Z03Tir5GO5tNlHBpIYfRYAILiTxKVfIDtvh0/gYnPvzunbDabCzgcUCSZV6pVsEkWQhckiSk0xlE42nMLUREKGryN+AffviMUALnHIwxZtZktDbYeEfAjlXtDrQ2mHF6MsGfPzU3Eksk/jqVSj6RLRaz7wT4uCJHYoy5nc41Hrf3K1tXBG7Y0u9XJkIZPHUqhAvBFOKZIphEaRkkMBHrKemSEKOJNEwWKxLpLNas7OGcMejVqvha8oxypQKzSQWv6ijkc1A0DeFERvy5qiigZKxzzotlHaPzKYzOJ3F2yoplrS4MdjjY5+4cWPnC8Pz9x8+HfzeRiv/XUDx+jL7+35USNFXRmn2NH2/0u//n3Vs6AvmyjgOnQxiejiORLQuHrCtAJuFLjJQmlMG5LoQcTWRRKhXhdtkhyRLKlJlJubIivEFVZKSSaWgmDQ1uJ85PhCAzBlU1lFCtKU3XDc+Yj+eE112YTWKw0411nW5ly7LA3sdfndyvmCx/GQovfKdUqeT/XSjBbbE0ePyNX9nYG/joroGANhLM4LVLUUyE0ga6YYawZZkehUOWJEgSCZ+snARmeMZCNAkKJ1ySYbGYYSWlMQmcMyHgcqkIWZHBwKEoCnKFcu19dBGuGGTxPFwihejiPXVdx0KigFBijo0vZPiG3gZ8dE+X99CI46vPn1KviUdCf5LM52Z/q5Xgb2jocjs999+ysXVPa4ONHRwO4fhYFNlCVVgzhShZkkgXkKU3FUFRqKJXhdWSAE0MSKbSzNTgQiKVYwG/l5Py6M91gHG9CpNm4ppmAhMpW0elXCIPBOUO8giFYCuFJVKAQl7HUKnqQhH0uhhMsvlYloeTBWzp8yht7p73/fiQ3GuORz8ZSiROX87YZMSAt/mQEANeX7/X1/jI+7Z37LWbFPbLE3N4dSSEbNFQgPwrL1WRa0LShZWSBfudFnT4HfA5VDDGDYHrVUzOhoSyVE2Doqq179VQLpeRz2UBCl/ling/s8LQ5nWwJrcFJgUij9B7U55QZIlRPaHIEnkOV2SZZ4pVvDg8j1+eXEBJ5+xje7s3BgKNj3U0Nm29LIKqncvy3h6nq8vvafjn929t2Zwp6njm1AJOT8VR5YAsy8ZLkqDIMiQG8gZhjWZVRoffjiaXGflSBVPhNGYjGWRyJaiqysEkPnJpmgIVI2EKJSoqqtUyp7xRLZcQjydZvlCC02lHLF3AdCTDQ8k8HBYNvc1uNDjMQlHkFRIT+UoUd6Q0Ug6FuFPjMTx3Ooy5ZAkf29PZ52nwPNLq82/Db0s4clqtfp/X/727tndsjOcqeOlcBJPhTA3xSIbwFYr7FM/JySkvcLT6HShXOWajGRRLZaEU+jMRBhiDz2NHKpXFfCQJvZYvSJmlUomXyhVUymUUCkXhBcHgAutqC1ANwSvVCqo6w1yMQ5ELzGnV0BlwIJEpIJmrCEAgMQpqivBCTiUIGMbmUihXqrh2pQ8f2dXW8dCL/OGCXrk7HIufvqo9waJpVq+/6Wu3b2y5Nl+osEMjYTYVyRrCF6FHFuFAJErCn5zD7zTzZq8TM+E0phYSICuu6jqquoFo6om5p7MNc6EYxieDCMeSKJXK4s8p5qdTGeRyBaEcioW5XB6dzd7F5EuvSqVCyuWxVJ6PzSe5pCi8q9EBk2I8jwhLEjO8QjynhOlwFs+fCWEiWsBHrm3r8Xl8D3pstnZcrUqQJUnyewOf27HMd4/drLBD56N8IpTljElMoRiukNtLMKmKQESUG3uaPcgVKxifiyNfqoqQQnCy5iDCCyhPWMwa/D4P8oUistkchs+N8VKxJPIAfXEuX0C+WBLCL5dKvFzMU2jjrU0NRtEHqrKrNaXpqFQ5FmIZTCyk0OixwW1TwXXKRVRXUFiSBbQlRQSjebw8EmORdBnv296+zuPxf8OqmRy4GpXgdbj29rQ0/Mn6ngbl6GgSF+bSwtVlmfF6Aibj1zlnDgoJjS5MLiR4PJUHhZNqtcJI2GSRIkxxCK9hTML61X2IRKLi55jNGl45OoRytcoKxRLLZPOMEnYhn0c+nxeCJrJvPjjHbrp2I3QumL564Vzziqp4FctVXJqNCSqkPeCseR1EfqDnIEWQ91JRefhSgiAy27225daAN/B5hTT0Np235Y3sVmuTx9vwV7evCziPTaRxejJei/VG8pVlJj4UVcFuu4k7LSrOT4WRzRcFFDXqAUqURo1QP1ynuoHh9uu34vS5MZGMNVXFqeGLRFHwTCbLs7k8J4Gmk0kUiyWRRcwmlefSSb59fT8afe5F2oLes1Y9C7RFIapSqfK5WAaRRBadfjtII/QIRngyjEeSGD8zlcChkQg293nknjbPH7lc7htwtShBkiTJ5fb90Y1rmgaDqTKOj8VQrPIaDDXwuaqqAv17nGZYTCrG5hLUAxDCQA2Ak2ERAVcqV0kZnDESB8fm9SugqYzFkxmYNOrrSEhncjhy7CyymaxIxrlsDsVSiVASLJoChUoPriM6F8R9994kFElhzfAwehIOTWGwaLLIP6QMStTzyQK6m90GF0IhVjFQHL3ou46PxXH4Qgz3bGlx+hu8f+Wy2ZquCiV4nO6NnQHnJ1obrOzkRArhVKGGgBRomiriPxVTbpsJdosJU6GkSKb04cnSzZoi4rbwhtp7cnDGdc68Hic+fM+NODF0QUBJs0khRcBqNuGpg0eJ6GPJRIolEikUCwUoDHDYNJhUiusSCpkEVrQ6cedNW998YOEPHJtWtmP3pj56PrJ9AQQy+RJBWvS0uIWiyHdUVWXCGxiEcb0xGsN4KItb1jeucjk9X3g7wtJv9AaqLKt2u/M/3zgYcA/PZDASTBoUhIB9IlkbFa8iURjC6EwE1QqhFUPcXpcVd92wASb1zRAk0Ey1ClWV8YefuAszwQUQctQ0BSaTBrNZFbF6bi6EMxemkCuWeDQaFfmAvI88z2Y1QZYAmQHpWAj33LAON+xab1i4+Jvh9KU5HB2aoNqFiUqaisGqjnyxjNlwGq0+uzAgcgIKqYpCBiUhGM/j+EQK/U0O1tbo+ni3z7fhHVWC1+3Z2t/qvoUe8txsihFLSUKXagWZgfWBtoALo7NR0H/S7xESoUMh4OjQKAqlSq1mMPA/Cfkz970bVrOGSDQliiubxcJNmsZdTjtsVhUWi4p9zx7mlCNS6RyKRaotOCxmM0xmDU6HTRiCXq2gFF/Ap+7agXvu2LXIS6XyZUQzReEVxs8l1GQk7Ey+jGyhggY7MbMV4YWEmIw6R8b5YBpD02l2x8Ymd9Vk/Zxq8C1XXgmqoih2h/sPd63w2s4G8ywYz3OyfMMTjERMp8ljYVT1VkkBhFSIsKOWJPFDVR2jM9EaW0SyJmGb8Pnfuwerl3ciGkuI8GGxWkDf4rBZYdFUWEya+LVNZezU2TFYrWYSnqACiVV12u2QGSlEg9Ho5NAzUXz0lvX44qffhwaX3TAEkR9qfDhRfBKRgTXWNpWHJjODIqeAJeCrARxyxSqGpohQ1NDVaL894HKtfUeU4LZY+gIu816ykEtzCV4m+YpcYPQB6MGdNhPFWp5M52sUskEl2ywaXHarUUxxgVyEoFqb/fjyn36CDQ50sVg8IZCV1aLBbjPD6bALD6HYbbfZ0NToZf19XfyhHz0Nj7+RaWazAYM5KUNnlDcEKKC8ZNTOQC6Ovav9+Ic/+zj27Fhbe07jWSn0OGwWkavoGQk2zyUKaPLYhDdRjqBap15wjs9n+PBkDHtX+Z0Wu+tj8q/CuiulBLPNcc+W5T43tSNnYznxaUQ1XMt9hN39LgtCiTyqgpgzoCh9XTZfQCpLVL0hBXr8m3ZvZl/78mdYV3sApWKREznncFjhcthht1kp/zBKkjablUmyxK7dvpHvO3AUqUwO3/3nfXzNujWcapJyucJ1nXPNpDGy4jrWV2BYs1QtoN1exZ/93k34m//+Sawb7BPKI6gcTxlNNSNAUd+6glgqxyifGRLWhYfT+1GNcX4uC4dFhctuerfTYm39dWWp/LotSrPFckeLx4ynT4VRKOuGB5BLEwnGdbisqsDexAMR5y8+mEBAXKAiwf1IwKqVPfjY79zG1q/u5bxaRSabY2ZNBbMzUQmrZHmyxKhpT7SD2aRhcGUve/7QCcwtRIVwRifm8S/7D+Gu6zcgEpwS9EUqk+NepxUWlUKkBIlYV8rU0IVVW1WOHSu92Db4UZy4EMRPnj6CI8fPCX6KMpZhMEyQgO0+O5K1jh+gi14TKW4qmsdEJI9tyxoaI7HkrfFs5oErpwSzZWV3o3NFusgxnyiIh6M4r6hGKCJ3dtrMoh4w0I4OndRASVkUUyasWdWLe951HTZvGGCUB8iDCsUiNe951WKBLBUFfCVagxK3qDskhq6OZsxFkjjw8vFaC9Sw0SPHRkSsv2nrCsTmg8wkg1N813Uwi1nh9F7kCopmgkbvr2ow22xQTWZct6EL21a3YXYhjuPnpvDG8BhGLs0gFEkinc1jNpIWRSL15mRJ5lWqQ5iOdL6MsVAW1w94JdVseY9JUR4qUtFxJZRgtdl3D3S4bdPRAiLpgkGAUbwk5KHrsJgUls4WuUHCUReLw+tyoKezGZvWrcDGtcvQ2uKHw2qhBMsVRWGCwaQEShyOIsNSNiOXyyGdJwRDhJ5OeQCZXJE9/KOnao5leBZZOln3Lw++IZRy+65VCE2MUr+Am00m4q9gMqkwmS0kRSgmk+hZy5pJeC4pRFGK6AyYETD5sWeFC4X8SkSSecylKhiajGPfweMo5gpCGRSSqlUJks4RjOXBJQUBt2X9QtjUXKxUpi+7EhRZljWzdU+z147zwyHR66X4LUxOJizCYNVkvhDPwOdx4Y6bt6O7o4m5XTYu4rMkMZvVxGlQi6pnqYaoKMEzZhENG1nOi34xeUC5RNRCBS1NAczMR/C9H/6S4r5B8tRChJHWjemM/Qdeo5DCP3DbFpZamEY2l4MqAyWLBotdgclshlYTPmF/WTMLJfJqyWgwEXBjRRSTMwiNzCCcKPG9y1qx9fdvw1e//xxbiKY45T6hCF0XkWAqmsPyZrtvbMa6Pp3NTl/2xGw1mdxum2mAkFAkVSRKRgBvIRNGLUMjfAwOLMNnP3m3QEj5XB6xaILlsnkakyDczRRZFt+ryBJTVYXXGz1Gg4ejVKmiRNwO5ywQ8PGhkSn+nYf3CbaUGi9C4oyJCpqU+6vn5dfO4Os/eJqrnhZIJotgXykkCuVJMiQSvkwdO6FGI+GShSuKgZgqRSyEkjgfzCLDNBwfnoAcGcUXP7KbU+1C7yV64wByJR3TkSx6G62SWTNv+3Ug0ltWgqwoLV6H2Zct6lRsGSCci8Y5Mwg0hp3b1uPm6zbg2Mkz1OmiKQmuKDKn6pOoDIOC0ESGrlQqnBoygrRgTFhXsVxGhmhp8WEl/vCPnsEjTzwjCik69QEASuR5orFJyLWqlz4SGcjF8Vl8+RuPYShYguz0I5HJI51MoVQoCBqF6glJNYn304lCp0KtXEK1XMDcVBCvnQli+apebFweYFPxIpsYD8KaCuILH94jQAjVPMJgOEcqVxVAxGzSNtlJk2/xvOVvkCW5u81nM8fTRWSLBp9v6JLzFX3t2L1tLZKpFE4NnwdVt7IsiWa+WUBGTVhPtarzVDorpiaciiKSLr1NoZBHIplCJpMVEPDYyQt4/OcvIpnOGRYqnsAg44xDdYaO3s4morT5QiQpYCjpggSUyRVw/z/vx5GV3fjgnbtgdpgQj8agl4tw+TksTpcRDkmB5SIq+QxC00GMBnNYuWkjtqzvYko5Dd3swoWRCRYMhnigRcLnP7gL33rsVaQyBWF00XRBeJjNonRAVW2oVJKXVQmaqvX4XWZpIU7cPWc9nU3obm9Ca5NXDFcdGzovulNOh1XgaZqCqFQqjNhOi9nIARQKbDYrTJqCaqWKRDwhGjSENpKpDI4eP4efPXUIkzOhWiOAnISCtZG4jQhoqMTjtLFPf/BWmC0m9t1HnubD58YEbUKHoDB9LVXVZy5MYtuGlXjXnrUY0GzQF+aQTcShagr0Yh5cr0DmJTR3NIuexcEXTuGEIsFpruLwKydx7bp2+HytmJyYh5xK4DPv3ojnjk3g+PlZpAsVQXUEXOaGabPJm87nL68SICttbquK0fkMqsTzGA1yPhsMCcsjbySLJw+gMCXKfVXlYjaorDPKF3oyjWQqKSpQJsmilxsKx/D6yYt45bVhTAfDhrBFtDYqcUOqhlCNjpvIQ6yqVzE1HcSqgX589r472JNPv8oPvHJS8ECCnaXvozBX1fHK68M4emIEy3rbsHvTMqzu8qLBVEUhHMTsXBgVMNbX7uPLV/dj794NOD00ymNcZrs3dfO+vhbkswVGhlUp5hFNzfMN3R4UC3lcmIkhlavAY9NsnDGit8cumxIMVlFuIoaThnfJSC+MBzE+HUaD24GWgBs+jw1Ol4OoCWI+mULkl6aJGJyjvnCK9je4SN6RWArj0/M4f3EaU8GwKM4Ep1NDPvVdgnqlbcBRJsYdt6zth8Nm4oeOjeCZQyfZ+fEgOtsaBfO6ddMq/O13HkMqk6cMIRRRrycor5y/NI3R8VmosoRGrwNdfjP6G61Yv7qHH5uMwOaOoL27DasGOUILcb5i7WrEo1FMRkq4lNFwJgp+YTaGqXlqy1ZE27ZQATx2TTFxoQRcPk9gTFJU2UGJlSrhmmpEmEhlsgK5hEJhxLMVUZ3abVZOJJpBfMnUkuT5XAG5QgmZbF54AF/kjsTbCwUsMqo17GL8t/F7op7gOvq6WtHZ4kMkkQYNCd9xw2YEWlr5wZdexx237EKD24UvfeV+5ESdYXgUHXJQ8iz6WYVSGflCAWu6O9idt+6EzdfCf/zyw3j66Kvw2E2wyACrlpHbd1aEylKxgGgBgiEg0KCTigWY4ChWdditJlaQZPtlVYLCmKRKko1CCP1goQIxjWg8DCEMl8OJUCKMUlkSgjZCRz2UGKIkXr4eUniNQBNCJ5LNiB5vjkCKPrP4XaOQoz6FIrNXj53hEluJZr8byzub+Ya1A7A67Pjlc6/iwAtH8J7b9+Czn7wLf/3NfxEeRd8nemrc8Ab6i3rd79u7mt1x+14EWlvxze/vY4ePj8BusyCZK3GaviBjIsUTZA44zajm8pxyDk368UrF+CxcF51CzUqfDQbkulwQlVgTziQCyqiQEoziQAiYGE9DEUQRGwqpC1ooQMR1o3Ffn6gwWss6r/eYxV+iDVkrwgQxWUPeta/ZsWkl7r1tO6ci8bWhUfhcdjQ1B+DyehGcC4uJi0g0gZm5ED509w248+bt9Ua/gaZrw8EmTcJ1G/tw7c7N6FnWz984Pc4f+vGzgtYqlCqc6JJcqYJkroxsmYOGmJkkGc3RWj9ENIMWR+9FC5Srvwab+tbCEclL5xVJ1Qh6CLMiCZKABWwkiyM6mXJy1XhASo4itlOirHPHhrXThDtqnsCZGC418o5JM1jKen4QYaoWjog1Xdm7FsH5KBoaXHA47aLjNjo+LSz4+PBFrF3RxU6dviB6E3/6hx/C7HyUHz121hg406vobPXjll2DIpz1r1yOVK6Mr//jT8SySZ0up4Rf5WJCkFhdY/KDUJcABgZKWwQIFOaoBarITKNvvJyeULPXvGoy1atUIRmyenJZKrRq0V1YPn01cTbLe9sFIiG0RBR3zfLF7C6TaCzY8CkaDBjobcHerQMiaS4OiRmlnFDQ6fOTePTnLzH6wER7dHe2Yuf2jVi7qh9D58ZhNWk4d2kaR4+f5fueOcpTqTS+8Kn3CXKRBOdx23H9tpVY0d/JbrtlDzNZbPzP//YHOHb6knhmMhiaBCQam4YO6NdUoRv2YPSi6WnrI/a1di6j9y9WOC+oSF9WT2DCDyrJaqVC3S0x/1wPI/RgqqKiWBJDVsxpt/J33bgd61d3YXl/N2LxFE4Mj+JbD5LFlWteISLXm7ifQ8DcBNHJ9UGhRbxfcx/OMRmMYN1AD7o7W9DZ3gSL2YRIPI3HnnwGLocVwVCMR+NJeGNRTIxP8fbOdqZIMueSjt1bBlhHewu2bV7DfQE/Hnp0P57Yf6gWJlHzYDIm2oujgQQKtQawqMMDWo0wwqtBOhIL4LKZEE3mdF3XE5dVCRXadqlWwkQT2Exvhgh6YKNLpgvIZreZ+Oc+eTfcTgtbsayHf/uhJ8RA1gfuuhHL+zrwX7/yXQFPxaACNwQsYirnmJ6PY2Yhvpj4hSJqIYCgqSQz7Ng0gIDPxdeuWw1fg0tY+N9864diXaq10S0s1+9xIZfPC+WDBXlPux/Lu5vgdduwbfM63tbWhvOj0/ja/U8Y4/PCC4TGRbKl5yIUKCbFa+iMC7bW+Nj0fEai5zTdwa1mFZdmSwQAw5edO2KV6nQyU+LUoqyjFmEdVWOfIJkt4l03bIci6Xjl6GkcO3kWt920E5FYHGdHLrH+7hZ84dMfWLR+ZvB/iwIXWzUivdRzRy0cieQOrB/oZt3tjdi4fhCrl3Wgs70Fj+9/GU/se0n0rPP5Ig0Ji6GD4EIMB14+hhcPvYEd63vR29XMbr5hF+/ubEYymcLf3f+4WMUylFx7BhFyKAkzVCgGSTSpTSGHho/LorgkdRgkovH5rZrErJqMcCKbhV65/EooVMtjC8mCThZF9IRIzmQVOhdjJyv72+H3ODA8IopG/uyLr8FhNaGnuwMT0/MIzgaxd+da9HU1i/xQFbjf+DBvjvAYyjHmkQyFULdt+8aVaG/ywtfgRm9vJ7OYLZiYmMa3H3oSdquZmDiWzBQYIaeJmXk2Nh0SXTmHzcRWrVrGbr95NzasXQmdS3j6xWN45kXqP9SsuwYAFr26Nl1MaIGQH43l0G/VhpR5HSGRF7rsJjhsKqKZwly+WIhfdiVI1croTDhdCjQ4BM4myyX6WlNlduO1G9l7btoKq80Cv9/HnHYL3E4Du99+8y7QbsF8KCp2EqinXOt5wuCEaj3omhDqOI/CAe2p7d2xBoEGO2tu9lNzh1O1TG/0s/3PwyLTRB0RdkWeyeV5i8/J+zsbKW+wgNeJdWtWYM9129DX2yko8yd+8Qr+19/8QFTtRmoSo0eisKQ/r7uF8MKax7psZrGCtYgialCVful1mEUDK5MtjRQK5dxblelb5o7ypdLsbDgZdlq1Dlq4CMUy2LJhhZhyW97XzfcfOIprt64FTc+NnL+E8SkZk9NBsbLk8bgwPRtmyVQW264ZxC8PHMHoRLBmhYbYCQrWrZCsjARz+95rkE6l2eCKLmzdvB4ulwMWixWpVAbjE9N4987lIMR28MQEuhpd6O1oZrFsASaLDRvXrcTObevgcDhFc+hbD/0Uf/fA47VxSzFnRwUnaYGLvQgR9t58nvoCC/VFJufztd0IwzgoK6iKzJq8dh5Ll3m+VDpM0yWXPxwVi/FkJn+G+gA+t03M/Nx7xx7s2rKWL4SiKBeLePDR/bA77Hzlil50tDXC43JifGIKq5Z3C2+Ymwvz5kYv/9B79+D2G7cuEnP0Iqxv5ADRbsd1WwZZsZBnBHU7O1rR292O1pYmwekPnbkoesBnZ1OoyBZcs7ob4VQOkUyRL1/ehztu38N3btvAHQ4np1bp9374C/b1B54Q8FNcikFc2CJlAiaU/+YspgAaZO1ENBLjS6hIeL5ATcYSC01itAccODcVK5eKhWNvVZ6/lhIqul6tVEovTM8n0Oqzio3KeCKJl48MYf9zh3F+dAqxWByPPPEUWlqa0BTwwOdzYT4Uw+qBfsRTGZwYPo9KsYCxyVl85r73YvXKbhDspXhLfYZ6Um7ye0A7BiSI3q5WWCwWkXzp0M888sYQaN6pu6sN8VwZVpcH1+64Btfv3oKbrt/OVy3rht1uQyKRxCtHTvJHfvIMJ8p8kV2tyVyU7LW6uk6XkFYoF9C/rSYZ2TyFIkJNb86zUght8zvR5rURkzpb0XPnr4gS6GTT6eeOXZjPdDW5YTOrOHdpSryIh28OuOH12HH49WFEYgnW0toCr9cjqtGmQIMIA9TjPTY0gvlQHKeGzuH3P3aXMbH3K55Mwhjo7xCb+q0tjejv7UBbWzNMJpNQ1sj5URRyOfR3NmP3tZtFyKGO3p5rN2P9+kH4GjzCW6ib9uqR4xg6O47x6YVaBb/4UwQkpZmcRpeZ5otEt7aO3OhF1HzAbcVCLC0MpU6/1GuXtkYnypwjmcm/nM2UjCWKKzFtkc3nzk/MJ07brPZtH7n3Nub1OKDzqpgDpQ8wF0rg0NFTeO7F1/Ghu2+G3+tGJpMXE26SrOL40AXctHcrVq3oxuFjZ/Ghe27CNRsG8OrrpxdxOL1Po88lpt9amgO8rbVFDIJRzqBLRggqrhtchoYGDwZW9EFRNdG8V2UGTdOojBeNotNnqMPnwIFXTtSe3kCXYty9BomJWb1msBs0KXjo1AR0biwliq/mOqxWK7JzKeEE5VJZhCOyF7/Lir5mJz9xIVRJ59JPlGkF6EopoVypFHLZ3OOvT6S3bl7XyqaCIUSjCUInxqxmLg+n04bTZy7x6p3XwdvgRiSWFjwMhaeLo1P45TMvY+O6AWzZOICDr5zAbXu3ENWwuOhNtIjFpDJNtXBalSK6QavNhZJ1UqXc0uyH2+kQSdfophFU1gXXQz0PUoLbaUMwksGFsdoQRO2qBofdimQ6uziKefC1i8a4vkBmxjQheVLAYxFzTvQzi8VaL7v26m1pQF9bA/YdvnghmUwfwpUeg8zm0j85NnRhjjA5lfR0KQhRBXOhqBgvpyW/S+NTGJ1aoCQNh90swkgPxe9klgobHHzlGF59bVhMTORyWVCP2hhT18WOgZnmSxWFUVyvRytKnprJBK/Xh8aAHw6Ho8a2chEuCtTIJ6IvlQItlszFcviLv/0BymXqtNEx5k6pjSpCjiCFgGyhhEK5aqxp0cQe7SbIEgJeN2KpvIDhpGgyEnoQu0XF8nY35iMpnsmnH8/l8m+ZrqifX3ukO5PNzCaSyZ+NTsx8ZmBZJ8YmZgSr2dnix1QwCimVRt5iwWsnL+LdN10j5nkIZ/d0tQp+KBxNixA2uxBn58dnOSVht8MKp90Kh80Mn8dBiyLIl4rIpDNi2U9YuBhRJ3aTqlpic424LhYCBeoBYtEoZhdiePCRX+DZF4+hWDEYXKPqNoYK6CwOD4j9OFEJG0Qh7UhLkkA9lAuoiKQQVKb7McTenY5lHY0Y7GvC4wdOh6Lx2KN1muWKegLBiHQqdv+REyMxspDmpgB2XLMGrW0tuPfO3VjZ146ArwFDp8/BbLbCZncglUygtdEnxtiT6YzwDJvVLKYy6IPSJPX6VT0ihFCYSaTSSGeymA/HWThiXKFDP0u0WWXaOzaEK/YghAdVEY5G8YsDr+HTX/wa9j131ICj9SExA46KKxlI8lTgGc0eQ7CCya1t/NutGmxWC0JRIxeQemgogV52s4Z1y5qpw8ZH5yKPRaKZ0XdsmTybzZyNxxI/Gjo79un+7mZGVnrw0Em43Q686/rNYo/sjaFLiCYycLucYjq7weMkGoP7vB5GixyaKnExAEwhIp1hFFKIgiA2NBJLQlUlGofkhKpEhforLVCa1qN6gkJUNpfFzOw8vvODn+MXz71mdNNqexAkQtoKiiVz9f41Z5yz5oCHk5BzhZL4TTIEIu1o2b2/3Y8LU2HROqTcUq1yQeSRslb3NmLDsiY8vP/4fC6f+mbVKBreGSVUq1U9Hg1946XDp97b2RZo0nWd9XY180NHh6BKHFs2DLDJYISfPH0eWzeswEI4CofDLpY+iIkl5tFld4jwQzOeLoeVqla6Ug3ZXF4MgfU0tvLmRh+cDgdUukCkLlaDwBECop7BkTeG8fX7n8DY1NxitbtYeXEurmaoNWAER0FdsqlghFGfyrgFxhh5J6jc3exBKJ4RcNoIFsZyIfVNWv0ubF/bibHZuH5hauFbM8HYJfyG5zdeeoun4heT6eTXXj56uloolviynnas7OvAyTOjlBs4bdxQZetyOQ1351UKP2IAmMKHpiqwWWlGyZi6JhsvVSo8V8jz1mYfVg30ieURj9spoKdBeRsNf6ohJqdm8c0Hn8SX/vcDGJuaXww7i/xTTQ9Up9TGX4wsIoo+8XWCsiC9UQnQ7CeP5QjFjN4MQVLqL5Nn0bNuHexkva0eygXHItHId2uF3jt7twU9RDIe+t65i6Z3rVretWtZbyf6utvgcNjw1MHDePdNO8T+QCKZgc1mE/h+/epe7H/2MPIWjReKRWYxm7i3wSW8I51OM09Vh81u4wMrejm9F5GAVgsN8tIqLheL4zReSQXfdx7+VzHGTof2jTnXhfgpT1o0JiYjRJO13g8X6UH0J4T8RPHFdTFPREMDDqtZLDiKuVpacqQQRErQdWxc3Y0da9r5k8+fSc1HQv8lk8nEcLVcMJJKZxKKFv7isy8f//meHRt83gYnBgd6KcbjpSND6O5owYkzo9iybhmqlRKW9bTy0u7NePnwKSiqCS6nFX097aLiLpeKnBTldjtFfUFbOsRPaarB2FKiDs4t4Ec/PYjH970kRloMIq4+XWQkapMq4ebtAzgxMo1puuCqPlpTi1SMEaVY6w1LCloDLrFTcWkqLBQjkrC4gkEXfYW+dh/bua6Dj83G9OMjM9+MxWMv4W06b9vVAIlY7I2FhYU///5jz5SbGn2iuLrrXbuhmhRRFA0NXxCUA8E8m82OrRtWCrgaiSXFapTH40RPVwvWDK7A2sEVgqjzeT1iC9NEFTA4cllDAd/5/s/wT48/JxRgLHobSNXogBr9beNGMTHKX2eK6o6w+MyiwlZkErDYFJ2cSywOJZPwqb1JuaDJ68DeTb28wWnlP33h9IH5hdn/V6bBo6tNCTrXeTgUfPDYieEHnnr+jaqqaujpbMFH7r0N4WgC0VgSEzMhgTL6e1pxZmQMn/v0+8V+w6WJWZbL5WhgmBGKIshKHkDjlITZCaMTCTc2PolvP/RT7H/uaG3XzRCYIH/qKL9m7FR4PX3oHCbmErVcYNDPtQlyoRzivQZ6mkT9MrOQMFqZuk5NIUZKIAjc4LTgxi3LsLa/Cf/45NGh4Nzs72eyubc0a3pFr9oplUrFhYXZ//bzp1/5yc+eeVU0OmhSjrihXD6H5144IoTb1tworLSQzeJLn/sIhkcm+MGX3qAdNJ7J0JikYWRkhZlMDuOTM3j85y/gj7/8APY9e6Q+FGzEdGNshlHCph4GCdcITBLyFaM/XO/ciX/SRVOKhK6WBvR3+DERjCGSMH5muVJlhIhKpTIvEeVhN+GWHSuxc107Hnjy6NR8JPypWDw58XbK7M0ne5uPx+X2Nja33//h9992x1237ZSoWTJ05hKe2PcCPvG778WGwV4Mj4wTsmF7dm3C2HSI/9+/exjNAQ9uu34Lli/rgd1iRigSo4tFxBrsX/3DvyCRzBpj9ERdqHQts4RsqT7mU9/Xqf271q40BsqMC6pomzPgsaHBaUUkkTO2NRlxYVXoNGWh07iLkYR9bitu3rYCu9a243s/eyN4YXLmY+MTkwcux02dl+1yWqvV5mnr6Pn7O2+57n33vf9mORJPsjNnL+GFV4fYlz73u2hp8uPAC4fR2tqI5X2dPJHMsAcf/QWOvDbEqaAjYdPw1t4d67BqYBk+8KkvC6+oDwvTRSDU7Tp5cU4k1npLQNwwttgVo8qa9gY0NHldsFlUhOMZcb2nkUtovsgoxMS8ES2hcI7OZg/2bl6G9csa8eDPXpsen134+PjExPN1D/2tuiHYarPZA42t/2PbNev/4PP/6R4zJVYaACDrpvGXzrZmvH5iWNwATLNJRG1Qo+fRJ57F0WPDIIr8ui2rRYH31b9/VFSyhSINZdWvYTAmIsjKDXJUN+oOi4mGc4XQKa/QbWLGOq8xO0o7y6JzJq56Lr85mCwBa5e3UxKm6+D4t3986OzY1NQnQ+Hw65fzrtrLfk2zqqhqU1PLB9s7Or/y2U/e3dTR6sf5ixM4d3EKjY1+bN80AOo504YObe43NTWK7X2qGSgX5Ao6vv6dH2JuIcbsFo277Ea9IC6dFaOIRgCqoyGCllRcUbLN5un6nfqofQ3ECghLVk9341HypUGFMvweO7YMdgsFBEMJ/bFnTxycnJ7+g1gi/hvxQlfNXdkUIgKBxvVuT+D/3Hr9tj3vuXWnEo7ExXJHOBqHy+1Ef3c7+rtbwSUVC6EQ9j17FAdefgOZbG20naprjWaRq8Ly67QFzQOJkUTRA6B8YaCfGlwVQ4OCAq8Vz4R4KP4b6KcidqVX97Vg27ourOxuxM+fH8oeOnHxu6HIwlczmUz8isgHV/BYzBabz994X1dX1x9/+N5b23dsHmSJeJzmlZjD4RAc0cilSfzlNx7BuQuT9Rskmbg5RpLE6mp9a1IIvfbrxWWS2tg9Cbe+zyYuuhUCN7zGmHWqijmm/k4/W7eijRP8nA8n9ScODJ2YmZv/i1Bo4amK8Sb/Pm+NJ8G6XO7Opua2z/Z1d35w19ZBn9/nZdlERISok8MXMTWfECGF0I2werp6Wew3CzxKSiH7FvcZ1ZsCYm6p3vsVt7VQnDeEXp+gpi9vcFnR3ebHqt4mrO7xg5be9790ZmE6nPzezMzU38cTiegVlwneoSPLMnM4nP0N3sb7GJPu3dDr79i5plWi0EI3a02FMpiLpFg4keX5QlnMfpJXLIaaGtn3qx/DMF5jTN/IFQRlZbgdFjT7XWhrdGN5lx/tjR4arefPHLkYnpyL/CgcDn0zkUqOVilB/Ef837lINKZodwQcductTofzQy1e+6ZNy5sdPW0+VixXQLf70t0SiUxZrNKmc0WanGOFYoW2QhfHY4gDEvvRGjVkDHREo5otXhvamzzwuKzijr2T5+dLJ87PXIrEEo9GI5EfJTPpCaLk30kZvONK+NUHMZs0zWq3LjeZ7De6nI5bAx7H4IrOgLu3zSN7HCa6lp8Vy1xAzWyJ0x2oAuUIBWiamIGi9iiRdzRDWqoAs6EkPz82nx8LxiYSyczzyUxqXzqdej2dTifq18C90+eqUcK/PSaTolqt9marybyKKaZrrBbLoNNm6XM7zH6nzWTzuJ2aw6rKlDMo/pfKOs/mipVoIl1IZ4uxZCY3k8nlzxWLxWG6P7BSLl6IxmLpdyrk/FYq4d8eWZYlVdXMqqo4HFar2+50um02k81isZkK2TwtUhbjyWQimUjFK5VyIp8vZAnhXMH/Ich/3CO9fRf1Lp2ls3SWztJZOktn6SydpbN0ls7SWTpLB1fT+f9WDhPrfeUbcgAAAABJRU5ErkJggg==" id="image39c0c30787" transform="scale(1 -1) translate(0 -29.1)" x="13.4784" y="-12.66" width="29.1" height="29.1"/> - @@ -47,300 +47,219 @@ z - 1 + 1 - 2 + 2 - 4 + 4 - 8 + 8 - 16 + 16 - 32 + 32 - Questions per request (log₂ scale) + Questions per request - + - 0 + 0 - + - 75 + 50 - + - 150 + 100 - + - 225 + 150 - + - 300 + 200 - - Request latency (ms) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + 250 + - + - +" style="stroke: #7b9299"/> - - - - - - - + + + + + + + - + - +" style="stroke: #a36a40"/> - - - - - - - + + + + + + + - + - + - Sol 2B + Nox · v1.1 · previous - + - + - Nox 4B - - - - - - - - - Laya - - - - - - - - - Decider + Nox · v1.2 · this release - - + + - - + + D E C I S I O N 1 . 0 - - D E C I S I O N 1 . 0 - - - Question scaling + + More questions, measured - - Median request latency · fixed question length + + Choice · median request latency (ms) - - + + - - + + diff --git a/assets/readout.pdf b/assets/readout.pdf deleted file mode 100644 index 27a0bf95bb709acd674f6fa46804a2429c749d59..0000000000000000000000000000000000000000 --- a/assets/readout.pdf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d489b0d1814e37495a8612c12fabd3bf4a6e73d11c7e356bf1ce0cedded8a745 -size 123008 diff --git a/assets/readout.svg b/assets/readout.svg deleted file mode 100644 index 918d0ba310c0a4fd21295b7b9deeb582bc529cf7..0000000000000000000000000000000000000000 --- a/assets/readout.svg +++ /dev/null @@ -1,92 +0,0 @@ -Decision decoder — contextual endpoints and typed numerical outputsActual endpoint selection and numerical output mapping. The diagram expands the readout of the causal decoder; it does not add a second backbone or three learned task heads. - - -Contextual endpoints → one numerical decision distribution - - - -Choice -key[argmaxᵢ pᵢ] - - - -Noul -p(key = true) - - - -Score -Σᵢ i · pᵢ (ordered levels) - - - - - - - - -Mask padded candidates; softmax over K -pᵢ = exp(sᵢ / T) / Σⱼ exp(sⱼ / T) - -T > 0; no generated tokens - - -Shared bilinear + GELU scorer -Same weights for every candidate · see architecture panel (b) - - - - -Candidate LayerNorm -cᵢ = LNc(H[tᵢ]) - - - -Query LayerNorm -q = LNq(H[tq]) - - - - - -Gather candidate endpoints -H[t₁], …, H[tK] - - - -Gather final query token -H[tq] - - - - - -State + instructions - - - -Candidate 1 - - - -… - - - -Candidate K - - - -Global query suffix - - - - - - - - - -H ∈ Rᴸˣᵈ after the final backbone RMSNorm -One complete causal sequence per question; external candidate keys are preserved. - diff --git a/backbone/model-00001-of-00003.safetensors b/backbone/model-00001-of-00003.safetensors index 2ecf59f41025b6e21a7c6d2631eff61e2dba241b..ffad362324231d21d942c6d3ef6dbcbd14863b89 100644 --- a/backbone/model-00001-of-00003.safetensors +++ b/backbone/model-00001-of-00003.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8fe3a641db8854b36595b094abc8f8ff1e94ce9328cdbb64c36b72c92ca2a9d6 +oid sha256:5f9c4bc396605b551b5983a1699de8e96755fafdb6afdb228a5cc4d9b438a5da size 3991295368 diff --git a/backbone/model-00002-of-00003.safetensors b/backbone/model-00002-of-00003.safetensors index fb78d03961ca61eba1751f07ac3fc51c9564b047..6d055fa229ced6e9575bcabe420c5f1fac07a6ff 100644 --- a/backbone/model-00002-of-00003.safetensors +++ b/backbone/model-00002-of-00003.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e646f2549824a57e46e80c29260ae972d56104d37794af596a6e2330666cc73f +oid sha256:656cc757ba03f87cedc0e4c88237db1ae8215686f9b6477334a34dc0a32136db size 3979828128 diff --git a/backbone/model-00003-of-00003.safetensors b/backbone/model-00003-of-00003.safetensors index da4f1436981d70b8662be3b448644b0eb8224b58..ee071df65338c50bc0cc2658d1f65c78d8dd29e1 100644 --- a/backbone/model-00003-of-00003.safetensors +++ b/backbone/model-00003-of-00003.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:893a513659cc7e917261336fcb81ea08e7e145f7409d6e7feaa13554a4faf52e +oid sha256:0a30a07b6efbaa37d016008848c1fa6dadcb46ae451a0e5e4b927ac9b5771804 size 440425856 diff --git a/bundle-manifest.json b/bundle-manifest.json index 15c658eaa7121645a173d1f9f3d741341aac8199..3c34fce110fb82c819bef08f2bf1928c5fecb015 100644 --- a/bundle-manifest.json +++ b/bundle-manifest.json @@ -1,7 +1,22 @@ { "format": "research-pointer-bundle-v1", - "status": "candidate-export-awaiting-independent-reload-and-quality-gates", + "status": "profile-bound-candidate-awaiting-default-public-entrypoint-offline-proof", "files": [ + { + "file": "NORMALIZATION_RUNTIME.md", + "bytes": 756, + "sha256": "cf8e6ce1f07687a68b6adeb98e6704b8f292cb6adc7b10e4b84e8e70aa5472e5" + }, + { + "file": "RUNTIME_BINDING.json", + "bytes": 2621, + "sha256": "65a7772ed57e811faefe6b3b3868e174c5e4a3d7f3aa0138a3838464bb2612fe" + }, + { + "file": "SOURCE_BUNDLE_MANIFEST.json", + "bytes": 5637, + "sha256": "6d402d6d55e734851cb0d6417d6f3b416b25ae807c3a0d6dcf12e40acb6ac546" + }, { "file": "backbone/config.json", "bytes": 1978, @@ -10,17 +25,17 @@ { "file": "backbone/model-00001-of-00003.safetensors", "bytes": 3991295368, - "sha256": "8fe3a641db8854b36595b094abc8f8ff1e94ce9328cdbb64c36b72c92ca2a9d6" + "sha256": "5f9c4bc396605b551b5983a1699de8e96755fafdb6afdb228a5cc4d9b438a5da" }, { "file": "backbone/model-00002-of-00003.safetensors", "bytes": 3979828128, - "sha256": "e646f2549824a57e46e80c29260ae972d56104d37794af596a6e2330666cc73f" + "sha256": "656cc757ba03f87cedc0e4c88237db1ae8215686f9b6477334a34dc0a32136db" }, { "file": "backbone/model-00003-of-00003.safetensors", "bytes": 440425856, - "sha256": "893a513659cc7e917261336fcb81ea08e7e145f7409d6e7feaa13554a4faf52e" + "sha256": "0a30a07b6efbaa37d016008848c1fa6dadcb46ae451a0e5e4b927ac9b5771804" }, { "file": "backbone/model.safetensors.index.json", @@ -34,8 +49,8 @@ }, { "file": "code/decision_api.py", - "bytes": 6724, - "sha256": "147b2fec32cbbbbb1b92cf2a19bb887f9d945e4974e141ca7ea93b8c542f5b21" + "bytes": 8251, + "sha256": "1b068eccdffd3c3b67bfa52f8f526e6b482d92551c927668ad28a794767f8a40" }, { "file": "code/decision_model.py", @@ -47,6 +62,16 @@ "bytes": 3164, "sha256": "02352e8385ab47157b6459910da54d962e5da4bb4940d571faedc86bc5da9aee" }, + { + "file": "code/profile_guard.py", + "bytes": 6184, + "sha256": "061d6ba3edf032038b074b061879a1ff81cfd2928fc0131c66b15563f7822509" + }, + { + "file": "code/runtime_profile.py", + "bytes": 2857, + "sha256": "3e26ed65f2cd706def209761421cb8fc825281e5c1ed8f168350365ad803dc96" + }, { "file": "decision_config.json", "bytes": 753, @@ -55,17 +80,47 @@ { "file": "decision_head.safetensors", "bytes": 10529624, - "sha256": "f4f1df3f8c3bf507300bb810e098fb8854f182c715e1dd9cc57d44361f949fde" + "sha256": "8b8e342445035e503b4963e16c5f7a4fef95c08368e27eace312145fc3660772" + }, + { + "file": "pyproject.toml", + "bytes": 436, + "sha256": "135a9516e87ca2fb41ef54a974e29132256b6c2d528a1cbdea1001e28f306946" + }, + { + "file": "runtime-profile/l2norm_fwd_kernel.json", + "bytes": 26330, + "sha256": "d7ed7c9962a48efdfa76ed695c8bdc0f56afe1397c202c936eb78315e87a26df" + }, + { + "file": "runtime-profile/profile.json", + "bytes": 35040, + "sha256": "be32858d15233e0a3fbee0e4257fb02be0b3439deee4eb9c3f61151df7b73850" }, { "file": "runtime.json", - "bytes": 378, - "sha256": "c5d3521358b2817f4e56ea8150c5c612139b4e2b2c5bb09f512d9a7c0b5298b9" + "bytes": 1112, + "sha256": "78a2c21f8138beeffb3e6b1c11e03cb48972a78ca5ef3d1f58c50651031c34b6" + }, + { + "file": "src/decision/__init__.py", + "bytes": 167, + "sha256": "70de37df98b6fc8e3b9f9d43935ba31a32350496214c11adf8c5fba72c433313" + }, + { + "file": "src/decision/example.py", + "bytes": 3581, + "sha256": "a54dec885f92c2d38d07ef2333dff51965be52bc119f772cc74200ed314e69b6" + }, + { + "file": "src/decision/model.py", + "bytes": 9415, + "sha256": "ba240d7493fc29203fe036966f0ab911200cd4a9252b50b05423409977639ee0" }, { "file": "temperature.json", - "bytes": 13765, - "sha256": "09a7559b6b85cc8e7e7f3dff176b9b2433f165019ceb6e9215bf467d28bb8256" + "bytes": 6095, + "sha256": "1d652e75f49b5c232d1e742b189c1c826ad52ced1a931a124e29e65e7d7fb569" }, { "file": "tokenizer.json", @@ -117,37 +172,37 @@ { "file": "backbone/model-00001-of-00005.safetensors", "bytes": 3992269864, - "sha256": "55da9bdc3cd9e0a1b694485a42bfc6f93f29fab27ba5a1ef5acaf1812f306937" + "sha256": "8adb3dbf9db30b3ecd1f29ce6a853bc6d7e3f465857471cfe579c570537250f8" }, { "file": "backbone/model-00002-of-00005.safetensors", "bytes": 3990302320, - "sha256": "036bfc624525cc967d04bf8c4d118f74a97c9c78dc67f110065606b084caae77" + "sha256": "8aa80bed34a466a7208ad177d5e5742773e4140c55cba69911fcc805633afa5e" }, { "file": "backbone/model-00003-of-00005.safetensors", "bytes": 3937871784, - "sha256": "2a63a31ea3e064766de69ed36d02f660ac23c57189f951d52f1b8777d701819d" + "sha256": "9af681e845c99e0d933dc3fa643271ee9f5bb382f98454ceb41eb1faf0979f87" }, { "file": "backbone/model-00004-of-00005.safetensors", "bytes": 3926578128, - "sha256": "50339e559dcd750f1381383e1b93e8ae1900220700f702e3cf6d60b28dce369b" + "sha256": "34de5c39c2338e1c7110beaf2834c856f3830411002893f7946045c0a083cc92" }, { "file": "backbone/model-00005-of-00005.safetensors", "bytes": 976029360, - "sha256": "ee2155ceaef06e9f9c36c5d0f3f6a97377e18dd748dbba454a84c4de9a418b1a" + "sha256": "ce20083dd8deb786b14fe1126a12cdbf678a8a3c25367a6676c0ca55d9ae7980" }, { "file": "decision_config.json", - "bytes": 1118, - "sha256": "e6341d9d4d6bd17c80ae0d1a68e61cbee3d55614c854c79e8e3d62da82f9de67" + "bytes": 1110, + "sha256": "a12bd49b459771f22d22f29c59fe3cb6779c52a5d39b40cdbe5e247c5181b3fc" }, { "file": "decision_head.safetensors", "bytes": 10529624, - "sha256": "f4f1df3f8c3bf507300bb810e098fb8854f182c715e1dd9cc57d44361f949fde" + "sha256": "8b8e342445035e503b4963e16c5f7a4fef95c08368e27eace312145fc3660772" }, { "file": "tokenizer.json", @@ -161,13 +216,16 @@ } ], "source_model_code_sha256": "d3e28489c09f3bd7130e2d43d92e0b5c4a08e25b09b21303904defb0ff1c3646", - "source_api_code_sha256": "147b2fec32cbbbbb1b92cf2a19bb887f9d945e4974e141ca7ea93b8c542f5b21", - "dev_sha256": "5ba3bd7527da831d77cec8c6a2f260a71e10a57742e356510c719f8f30d61a70", - "production_predictions_sha256": "95ba29cb751506b796518940227d12fa01d4c0dda676c0e9ec85164e9b781c79", - "temperature_sha256": "09a7559b6b85cc8e7e7f3dff176b9b2433f165019ceb6e9215bf467d28bb8256", - "runtime_sha256": "c5d3521358b2817f4e56ea8150c5c612139b4e2b2c5bb09f512d9a7c0b5298b9", + "source_api_code_sha256": "1b068eccdffd3c3b67bfa52f8f526e6b482d92551c927668ad28a794767f8a40", + "dev_sha256": "45b4cd46acc4be53b95c7ed8f333b3533972296a4d0557c7b8c48c9cab6ced61", + "production_predictions_sha256": "cf0b1a17008bddc2e3e18289866993770c41d4a5af5f6f4219508bb34fcabefa", + "temperature_sha256": "1d652e75f49b5c232d1e742b189c1c826ad52ced1a931a124e29e65e7d7fb569", + "runtime_sha256": "78a2c21f8138beeffb3e6b1c11e03cb48972a78ca5ef3d1f58c50651031c34b6", "production_batch_size": 8, "input_length_limit": 16384, - "original_checkpoint_name": "checkpoint-001563", - "no_publication_performed": true + "original_checkpoint_name": "winner", + "no_publication_performed": true, + "source_bundle_manifest_sha256": "6d402d6d55e734851cb0d6417d6f3b416b25ae807c3a0d6dcf12e40acb6ac546", + "normalization_profile_sha256": "be32858d15233e0a3fbee0e4257fb02be0b3439deee4eb9c3f61151df7b73850", + "public_wrapper_included": true } diff --git a/code/decision_api.py b/code/decision_api.py index c1b04d342a27beb6358671e6bbedc8f8d31632ca..6b22d88efcd190704b44207676e9394e6aa4902a 100644 --- a/code/decision_api.py +++ b/code/decision_api.py @@ -10,6 +10,32 @@ import math from pathlib import Path +def prepare_runtime_profile(checkpoint, device='cuda:0'): + # This profile is verified before any dependency import can choose kernels. + import hashlib, json, sys + root=Path(checkpoint);runtime=json.loads((root/'runtime.json').read_text()) + spec=runtime.get('normalization_profile') + if spec is None: + if '_decision_process_normalization_profile_v1' in sys.modules: + raise RuntimeError('Use separate processes for profiled and unprofiled models') + return None + import torch + target=torch.device(device) + if target.type!='cuda' or not torch.cuda.is_available(): + raise RuntimeError('The bound profile requires a ROCm CUDA device') + arch=getattr(torch.cuda.get_device_properties(target),'gcnArchName','').split(':')[0] + if arch!=spec['validated_arch']: + raise RuntimeError('Target GPU architecture does not match the bound profile: '+arch) + relative=Path(spec['loader_file']) + if relative.is_absolute() or '..' in relative.parts:raise ValueError('Unsafe profile loader path') + path=root/relative + if hashlib.sha256(path.read_bytes()).hexdigest()!=spec['loader_sha256']: + raise ValueError('Bound runtime profile loader changed') + definition=importlib.util.spec_from_file_location('decision_bundle_runtime_profile',path) + module=importlib.util.module_from_spec(definition);definition.loader.exec_module(module) + return module.ensure_profile(root) + + def question_row(state, name, question): kind=question.get('type') if kind not in {'choice','noul','score'}:raise ValueError('Unknown question type') @@ -61,6 +87,7 @@ def typed_answer(row, probabilities): class DecisionEngine: def __init__(self, checkpoint, model_code, *, device='cuda:0', max_length=16384, batch_size=8, temperatures=None, model_name='local-decision-research'): + self.normalization_profile=prepare_runtime_profile(checkpoint, device=device) import torch path=Path(model_code)/'decision_model.py' spec=importlib.util.spec_from_file_location('research_decision_runtime',path) diff --git a/code/profile_guard.py b/code/profile_guard.py new file mode 100644 index 0000000000000000000000000000000000000000..03f2e2021cd62c8cafde60b4cd7813232510a926 --- /dev/null +++ b/code/profile_guard.py @@ -0,0 +1,82 @@ +"""Fail-closed official FLA strict-config setup for one isolated process. + +No model/prompt/head changes. The guard prevents FLA STRICT's ordinary missing +key fallback and records actual configured calls. This is a diagnostic module, +not an installed change to the published wrapper or dependency environment. +""" +import hashlib,importlib,json,os,sys +from pathlib import Path + +def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest() +def serialized(key):return json.dumps(key,separators=(',',':'),sort_keys=True) +def config_fields(config): + if isinstance(config,dict):return {k:config.get(k) for k in ['kwargs','num_warps','num_stages','num_ctas','maxnreg','ir_override']} + return {k:getattr(config,k,None) for k in ['kwargs','num_warps','num_stages','num_ctas','maxnreg','ir_override']} +def validate_profile(path,expected_sha): + path=Path(path).resolve() + if sha(path)!=expected_sha:raise ValueError('Profile hash changed') + profile=json.loads(path.read_text()) + if profile['format']!='decision-fla-l2norm-profile-v1' or profile.get('model_family')!='Qwen/Qwen3.5-4B' or profile['cache_mode']!='strict':raise ValueError('Profile format/mode unsupported') + if len(profile['files'])!=1 or profile['files'][0]['file']!='l2norm_fwd_kernel.json':raise ValueError('Unexpected profile file set') + f=path.parent/'l2norm_fwd_kernel.json' + if sha(f)!=profile['files'][0]['sha256']:raise ValueError('Explicit kernel config changed') + data=json.loads(f.read_text());entries={} + if data.get('default_config') is not None:raise ValueError('Implicit fallback defaults forbidden') + for h,item in data['autotune_entries'].items(): + key=item['autotune_key'];encoded=serialized(key) + if hashlib.md5(encoded.encode()).hexdigest()!=h or encoded in entries:raise ValueError('Invalid/duplicate key') + if len(key)!=5 or key[0]!=128 or type(key[1]) is not int or not 1<=key[1]<=64 or key[2:]!=['torch.bfloat16','torch.bfloat16','torch.float32']:raise ValueError('Unsupported numerical key') + c=item['config'] + if c['kwargs'].keys()!={'BT'} or c['kwargs']['BT'] not in [8,16,32,64] or c['num_warps'] not in [1,2,4,8,16] or c['num_stages']!=3 or c['num_ctas']!=1 or any(c.get(x) is not None for x in ['maxnreg','pre_hook','ir_override']):raise ValueError('Unexpected launch configuration') + entries[encoded]=c + if {json.loads(k)[1] for k in entries}!=set(range(1,65)):raise ValueError('Incomplete legal NB coverage') + return path,profile,entries + +def attach_guard(kernel,cache_module,entries,telemetry): + original=kernel.run + def guarded(*args,**kwargs): + if cache_module.FLA_CACHE_MODE is not cache_module.FlaCacheMode.STRICT:raise RuntimeError('FLA strict mode changed') + key=cache_module.AutotuneKey.build(kernel.arg_names,kernel.keys,args,kwargs);encoded=serialized(list(key.autotune_key)) + if encoded not in entries:raise RuntimeError('Uncontracted FLA l2norm key: '+encoded) + expected=entries[encoded];loaded=cache_module.load_cached_config(kernel.kernel_name,key) + if config_fields(loaded)!=config_fields(expected):raise RuntimeError('FLA exact config lookup mismatch') + if key.autotune_key in kernel.cache and config_fields(kernel.cache[key.autotune_key])!=config_fields(expected):raise RuntimeError('A conflicting in-process kernel cache exists') + # Explicit official configuration load guarantees that the following original + # run finds this exact cache entry and cannot perform timing-based autotune. + kernel.maybe_load_cached_config(key) + if key.autotune_key not in kernel.cache or config_fields(kernel.cache[key.autotune_key])!=config_fields(expected):raise RuntimeError('Official strict config did not load') + result=original(*args,**kwargs) + if config_fields(kernel.cache[key.autotune_key])!=config_fields(expected):raise RuntimeError('Kernel config changed during call') + telemetry['calls']+=1;telemetry['keys'][encoded]=telemetry['keys'].get(encoded,0)+1 + return result + kernel.run=guarded + return original + +def install(profile_path,expected_sha): + path,profile,entries=validate_profile(profile_path,expected_sha) + if any(n=='fla' or n.startswith('fla.') for n in sys.modules):raise RuntimeError('Install profile before importing FLA; use a fresh isolated process') + for name,wanted in {'FLA_CACHE_MODE':'strict','FLA_CONFIG_DIR':str(path.parent)}.items(): + actual=os.environ.get(name) + if actual is not None and actual!=wanted:raise RuntimeError('Conflicting '+name) + os.environ[name]=wanted + import torch,triton,fla + actual={'torch':str(torch.__version__),'hip':torch.version.hip,'triton':triton.__version__,'fla':fla.__version__} + for name,value in actual.items(): + if value!=profile['runtime'][name]:raise RuntimeError('Runtime mismatch: '+name) + if not torch.cuda.is_available():raise RuntimeError('Profile is only qualified for the specified ROCm GPU') + arch=torch.cuda.get_device_properties(0).gcnArchName.split(':')[0] + if arch!=profile['runtime']['gpu_arch']:raise RuntimeError('Unsupported GPU architecture '+arch) + module=importlib.import_module('fla.modules.l2norm');cache_module=importlib.import_module('fla.ops.utils.cache');root=Path(fla.__file__).parent + for name,value in profile['fla_source_sha256'].items(): + if sha(root/name)!=value:raise RuntimeError('Pinned FLA source changed: '+name) + kernel=module.l2norm_fwd_kernel + if kernel.kernel_name!='l2norm_fwd_kernel' or kernel.keys!=['D','NB'] or kernel.cache:raise RuntimeError('Kernel identity or fresh-cache precondition failed') + telemetry={'profile_sha256':expected_sha,'status':'installed','calls':0,'keys':{},'strict_guard':True,'unknown_keys':'raise','autotune_fallback_permitted':False,'runtime':actual,'gpu_arch':arch,'process_scope':'one explicitly profiled Nox model; other model loading in this process is not supported'} + attach_guard(kernel,cache_module,entries,telemetry) + # The validated inference path only uses the vectorized D128 forward kernel. + # Other dimensions/backward must not silently enter a different autotuner. + for name in ['l2norm_fwd_kernel1','l2norm_bwd_kernel','l2norm_bwd_kernel1']: + other=getattr(module,name) + def reject(*args,_name=name,**kwargs):raise RuntimeError('Uncontracted normalization kernel: '+_name) + other.run=reject + return telemetry diff --git a/code/runtime_profile.py b/code/runtime_profile.py new file mode 100644 index 0000000000000000000000000000000000000000..62fc7c1a8dbdf0e496f35a718edf701ae489508c --- /dev/null +++ b/code/runtime_profile.py @@ -0,0 +1,36 @@ +"""Bundle-local automatic launch-profile binding, before importing FLA. + +One profile is active per Python process. Repeated loading of the same verified +profile is allowed; mixing with an unprofiled/different-profile model is not. +""" +import hashlib,importlib.util,json,os,sys,types +from pathlib import Path +STATE='_decision_process_normalization_profile_v1' +def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest() +def safe(root,relative): + p=Path(relative) + if p.is_absolute() or '..' in p.parts:raise ValueError('Unsafe runtime-profile path') + return root/p + +def ensure_profile(bundle): + bundle=Path(bundle).resolve();runtime=json.loads((bundle/'runtime.json').read_text());spec=runtime.get('normalization_profile');active=sys.modules.get(STATE) + if spec is None: + if active is not None:raise RuntimeError('Load unprofiled and profiled Decision models in separate processes') + return None + if spec.get('kind')!='decision-fla-l2norm-profile-v1' or spec.get('validated_arch')!='gfx942':raise ValueError('Unknown normalization profile contract') + profile=safe(bundle,spec['profile_file']);guard=safe(bundle,spec['guard_file']) + if sha(profile)!=spec['profile_sha256'] or sha(guard)!=spec['guard_sha256']:raise ValueError('Bound runtime profile bytes changed') + if json.loads((bundle/'decision_config.json').read_text()).get('base_model')!='Qwen/Qwen3.5-4B':raise ValueError('This bundle profile is bound to the validated Nox family only') + if active is not None: + if active.profile_sha256!=spec['profile_sha256'] or active.guard_sha256!=spec['guard_sha256']:raise RuntimeError('Different Decision normalization profile is already active; use a separate process') + active.guard.validate_profile(profile,spec['profile_sha256']) + if os.environ.get('FLA_CACHE_MODE')!='strict' or os.environ.get('FLA_CONFIG_DIR')!=active.profile_dir:raise RuntimeError('Active FLA profile environment changed') + return {'profile_sha256':active.profile_sha256,'guard_sha256':active.guard_sha256,'validated_arch':'gfx942','automatic_bundle_binding':True,'scope':'single profile per process'} + module_spec=importlib.util.spec_from_file_location('decision_profile_guard_'+spec['guard_sha256'][:16],guard);module=importlib.util.module_from_spec(module_spec);module_spec.loader.exec_module(module) + telemetry=module.install(profile,spec['profile_sha256']) + state=types.ModuleType(STATE);state.profile_sha256=spec['profile_sha256'];state.guard_sha256=spec['guard_sha256'];state.profile_dir=str(profile.parent);state.guard=module;state.telemetry=telemetry;sys.modules[STATE]=state + return {'profile_sha256':state.profile_sha256,'guard_sha256':state.guard_sha256,'validated_arch':'gfx942','automatic_bundle_binding':True,'scope':'single profile per process'} + +def active_telemetry(): + state=sys.modules.get(STATE) + return None if state is None else dict(state.telemetry) diff --git a/decision_head.safetensors b/decision_head.safetensors index d9dd833fc7d0636b9948cb5bcb339ef2d3cace44..d99928ffb4d97de833dd0cba5799c06915869d2e 100644 --- a/decision_head.safetensors +++ b/decision_head.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f4f1df3f8c3bf507300bb810e098fb8854f182c715e1dd9cc57d44361f949fde +oid sha256:8b8e342445035e503b4963e16c5f7a4fef95c08368e27eace312145fc3660772 size 10529624 diff --git a/metrics/comparator-coverage.json b/metrics/comparator-coverage.json new file mode 100644 index 0000000000000000000000000000000000000000..c398caf4837c4cc0e63121a91c9df0f57d85e6eb --- /dev/null +++ b/metrics/comparator-coverage.json @@ -0,0 +1,440 @@ +{ + "Jev \u00b7 1.13.0": { + "old_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 0, + "retention_unknown": 880, + "raw_sha256": "15fc57a65b0571eafcb4a1ad695fb224aeae44b65e1a61ecd4d348f2e8fd3a60", + "normalized_sha256": "069e48e1fdf1b5f3d95cc0097e4179ac3faab97bb357b0ce897cfb463b87cbaf", + "requests_sha256": "7c46df786001f7bd49c4e43d0998c494c169b40164e500e5c523016350db312f", + "raw_to_normalized_reverified": true, + "service_versions": [ + "jev-1.13.0" + ], + "first_request_utc": "2026-09-21T09:49:11.121634+00:00", + "last_request_utc": "2026-09-21T09:55:51.282211+00:00", + "exact_weight_snapshot_available": false + }, + "v3_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 0, + "retention_unknown": 880, + "raw_sha256": "48c1b0d8729c88fdc4a4f89fbf5eda045925243c997c4aae1e6172ba63ab13b9", + "normalized_sha256": "54707354bfe5d96b5de56e6dc1b38cba0a610eea516b28fbff013b94f422b397", + "requests_sha256": "bd5dfe8ab0aff17435b5826126ebcee502c4cc44673dbe0be308bff489833095", + "raw_to_normalized_reverified": true, + "service_versions": [ + "jev-1.13.0" + ], + "first_request_utc": "2026-09-21T14:53:06.592910+00:00", + "last_request_utc": "2026-09-21T15:06:45.698712+00:00", + "exact_weight_snapshot_available": false + }, + "v4": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 0, + "retention_unknown": 480, + "raw_sha256": "246ddbff360a15adc3e4679c86aed44df8541afee53b0205da1892eb0fe8a811", + "normalized_sha256": "d6583f903e974d70747fae2944014828954dc4d7344ceddcbb2cd1b06fbfc799", + "requests_sha256": "8e7397c7086fc867d2aafaac40f49db156be0f10e2b2728ef041fa04afd59dbe", + "raw_to_normalized_reverified": true, + "service_versions": [ + "jev-1.13.0" + ], + "first_request_utc": "2026-09-21T16:22:16.227785+00:00", + "last_request_utc": "2026-09-21T16:28:05.090381+00:00", + "exact_weight_snapshot_available": false + }, + "v5": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 0, + "retention_unknown": 480, + "raw_sha256": "a0a6d0699aed767bad897b71191d18f9ee59a49bc0a71c5fbb5003f6d966133b", + "normalized_sha256": "f883164c4c83db8e42ebe03f7e165afbd2f81f2b541bcf8008a2cc06a60e1ee7", + "requests_sha256": "c83a7b9611c740c5232ebeb15b00377798c76806c509e62e2f63b08369c898bc", + "raw_to_normalized_reverified": true, + "service_versions": [ + "jev-1.13.0" + ], + "first_request_utc": "2026-09-21T19:47:20.496709+00:00", + "last_request_utc": "2026-09-21T19:50:20.927928+00:00", + "exact_weight_snapshot_available": false + } + }, + "Laya \u00b7 EN/ML": { + "old_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 112, + "retention_unknown": 0, + "raw_sha256": "cc1df0126bd7d790b55b40c39028a3dcea2bb20d8c8115b7a7f4042ec1314d6e", + "normalized_sha256": "2503adb3d7a3476a9b84f6647f2a69e7986e96eea878c191568a33597800a3dc", + "requests_sha256": "7c46df786001f7bd49c4e43d0998c494c169b40164e500e5c523016350db312f", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "repo": "convaiinnovations/laya", + "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", + "router_source_revision": "42626c348753fbb17572a813127df2278a1ec527", + "adapter_sha256": "a5484e30d9fad04ca578265e5f18df2a10731f5055bef070311102c3b35643fc", + "mode": "routed", + "weight_fingerprints": { + "english": "5bf5d0f9462d860e4c09000fce3131a070578d3574777ae7981077f11cd68185", + "multilingual": "a472bfcc4aa52a25dda0b2864b3e7c7c1267ddd1752b04a1e70bf6ba7e4b514c" + }, + "routing": "Unmodified upstream Router.route(state,questions), no explicit lang, no automatic workflow detection, default english" + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v3_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 40, + "retention_unknown": 0, + "raw_sha256": "c5e7703af1d200cd662d1eef8e87f6d30a237c451233893b295e0eac0f92e0fa", + "normalized_sha256": "fd5053fb037d43de5383140b33cb3a363ad5cab56387354f1bac0414d52f7da2", + "requests_sha256": "bd5dfe8ab0aff17435b5826126ebcee502c4cc44673dbe0be308bff489833095", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "repo": "convaiinnovations/laya", + "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", + "router_source_revision": "42626c348753fbb17572a813127df2278a1ec527", + "adapter_sha256": "a5484e30d9fad04ca578265e5f18df2a10731f5055bef070311102c3b35643fc", + "mode": "routed", + "weight_fingerprints": { + "english": "5bf5d0f9462d860e4c09000fce3131a070578d3574777ae7981077f11cd68185", + "multilingual": "a472bfcc4aa52a25dda0b2864b3e7c7c1267ddd1752b04a1e70bf6ba7e4b514c" + }, + "routing": "Unmodified upstream Router.route(state,questions), no explicit lang, no automatic workflow detection, default english" + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v4": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "550cc73b98da4ff7efc4b4cf07703f606831d59ad90ea5ea223ac5f587d51fe5", + "normalized_sha256": "a10922609e2223ca0a818667bc1425d476aec58a4d881002e0e322596c1d0b8d", + "requests_sha256": "8e7397c7086fc867d2aafaac40f49db156be0f10e2b2728ef041fa04afd59dbe", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "repo": "convaiinnovations/laya", + "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", + "router_source_revision": "42626c348753fbb17572a813127df2278a1ec527", + "adapter_sha256": "a5484e30d9fad04ca578265e5f18df2a10731f5055bef070311102c3b35643fc", + "mode": "routed", + "weight_fingerprints": { + "english": "5bf5d0f9462d860e4c09000fce3131a070578d3574777ae7981077f11cd68185", + "multilingual": "a472bfcc4aa52a25dda0b2864b3e7c7c1267ddd1752b04a1e70bf6ba7e4b514c" + }, + "routing": "Unmodified upstream Router.route(state,questions), no explicit lang, no automatic workflow detection, default english" + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v5": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 2, + "retention_unknown": 0, + "raw_sha256": "31064d69fd8a2b1bb86745a0525af40ef3f8feb91a0aa462f3035770fc84e6b3", + "normalized_sha256": "d1334fc4b9106621aa754a8567079201f3ae8bb270ceab981f7351a4a4479c62", + "requests_sha256": "c83a7b9611c740c5232ebeb15b00377798c76806c509e62e2f63b08369c898bc", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "repo": "convaiinnovations/laya", + "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", + "router_source_revision": "42626c348753fbb17572a813127df2278a1ec527", + "adapter_sha256": "a5484e30d9fad04ca578265e5f18df2a10731f5055bef070311102c3b35643fc", + "mode": "routed", + "weight_fingerprints": { + "english": "5bf5d0f9462d860e4c09000fce3131a070578d3574777ae7981077f11cd68185", + "multilingual": "a472bfcc4aa52a25dda0b2864b3e7c7c1267ddd1752b04a1e70bf6ba7e4b514c" + }, + "routing": "Unmodified upstream Router.route(state,questions), no explicit lang, no automatic workflow detection, default english" + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + } + }, + "Decider \u00b7 2B": { + "old_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "5b6e107f9b0eea7686d0e2e3e983f5950254a7dffd875ecaca0a6a9ec5e4e590", + "normalized_sha256": "e1b14f0f9ec521bffbde5945684bc2410a413b6b6418d65ac4fc702a7b5e7937", + "requests_sha256": "7c46df786001f7bd49c4e43d0998c494c169b40164e500e5c523016350db312f", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "model_id": "Mapika/decider-2b", + "original_native_adapter_sha256": "983a2a0d582a1c5e7e58fe3fe4e7c1cd254dd0fe52f65449c68549108d0af387", + "wrapper_sha256": "95b9fcc132d1ae2f0818ea8bb4720f4b5342450e741fefb4315827225652e7f6", + "model_fingerprint": "35f7335c92cf1766c11f52fb4bcdb9c3894d3edad7408a00ae7b59c104b2d318", + "temperature": 1.3, + "layout": "state_first", + "independent": true, + "revision": "b37f7e1ba3fbc9238004cf531fabbee2619973fd" + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v3_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "eceb70e8c88117c288599b4a46eaea961f01a75f15fd48fe5b5075c070d680ef", + "normalized_sha256": "12a1f554decf1aff608743e7b4a44681289ea084ae25f383e519b1f34917d46f", + "requests_sha256": "bd5dfe8ab0aff17435b5826126ebcee502c4cc44673dbe0be308bff489833095", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "model_id": "Mapika/decider-2b", + "original_native_adapter_sha256": "983a2a0d582a1c5e7e58fe3fe4e7c1cd254dd0fe52f65449c68549108d0af387", + "wrapper_sha256": "95b9fcc132d1ae2f0818ea8bb4720f4b5342450e741fefb4315827225652e7f6", + "model_fingerprint": "35f7335c92cf1766c11f52fb4bcdb9c3894d3edad7408a00ae7b59c104b2d318", + "temperature": 1.3, + "layout": "state_first", + "independent": true, + "revision": "b37f7e1ba3fbc9238004cf531fabbee2619973fd" + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v4": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "6f4f5be5b606a0dd13cd2303d0562f1e7538bba92288f121ed9886ac5dc873ac", + "normalized_sha256": "626ef366ae471ec10dfb89ef2ff2f7b29a6c879976d3ff14fcbb2bafd7d9b045", + "requests_sha256": "8e7397c7086fc867d2aafaac40f49db156be0f10e2b2728ef041fa04afd59dbe", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "model_id": "Mapika/decider-2b", + "original_native_adapter_sha256": "983a2a0d582a1c5e7e58fe3fe4e7c1cd254dd0fe52f65449c68549108d0af387", + "wrapper_sha256": "95b9fcc132d1ae2f0818ea8bb4720f4b5342450e741fefb4315827225652e7f6", + "model_fingerprint": "35f7335c92cf1766c11f52fb4bcdb9c3894d3edad7408a00ae7b59c104b2d318", + "temperature": 1.3, + "layout": "state_first", + "independent": true, + "revision": "b37f7e1ba3fbc9238004cf531fabbee2619973fd" + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v5": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "90c05f48c5da7caeea4c4acfd58b1f1ff82c9e7af73e06036e30a4680fdc6c9a", + "normalized_sha256": "ae16b17ac8e4208940c0c3e9e04f25e94686b5d16bf46f64f53669baf41aeb71", + "requests_sha256": "c83a7b9611c740c5232ebeb15b00377798c76806c509e62e2f63b08369c898bc", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "model_id": "Mapika/decider-2b", + "original_native_adapter_sha256": "983a2a0d582a1c5e7e58fe3fe4e7c1cd254dd0fe52f65449c68549108d0af387", + "wrapper_sha256": "95b9fcc132d1ae2f0818ea8bb4720f4b5342450e741fefb4315827225652e7f6", + "model_fingerprint": "35f7335c92cf1766c11f52fb4bcdb9c3894d3edad7408a00ae7b59c104b2d318", + "temperature": 1.3, + "layout": "state_first", + "independent": true, + "revision": "b37f7e1ba3fbc9238004cf531fabbee2619973fd" + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + } + }, + "Qwen3.5 \u00b7 2B, untuned": { + "old_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "0f2e105e7204db3e94fce26f57af68d0fbc64fc47097c97207f5feb9c22ecb37", + "normalized_sha256": "c11a94d8f833d83427ea984a4f069aefaa3fa10a2b1085fb3a03542727f819cf", + "requests_sha256": "7c46df786001f7bd49c4e43d0998c494c169b40164e500e5c523016350db312f", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "base_repo": "Qwen/Qwen3.5-2B", + "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v3_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "75b93056650b33336800cb4ef4a2a1a230bb4d2aa95fdaef7d3a369d9c264a65", + "normalized_sha256": "cc51471716342ecd466b50d5dd9b4ba805f5a6873cd96075b56d7c2e8336175e", + "requests_sha256": "bd5dfe8ab0aff17435b5826126ebcee502c4cc44673dbe0be308bff489833095", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "base_repo": "Qwen/Qwen3.5-2B", + "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v4": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "780936e46e6296cd07a6bbefb774144a17464421c1be4251c0c3d0411057bc63", + "normalized_sha256": "93d84cdbe607691e2cef35ece3aa2dd5900f6410b0d72796b86d74c9156f3cdd", + "requests_sha256": "8e7397c7086fc867d2aafaac40f49db156be0f10e2b2728ef041fa04afd59dbe", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "base_repo": "Qwen/Qwen3.5-2B", + "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v5": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "deeeee3fa6a37cb9ebb0b891421159e7cdcb2137b38634e79e996b9b67d6bf7e", + "normalized_sha256": "75740c4279eb8dcc9c90b55f155916b3748c887e8c9d9e1d3a47edf7f4dbc34b", + "requests_sha256": "c83a7b9611c740c5232ebeb15b00377798c76806c509e62e2f63b08369c898bc", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "base_repo": "Qwen/Qwen3.5-2B", + "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + } + }, + "Qwen3.5 \u00b7 4B, untuned": { + "old_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "0f24febd8ea56174d29f3cf9642f1595905b6d55a29c2dce9258fec616c1cc68", + "normalized_sha256": "d20ad845066ff81caaf60ebee44df941872bdc642da78b5f3b3da4a33dd9a08d", + "requests_sha256": "7c46df786001f7bd49c4e43d0998c494c169b40164e500e5c523016350db312f", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "base_repo": "Qwen/Qwen3.5-4B", + "base_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v3_core": { + "requested": 880, + "recorded_outputs": 880, + "ok": 880, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "f27e7008cb7e7a73248ba119dd6d4279600c61f0169ed4514fb99ccec605dec3", + "normalized_sha256": "7541893d2b2ec0ec2ffc513f50c568eaef860b65bcef9a1747996c116fff9661", + "requests_sha256": "bd5dfe8ab0aff17435b5826126ebcee502c4cc44673dbe0be308bff489833095", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "base_repo": "Qwen/Qwen3.5-4B", + "base_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v4": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "1e09cd713ce91f9c4a5cc81939554a7351634865dc262d93db879de75537f2e5", + "normalized_sha256": "7a6d252ccaa72bf54b94445190bbad023fb465a3eaf4cf657532e2cdd4f750f5", + "requests_sha256": "8e7397c7086fc867d2aafaac40f49db156be0f10e2b2728ef041fa04afd59dbe", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "base_repo": "Qwen/Qwen3.5-4B", + "base_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + }, + "v5": { + "requested": 480, + "recorded_outputs": 480, + "ok": 480, + "truncated_true": 0, + "retention_unknown": 0, + "raw_sha256": "1f7068b586c6ca0dca6f7928acef2d539ba4746a2e7eeb73fd3a1a1ccb8697a5", + "normalized_sha256": "c01ad5daed65267e9aec40ed9e97ece1b6511e4cab17ebff09339b41524fb7f1", + "requests_sha256": "c83a7b9611c740c5232ebeb15b00377798c76806c509e62e2f63b08369c898bc", + "raw_to_normalized_reverified": true, + "adapter_identity": { + "base_repo": "Qwen/Qwen3.5-4B", + "base_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "runtime_fingerprint_sha256": "b5af3d997c95d0a6d2088a64f341726fc0daee5eebc730f851b2b65bf0de8d1b" + } + } +} diff --git a/metrics/expanded-quality.json b/metrics/expanded-quality.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b6849f8a04ead638c989fbb03720ee5065bbe1 --- /dev/null +++ b/metrics/expanded-quality.json @@ -0,0 +1,2410 @@ +{ + "models": { + "Decider \u00b7 2B": { + "old_core": { + "point": 0.6401413690476191, + "exact_fraction": "17207/26880", + "ci95": [ + 0.5992559523809524, + 0.6806184895833334 + ] + }, + "v3_core": { + "point": 0.4658333333333333, + "exact_fraction": "559/1200", + "ci95": [ + 0.4362500000000001, + 0.49499999999999994 + ] + }, + "v4": { + "point": 0.9203125, + "exact_fraction": "589/640", + "ci95": [ + 0.8903095518867924, + 0.9479299363057325 + ] + }, + "v5": { + "point": 0.84375, + "exact_fraction": "27/32", + "ci95": [ + 0.8130660699474849, + 0.8734622752789641 + ] + }, + "v3_joint": { + "point": 0.5529873511904762, + "exact_fraction": "148643/268800", + "ci95": [ + 0.5282024739583334, + 0.5777246093749999 + ] + }, + "expanded_joint": { + "point": 0.7175093005952381, + "exact_fraction": "385733/537600", + "ci95": [ + 0.7012097728950374, + 0.7336543123969403 + ] + }, + "families": { + "old_core": { + "ag_news": { + "correct": 111, + "requested": 128, + "accuracy_ci95": [ + 0.8046875, + 0.921875 + ] + }, + "boolean_constraints": { + "correct": 46, + "requested": 64, + "accuracy_ci95": [ + 0.578125, + 0.84375 + ] + }, + "dbpedia_14": { + "correct": 110, + "requested": 112, + "accuracy_ci95": [ + 0.9553571428571429, + 1.0 + ] + }, + "natural_intents": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + }, + "option_carrier": { + "correct": 8, + "requested": 96, + "accuracy_ci95": [ + 0.041666666666666664, + 0.13541666666666666 + ] + }, + "ordinal_rubric": { + "correct": 54, + "requested": 64, + "accuracy_ci95": [ + 0.71875, + 0.9375 + ] + }, + "relational_composition": { + "correct": 52, + "requested": 96, + "accuracy_ci95": [ + 0.3854166666666667, + 0.6979166666666666 + ] + }, + "scoped_evidence": { + "correct": 46, + "requested": 96, + "accuracy_ci95": [ + 0.2916666666666667, + 0.6666666666666666 + ] + }, + "state_tracking": { + "correct": 28, + "requested": 96, + "accuracy_ci95": [ + 0.13541666666666666, + 0.4583333333333333 + ] + }, + "unknown_rejection": { + "correct": 38, + "requested": 64, + "accuracy_ci95": [ + 0.375, + 0.796875 + ] + } + }, + "v3_core": { + "canonical_record_identity": { + "correct": 45, + "requested": 80, + "accuracy_ci95": [ + 0.45, + 0.675 + ] + }, + "capacitated_assignment": { + "correct": 41, + "requested": 80, + "accuracy_ci95": [ + 0.4375, + 0.5875 + ] + }, + "constraint_assignment": { + "correct": 19, + "requested": 80, + "accuracy_ci95": [ + 0.1625, + 0.3 + ] + }, + "massive_en": { + "correct": 108, + "requested": 120, + "accuracy_ci95": [ + 0.8416666666666667, + 0.9500000000000001 + ] + }, + "massive_zh": { + "correct": 106, + "requested": 120, + "accuracy_ci95": [ + 0.825, + 0.9333333333333333 + ] + }, + "multiset_reconciliation": { + "correct": 19, + "requested": 80, + "accuracy_ci95": [ + 0.1875, + 0.2875 + ] + }, + "ordinal_service_loss": { + "correct": 18, + "requested": 80, + "accuracy_ci95": [ + 0.175, + 0.275 + ] + }, + "paraconsistent_rule_closure": { + "correct": 21, + "requested": 80, + "accuracy_ci95": [ + 0.1875, + 0.3375 + ] + }, + "temporal_exclusion": { + "correct": 34, + "requested": 80, + "accuracy_ci95": [ + 0.2625, + 0.6125 + ] + }, + "transaction_recovery": { + "correct": 33, + "requested": 80, + "accuracy_ci95": [ + 0.3125, + 0.5125 + ] + } + }, + "v4": { + "boolq": { + "correct": 147, + "requested": 160, + "accuracy_ci95": [ + 0.875, + 0.95625 + ] + }, + "belebele_en": { + "correct": 149, + "requested": 160, + "accuracy_ci95": [ + 0.8910256410256411, + 0.9683544303797469 + ] + }, + "belebele_zh": { + "correct": 146, + "requested": 160, + "accuracy_ci95": [ + 0.8622754491017964, + 0.9559748427672956 + ] + } + }, + "v5": { + "cosmos_qa": { + "correct": 80, + "requested": 120, + "accuracy_ci95": [ + 0.5833333333333334, + 0.75 + ] + }, + "squad2_answerability": { + "correct": 101, + "requested": 120, + "accuracy_ci95": [ + 0.7857142857142857, + 0.896551724137931 + ] + }, + "snli": { + "correct": 109, + "requested": 120, + "accuracy_ci95": [ + 0.85, + 0.9583333333333334 + ] + }, + "qasc": { + "correct": 115, + "requested": 120, + "accuracy_ci95": [ + 0.9112903225806451, + 0.991869918699187 + ] + } + } + }, + "proper_scores": { + "old_core": 0.44246049270704674, + "v3_core": 0.5974556966463511, + "v4": 0.13737783476250795, + "v5": 0.23664932218586204, + "expanded_weighted_brier": 0.35348583657544197 + }, + "probability_coverage": { + "old_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v3_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v4": { + "requested": 480, + "valid_probability_rows": 480 + }, + "v5": { + "requested": 480, + "valid_probability_rows": 480 + } + }, + "brier_ci95": { + "old_core": [ + 0.40579693831982216, + 0.4787313843802038 + ], + "v3_core": [ + 0.5767822346729764, + 0.6189924929703139 + ], + "v4": [ + 0.10073496035335808, + 0.17666521563140353 + ], + "v5": [ + 0.2031026857962821, + 0.27234790035750356 + ], + "expanded_weighted_brier": [ + 0.3373790190368837, + 0.37038546676401063 + ] + } + }, + "Jev \u00b7 1.13.0": { + "old_core": { + "point": 0.7909598214285715, + "exact_fraction": "7087/8960", + "ci95": [ + 0.7616062127976191, + 0.819271763392857 + ] + }, + "v3_core": { + "point": 0.66375, + "exact_fraction": "531/800", + "ci95": [ + 0.6316666666666667, + 0.6945833333333333 + ] + }, + "v4": { + "point": 0.9453125, + "exact_fraction": "121/128", + "ci95": [ + 0.9197115384615384, + 0.9685135514495712 + ] + }, + "v5": { + "point": 0.8979166666666667, + "exact_fraction": "431/480", + "ci95": [ + 0.8713251366120218, + 0.9229854264081012 + ] + }, + "v3_joint": { + "point": 0.7273549107142857, + "exact_fraction": "65171/89600", + "ci95": [ + 0.7058627232142857, + 0.7482447916666666 + ] + }, + "expanded_joint": { + "point": 0.8244847470238095, + "exact_fraction": "443243/537600", + "ci95": [ + 0.8104711862785249, + 0.8378880015726611 + ] + }, + "families": { + "old_core": { + "ag_news": { + "correct": 109, + "requested": 128, + "accuracy_ci95": [ + 0.7890625, + 0.90625 + ] + }, + "boolean_constraints": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + }, + "dbpedia_14": { + "correct": 108, + "requested": 112, + "accuracy_ci95": [ + 0.9285714285714286, + 0.9910714285714286 + ] + }, + "natural_intents": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + }, + "option_carrier": { + "correct": 29, + "requested": 96, + "accuracy_ci95": [ + 0.2604166666666667, + 0.34375 + ] + }, + "ordinal_rubric": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + }, + "relational_composition": { + "correct": 54, + "requested": 96, + "accuracy_ci95": [ + 0.3645833333333333, + 0.75 + ] + }, + "scoped_evidence": { + "correct": 86, + "requested": 96, + "accuracy_ci95": [ + 0.78125, + 0.9895833333333334 + ] + }, + "state_tracking": { + "correct": 32, + "requested": 96, + "accuracy_ci95": [ + 0.16640625000000023, + 0.5106770833333295 + ] + }, + "unknown_rejection": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + } + }, + "v3_core": { + "canonical_record_identity": { + "correct": 61, + "requested": 80, + "accuracy_ci95": [ + 0.6375, + 0.875 + ] + }, + "capacitated_assignment": { + "correct": 55, + "requested": 80, + "accuracy_ci95": [ + 0.575, + 0.7875 + ] + }, + "constraint_assignment": { + "correct": 44, + "requested": 80, + "accuracy_ci95": [ + 0.4, + 0.675 + ] + }, + "massive_en": { + "correct": 110, + "requested": 120, + "accuracy_ci95": [ + 0.8666666666666667, + 0.9583333333333334 + ] + }, + "massive_zh": { + "correct": 106, + "requested": 120, + "accuracy_ci95": [ + 0.825, + 0.9416666666666667 + ] + }, + "multiset_reconciliation": { + "correct": 47, + "requested": 80, + "accuracy_ci95": [ + 0.5125, + 0.65 + ] + }, + "ordinal_service_loss": { + "correct": 34, + "requested": 80, + "accuracy_ci95": [ + 0.325, + 0.525 + ] + }, + "paraconsistent_rule_closure": { + "correct": 53, + "requested": 80, + "accuracy_ci95": [ + 0.5875, + 0.725 + ] + }, + "temporal_exclusion": { + "correct": 33, + "requested": 80, + "accuracy_ci95": [ + 0.3125, + 0.5125 + ] + }, + "transaction_recovery": { + "correct": 60, + "requested": 80, + "accuracy_ci95": [ + 0.625, + 0.8625 + ] + } + }, + "v4": { + "boolq": { + "correct": 148, + "requested": 160, + "accuracy_ci95": [ + 0.88125, + 0.9625 + ] + }, + "belebele_en": { + "correct": 155, + "requested": 160, + "accuracy_ci95": [ + 0.9390243902439024, + 0.9937106918238994 + ] + }, + "belebele_zh": { + "correct": 154, + "requested": 160, + "accuracy_ci95": [ + 0.930379746835443, + 0.9877300613496932 + ] + } + }, + "v5": { + "cosmos_qa": { + "correct": 104, + "requested": 120, + "accuracy_ci95": [ + 0.8, + 0.925 + ] + }, + "squad2_answerability": { + "correct": 109, + "requested": 120, + "accuracy_ci95": [ + 0.8620689655172413, + 0.9508196721311475 + ] + }, + "snli": { + "correct": 99, + "requested": 120, + "accuracy_ci95": [ + 0.75, + 0.8916666666666667 + ] + }, + "qasc": { + "correct": 119, + "requested": 120, + "accuracy_ci95": [ + 0.9745762711864406, + 1.0 + ] + } + } + }, + "proper_scores": { + "old_core": 0.24806216931976088, + "v3_core": 0.4403896043771044, + "v4": 0.07270531250000001, + "v5": 0.15378333333333333, + "expanded_weighted_brier": 0.22873510488254967 + }, + "probability_coverage": { + "old_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v3_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v4": { + "requested": 480, + "valid_probability_rows": 480 + }, + "v5": { + "requested": 480, + "valid_probability_rows": 480 + } + }, + "brier_ci95": { + "old_core": [ + 0.21912044641352113, + 0.27809132807795134 + ], + "v3_core": [ + 0.4130360873769258, + 0.4688325815363228 + ], + "v4": [ + 0.04802243933725408, + 0.100621887515886 + ], + "v5": [ + 0.12116203971412033, + 0.18851406142759564 + ], + "expanded_weighted_brier": [ + 0.21435641903529326, + 0.2433957477840872 + ] + } + }, + "Laya \u00b7 EN/ML": { + "old_core": { + "point": 0.5701264880952381, + "exact_fraction": "3065/5376", + "ci95": [ + 0.5321056547619049, + 0.6094494047619048 + ] + }, + "v3_core": { + "point": 0.3775, + "exact_fraction": "151/400", + "ci95": [ + 0.35458333333333336, + 0.39958333333333335 + ] + }, + "v4": { + "point": 0.5125, + "exact_fraction": "41/80", + "ci95": [ + 0.4661671277501309, + 0.557454268292683 + ] + }, + "v5": { + "point": 0.6375, + "exact_fraction": "51/80", + "ci95": [ + 0.6032948178983685, + 0.6716809713450388 + ] + }, + "v3_joint": { + "point": 0.47381324404761904, + "exact_fraction": "127361/268800", + "ci95": [ + 0.4519083891369048, + 0.4964252232142857 + ] + }, + "expanded_joint": { + "point": 0.5244066220238095, + "exact_fraction": "281921/537600", + "ci95": [ + 0.5065901244344904, + 0.5426481205211884 + ] + }, + "families": { + "old_core": { + "ag_news": { + "correct": 117, + "requested": 128, + "accuracy_ci95": [ + 0.859375, + 0.9609375 + ] + }, + "boolean_constraints": { + "correct": 28, + "requested": 64, + "accuracy_ci95": [ + 0.28125, + 0.609375 + ] + }, + "dbpedia_14": { + "correct": 94, + "requested": 112, + "accuracy_ci95": [ + 0.7678571428571429, + 0.9017857142857143 + ] + }, + "natural_intents": { + "correct": 55, + "requested": 64, + "accuracy_ci95": [ + 0.75, + 0.9375 + ] + }, + "option_carrier": { + "correct": 92, + "requested": 96, + "accuracy_ci95": [ + 0.8854166666666666, + 1.0 + ] + }, + "ordinal_rubric": { + "correct": 12, + "requested": 64, + "accuracy_ci95": [ + 0.0625, + 0.34375 + ] + }, + "relational_composition": { + "correct": 24, + "requested": 96, + "accuracy_ci95": [ + 0.17708333333333334, + 0.3333333333333333 + ] + }, + "scoped_evidence": { + "correct": 36, + "requested": 96, + "accuracy_ci95": [ + 0.23958333333333334, + 0.5208333333333334 + ] + }, + "state_tracking": { + "correct": 23, + "requested": 96, + "accuracy_ci95": [ + 0.11458333333333333, + 0.3854166666666667 + ] + }, + "unknown_rejection": { + "correct": 41, + "requested": 64, + "accuracy_ci95": [ + 0.4375, + 0.84375 + ] + } + }, + "v3_core": { + "canonical_record_identity": { + "correct": 38, + "requested": 80, + "accuracy_ci95": [ + 0.425, + 0.525 + ] + }, + "capacitated_assignment": { + "correct": 51, + "requested": 80, + "accuracy_ci95": [ + 0.55, + 0.725 + ] + }, + "constraint_assignment": { + "correct": 18, + "requested": 80, + "accuracy_ci95": [ + 0.15, + 0.3125 + ] + }, + "massive_en": { + "correct": 89, + "requested": 120, + "accuracy_ci95": [ + 0.6583333333333333, + 0.8166666666666667 + ] + }, + "massive_zh": { + "correct": 88, + "requested": 120, + "accuracy_ci95": [ + 0.6583333333333332, + 0.8083333333333333 + ] + }, + "multiset_reconciliation": { + "correct": 21, + "requested": 80, + "accuracy_ci95": [ + 0.25, + 0.2875 + ] + }, + "ordinal_service_loss": { + "correct": 16, + "requested": 80, + "accuracy_ci95": [ + 0.15, + 0.25 + ] + }, + "paraconsistent_rule_closure": { + "correct": 15, + "requested": 80, + "accuracy_ci95": [ + 0.1375, + 0.2375 + ] + }, + "temporal_exclusion": { + "correct": 10, + "requested": 80, + "accuracy_ci95": [ + 0.05, + 0.2125 + ] + }, + "transaction_recovery": { + "correct": 15, + "requested": 80, + "accuracy_ci95": [ + 0.1, + 0.2625 + ] + } + }, + "v4": { + "boolq": { + "correct": 111, + "requested": 160, + "accuracy_ci95": [ + 0.61875, + 0.7625 + ] + }, + "belebele_en": { + "correct": 61, + "requested": 160, + "accuracy_ci95": [ + 0.3062370621019109, + 0.4585987261146497 + ] + }, + "belebele_zh": { + "correct": 45, + "requested": 160, + "accuracy_ci95": [ + 0.2138364779874214, + 0.3503184713375796 + ] + } + }, + "v5": { + "cosmos_qa": { + "correct": 36, + "requested": 120, + "accuracy_ci95": [ + 0.21666666666666667, + 0.38333333333333336 + ] + }, + "squad2_answerability": { + "correct": 69, + "requested": 120, + "accuracy_ci95": [ + 0.5084745762711864, + 0.6416666666666667 + ] + }, + "snli": { + "correct": 87, + "requested": 120, + "accuracy_ci95": [ + 0.6416666666666667, + 0.8 + ] + }, + "qasc": { + "correct": 114, + "requested": 120, + "accuracy_ci95": [ + 0.9083333333333333, + 0.983739837398374 + ] + } + } + }, + "proper_scores": { + "old_core": 0.5709760997718798, + "v3_core": 0.700479457442883, + "v4": 0.6149664146731222, + "v5": 0.4747850267787647, + "expanded_weighted_brier": 0.5903017496666624 + }, + "probability_coverage": { + "old_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v3_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v4": { + "requested": 480, + "valid_probability_rows": 480 + }, + "v5": { + "requested": 480, + "valid_probability_rows": 480 + } + }, + "brier_ci95": { + "old_core": [ + 0.5307603405543191, + 0.611972788595099 + ], + "v3_core": [ + 0.677830213791377, + 0.7242813502662896 + ], + "v4": [ + 0.5601239092953858, + 0.6718388215324546 + ], + "v5": [ + 0.4441678879307976, + 0.5061559610657542 + ], + "expanded_weighted_brier": [ + 0.5709748439741565, + 0.6102902589236013 + ] + } + }, + "Nox-natural-selected": { + "old_core": { + "point": 0.8321056547619048, + "exact_fraction": "22367/26880", + "ci95": [ + 0.799775855654762, + 0.8639146205357143 + ] + }, + "v3_core": { + "point": 0.5108333333333334, + "exact_fraction": "613/1200", + "ci95": [ + 0.48540624999999993, + 0.5366666666666667 + ] + }, + "v4": { + "point": 0.78125, + "exact_fraction": "25/32", + "ci95": [ + 0.7348237223129993, + 0.825 + ] + }, + "v5": { + "point": 0.84375, + "exact_fraction": "27/32", + "ci95": [ + 0.8125286977325981, + 0.8745048035497992 + ] + }, + "v3_joint": { + "point": 0.671469494047619, + "exact_fraction": "180491/268800", + "ci95": [ + 0.6504053199404762, + 0.6917820870535715 + ] + }, + "expanded_joint": { + "point": 0.7419847470238096, + "exact_fraction": "398891/537600", + "ci95": [ + 0.7246839089844096, + 0.7585761485522727 + ] + }, + "families": { + "old_core": { + "ag_news": { + "correct": 109, + "requested": 128, + "accuracy_ci95": [ + 0.7890625, + 0.90625 + ] + }, + "boolean_constraints": { + "correct": 60, + "requested": 64, + "accuracy_ci95": [ + 0.859375, + 1.0 + ] + }, + "dbpedia_14": { + "correct": 108, + "requested": 112, + "accuracy_ci95": [ + 0.9285714285714286, + 0.9910714285714286 + ] + }, + "natural_intents": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + }, + "option_carrier": { + "correct": 96, + "requested": 96, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + }, + "ordinal_rubric": { + "correct": 57, + "requested": 64, + "accuracy_ci95": [ + 0.78125, + 0.984375 + ] + }, + "relational_composition": { + "correct": 52, + "requested": 96, + "accuracy_ci95": [ + 0.375, + 0.7083333333333334 + ] + }, + "scoped_evidence": { + "correct": 75, + "requested": 96, + "accuracy_ci95": [ + 0.6145833333333334, + 0.9270833333333334 + ] + }, + "state_tracking": { + "correct": 34, + "requested": 96, + "accuracy_ci95": [ + 0.19791666666666666, + 0.53125 + ] + }, + "unknown_rejection": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + } + }, + "v3_core": { + "canonical_record_identity": { + "correct": 40, + "requested": 80, + "accuracy_ci95": [ + 0.5, + 0.5 + ] + }, + "capacitated_assignment": { + "correct": 33, + "requested": 80, + "accuracy_ci95": [ + 0.35, + 0.475 + ] + }, + "constraint_assignment": { + "correct": 33, + "requested": 80, + "accuracy_ci95": [ + 0.3375, + 0.475 + ] + }, + "massive_en": { + "correct": 109, + "requested": 120, + "accuracy_ci95": [ + 0.8581250000000001, + 0.9583333333333333 + ] + }, + "massive_zh": { + "correct": 105, + "requested": 120, + "accuracy_ci95": [ + 0.8166666666666667, + 0.925 + ] + }, + "multiset_reconciliation": { + "correct": 23, + "requested": 80, + "accuracy_ci95": [ + 0.2125, + 0.375 + ] + }, + "ordinal_service_loss": { + "correct": 25, + "requested": 80, + "accuracy_ci95": [ + 0.2375, + 0.3625 + ] + }, + "paraconsistent_rule_closure": { + "correct": 27, + "requested": 80, + "accuracy_ci95": [ + 0.275, + 0.425 + ] + }, + "temporal_exclusion": { + "correct": 36, + "requested": 80, + "accuracy_ci95": [ + 0.3, + 0.5875 + ] + }, + "transaction_recovery": { + "correct": 49, + "requested": 80, + "accuracy_ci95": [ + 0.5125, + 0.7125 + ] + } + }, + "v4": { + "boolq": { + "correct": 137, + "requested": 160, + "accuracy_ci95": [ + 0.8, + 0.90625 + ] + }, + "belebele_en": { + "correct": 115, + "requested": 160, + "accuracy_ci95": [ + 0.6441717791411042, + 0.7922077922077922 + ] + }, + "belebele_zh": { + "correct": 111, + "requested": 160, + "accuracy_ci95": [ + 0.61875, + 0.7664670658682635 + ] + } + }, + "v5": { + "cosmos_qa": { + "correct": 82, + "requested": 120, + "accuracy_ci95": [ + 0.6, + 0.7666666666666667 + ] + }, + "squad2_answerability": { + "correct": 102, + "requested": 120, + "accuracy_ci95": [ + 0.7829998478393184, + 0.9137931034482759 + ] + }, + "snli": { + "correct": 104, + "requested": 120, + "accuracy_ci95": [ + 0.8, + 0.925 + ] + }, + "qasc": { + "correct": 117, + "requested": 120, + "accuracy_ci95": [ + 0.9435483870967742, + 1.0 + ] + } + } + }, + "proper_scores": { + "old_core": 0.22574338686556997, + "v3_core": 0.6216883113559425, + "v4": 0.3087204159143027, + "v5": 0.21034073450197993, + "expanded_weighted_brier": 0.3416232121594488 + }, + "probability_coverage": { + "old_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v3_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v4": { + "requested": 480, + "valid_probability_rows": 480 + }, + "v5": { + "requested": 480, + "valid_probability_rows": 480 + } + }, + "brier_ci95": { + "old_core": [ + 0.18384172890538428, + 0.26968518800214547 + ], + "v3_core": [ + 0.594235087513965, + 0.6497064829839175 + ], + "v4": [ + 0.24847257992866453, + 0.37369416170641057 + ], + "v5": [ + 0.1763606748543631, + 0.24678438691822258 + ], + "expanded_weighted_brier": [ + 0.3200581388236899, + 0.3637875894103542 + ] + } + }, + "Nox-v1.1-released": { + "old_core": { + "point": 0.8284598214285714, + "exact_fraction": "7423/8960", + "ci95": [ + 0.7947907366071428, + 0.861123511904762 + ] + }, + "v3_core": { + "point": 0.5125, + "exact_fraction": "41/80", + "ci95": [ + 0.48708333333333337, + 0.5383333333333333 + ] + }, + "v4": { + "point": 0.784375, + "exact_fraction": "251/320", + "ci95": [ + 0.7390625, + 0.8279707260614273 + ] + }, + "v5": { + "point": 0.7916666666666666, + "exact_fraction": "19/24", + "ci95": [ + 0.7571779119014277, + 0.8248930921052631 + ] + }, + "v3_joint": { + "point": 0.6704799107142857, + "exact_fraction": "2403/3584", + "ci95": [ + 0.6491663876488095, + 0.6914181547619047 + ] + }, + "expanded_joint": { + "point": 0.7292503720238095, + "exact_fraction": "78409/107520", + "ci95": [ + 0.7118337656360736, + 0.746332556571124 + ] + }, + "families": { + "old_core": { + "ag_news": { + "correct": 107, + "requested": 128, + "accuracy_ci95": [ + 0.7734375, + 0.8984375 + ] + }, + "boolean_constraints": { + "correct": 60, + "requested": 64, + "accuracy_ci95": [ + 0.859375, + 1.0 + ] + }, + "dbpedia_14": { + "correct": 108, + "requested": 112, + "accuracy_ci95": [ + 0.9285714285714286, + 0.9910714285714286 + ] + }, + "natural_intents": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + }, + "option_carrier": { + "correct": 96, + "requested": 96, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + }, + "ordinal_rubric": { + "correct": 57, + "requested": 64, + "accuracy_ci95": [ + 0.78125, + 0.984375 + ] + }, + "relational_composition": { + "correct": 53, + "requested": 96, + "accuracy_ci95": [ + 0.375, + 0.7291666666666666 + ] + }, + "scoped_evidence": { + "correct": 72, + "requested": 96, + "accuracy_ci95": [ + 0.5833333333333334, + 0.9166666666666666 + ] + }, + "state_tracking": { + "correct": 34, + "requested": 96, + "accuracy_ci95": [ + 0.19791666666666666, + 0.53125 + ] + }, + "unknown_rejection": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + } + }, + "v3_core": { + "canonical_record_identity": { + "correct": 40, + "requested": 80, + "accuracy_ci95": [ + 0.5, + 0.5 + ] + }, + "capacitated_assignment": { + "correct": 35, + "requested": 80, + "accuracy_ci95": [ + 0.3875, + 0.475 + ] + }, + "constraint_assignment": { + "correct": 34, + "requested": 80, + "accuracy_ci95": [ + 0.3625, + 0.4875 + ] + }, + "massive_en": { + "correct": 110, + "requested": 120, + "accuracy_ci95": [ + 0.8666666666666667, + 0.9583333333333334 + ] + }, + "massive_zh": { + "correct": 106, + "requested": 120, + "accuracy_ci95": [ + 0.825, + 0.9333333333333333 + ] + }, + "multiset_reconciliation": { + "correct": 25, + "requested": 80, + "accuracy_ci95": [ + 0.2375, + 0.425 + ] + }, + "ordinal_service_loss": { + "correct": 23, + "requested": 80, + "accuracy_ci95": [ + 0.225, + 0.35 + ] + }, + "paraconsistent_rule_closure": { + "correct": 28, + "requested": 80, + "accuracy_ci95": [ + 0.2875, + 0.4375 + ] + }, + "temporal_exclusion": { + "correct": 33, + "requested": 80, + "accuracy_ci95": [ + 0.2625, + 0.5625 + ] + }, + "transaction_recovery": { + "correct": 48, + "requested": 80, + "accuracy_ci95": [ + 0.5, + 0.7 + ] + } + }, + "v4": { + "boolq": { + "correct": 140, + "requested": 160, + "accuracy_ci95": [ + 0.81875, + 0.925 + ] + }, + "belebele_en": { + "correct": 114, + "requested": 160, + "accuracy_ci95": [ + 0.6387057909055577, + 0.7848217752582121 + ] + }, + "belebele_zh": { + "correct": 108, + "requested": 160, + "accuracy_ci95": [ + 0.5999709302325581, + 0.7484662576687117 + ] + } + }, + "v5": { + "cosmos_qa": { + "correct": 69, + "requested": 120, + "accuracy_ci95": [ + 0.48333333333333334, + 0.6583333333333333 + ] + }, + "squad2_answerability": { + "correct": 91, + "requested": 120, + "accuracy_ci95": [ + 0.6833333333333333, + 0.8305084745762712 + ] + }, + "snli": { + "correct": 103, + "requested": 120, + "accuracy_ci95": [ + 0.7916666666666666, + 0.9166666666666666 + ] + }, + "qasc": { + "correct": 117, + "requested": 120, + "accuracy_ci95": [ + 0.9435483870967742, + 1.0 + ] + } + } + }, + "proper_scores": { + "old_core": 0.2331125387762136, + "v3_core": 0.6105027919948053, + "v4": 0.3216080833637166, + "v5": 0.2866349864374189, + "expanded_weighted_brier": 0.3629646001430386 + }, + "probability_coverage": { + "old_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v3_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v4": { + "requested": 480, + "valid_probability_rows": 480 + }, + "v5": { + "requested": 480, + "valid_probability_rows": 480 + } + }, + "brier_ci95": { + "old_core": [ + 0.19013891719456238, + 0.2779526850258884 + ], + "v3_core": [ + 0.5846362480182131, + 0.637147854744837 + ], + "v4": [ + 0.2589502553113817, + 0.38874975412458873 + ], + "v5": [ + 0.24724384093998833, + 0.3276631365833609 + ], + "expanded_weighted_brier": [ + 0.340384554547154, + 0.3860851198040383 + ] + } + }, + "Qwen3.5 \u00b7 2B, untuned": { + "old_core": { + "point": 0.5712425595238095, + "exact_fraction": "3071/5376", + "ci95": [ + 0.535453869047619, + 0.607627418154762 + ] + }, + "v3_core": { + "point": 0.39, + "exact_fraction": "39/100", + "ci95": [ + 0.3733333333333334, + 0.40625 + ] + }, + "v4": { + "point": 0.7375, + "exact_fraction": "59/80", + "ci95": [ + 0.6928571428571428, + 0.782292158018868 + ] + }, + "v5": { + "point": 0.7229166666666667, + "exact_fraction": "347/480", + "ci95": [ + 0.6848579953997695, + 0.760121271215623 + ] + }, + "v3_joint": { + "point": 0.4806212797619048, + "exact_fraction": "129191/268800", + "ci95": [ + 0.46085509672619046, + 0.5003615141369048 + ] + }, + "expanded_joint": { + "point": 0.605414806547619, + "exact_fraction": "325471/537600", + "ci95": [ + 0.5881936025503637, + 0.6231347666599296 + ] + }, + "families": { + "old_core": { + "ag_news": { + "correct": 103, + "requested": 128, + "accuracy_ci95": [ + 0.734375, + 0.8671875 + ] + }, + "boolean_constraints": { + "correct": 24, + "requested": 64, + "accuracy_ci95": [ + 0.21875, + 0.53125 + ] + }, + "dbpedia_14": { + "correct": 104, + "requested": 112, + "accuracy_ci95": [ + 0.875, + 0.9732142857142857 + ] + }, + "natural_intents": { + "correct": 62, + "requested": 64, + "accuracy_ci95": [ + 0.921875, + 1.0 + ] + }, + "option_carrier": { + "correct": 9, + "requested": 96, + "accuracy_ci95": [ + 0.041666666666666664, + 0.15625 + ] + }, + "ordinal_rubric": { + "correct": 52, + "requested": 64, + "accuracy_ci95": [ + 0.671875, + 0.9375 + ] + }, + "relational_composition": { + "correct": 47, + "requested": 96, + "accuracy_ci95": [ + 0.375, + 0.6145833333333334 + ] + }, + "scoped_evidence": { + "correct": 35, + "requested": 96, + "accuracy_ci95": [ + 0.23958333333333334, + 0.5 + ] + }, + "state_tracking": { + "correct": 24, + "requested": 96, + "accuracy_ci95": [ + 0.15625, + 0.3541666666666667 + ] + }, + "unknown_rejection": { + "correct": 40, + "requested": 64, + "accuracy_ci95": [ + 0.4375, + 0.8125 + ] + } + }, + "v3_core": { + "canonical_record_identity": { + "correct": 37, + "requested": 80, + "accuracy_ci95": [ + 0.425, + 0.5 + ] + }, + "capacitated_assignment": { + "correct": 40, + "requested": 80, + "accuracy_ci95": [ + 0.5, + 0.5 + ] + }, + "constraint_assignment": { + "correct": 17, + "requested": 80, + "accuracy_ci95": [ + 0.15, + 0.275 + ] + }, + "massive_en": { + "correct": 85, + "requested": 120, + "accuracy_ci95": [ + 0.625, + 0.7833333333333334 + ] + }, + "massive_zh": { + "correct": 89, + "requested": 120, + "accuracy_ci95": [ + 0.6666666666666666, + 0.8166666666666667 + ] + }, + "multiset_reconciliation": { + "correct": 22, + "requested": 80, + "accuracy_ci95": [ + 0.25, + 0.3125 + ] + }, + "ordinal_service_loss": { + "correct": 16, + "requested": 80, + "accuracy_ci95": [ + 0.2, + 0.2 + ] + }, + "paraconsistent_rule_closure": { + "correct": 22, + "requested": 80, + "accuracy_ci95": [ + 0.25, + 0.3125 + ] + }, + "temporal_exclusion": { + "correct": 23, + "requested": 80, + "accuracy_ci95": [ + 0.2375, + 0.3375 + ] + }, + "transaction_recovery": { + "correct": 19, + "requested": 80, + "accuracy_ci95": [ + 0.2125, + 0.25 + ] + } + }, + "v4": { + "boolq": { + "correct": 104, + "requested": 160, + "accuracy_ci95": [ + 0.575, + 0.725 + ] + }, + "belebele_en": { + "correct": 134, + "requested": 160, + "accuracy_ci95": [ + 0.7791411042944786, + 0.8924050632911392 + ] + }, + "belebele_zh": { + "correct": 130, + "requested": 160, + "accuracy_ci95": [ + 0.7484276729559748, + 0.8742138364779874 + ] + } + }, + "v5": { + "cosmos_qa": { + "correct": 68, + "requested": 120, + "accuracy_ci95": [ + 0.48333333333333334, + 0.65 + ] + }, + "squad2_answerability": { + "correct": 99, + "requested": 120, + "accuracy_ci95": [ + 0.7631578947368421, + 0.8833333333333333 + ] + }, + "snli": { + "correct": 77, + "requested": 120, + "accuracy_ci95": [ + 0.55, + 0.725 + ] + }, + "qasc": { + "correct": 103, + "requested": 120, + "accuracy_ci95": [ + 0.7950819672131147, + 0.9173553719008265 + ] + } + } + }, + "proper_scores": { + "old_core": 0.5723777597574953, + "v3_core": 0.7477724847216414, + "v4": 0.31678065323351606, + "v5": 0.4151003191775817, + "expanded_weighted_brier": 0.5130078042225586 + }, + "probability_coverage": { + "old_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v3_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v4": { + "requested": 480, + "valid_probability_rows": 480 + }, + "v5": { + "requested": 480, + "valid_probability_rows": 480 + } + }, + "brier_ci95": { + "old_core": [ + 0.5306199339702083, + 0.613544919176592 + ], + "v3_core": [ + 0.7296525923390248, + 0.7668117066670981 + ], + "v4": [ + 0.27446950339258713, + 0.3607698891917115 + ], + "v5": [ + 0.37186021348012016, + 0.45924139988994434 + ], + "expanded_weighted_brier": [ + 0.4934107964402564, + 0.5319870117266131 + ] + } + }, + "Qwen3.5 \u00b7 4B, untuned": { + "old_core": { + "point": 0.6988839285714286, + "exact_fraction": "3131/4480", + "ci95": [ + 0.6632803199404762, + 0.7337797619047619 + ] + }, + "v3_core": { + "point": 0.43333333333333335, + "exact_fraction": "13/30", + "ci95": [ + 0.4179166666666667, + 0.4479166666666667 + ] + }, + "v4": { + "point": 0.8796875, + "exact_fraction": "563/640", + "ci95": [ + 0.8461463032298809, + 0.9125000000000001 + ] + }, + "v5": { + "point": 0.7979166666666667, + "exact_fraction": "383/480", + "ci95": [ + 0.7641944237372369, + 0.830579286387442 + ] + }, + "v3_joint": { + "point": 0.566108630952381, + "exact_fraction": "15217/26880", + "ci95": [ + 0.5467666480654761, + 0.5849108072916667 + ] + }, + "expanded_joint": { + "point": 0.7024553571428571, + "exact_fraction": "3147/4480", + "ci95": [ + 0.6873480253417569, + 0.7176086887612844 + ] + }, + "families": { + "old_core": { + "ag_news": { + "correct": 108, + "requested": 128, + "accuracy_ci95": [ + 0.78125, + 0.90625 + ] + }, + "boolean_constraints": { + "correct": 40, + "requested": 64, + "accuracy_ci95": [ + 0.484375, + 0.765625 + ] + }, + "dbpedia_14": { + "correct": 109, + "requested": 112, + "accuracy_ci95": [ + 0.9375, + 1.0 + ] + }, + "natural_intents": { + "correct": 64, + "requested": 64, + "accuracy_ci95": [ + 1.0, + 1.0 + ] + }, + "option_carrier": { + "correct": 70, + "requested": 96, + "accuracy_ci95": [ + 0.6354166666666666, + 0.8229166666666666 + ] + }, + "ordinal_rubric": { + "correct": 58, + "requested": 64, + "accuracy_ci95": [ + 0.8125, + 0.984375 + ] + }, + "relational_composition": { + "correct": 49, + "requested": 96, + "accuracy_ci95": [ + 0.3854166666666667, + 0.6354166666666666 + ] + }, + "scoped_evidence": { + "correct": 49, + "requested": 96, + "accuracy_ci95": [ + 0.3645833333333333, + 0.65625 + ] + }, + "state_tracking": { + "correct": 27, + "requested": 96, + "accuracy_ci95": [ + 0.16666666666666666, + 0.3958333333333333 + ] + }, + "unknown_rejection": { + "correct": 39, + "requested": 64, + "accuracy_ci95": [ + 0.421875, + 0.7816406249999943 + ] + } + }, + "v3_core": { + "canonical_record_identity": { + "correct": 40, + "requested": 80, + "accuracy_ci95": [ + 0.5, + 0.5 + ] + }, + "capacitated_assignment": { + "correct": 40, + "requested": 80, + "accuracy_ci95": [ + 0.5, + 0.5 + ] + }, + "constraint_assignment": { + "correct": 19, + "requested": 80, + "accuracy_ci95": [ + 0.1625, + 0.3125 + ] + }, + "massive_en": { + "correct": 110, + "requested": 120, + "accuracy_ci95": [ + 0.8666666666666667, + 0.9583333333333334 + ] + }, + "massive_zh": { + "correct": 104, + "requested": 120, + "accuracy_ci95": [ + 0.8, + 0.925 + ] + }, + "multiset_reconciliation": { + "correct": 22, + "requested": 80, + "accuracy_ci95": [ + 0.225, + 0.325 + ] + }, + "ordinal_service_loss": { + "correct": 17, + "requested": 80, + "accuracy_ci95": [ + 0.2, + 0.2375 + ] + }, + "paraconsistent_rule_closure": { + "correct": 20, + "requested": 80, + "accuracy_ci95": [ + 0.2125, + 0.2875 + ] + }, + "temporal_exclusion": { + "correct": 25, + "requested": 80, + "accuracy_ci95": [ + 0.25, + 0.375 + ] + }, + "transaction_recovery": { + "correct": 21, + "requested": 80, + "accuracy_ci95": [ + 0.25, + 0.2875 + ] + } + }, + "v4": { + "boolq": { + "correct": 134, + "requested": 160, + "accuracy_ci95": [ + 0.78125, + 0.89375 + ] + }, + "belebele_en": { + "correct": 149, + "requested": 160, + "accuracy_ci95": [ + 0.8910256410256411, + 0.967948717948718 + ] + }, + "belebele_zh": { + "correct": 146, + "requested": 160, + "accuracy_ci95": [ + 0.8633475672877846, + 0.95625 + ] + } + }, + "v5": { + "cosmos_qa": { + "correct": 73, + "requested": 120, + "accuracy_ci95": [ + 0.5166666666666667, + 0.6916666666666667 + ] + }, + "squad2_answerability": { + "correct": 98, + "requested": 120, + "accuracy_ci95": [ + 0.7540983606557377, + 0.8790322580645161 + ] + }, + "snli": { + "correct": 96, + "requested": 120, + "accuracy_ci95": [ + 0.725, + 0.8666666666666667 + ] + }, + "qasc": { + "correct": 116, + "requested": 120, + "accuracy_ci95": [ + 0.9333333333333333, + 0.991869918699187 + ] + } + } + }, + "proper_scores": { + "old_core": 0.427127994572904, + "v3_core": 0.6845537392617043, + "v4": 0.19639894537679964, + "v5": 0.2904012063572707, + "expanded_weighted_brier": 0.39962047139216966 + }, + "probability_coverage": { + "old_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v3_core": { + "requested": 880, + "valid_probability_rows": 880 + }, + "v4": { + "requested": 480, + "valid_probability_rows": 480 + }, + "v5": { + "requested": 480, + "valid_probability_rows": 480 + } + }, + "brier_ci95": { + "old_core": [ + 0.377528048361006, + 0.4790590681446513 + ], + "v3_core": [ + 0.6674510320346829, + 0.7036301143486854 + ], + "v4": [ + 0.1623582012987519, + 0.23277732548629693 + ], + "v5": [ + 0.24711830906655044, + 0.33490954796052225 + ], + "expanded_weighted_brier": [ + 0.38062758349672393, + 0.4193732417566481 + ] + } + } + }, + "paired_comparisons": { + "Nox-natural-selected minus Nox-v1.1-released": { + "panels": { + "old_core": { + "delta": 0.0036458333333333334, + "exact_fraction": "7/1920", + "paired_ci95": [ + -0.002343750000000089, + 0.011979166666666652 + ] + }, + "v3_core": { + "delta": -0.0016666666666666668, + "exact_fraction": "-1/600", + "paired_ci95": [ + -0.012916666666666465, + 0.009583333333333277 + ] + }, + "v4": { + "delta": -0.003125, + "exact_fraction": "-1/320", + "paired_ci95": [ + -0.02179487179487172, + 0.014091990520971005 + ] + }, + "v5": { + "delta": 0.052083333333333336, + "exact_fraction": "5/96", + "paired_ci95": [ + 0.0255676834130781, + 0.0798375706214689 + ] + }, + "v3_joint": { + "delta": 0.0009895833333333334, + "exact_fraction": "19/19200", + "paired_ci95": [ + -0.005546874999999954, + 0.00789062500000004 + ] + }, + "expanded_joint": { + "delta": 0.012734375, + "exact_fraction": "163/12800", + "paired_ci95": [ + 0.004166573619049133, + 0.021864398132660924 + ] + } + }, + "brier": { + "old_core": { + "delta": -0.007369151910643634, + "paired_ci95": [ + -0.013906686611963625, + -0.0019409726778719298 + ] + }, + "v3_core": { + "delta": 0.011185519361137208, + "paired_ci95": [ + 0.006392271109264711, + 0.015911233958325322 + ] + }, + "v4": { + "delta": -0.012887667449413887, + "paired_ci95": [ + -0.025660045932955716, + 0.00020353457611990694 + ] + }, + "v5": { + "delta": -0.07629425193543898, + "paired_ci95": [ + -0.09835822974417568, + -0.05511732226877328 + ] + }, + "expanded_weighted_brier": { + "delta": -0.02134138798358981, + "paired_ci95": [ + -0.027988975580296924, + -0.014743547095748813 + ] + } + } + } + }, + "method": { + "seed": 202609225801, + "repeats": 10000, + "weights": { + "old_core": "1/4", + "v3_core": "1/4", + "v4": "1/4", + "v5": "1/4" + }, + "within_panel_weights": { + "old_core": { + "ag_news": "1/10", + "boolean_constraints": "1/10", + "dbpedia_14": "1/10", + "natural_intents": "1/10", + "option_carrier": "1/10", + "ordinal_rubric": "1/10", + "relational_composition": "1/10", + "scoped_evidence": "1/10", + "state_tracking": "1/10", + "unknown_rejection": "1/10" + }, + "v3_core": { + "canonical_record_identity": "1/10", + "capacitated_assignment": "1/10", + "constraint_assignment": "1/10", + "massive_en": "1/10", + "massive_zh": "1/10", + "multiset_reconciliation": "1/10", + "ordinal_service_loss": "1/10", + "paraconsistent_rule_closure": "1/10", + "temporal_exclusion": "1/10", + "transaction_recovery": "1/10" + }, + "v4": { + "boolq": "1/2", + "belebele_en": "1/4", + "belebele_zh": "1/4" + }, + "v5": { + "cosmos_qa": "1/4", + "squad2_answerability": "1/4", + "snli": "1/4", + "qasc": "1/4" + } + }, + "v3_joint_unchanged": "(old_core+v3_core)/2", + "exact_accuracy": "Reduced Fraction from integer family correct/total counts; no rounded eligibility.", + "native_decision": "Use recorded prediction; never replace it with FP64-softmax argmax.", + "brier": "Recorded valid probability vector normalized once as specified by shared score.assess; sum squared category errors, no division by K. Complete coverage required for a panel proper-score point.", + "clusters": "Original V3 within-family groups and coupled MASSIVE translations; original V4 shared passages/translations; original V5 complete source/near components. Identical panel seeds reused across models and accuracy/Brier. Panels sampled independently.", + "interval_role": "Report only; no new CI/family gate.", + "native_supplements": "Both 220-answer native panels remain outside the quality average, fully reported." + } +} diff --git a/metrics/figure-evidence.json b/metrics/figure-evidence.json new file mode 100644 index 0000000000000000000000000000000000000000..82a45d54d6b90229457408c9142a426ac1180135 --- /dev/null +++ b/metrics/figure-evidence.json @@ -0,0 +1,428 @@ +{ + "format": "decision-expanded-figure-evidence-v1", + "kind": "measured", + "manifest_sha256": "837ef051c424ff947e307d2b862e1f771aa3ba0a56e7f20ff4f453c665951ac5", + "statistics_sha256": "d8f37dcb8fdb274a4d3c11bd4d059c4d52f018087e33ca6b340ca8d64b2b6cce", + "public_projection_sha256": "3f8e6d9be291dae6532d6e7fb16232c12c6017cdbf64c4055ddb0d41a1ee363f", + "base_style_sha256": "93fa22aa96a6dd5f06fe9a5670d84691f2092f84a91bac600c9777a29a36123a", + "branding_sha256": "d83cf7878c3839f3085feb8c02334217ad2fef0a615b805e1726b292552c46e8", + "source_sha256": { + "render.py": "f6ecc60bfa616eb41c422bc30aaebd9f30a123318991bbedb56a9f9920dfa3c0", + "contract.py": "55256a32b7759d993b5aff10a39d185b3ad67191495f840f9ef882a2001ff325" + }, + "display_order": [ + "Jev \u00b7 1.13.0", + "Nox-natural-selected", + "Nox-v1.1-released", + "Decider \u00b7 2B", + "Qwen3.5 \u00b7 4B, untuned", + "Qwen3.5 \u00b7 2B, untuned", + "Laya \u00b7 EN/ML" + ], + "display_labels": { + "Decider \u00b7 2B": { + "label": "Decider", + "detail": "2B" + }, + "Jev \u00b7 1.13.0": { + "label": "Jev", + "detail": "1.13.0" + }, + "Laya \u00b7 EN/ML": { + "label": "Laya", + "detail": "Upstream default" + }, + "Qwen3.5 \u00b7 2B, untuned": { + "label": "Qwen3.5", + "detail": "2B \u00b7 untuned" + }, + "Qwen3.5 \u00b7 4B, untuned": { + "label": "Qwen3.5", + "detail": "4B \u00b7 untuned" + }, + "Nox-natural-selected": { + "label": "Nox", + "detail": "v1.2 \u00b7 this release" + }, + "Nox-v1.1-released": { + "label": "Nox", + "detail": "v1.1 \u00b7 previous" + } + }, + "model_versions": { + "Jev \u00b7 1.13.0": { + "old_core": [ + "jev-1.13.0" + ], + "v3_core": [ + "jev-1.13.0" + ], + "v4": [ + "jev-1.13.0" + ], + "v5": [ + "jev-1.13.0" + ] + }, + "Laya \u00b7 EN/ML": { + "old_core": { + "repo": "convaiinnovations/laya", + "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", + "router_source_revision": "42626c348753fbb17572a813127df2278a1ec527", + "adapter_sha256": "a5484e30d9fad04ca578265e5f18df2a10731f5055bef070311102c3b35643fc", + "mode": "routed", + "weight_fingerprints": { + "english": "5bf5d0f9462d860e4c09000fce3131a070578d3574777ae7981077f11cd68185", + "multilingual": "a472bfcc4aa52a25dda0b2864b3e7c7c1267ddd1752b04a1e70bf6ba7e4b514c" + }, + "routing": "Unmodified upstream Router.route(state,questions), no explicit lang, no automatic workflow detection, default english" + }, + "v3_core": { + "repo": "convaiinnovations/laya", + "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", + "router_source_revision": "42626c348753fbb17572a813127df2278a1ec527", + "adapter_sha256": "a5484e30d9fad04ca578265e5f18df2a10731f5055bef070311102c3b35643fc", + "mode": "routed", + "weight_fingerprints": { + "english": "5bf5d0f9462d860e4c09000fce3131a070578d3574777ae7981077f11cd68185", + "multilingual": "a472bfcc4aa52a25dda0b2864b3e7c7c1267ddd1752b04a1e70bf6ba7e4b514c" + }, + "routing": "Unmodified upstream Router.route(state,questions), no explicit lang, no automatic workflow detection, default english" + }, + "v4": { + "repo": "convaiinnovations/laya", + "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", + "router_source_revision": "42626c348753fbb17572a813127df2278a1ec527", + "adapter_sha256": "a5484e30d9fad04ca578265e5f18df2a10731f5055bef070311102c3b35643fc", + "mode": "routed", + "weight_fingerprints": { + "english": "5bf5d0f9462d860e4c09000fce3131a070578d3574777ae7981077f11cd68185", + "multilingual": "a472bfcc4aa52a25dda0b2864b3e7c7c1267ddd1752b04a1e70bf6ba7e4b514c" + }, + "routing": "Unmodified upstream Router.route(state,questions), no explicit lang, no automatic workflow detection, default english" + }, + "v5": { + "repo": "convaiinnovations/laya", + "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", + "router_source_revision": "42626c348753fbb17572a813127df2278a1ec527", + "adapter_sha256": "a5484e30d9fad04ca578265e5f18df2a10731f5055bef070311102c3b35643fc", + "mode": "routed", + "weight_fingerprints": { + "english": "5bf5d0f9462d860e4c09000fce3131a070578d3574777ae7981077f11cd68185", + "multilingual": "a472bfcc4aa52a25dda0b2864b3e7c7c1267ddd1752b04a1e70bf6ba7e4b514c" + }, + "routing": "Unmodified upstream Router.route(state,questions), no explicit lang, no automatic workflow detection, default english" + } + }, + "Decider \u00b7 2B": { + "old_core": { + "model_id": "Mapika/decider-2b", + "original_native_adapter_sha256": "983a2a0d582a1c5e7e58fe3fe4e7c1cd254dd0fe52f65449c68549108d0af387", + "wrapper_sha256": "95b9fcc132d1ae2f0818ea8bb4720f4b5342450e741fefb4315827225652e7f6", + "model_fingerprint": "35f7335c92cf1766c11f52fb4bcdb9c3894d3edad7408a00ae7b59c104b2d318", + "temperature": 1.3, + "layout": "state_first", + "independent": true, + "revision": "b37f7e1ba3fbc9238004cf531fabbee2619973fd" + }, + "v3_core": { + "model_id": "Mapika/decider-2b", + "original_native_adapter_sha256": "983a2a0d582a1c5e7e58fe3fe4e7c1cd254dd0fe52f65449c68549108d0af387", + "wrapper_sha256": "95b9fcc132d1ae2f0818ea8bb4720f4b5342450e741fefb4315827225652e7f6", + "model_fingerprint": "35f7335c92cf1766c11f52fb4bcdb9c3894d3edad7408a00ae7b59c104b2d318", + "temperature": 1.3, + "layout": "state_first", + "independent": true, + "revision": "b37f7e1ba3fbc9238004cf531fabbee2619973fd" + }, + "v4": { + "model_id": "Mapika/decider-2b", + "original_native_adapter_sha256": "983a2a0d582a1c5e7e58fe3fe4e7c1cd254dd0fe52f65449c68549108d0af387", + "wrapper_sha256": "95b9fcc132d1ae2f0818ea8bb4720f4b5342450e741fefb4315827225652e7f6", + "model_fingerprint": "35f7335c92cf1766c11f52fb4bcdb9c3894d3edad7408a00ae7b59c104b2d318", + "temperature": 1.3, + "layout": "state_first", + "independent": true, + "revision": "b37f7e1ba3fbc9238004cf531fabbee2619973fd" + }, + "v5": { + "model_id": "Mapika/decider-2b", + "original_native_adapter_sha256": "983a2a0d582a1c5e7e58fe3fe4e7c1cd254dd0fe52f65449c68549108d0af387", + "wrapper_sha256": "95b9fcc132d1ae2f0818ea8bb4720f4b5342450e741fefb4315827225652e7f6", + "model_fingerprint": "35f7335c92cf1766c11f52fb4bcdb9c3894d3edad7408a00ae7b59c104b2d318", + "temperature": 1.3, + "layout": "state_first", + "independent": true, + "revision": "b37f7e1ba3fbc9238004cf531fabbee2619973fd" + } + }, + "Qwen3.5 \u00b7 2B, untuned": { + "old_core": { + "base_repo": "Qwen/Qwen3.5-2B", + "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "v3_core": { + "base_repo": "Qwen/Qwen3.5-2B", + "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "v4": { + "base_repo": "Qwen/Qwen3.5-2B", + "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "v5": { + "base_repo": "Qwen/Qwen3.5-2B", + "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + } + }, + "Qwen3.5 \u00b7 4B, untuned": { + "old_core": { + "base_repo": "Qwen/Qwen3.5-4B", + "base_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "v3_core": { + "base_repo": "Qwen/Qwen3.5-4B", + "base_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "v4": { + "base_repo": "Qwen/Qwen3.5-4B", + "base_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + }, + "v5": { + "base_repo": "Qwen/Qwen3.5-4B", + "base_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "adapter_sha256": "e08336ecd148e9240792dc3d915536fc52998364f248f9f2016b8b83e289b989", + "prompt_version": "decision-letter-v1", + "temperature": 1.0, + "readout": "Frozen pretrained next-token LM logits restricted to A..Z; conditional softmax over listed candidates", + "new_trainable_head": false, + "task_finetuning": false + } + }, + "Nox-natural-selected": { + "bundle_manifest_sha256": "7762761b5da5d51d33e06d322c037f7cfdee54f910e6e83d3e8c7ce02ae42b2e" + }, + "Nox-v1.1-released": { + "repo_id": "llm-semantic-router/Decision-1.0-Nox", + "revision": "5bee3061a7abf675813963e4e05267cb1a28f92f" + } + }, + "weights": { + "old_core": "1/4", + "v3_core": "1/4", + "v4": "1/4", + "v5": "1/4" + }, + "within_panel_weights": { + "old_core": { + "ag_news": "1/10", + "boolean_constraints": "1/10", + "dbpedia_14": "1/10", + "natural_intents": "1/10", + "option_carrier": "1/10", + "ordinal_rubric": "1/10", + "relational_composition": "1/10", + "scoped_evidence": "1/10", + "state_tracking": "1/10", + "unknown_rejection": "1/10" + }, + "v3_core": { + "canonical_record_identity": "1/10", + "capacitated_assignment": "1/10", + "constraint_assignment": "1/10", + "massive_en": "1/10", + "massive_zh": "1/10", + "multiset_reconciliation": "1/10", + "ordinal_service_loss": "1/10", + "paraconsistent_rule_closure": "1/10", + "temporal_exclusion": "1/10", + "transaction_recovery": "1/10" + }, + "v4": { + "boolq": "1/2", + "belebele_en": "1/4", + "belebele_zh": "1/4" + }, + "v5": { + "cosmos_qa": "1/4", + "squad2_answerability": "1/4", + "snli": "1/4", + "qasc": "1/4" + } + }, + "families": 27, + "quality_decisions_per_model": 2720, + "exact_bold_ties": true, + "no_raw_examples_or_predictions_read": true, + "background": "white", + "footnotes_in_image": false, + "qualification_or_publication_performed": false, + "files": [ + { + "file": "TABLES.md", + "bytes": 4683, + "sha256": "ba6f316af904830c323818df2f3006ac7f2b6c7888b36630238d6a0444aa341a" + }, + { + "file": "decision-expanded-old_core-600px.png", + "bytes": 140155, + "sha256": "519923f36d62df99c6769731416d70413c9f5ec3991ba43a75de0714767a12cc" + }, + { + "file": "decision-expanded-old_core.pdf", + "bytes": 44043, + "sha256": "29e7f84737b7bfe2d0a444db76ce9d0b867fb206cb4253192cd19594bba0f9c0" + }, + { + "file": "decision-expanded-old_core.png", + "bytes": 261350, + "sha256": "c59e464a07320a13a182ec841fef0a826eed777d03264260d3a3d6bcf99a2819" + }, + { + "file": "decision-expanded-old_core.svg", + "bytes": 72452, + "sha256": "593cada5637013c20daf90c8c6d8656554666c134e0123c5099d703f6a750be6" + }, + { + "file": "decision-expanded-overview-600px.png", + "bytes": 83667, + "sha256": "f1e659e6ba773c872d6f6e64ca783acface0a29c7db57f0e2d06758bbcf1a397" + }, + { + "file": "decision-expanded-overview.pdf", + "bytes": 40642, + "sha256": "b0427a480be62113f7c14f60c882673eb0c822b4b23b2444411f939a1fab638d" + }, + { + "file": "decision-expanded-overview.png", + "bytes": 167930, + "sha256": "fa9972beb8e9081bb5a8c59e487e69f31567470fd4af6b0d6d36bdb82d60cdf1" + }, + { + "file": "decision-expanded-overview.svg", + "bytes": 44991, + "sha256": "c8c4dff1759965471229ccd5abb782e4050099adc328c28a18fb451694683f5a" + }, + { + "file": "decision-expanded-ranking-600px.png", + "bytes": 75990, + "sha256": "80d65d7fbe04b86fa6ca45c3659b18b968f767ff1102ca5b23de2da2fbd91a59" + }, + { + "file": "decision-expanded-ranking.pdf", + "bytes": 37284, + "sha256": "38ef0ccf62b82ec3da5076241646f42c21f2387c864e186165a3a9c1b2b61e37" + }, + { + "file": "decision-expanded-ranking.png", + "bytes": 171628, + "sha256": "038b995974df964403278b2d121bf9b888c52387aeed4e1726ddf5cbbb18f0b3" + }, + { + "file": "decision-expanded-ranking.svg", + "bytes": 36214, + "sha256": "236c5add9d316a3ecbd328beb5c171c71d4b1501a84dee5a83581e808c5f6fa3" + }, + { + "file": "decision-expanded-v3_core-600px.png", + "bytes": 143107, + "sha256": "ba4b96800e5f24d1029911d75fae19d7cf9dfd0f2ded82c05fef3b4997259e8a" + }, + { + "file": "decision-expanded-v3_core.pdf", + "bytes": 44142, + "sha256": "f253e527c21c26544ab23a024ad31e28cab1336b6fdc14d7b8e42b533e02a13d" + }, + { + "file": "decision-expanded-v3_core.png", + "bytes": 269511, + "sha256": "8c9b3b703cacfca5eed2099665cfcd4fab210c2b0dc6e7540e8c6a9b84b2096f" + }, + { + "file": "decision-expanded-v3_core.svg", + "bytes": 72477, + "sha256": "27fc95d04dc6fd6dbaa1a353fe30588a0c1832650350509cbb5c88b6a60b7476" + }, + { + "file": "decision-expanded-v4-600px.png", + "bytes": 66445, + "sha256": "b22c8b55bc594fccfeda1b33132cfd94932fb9dd101749699e7c91899a3f1166" + }, + { + "file": "decision-expanded-v4.pdf", + "bytes": 38130, + "sha256": "005cac2146b983187002fab407d611f9b506f9cc53fec9ece370c6cf3a638eec" + }, + { + "file": "decision-expanded-v4.png", + "bytes": 135301, + "sha256": "3b392a7f29d9cf0d87920658d2d57ddd2d5fc500f9658b916a518a9570d2fd08" + }, + { + "file": "decision-expanded-v4.svg", + "bytes": 39927, + "sha256": "1210b42e075012c39001ef33324ff2391681e86592212878d26086ebe3a496f2" + }, + { + "file": "decision-expanded-v5-600px.png", + "bytes": 79064, + "sha256": "ef46e061eb5411360b846e4df64eafca18ce4561d5fb7eaf5dafd21681e7a193" + }, + { + "file": "decision-expanded-v5.pdf", + "bytes": 40093, + "sha256": "9988a0ba3538f0ddf1c83430dd0763963dc4b6eba1e80969b4c81311389e2018" + }, + { + "file": "decision-expanded-v5.png", + "bytes": 161808, + "sha256": "aef547fd323bf044b1f81edf89f07e1e0175f9a52b9742b14eda6890e7fa0f4d" + }, + { + "file": "decision-expanded-v5.svg", + "bytes": 44951, + "sha256": "27bd11dbac7c6bdcc35dff7b9c7cc2c3ed984454121045694e3555dfc73f7b1a" + } + ] +} diff --git a/metrics/materials-provenance.json b/metrics/materials-provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c9b08ca75db8c35b35fe7c3964a59e61145bf09c --- /dev/null +++ b/metrics/materials-provenance.json @@ -0,0 +1,11 @@ +{ + "candidate_bundle_manifest_sha256": "7762761b5da5d51d33e06d322c037f7cfdee54f910e6e83d3e8c7ce02ae42b2e", + "qualification_sha256": "49258b41a55935e73d01f86966f7d3167047dfe3550a1d5a4a4b1e32bc9cc0d3", + "evaluation_receipt_sha256": "f7d59d85873d2a612d7d4926e693246dba7bc01da22284e1a85a86f719a348af", + "statistics_sha256": "d8f37dcb8fdb274a4d3c11bd4d059c4d52f018087e33ca6b340ca8d64b2b6cce", + "public_projection_sha256": "3f8e6d9be291dae6532d6e7fb16232c12c6017cdbf64c4055ddb0d41a1ee363f", + "efficiency_receipt_sha256": "89829d047ebb4fd6d2b511bd74307b584feb9b0af8c9cffbe1b58bdf3e2bb2b3", + "release_tag": "v1.2", + "source_sha256": "5706c0b36cb6c31c157ba1d043ee13873f7383278adb015e3e99694f1636a8a8", + "publication_performed": false +} diff --git a/metrics/natural-confirmation.json b/metrics/natural-confirmation.json deleted file mode 100644 index 6024743de4d4c2e5da75092623b82295b7a65adf..0000000000000000000000000000000000000000 --- a/metrics/natural-confirmation.json +++ /dev/null @@ -1,791 +0,0 @@ -{ - "version": "nox-v4-natural-confirmation-public-1", - "requested_examples": 480, - "families": { - "boolq": { - "requested": 160, - "false": 80, - "true": 80, - "weight": "1/2" - }, - "belebele_en": { - "requested": 160, - "weight": "1/4" - }, - "belebele_zh": { - "requested": 160, - "weight": "1/4" - } - }, - "score_definition": "0.5 * BoolQ accuracy + 0.25 * Belebele English accuracy + 0.25 * Belebele Chinese accuracy; all 480 requested examples retained.", - "models": { - "nox-update": { - "label": "Nox (this update)", - "weighted_accuracy": 0.784375, - "weighted_accuracy_exact": { - "numerator": 251, - "denominator": 320 - }, - "weighted_accuracy_ci95": [ - 0.7399293154761905, - 0.8265123552522746 - ], - "families": { - "boolq": { - "correct": 140, - "requested": 160, - "accuracy": 0.875, - "ci95": [ - 0.81875, - 0.925 - ] - }, - "belebele_en": { - "correct": 114, - "requested": 160, - "accuracy": 0.7125, - "ci95": [ - 0.6380368098159509, - 0.782608695652174 - ] - }, - "belebele_zh": { - "correct": 108, - "requested": 160, - "accuracy": 0.675, - "ci95": [ - 0.5975609756097561, - 0.7469135802469136 - ] - } - }, - "unweighted": { - "correct": 362, - "requested": 480, - "accuracy": 0.7541666666666667 - }, - "coverage": { - "requested": 480, - "accepted": 480, - "finite_valid_outputs": 480, - "full_input_verified": 480, - "truncated": 0, - "retention": "Complete frozen inputs verified." - }, - "typed_probability_metrics": { - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.875, - "brier": 0.21024758152305395, - "nll": 0.4175424597184884, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.875, - "recall_true": 0.875, - "balanced_accuracy": 0.875, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "choice": { - "n": 320, - "valid_probability_coverage": 320, - "missing_or_invalid": 0, - "accuracy": 0.69375, - "brier": 0.4329685852043793, - "nll": 1.0936456725285504, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - } - } - }, - "sol-v1.0": { - "label": "Sol v1.0", - "weighted_accuracy": 0.69375, - "weighted_accuracy_exact": { - "numerator": 111, - "denominator": 160 - }, - "weighted_accuracy_ci95": [ - 0.6437499999999999, - 0.7417484177215189 - ], - "families": { - "boolq": { - "correct": 119, - "requested": 160, - "accuracy": 0.74375, - "ci95": [ - 0.675, - 0.8125 - ] - }, - "belebele_en": { - "correct": 106, - "requested": 160, - "accuracy": 0.6625, - "ci95": [ - 0.5870967741935483, - 0.7349397590361446 - ] - }, - "belebele_zh": { - "correct": 100, - "requested": 160, - "accuracy": 0.625, - "ci95": [ - 0.546583850931677, - 0.7005988023952096 - ] - } - }, - "unweighted": { - "correct": 325, - "requested": 480, - "accuracy": 0.6770833333333334 - }, - "coverage": { - "requested": 480, - "accepted": 480, - "finite_valid_outputs": 480, - "full_input_verified": 480, - "truncated": 0, - "retention": "Complete frozen inputs verified." - }, - "typed_probability_metrics": { - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.74375, - "brier": 0.43895629019878823, - "nll": 0.9002462702649545, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.575, - "recall_true": 0.9125, - "balanced_accuracy": 0.7437499999999999, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "choice": { - "n": 320, - "valid_probability_coverage": 320, - "missing_or_invalid": 0, - "accuracy": 0.64375, - "brier": 0.5085778581327408, - "nll": 1.2810086045362705, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - } - } - }, - "jev-1.13.0": { - "label": "Jev 1.13.0", - "weighted_accuracy": 0.9453125, - "weighted_accuracy_exact": { - "numerator": 121, - "denominator": 128 - }, - "weighted_accuracy_ci95": [ - 0.9200550899983875, - 0.9676459580838324 - ], - "families": { - "boolq": { - "correct": 148, - "requested": 160, - "accuracy": 0.925, - "ci95": [ - 0.88125, - 0.9625 - ] - }, - "belebele_en": { - "correct": 155, - "requested": 160, - "accuracy": 0.96875, - "ci95": [ - 0.9386503067484663, - 0.9936708860759493 - ] - }, - "belebele_zh": { - "correct": 154, - "requested": 160, - "accuracy": 0.9625, - "ci95": [ - 0.9308176100628931, - 0.9877300613496932 - ] - } - }, - "unweighted": { - "correct": 457, - "requested": 480, - "accuracy": 0.9520833333333333 - }, - "coverage": { - "requested": 480, - "accepted": 480, - "finite_valid_outputs": 480, - "full_input_verified": null, - "truncated": null, - "retention": "Unknown inside the closed API." - }, - "typed_probability_metrics": { - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.925, - "brier": 0.10445375000000001, - "nll": 0.19516064889282234, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.925, - "recall_true": 0.925, - "balanced_accuracy": 0.925, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "choice": { - "n": 320, - "valid_probability_coverage": 320, - "missing_or_invalid": 0, - "accuracy": 0.965625, - "brier": 0.040956875, - "nll": 0.07843730477267183, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - } - } - }, - "laya-routed": { - "label": "Laya EN/ML routed", - "weighted_accuracy": 0.5125, - "weighted_accuracy_exact": { - "numerator": 41, - "denominator": 80 - }, - "weighted_accuracy_ci95": [ - 0.465988068600994, - 0.5583870648734177 - ], - "families": { - "boolq": { - "correct": 111, - "requested": 160, - "accuracy": 0.69375, - "ci95": [ - 0.61875, - 0.7625 - ] - }, - "belebele_en": { - "correct": 61, - "requested": 160, - "accuracy": 0.38125, - "ci95": [ - 0.30538922155688625, - 0.4591194968553459 - ] - }, - "belebele_zh": { - "correct": 45, - "requested": 160, - "accuracy": 0.28125, - "ci95": [ - 0.2147239263803681, - 0.3515151515151515 - ] - } - }, - "unweighted": { - "correct": 217, - "requested": 480, - "accuracy": 0.45208333333333334 - }, - "coverage": { - "requested": 480, - "accepted": 480, - "finite_valid_outputs": 480, - "full_input_verified": 480, - "truncated": 0, - "retention": "Complete frozen inputs verified." - }, - "typed_probability_metrics": { - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.69375, - "brier": 0.41904120474999995, - "nll": 0.6464549663099783, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.525, - "recall_true": 0.8625, - "balanced_accuracy": 0.6937500000000001, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "choice": { - "n": 320, - "valid_probability_coverage": 320, - "missing_or_invalid": 0, - "accuracy": 0.33125, - "brier": 0.8108916245962444, - "nll": 1.5869034433374543, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - } - } - }, - "decider": { - "label": "Decider 2B", - "weighted_accuracy": 0.9203125, - "weighted_accuracy_exact": { - "numerator": 589, - "denominator": 640 - }, - "weighted_accuracy_ci95": [ - 0.8916901676829269, - 0.9464989496201366 - ], - "families": { - "boolq": { - "correct": 147, - "requested": 160, - "accuracy": 0.91875, - "ci95": [ - 0.875, - 0.95625 - ] - }, - "belebele_en": { - "correct": 149, - "requested": 160, - "accuracy": 0.93125, - "ci95": [ - 0.8902439024390244, - 0.967948717948718 - ] - }, - "belebele_zh": { - "correct": 146, - "requested": 160, - "accuracy": 0.9125, - "ci95": [ - 0.8630864845938376, - 0.9559748427672956 - ] - } - }, - "unweighted": { - "correct": 442, - "requested": 480, - "accuracy": 0.9208333333333333 - }, - "coverage": { - "requested": 480, - "accepted": 480, - "finite_valid_outputs": 480, - "full_input_verified": 480, - "truncated": 0, - "retention": "Complete frozen inputs verified." - }, - "typed_probability_metrics": { - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.91875, - "brier": 0.14787474037499998, - "nll": 0.2631311328165121, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.9375, - "recall_true": 0.9, - "balanced_accuracy": 0.91875, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "choice": { - "n": 320, - "valid_probability_coverage": 320, - "missing_or_invalid": 0, - "accuracy": 0.921875, - "brier": 0.12688092915001592, - "nll": 0.24728184456069316, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - } - } - }, - "qwen35-2b-lm-head": { - "label": "Qwen3.5 2B + LM-head adapter", - "weighted_accuracy": 0.7375, - "weighted_accuracy_exact": { - "numerator": 59, - "denominator": 80 - }, - "weighted_accuracy_ci95": [ - 0.6917801490514904, - 0.7823309748427674 - ], - "families": { - "boolq": { - "correct": 104, - "requested": 160, - "accuracy": 0.65, - "ci95": [ - 0.575, - 0.725 - ] - }, - "belebele_en": { - "correct": 134, - "requested": 160, - "accuracy": 0.8375, - "ci95": [ - 0.7784810126582279, - 0.89171974522293 - ] - }, - "belebele_zh": { - "correct": 130, - "requested": 160, - "accuracy": 0.8125, - "ci95": [ - 0.746986092518221, - 0.8727272727272727 - ] - } - }, - "unweighted": { - "correct": 368, - "requested": 480, - "accuracy": 0.7666666666666667 - }, - "coverage": { - "requested": 480, - "accepted": 480, - "finite_valid_outputs": 480, - "full_input_verified": 480, - "truncated": 0, - "retention": "Complete frozen inputs verified." - }, - "typed_probability_metrics": { - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.65, - "brier": 0.4013130395712821, - "nll": 0.5740625822372918, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.9375, - "recall_true": 0.3625, - "balanced_accuracy": 0.65, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "choice": { - "n": 320, - "valid_probability_coverage": 320, - "missing_or_invalid": 0, - "accuracy": 0.825, - "brier": 0.23224826689575004, - "nll": 0.47036782610781563, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - } - } - }, - "qwen35-4b-lm-head": { - "label": "Qwen3.5 4B + LM-head adapter", - "weighted_accuracy": 0.8796875, - "weighted_accuracy_exact": { - "numerator": 563, - "denominator": 640 - }, - "weighted_accuracy_ci95": [ - 0.8453125000000001, - 0.9125 - ], - "families": { - "boolq": { - "correct": 134, - "requested": 160, - "accuracy": 0.8375, - "ci95": [ - 0.78125, - 0.89375 - ] - }, - "belebele_en": { - "correct": 149, - "requested": 160, - "accuracy": 0.93125, - "ci95": [ - 0.89171974522293, - 0.967948717948718 - ] - }, - "belebele_zh": { - "correct": 146, - "requested": 160, - "accuracy": 0.9125, - "ci95": [ - 0.8647011385199241, - 0.9559748427672956 - ] - } - }, - "unweighted": { - "correct": 429, - "requested": 480, - "accuracy": 0.89375 - }, - "coverage": { - "requested": 480, - "accepted": 480, - "finite_valid_outputs": 480, - "full_input_verified": 480, - "truncated": 0, - "retention": "Complete frozen inputs verified." - }, - "typed_probability_metrics": { - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.8375, - "brier": 0.2685250413432339, - "nll": 0.43323629871265307, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.7875, - "recall_true": 0.8875, - "balanced_accuracy": 0.8374999999999999, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "choice": { - "n": 320, - "valid_probability_coverage": 320, - "missing_or_invalid": 0, - "accuracy": 0.921875, - "brier": 0.12427284941036536, - "nll": 0.26369763063759766, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - } - } - } - }, - "released_nox_reference": { - "label": "Nox v1.0", - "weighted_accuracy": 0.7546875, - "weighted_accuracy_exact": { - "numerator": 483, - "denominator": 640 - }, - "weighted_accuracy_ci95": [ - 0.7070792189209251, - 0.8001929012345679 - ], - "families": { - "boolq": { - "correct": 132, - "requested": 160, - "accuracy": 0.825, - "ci95": [ - 0.7625, - 0.88125 - ] - }, - "belebele_en": { - "correct": 112, - "requested": 160, - "accuracy": 0.7, - "ci95": [ - 0.6234567901234568, - 0.7721518987341772 - ] - }, - "belebele_zh": { - "correct": 107, - "requested": 160, - "accuracy": 0.66875, - "ci95": [ - 0.5903614457831325, - 0.7421383647798742 - ] - } - }, - "unweighted": { - "correct": 351, - "requested": 480, - "accuracy": 0.73125 - }, - "coverage": { - "requested": 480, - "accepted": 480, - "finite_valid_outputs": 480, - "full_input_verified": 480, - "truncated": 0, - "retention": "Complete frozen inputs verified." - }, - "typed_probability_metrics": { - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.825, - "brier": 0.29269334225936294, - "nll": 1.2531297142985633, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.8125, - "recall_true": 0.8375, - "balanced_accuracy": 0.825, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "choice": { - "n": 320, - "valid_probability_coverage": 320, - "missing_or_invalid": 0, - "accuracy": 0.684375, - "brier": 0.5462488185347467, - "nll": 2.6090008676531964, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - } - } - }, - "nox_update_comparison": { - "weighted_accuracy_delta": 0.0296875, - "exact_delta": { - "numerator": 19, - "denominator": 640 - }, - "paired_ci95": [ - 0.007691761363636341, - 0.05318518595020634 - ] - }, - "bootstrap": { - "resamples": 10000, - "seed": 202609215404, - "weights": { - "boolq": "1/2", - "belebele_en": "1/4", - "belebele_zh": "1/4" - }, - "clusters": "BoolQ complete normalized passage; Belebele source link with all question IDs and both translations coupled. Identical random draws shared across models.", - "point_denominator": "All requested rows; invalid/unsupported count as incorrect.", - "readiness": "Descriptive confirmation only; no change to active V3 update policy." - }, - "candidate_bundle_manifest_sha256": "a8cfd42bc908fc93d5a2a20be527d09704b6e3f6c1217f27307c32cc2d822971", - "released_nox_revision": "43d60043795a1ddd840c0edd5bbdc5a399c9715b", - "sol_v1_revision": "b6a4ce881ca625dcf19663a277d27577cfe4540a", - "sources": [ - { - "name": "BoolQ validation", - "revision": "35b264d03638db9f4ce671b711558bf7ff0f80d5", - "license": "CC BY-SA 3.0", - "url": "https://huggingface.co/datasets/google/boolq/tree/35b264d03638db9f4ce671b711558bf7ff0f80d5" - }, - { - "name": "Belebele English/Chinese test", - "revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", - "license": "CC BY-SA 4.0", - "url": "https://huggingface.co/datasets/facebook/belebele/tree/7899cdfa4e1e0d733fd77c848e2c273cb1d32be2" - } - ], - "independence": { - "excluded_from_custom_train_select_cal": true, - "original_human_annotations": true, - "new_human_adjudication": false, - "upstream_pretraining_exposure": "Unknown", - "first_global_unseal_at": "2026-09-21T16:40:29.779741+00:00", - "candidate_frozen_before_first_unseal": true, - "subsequent_use": "Observed regression for candidates not frozen before first exposure.", - "source_parent_clusters": { - "boolq_passages": 160, - "belebele_shared_passages": 144 - }, - "overlap_audit": "No exact or character-5-gram Jaccard >=0.5 overlap across 51 custom-data and earlier-panel files; this is not proof of semantic or upstream-pretraining independence.", - "boolq_grouping_limit": "Passage grouping only; this distribution lacks article titles, so article-level independence is not established." - }, - "scope": "Evidence yes/no and four-choice reading comprehension; no ordinal Score or all-capability claim.", - "changes_release_gate": false, - "source_sha256": { - "aggregate": "c55a218a533751cf6fd8e783182bb2edfa1731f1a7aa3446fbed4575d6291dcb", - "statistics": "23105cc6fbffb15b1fac390b64bf31b70403091b5f0d4a3de06a78a27b3c5474", - "evaluation_receipt": "e3220736d2da9c87d7b08808fa8903af0ce1c90961f88e7a54a8348abca29794", - "suite_manifest": "969a92c4c5a2f395a3ba55d3a4aff28be060d89327c3b09de3e1f5188cf74830", - "metrics_policy": "259b907f3634471f7c077acbe6cb4eba6ebb85d24ba305c86ea6a679349491c3" - } -} diff --git a/metrics/question-scaling.json b/metrics/question-scaling.json index 8314bbc91fd4b42373a0e236190fd00c06833fe1..24f00fdef15e02fe31e9f48db8324fb1986306c8 100644 --- a/metrics/question-scaling.json +++ b/metrics/question-scaling.json @@ -1,327 +1,467 @@ { - "scope": "Fixed same question text and819-characterstate, K4; actual request latency, no concurrent serving claim", - "original_timing_sha256": "a0e5f1c94277b31a922ae1e1a708ce70a2f03b5e49996a688112fc1e9697615a", - "additive_protocol_sha256": "391114fe1d4bd3641957a28920a14347418d4e27b8ff495e743627a73aef0135", - "model_runtime_keys": { - "Decision-1.0-Sol": "aa382311fc7d9249e78ae4459d9238d10b1b2e02a7b454949be4ec87c01c971d", - "Decision-1.0-Nox": "34be384b21efb03b65b90d28463bdab1fb180438a9c7538dbf0fadb0e049c230", - "laya-routed": "4c6fdee5192ac14c2842de9fef374aa6d2fe636dbd593ad182a75156422e663f", - "decider": "417e15690883f8700b02327aacb6ef2f39c5a14737ee081163c239183e5f27eb", - "base-2b": "4cfa0428c0f0332610cc36e7a92c97d1c039b08961f1c16aea4d552165097ea5", - "base-4b": "2edfae75b3a649b147bd49096006f70e5eb29515cb8fa134e79b9255f198c278" - }, - "points": { - "Decision-1.0-Sol": [ - { - "q": 1, - "p50_ms": 21.03651, - "p95_ms": 21.4133929, - "round": "original", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 2, - "p50_ms": 21.4783635, - "p95_ms": 21.808785, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 4, - "p50_ms": 23.473785, - "p95_ms": 31.8870149, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 8, - "p50_ms": 37.294218, - "p95_ms": 37.50742325, - "round": "original", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 16, - "p50_ms": 73.88406900000001, - "p95_ms": 74.45837850000001, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 32, - "p50_ms": 147.50142499999998, - "p95_ms": 147.90909245, - "round": "original", - "samples": 30, - "tokens_per_native_row": 309 - } - ], - "Decision-1.0-Nox": [ - { - "q": 1, - "p50_ms": 27.374125, - "p95_ms": 28.94694755, - "round": "original", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 2, - "p50_ms": 28.1707225, - "p95_ms": 28.5395078, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 4, - "p50_ms": 40.8942485, - "p95_ms": 41.03577595, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 8, - "p50_ms": 69.247188, - "p95_ms": 69.84472105, - "round": "original", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 16, - "p50_ms": 137.64264550000001, - "p95_ms": 137.81669829999998, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 309 - }, - { - "q": 32, - "p50_ms": 275.653244, - "p95_ms": 278.2649851, - "round": "original", - "samples": 30, - "tokens_per_native_row": 309 - } - ], - "laya-routed": [ - { - "q": 1, - "p50_ms": 11.539303, - "p95_ms": 12.21025405, - "round": "original", - "samples": 30, - "tokens_per_native_row": 228 - }, - { - "q": 2, - "p50_ms": 12.602103499999998, - "p95_ms": 13.020027549999998, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 228 - }, - { - "q": 4, - "p50_ms": 13.366638, - "p95_ms": 13.68192385, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 228 - }, - { - "q": 8, - "p50_ms": 14.559828, - "p95_ms": 15.0213171, - "round": "original", - "samples": 30, - "tokens_per_native_row": 228 - }, - { - "q": 16, - "p50_ms": 22.471257, - "p95_ms": 22.797051449999998, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 228 - }, - { - "q": 32, - "p50_ms": 37.885172999999995, - "p95_ms": 38.3012614, - "round": "original", - "samples": 30, - "tokens_per_native_row": 228 - } - ], - "decider": [ - { - "q": 1, - "p50_ms": 23.3526305, - "p95_ms": 23.906231950000002, - "round": "original", - "samples": 30, - "tokens_per_native_row": 250 - }, - { - "q": 2, - "p50_ms": 23.515689000000002, - "p95_ms": 23.8890495, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 250 - }, - { - "q": 4, - "p50_ms": 24.045369, - "p95_ms": 24.310676, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 250 - }, - { - "q": 8, - "p50_ms": 31.591898999999998, - "p95_ms": 31.7650492, - "round": "original", - "samples": 30, - "tokens_per_native_row": 250 - }, - { - "q": 16, - "p50_ms": 53.1165, - "p95_ms": 53.321273049999995, - "round": "additive", - "samples": 30, - "tokens_per_native_row": 250 - }, - { - "q": 32, - "p50_ms": 96.6671255, - "p95_ms": 97.02467545, - "round": "original", - "samples": 30, - "tokens_per_native_row": 250 - } - ], - "base-2b": [ - { - "q": 1, - "p50_ms": 20.508525, - "p95_ms": 20.714382399999998, - "round": "original", - "samples": 30, - "tokens_per_native_row": 296 - }, - { - "q": 8, - "p50_ms": 37.320352, - "p95_ms": 37.61330755, - "round": "original", - "samples": 30, - "tokens_per_native_row": 296 - }, - { - "q": 32, - "p50_ms": 146.8754105, - "p95_ms": 147.15678465000002, - "round": "original", - "samples": 30, - "tokens_per_native_row": 296 - } - ], - "base-4b": [ - { - "q": 1, - "p50_ms": 27.469792499999997, - "p95_ms": 28.755623, - "round": "original", - "samples": 30, - "tokens_per_native_row": 296 - }, - { - "q": 8, - "p50_ms": 69.02867599999999, - "p95_ms": 69.45783845, - "round": "original", - "samples": 30, - "tokens_per_native_row": 296 - }, - { - "q": 32, - "p50_ms": 274.68847300000004, - "p95_ms": 275.19135435000004, - "round": "original", - "samples": 30, - "tokens_per_native_row": 296 + "all_types": { + "Nox-natural-selected": { + "choice-q1": { + "samples": 30, + "p50_ms": 32.107608, + "p95_ms": 33.04698645, + "requests_per_second_from_median": 31.14526625589798, + "questions_per_second_from_median": 31.14526625589798, + "measured_requests_per_second": 30.8760998861839, + "measured_questions_per_second": 30.8760998861839, + "max_allocated_bytes": 8649496576, + "max_reserved_bytes": 8923381760, + "input_tokens": 309 + }, + "choice-q2": { + "samples": 30, + "p50_ms": 33.0248195, + "p95_ms": 34.04666545, + "requests_per_second_from_median": 30.280256338721244, + "questions_per_second_from_median": 60.56051267744249, + "measured_requests_per_second": 30.32447258444086, + "measured_questions_per_second": 60.64894516888172, + "max_allocated_bytes": 8694344704, + "max_reserved_bytes": 8938061824, + "input_tokens": 618 + }, + "choice-q4": { + "samples": 30, + "p50_ms": 41.706045, + "p95_ms": 42.00540085, + "requests_per_second_from_median": 23.977339495989128, + "questions_per_second_from_median": 95.90935798395651, + "measured_requests_per_second": 23.96644908033505, + "measured_questions_per_second": 95.8657963213402, + "max_allocated_bytes": 8783385600, + "max_reserved_bytes": 9141485568, + "input_tokens": 1236 + }, + "choice-q8": { + "samples": 30, + "p50_ms": 69.48287, + "p95_ms": 70.71858545, + "requests_per_second_from_median": 14.392036483236803, + "questions_per_second_from_median": 115.13629186589442, + "measured_requests_per_second": 14.32914310038675, + "measured_questions_per_second": 114.633144803094, + "max_allocated_bytes": 8963302400, + "max_reserved_bytes": 9376366592, + "input_tokens": 2472 + }, + "choice-q16": { + "samples": 30, + "p50_ms": 141.298589, + "p95_ms": 141.920981, + "requests_per_second_from_median": 7.0772115070448445, + "questions_per_second_from_median": 113.23538411271751, + "measured_requests_per_second": 7.084327224101568, + "measured_questions_per_second": 113.34923558562508, + "max_allocated_bytes": 8962254336, + "max_reserved_bytes": 9244246016, + "input_tokens": 4944 + }, + "choice-q32": { + "samples": 30, + "p50_ms": 282.326369, + "p95_ms": 284.8558578, + "requests_per_second_from_median": 3.542000003549084, + "questions_per_second_from_median": 113.34400011357069, + "measured_requests_per_second": 3.5446644265419387, + "measured_questions_per_second": 113.42926164934204, + "max_allocated_bytes": 8963302912, + "max_reserved_bytes": 9141485568, + "input_tokens": 9888 + }, + "noul-q1": { + "samples": 30, + "p50_ms": 32.281844500000005, + "p95_ms": 32.977001, + "requests_per_second_from_median": 30.977164269532363, + "questions_per_second_from_median": 30.977164269532363, + "measured_requests_per_second": 31.030133981643576, + "measured_questions_per_second": 31.030133981643576, + "max_allocated_bytes": 8645022720, + "max_reserved_bytes": 8917090304, + "input_tokens": 262 + }, + "noul-q2": { + "samples": 30, + "p50_ms": 32.572923, + "p95_ms": 33.97754645, + "requests_per_second_from_median": 30.700345805625116, + "questions_per_second_from_median": 61.40069161125023, + "measured_requests_per_second": 30.42684666972788, + "measured_questions_per_second": 60.85369333945576, + "max_allocated_bytes": 8686444544, + "max_reserved_bytes": 9216983040, + "input_tokens": 524 + }, + "noul-q4": { + "samples": 30, + "p50_ms": 38.9671165, + "p95_ms": 39.276742999999996, + "requests_per_second_from_median": 25.662663543503403, + "questions_per_second_from_median": 102.65065417401361, + "measured_requests_per_second": 25.722971332785946, + "measured_questions_per_second": 102.89188533114378, + "max_allocated_bytes": 8767323136, + "max_reserved_bytes": 9191817216, + "input_tokens": 1048 + }, + "noul-q8": { + "samples": 30, + "p50_ms": 64.496848, + "p95_ms": 65.0679283, + "requests_per_second_from_median": 15.50463365279494, + "questions_per_second_from_median": 124.03706922235952, + "measured_requests_per_second": 15.492551650332626, + "measured_questions_per_second": 123.940413202661, + "max_allocated_bytes": 8932226048, + "max_reserved_bytes": 9141485568, + "input_tokens": 2096 + }, + "noul-q16": { + "samples": 30, + "p50_ms": 129.420839, + "p95_ms": 130.16497075, + "requests_per_second_from_median": 7.726730932411897, + "questions_per_second_from_median": 123.62769491859035, + "measured_requests_per_second": 7.723849562904405, + "measured_questions_per_second": 123.58159300647048, + "max_allocated_bytes": 8931440128, + "max_reserved_bytes": 9244246016, + "input_tokens": 4192 + }, + "noul-q32": { + "samples": 30, + "p50_ms": 260.114736, + "p95_ms": 261.01741085000003, + "requests_per_second_from_median": 3.8444573167127296, + "questions_per_second_from_median": 123.02263413480735, + "measured_requests_per_second": 3.8574251583945887, + "measured_questions_per_second": 123.43760506862684, + "max_allocated_bytes": 8932226560, + "max_reserved_bytes": 9244246016, + "input_tokens": 8384 + }, + "score-q1": { + "samples": 30, + "p50_ms": 32.600662, + "p95_ms": 33.154171049999995, + "requests_per_second_from_median": 30.674223732021147, + "questions_per_second_from_median": 30.674223732021147, + "measured_requests_per_second": 30.70316656105357, + "measured_questions_per_second": 30.70316656105357, + "max_allocated_bytes": 8648972288, + "max_reserved_bytes": 9244246016, + "input_tokens": 307 + }, + "score-q2": { + "samples": 30, + "p50_ms": 33.2208605, + "p95_ms": 33.62534145, + "requests_per_second_from_median": 30.101568260099704, + "questions_per_second_from_median": 60.20313652019941, + "measured_requests_per_second": 30.219942037607215, + "measured_questions_per_second": 60.43988407521443, + "max_allocated_bytes": 8693689344, + "max_reserved_bytes": 9141485568, + "input_tokens": 614 + }, + "score-q4": { + "samples": 30, + "p50_ms": 41.775667999999996, + "p95_ms": 42.3791635, + "requests_per_second_from_median": 23.937379050408005, + "questions_per_second_from_median": 95.74951620163202, + "measured_requests_per_second": 23.877435233851767, + "measured_questions_per_second": 95.50974093540707, + "max_allocated_bytes": 8782337024, + "max_reserved_bytes": 9244246016, + "input_tokens": 1228 + }, + "score-q8": { + "samples": 30, + "p50_ms": 69.53526400000001, + "p95_ms": 70.1399854, + "requests_per_second_from_median": 14.381192253760622, + "questions_per_second_from_median": 115.04953803008497, + "measured_requests_per_second": 14.403771734792818, + "measured_questions_per_second": 115.23017387834254, + "max_allocated_bytes": 8963302400, + "max_reserved_bytes": 9376366592, + "input_tokens": 2456 + }, + "score-q16": { + "samples": 30, + "p50_ms": 140.446957, + "p95_ms": 141.9686554, + "requests_per_second_from_median": 7.120125785281343, + "questions_per_second_from_median": 113.92201256450149, + "measured_requests_per_second": 7.106122379909049, + "measured_questions_per_second": 113.69795807854479, + "max_allocated_bytes": 8963302912, + "max_reserved_bytes": 9244246016, + "input_tokens": 4912 + }, + "score-q32": { + "samples": 30, + "p50_ms": 281.5755855, + "p95_ms": 284.90178425, + "requests_per_second_from_median": 3.5514442710801006, + "questions_per_second_from_median": 113.64621667456322, + "measured_requests_per_second": 3.5406794630737664, + "measured_questions_per_second": 113.30174281836052, + "max_allocated_bytes": 8963302912, + "max_reserved_bytes": 9244246016, + "input_tokens": 9824 } - ] - }, - "Q1_bridge": { - "Decision-1.0-Sol": { - "original_p50_ms": 21.03651, - "bridge_p50_ms": 20.8673795, - "delta_ms": -0.1691305000000014, - "delta_fraction": -0.008039855470322821, - "flag": false, - "pooled_or_selected": false - }, - "Decision-1.0-Nox": { - "original_p50_ms": 27.374125, - "bridge_p50_ms": 27.164138, - "delta_ms": -0.20998699999999815, - "delta_fraction": -0.007671003182750047, - "flag": false, - "pooled_or_selected": false - }, - "laya-routed": { - "original_p50_ms": 11.539303, - "bridge_p50_ms": 12.141239, - "delta_ms": 0.6019360000000002, - "delta_fraction": 0.05216398252130139, - "flag": false, - "pooled_or_selected": false }, - "decider": { - "original_p50_ms": 23.3526305, - "bridge_p50_ms": 23.177763, - "delta_ms": -0.17486750000000129, - "delta_fraction": -0.007488128585771192, - "flag": false, - "pooled_or_selected": false + "Nox-v1.1-released": { + "choice-q1": { + "samples": 30, + "p50_ms": 28.9757945, + "p95_ms": 30.240221699999996, + "requests_per_second_from_median": 34.511564471510866, + "questions_per_second_from_median": 34.511564471510866, + "measured_requests_per_second": 34.32893449270674, + "measured_questions_per_second": 34.32893449270674, + "max_allocated_bytes": 8648972288, + "max_reserved_bytes": 8929673216, + "input_tokens": 309 + }, + "choice-q2": { + "samples": 30, + "p50_ms": 29.3374375, + "p95_ms": 29.953046999999998, + "requests_per_second_from_median": 34.08613993638674, + "questions_per_second_from_median": 68.17227987277349, + "measured_requests_per_second": 33.9753724393917, + "measured_questions_per_second": 67.9507448787834, + "max_allocated_bytes": 8694475776, + "max_reserved_bytes": 8713666560, + "input_tokens": 618 + }, + "choice-q4": { + "samples": 30, + "p50_ms": 42.0405665, + "p95_ms": 42.4125123, + "requests_per_second_from_median": 23.78654911798108, + "questions_per_second_from_median": 95.14619647192433, + "measured_requests_per_second": 23.793745836748805, + "measured_questions_per_second": 95.17498334699522, + "max_allocated_bytes": 8783385600, + "max_reserved_bytes": 9256828928, + "input_tokens": 1236 + }, + "choice-q8": { + "samples": 30, + "p50_ms": 70.1055465, + "p95_ms": 70.8502012, + "requests_per_second_from_median": 14.264206613095869, + "questions_per_second_from_median": 114.11365290476695, + "measured_requests_per_second": 14.24194480835554, + "measured_questions_per_second": 113.93555846684433, + "max_allocated_bytes": 8963302400, + "max_reserved_bytes": 9126805504, + "input_tokens": 2472 + }, + "choice-q16": { + "samples": 30, + "p50_ms": 141.3689395, + "p95_ms": 142.33760285, + "requests_per_second_from_median": 7.073689620484137, + "questions_per_second_from_median": 113.17903392774619, + "measured_requests_per_second": 7.082735221822023, + "measured_questions_per_second": 113.32376354915237, + "max_allocated_bytes": 8963302912, + "max_reserved_bytes": 9256828928, + "input_tokens": 4944 + }, + "choice-q32": { + "samples": 30, + "p50_ms": 283.68131300000005, + "p95_ms": 286.0126798, + "requests_per_second_from_median": 3.5250823870798986, + "questions_per_second_from_median": 112.80263638655676, + "measured_requests_per_second": 3.524747236128377, + "measured_questions_per_second": 112.79191155610806, + "max_allocated_bytes": 8963302912, + "max_reserved_bytes": 9256828928, + "input_tokens": 9888 + }, + "noul-q1": { + "samples": 30, + "p50_ms": 28.9925315, + "p95_ms": 29.37066045, + "requests_per_second_from_median": 34.49164140771909, + "questions_per_second_from_median": 34.49164140771909, + "measured_requests_per_second": 34.58104068093915, + "measured_questions_per_second": 34.58104068093915, + "max_allocated_bytes": 8645022720, + "max_reserved_bytes": 8919187456, + "input_tokens": 262 + }, + "noul-q2": { + "samples": 30, + "p50_ms": 29.196136, + "p95_ms": 30.45851935, + "requests_per_second_from_median": 34.25110774932683, + "questions_per_second_from_median": 68.50221549865365, + "measured_requests_per_second": 34.08654368923435, + "measured_questions_per_second": 68.1730873784687, + "max_allocated_bytes": 8686018560, + "max_reserved_bytes": 9196011520, + "input_tokens": 524 + }, + "noul-q4": { + "samples": 30, + "p50_ms": 39.3726845, + "p95_ms": 39.57324895, + "requests_per_second_from_median": 25.398318979240546, + "questions_per_second_from_median": 101.59327591696218, + "measured_requests_per_second": 25.417992661917022, + "measured_questions_per_second": 101.67197064766809, + "max_allocated_bytes": 8767454208, + "max_reserved_bytes": 9170845696, + "input_tokens": 1048 + }, + "noul-q8": { + "samples": 30, + "p50_ms": 64.3817725, + "p95_ms": 65.2974674, + "requests_per_second_from_median": 15.532346519350645, + "questions_per_second_from_median": 124.25877215480516, + "measured_requests_per_second": 15.53924018787304, + "measured_questions_per_second": 124.31392150298431, + "max_allocated_bytes": 8932226048, + "max_reserved_bytes": 9191817216, + "input_tokens": 2096 + }, + "noul-q16": { + "samples": 30, + "p50_ms": 129.92605450000002, + "p95_ms": 130.92555945, + "requests_per_second_from_median": 7.696685655916688, + "questions_per_second_from_median": 123.146970494667, + "measured_requests_per_second": 7.715408426351679, + "measured_questions_per_second": 123.44653482162687, + "max_allocated_bytes": 8931440128, + "max_reserved_bytes": 9256828928, + "input_tokens": 4192 + }, + "noul-q32": { + "samples": 30, + "p50_ms": 259.700112, + "p95_ms": 262.4176393, + "requests_per_second_from_median": 3.8505951818765487, + "questions_per_second_from_median": 123.21904582004956, + "measured_requests_per_second": 3.841294712746642, + "measured_questions_per_second": 122.92143080789255, + "max_allocated_bytes": 8931440128, + "max_reserved_bytes": 9158262784, + "input_tokens": 8384 + }, + "score-q1": { + "samples": 30, + "p50_ms": 28.816428000000002, + "p95_ms": 29.11869395, + "requests_per_second_from_median": 34.70242737927129, + "questions_per_second_from_median": 34.70242737927129, + "measured_requests_per_second": 34.6747983353027, + "measured_questions_per_second": 34.6747983353027, + "max_allocated_bytes": 8649496576, + "max_reserved_bytes": 9256828928, + "input_tokens": 307 + }, + "score-q2": { + "samples": 30, + "p50_ms": 29.3393325, + "p95_ms": 29.776925, + "requests_per_second_from_median": 34.08393834454141, + "questions_per_second_from_median": 68.16787668908282, + "measured_requests_per_second": 34.03598008577042, + "measured_questions_per_second": 68.07196017154084, + "max_allocated_bytes": 8694344704, + "max_reserved_bytes": 9198108672, + "input_tokens": 614 + }, + "score-q4": { + "samples": 30, + "p50_ms": 41.9650575, + "p95_ms": 42.2195708, + "requests_per_second_from_median": 23.82934897682435, + "questions_per_second_from_median": 95.3173959072974, + "measured_requests_per_second": 23.82153615943987, + "measured_questions_per_second": 95.28614463775948, + "max_allocated_bytes": 8782861312, + "max_reserved_bytes": 9198108672, + "input_tokens": 1228 + }, + "score-q8": { + "samples": 30, + "p50_ms": 69.58278200000001, + "p95_ms": 70.67227025, + "requests_per_second_from_median": 14.371371354482491, + "questions_per_second_from_median": 114.97097083585993, + "measured_requests_per_second": 14.322159646664494, + "measured_questions_per_second": 114.57727717331595, + "max_allocated_bytes": 8963302400, + "max_reserved_bytes": 9355395072, + "input_tokens": 2456 + }, + "score-q16": { + "samples": 30, + "p50_ms": 141.1238315, + "p95_ms": 142.7912914, + "requests_per_second_from_median": 7.085975411601548, + "questions_per_second_from_median": 113.37560658562477, + "measured_requests_per_second": 7.091980381610694, + "measured_questions_per_second": 113.4716861057711, + "max_allocated_bytes": 8963302912, + "max_reserved_bytes": 9355395072, + "input_tokens": 4912 + }, + "score-q32": { + "samples": 30, + "p50_ms": 283.68416249999996, + "p95_ms": 286.0974013, + "requests_per_second_from_median": 3.5250469789620356, + "questions_per_second_from_median": 112.80150332678514, + "measured_requests_per_second": 3.5312236579984573, + "measured_questions_per_second": 112.99915705595063, + "max_allocated_bytes": 8963302912, + "max_reserved_bytes": 9256828928, + "input_tokens": 9824 + } } }, - "all_bridge_drift_checks_pass": true, - "raw_additive_measurement_sha256": { - "block-0/Decision-1.0-Sol/measurements.jsonl": "e186d2eb157df15eca30befa39b92c02ebb47288dffbca82d601f4ffe6ab8a6e", - "block-1/Decision-1.0-Sol/measurements.jsonl": "715ef039bcd8a35bc75d25d94adc64dab02cfdb37697c9f0274690a06321f9e2", - "block-2/Decision-1.0-Sol/measurements.jsonl": "333007726ded345cf37281b6df4afee351cd9bf099ca59a807daeba272b4bf6e", - "block-0/Decision-1.0-Nox/measurements.jsonl": "6770f8d60ea448ee6c02df901f90eea489f0e8d198c245211ae8520ddfe11c86", - "block-1/Decision-1.0-Nox/measurements.jsonl": "3893b592c8af7ae163453891cbd1373c2edb5289db4d27cfd8ab1276e6b415b4", - "block-2/Decision-1.0-Nox/measurements.jsonl": "b0a73b6436323613b27b09f3b609a640ad9d1f00317a1161dff65f4444520521", - "block-0/laya-routed/measurements.jsonl": "2cf05c1438bb62d7510c00e00f4da2e134472661635c420fe58cba4b8bd4dd59", - "block-1/laya-routed/measurements.jsonl": "18582d373500d0826e4fa03f23f24d15c4f2a506b99f837723e07d19ca44fb95", - "block-2/laya-routed/measurements.jsonl": "20e699577a1fe5edccb3fb1dfa47b491930ffc3875cf7b62a2f9409881b73aca", - "block-0/decider/measurements.jsonl": "0116ea8a843afe0496762442dd0f3ef05c7b3639664bde2d6de47071d67463ea", - "block-1/decider/measurements.jsonl": "2d99b9ef948b16cf2c2b39d971c79ae15b05819ce4f1a89c1c85e22534384b9a", - "block-2/decider/measurements.jsonl": "a8a84c27bac9d3050f75ea65c97b35c8ba7c515b9c7ed45f77f7374ad0b277c1" - }, - "logo_sha256": "d83cf7878c3839f3085feb8c02334217ad2fef0a615b805e1726b292552c46e8", - "logo_modified": false, - "renderer_sha256": "cd9ccebdcce07cfda8aa301be71ab8c23a748d9e1b79c020873322d2f085c0b0", - "figure_footer_annotations": false, - "canvas_width_px": 792, - "designed_card_width_px": 600, - "axis_legend_effective_css_px_at_600": 12.152777777777779, - "display_label_aliases": { - "Laya": "Fixed upstream EN/ML routed Laya; English branch used for these English inputs", - "Decider": "Mapika/decider-2b" - } + "choice_q_counts": [ + 1, + 2, + 4, + 8, + 16, + 32 + ], + "candidate_p50_ms": [ + 32.107608, + 33.0248195, + 41.706045, + 69.48287, + 141.298589, + 282.326369 + ], + "reference_p50_ms": [ + 28.9757945, + 29.3374375, + 42.0405665, + 70.1055465, + 141.3689395, + 283.68131300000005 + ], + "geomean_latency_ratio": 1.0336937058472013, + "raw_observations": 1404, + "claim_boundary": "Sequential whole requests with fixed state/question bytes; not concurrent-serving throughput." } diff --git a/model-card-example.json b/model-card-example.json index 1d11916afd0745db661027b9f07c2664f15221d1..9155d6d529c17b37eeb9800b8f7ed438725c768d 100644 --- a/model-card-example.json +++ b/model-card-example.json @@ -31,25 +31,25 @@ "destination": { "type": "choice", "probabilities": { - "billing": 0.9993942525743535, - "technical": 0.0006057474256464916 + "billing": 0.9995198700709675, + "technical": 0.00048012992903245 }, - "confidence": 0.9987885051487071, + "confidence": 0.9990397401419351, "choice": "billing" }, "refund_requested": { "type": "noul", - "noul": 0.9998684086292363 + "noul": 0.9997257333959462 }, "urgency": { "type": "score", "probabilities": { - "0": 0.027512233426781958, - "1": 0.9724599620944665, - "2": 2.7804478751561622e-05 + "0": 0.01145875993979324, + "1": 0.9885278778333887, + "2": 1.3362226818050175e-05 }, - "confidence": 0.9586899431416998, - "score": 0.9725155710519696, + "confidence": 0.9827918167500831, + "score": 0.9885546022870247, "legend": { "0": "Routine information request with no payment problem or outage", "1": "A payment or billing problem, with no product outage", @@ -65,7 +65,7 @@ "direct_engine_exact_response": true, "overflow_rejected": true, "overflow_message": "check: 40083 tokens exceeds max_length=16384; no truncation allowed", - "bundle_manifest_sha256": "a8cfd42bc908fc93d5a2a20be527d09704b6e3f6c1217f27307c32cc2d822971", + "bundle_manifest_sha256": "7762761b5da5d51d33e06d322c037f7cfdee54f910e6e83d3e8c7ce02ae42b2e", "runtime": { "actual": { "torch": "2.12.0+git6bbd260", @@ -78,10 +78,36 @@ "gated_delta": "fla.ops.gated_delta_rule.chunk" }, "differences": {}, - "matches_validated_runtime": true + "matches_validated_runtime": true, + "normalization_profile": { + "profile_sha256": "be32858d15233e0a3fbee0e4257fb02be0b3439deee4eb9c3f61151df7b73850", + "guard_sha256": "061d6ba3edf032038b074b061879a1ff81cfd2928fc0131c66b15563f7822509", + "validated_arch": "gfx942", + "automatic_bundle_binding": true, + "scope": "single profile per process" + } }, "revision": null, "device": "cuda:0", "model_name": "Decision-1.0-Nox", - "example_source_sha256": "d52c20b602bd1049aecda55244e71c27d8ce44bd1aa8b1941db62ae41d31ba12" + "example_source_sha256": "a54dec885f92c2d38d07ef2333dff51965be52bc119f772cc74200ed314e69b6", + "normalization_telemetry": { + "profile_sha256": "be32858d15233e0a3fbee0e4257fb02be0b3439deee4eb9c3f61151df7b73850", + "status": "installed", + "calls": 96, + "keys": { + "[128,1,\"torch.bfloat16\",\"torch.bfloat16\",\"torch.float32\"]": 96 + }, + "strict_guard": true, + "unknown_keys": "raise", + "autotune_fallback_permitted": false, + "runtime": { + "torch": "2.12.0+git6bbd260", + "hip": "7.2.53211", + "triton": "3.7.1", + "fla": "0.5.2" + }, + "gpu_arch": "gfx942", + "process_scope": "one explicitly profiled Nox model; other model loading in this process is not supported" + } } diff --git a/pyproject.toml b/pyproject.toml index 9a64ed0009324fdb587ee152f3224d5f5a14c4d1..414ada1d87dcac10beb7ed3f47972dd7838f569d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "decision-local" -version = "1.0.0" +version = "1.1.0" description = "Local typed inference for exported Decision decoder models" requires-python = ">=3.10" dependencies = [] diff --git a/quality-metrics.json b/quality-metrics.json deleted file mode 100644 index 9ae56d2ceb86d3939ef3a28c8cd879920cbadbdf..0000000000000000000000000000000000000000 --- a/quality-metrics.json +++ /dev/null @@ -1,38062 +0,0 @@ -{ - "created_at": "2026-09-21T16:40:35.605010+00:00", - "policy_sha256": "359de2b6b81be91a66c6c538b3655bed06f4728357328cd318b1a765e64565cd", - "candidate": "Decision-1.0-Nox-stage4-selected", - "released_reference": "Decision-1.0-Nox", - "uniform_control": { - "old": { - "overall": { - "n": 880, - "expected_random_accuracy": 0.24181818181818182, - "fixed_tie_accuracy": 0.18522727272727274, - "nll": 1.530582043847413, - "brier": 0.7581818181818182, - "normalized_rps": 0.26093750000000004 - }, - "family_macro_expected_random_accuracy": 0.2479761904761905, - "family_macro_fixed_tie_accuracy": 0.1849330357142857, - "families": { - "ag_news": { - "n": 128, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.2421875, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - }, - "boolean_constraints": { - "n": 64, - "expected_random_accuracy": 0.5, - "fixed_tie_accuracy": 0.375, - "nll": 0.6931471805599453, - "brier": 0.5, - "normalized_rps": null - }, - "dbpedia_14": { - "n": 112, - "expected_random_accuracy": 0.07142857142857142, - "fixed_tie_accuracy": 0.044642857142857144, - "nll": 2.6390573296152584, - "brier": 0.9285714285714286, - "normalized_rps": null - }, - "natural_intents": { - "n": 64, - "expected_random_accuracy": 0.125, - "fixed_tie_accuracy": 0.09375, - "nll": 2.0794415416798357, - "brier": 0.875, - "normalized_rps": null - }, - "option_carrier": { - "n": 96, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.0, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - }, - "ordinal_rubric": { - "n": 64, - "expected_random_accuracy": 0.2, - "fixed_tie_accuracy": 0.0625, - "nll": 1.6094379124341003, - "brier": 0.8, - "normalized_rps": 0.26093750000000004 - }, - "relational_composition": { - "n": 96, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.21875, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - }, - "scoped_evidence": { - "n": 96, - "expected_random_accuracy": 0.3333333333333333, - "fixed_tie_accuracy": 0.34375, - "nll": 1.0986122886681098, - "brier": 0.6666666666666666, - "normalized_rps": null - }, - "state_tracking": { - "n": 96, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.28125, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - }, - "unknown_rejection": { - "n": 64, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.1875, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - } - }, - "by_type": { - "choice": { - "n": 752, - "expected_random_accuracy": 0.22340425531914893, - "fixed_tie_accuracy": 0.17952127659574468, - "nll": 1.5951419582900326, - "brier": 0.776595744680851, - "normalized_rps": null - }, - "noul": { - "n": 64, - "expected_random_accuracy": 0.5, - "fixed_tie_accuracy": 0.375, - "nll": 0.6931471805599453, - "brier": 0.5, - "normalized_rps": null - }, - "score": { - "n": 64, - "expected_random_accuracy": 0.2, - "fixed_tie_accuracy": 0.0625, - "nll": 1.6094379124341003, - "brier": 0.8, - "normalized_rps": 0.26093750000000004 - } - }, - "tie_policy": "Choice/Score first option; Noul p_true=.5 returns true per native threshold. Expected random accuracy is analytic 1/K and separate.", - "train_prior": "Omitted: arbitrary dynamic candidate IDs/semantics do not define one transferable TRAIN label prior. No prior fitted on final." - }, - "fresh": { - "overall": { - "n": 880, - "expected_random_accuracy": 0.24123376623376622, - "fixed_tie_accuracy": 0.2409090909090909, - "nll": 1.6363281844404418, - "brier": 0.7587662337662339, - "normalized_rps": 0.2 - }, - "family_macro_expected_random_accuracy": 0.25857142857142856, - "family_macro_fixed_tie_accuracy": 0.25833333333333336, - "families": { - "canonical_record_identity": { - "n": 80, - "expected_random_accuracy": 0.5, - "fixed_tie_accuracy": 0.5, - "nll": 0.6931471805599453, - "brier": 0.5, - "normalized_rps": null - }, - "capacitated_assignment": { - "n": 80, - "expected_random_accuracy": 0.5, - "fixed_tie_accuracy": 0.5, - "nll": 0.6931471805599453, - "brier": 0.5, - "normalized_rps": null - }, - "constraint_assignment": { - "n": 80, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.25, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - }, - "massive_en": { - "n": 120, - "expected_random_accuracy": 0.06785714285714285, - "fixed_tie_accuracy": 0.06666666666666667, - "nll": 2.690801983230472, - "brier": 0.9321428571428572, - "normalized_rps": null - }, - "massive_zh": { - "n": 120, - "expected_random_accuracy": 0.06785714285714285, - "fixed_tie_accuracy": 0.06666666666666667, - "nll": 2.690801983230472, - "brier": 0.9321428571428572, - "normalized_rps": null - }, - "multiset_reconciliation": { - "n": 80, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.25, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - }, - "ordinal_service_loss": { - "n": 80, - "expected_random_accuracy": 0.2, - "fixed_tie_accuracy": 0.2, - "nll": 1.6094379124341003, - "brier": 0.8, - "normalized_rps": 0.2 - }, - "paraconsistent_rule_closure": { - "n": 80, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.25, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - }, - "temporal_exclusion": { - "n": 80, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.25, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - }, - "transaction_recovery": { - "n": 80, - "expected_random_accuracy": 0.25, - "fixed_tie_accuracy": 0.25, - "nll": 1.3862943611198906, - "brier": 0.75, - "normalized_rps": null - } - }, - "by_type": { - "choice": { - "n": 640, - "expected_random_accuracy": 0.1816964285714286, - "fixed_tie_accuracy": 0.18125, - "nll": 1.8754847194113586, - "brier": 0.8183035714285714, - "normalized_rps": null - }, - "noul": { - "n": 160, - "expected_random_accuracy": 0.5, - "fixed_tie_accuracy": 0.5, - "nll": 0.6931471805599453, - "brier": 0.5, - "normalized_rps": null - }, - "score": { - "n": 80, - "expected_random_accuracy": 0.2, - "fixed_tie_accuracy": 0.2, - "nll": 1.6094379124341003, - "brier": 0.8, - "normalized_rps": 0.2 - } - }, - "tie_policy": "Choice/Score first option; Noul p_true=.5 returns true per native threshold. Expected random accuracy is analytic 1/K and separate.", - "train_prior": "Omitted: arbitrary dynamic candidate IDs/semantics do not define one transferable TRAIN label prior. No prior fitted on final." - } - }, - "train_prior_control": "Not defined across arbitrary dynamically generated candidate IDs; never fit a prior to heldout.", - "scope": "Old core880 is observed regression. The historical fresh880 key identifies the independent construction panel; consult global exposure provenance for whether it was unobserved at this candidate freeze. JOINT is equally weighted across20 task families; native220-answer panels are separate descriptive diagnostics.", - "exposure": { - "global_first_unseal": { - "policy_sha256": "359de2b6b81be91a66c6c538b3655bed06f4728357328cd318b1a765e64565cd", - "first_global_unseal_at": "2026-09-21T16:40:35.605010+00:00", - "first_candidate": "Decision-1.0-Nox-stage4-selected", - "first_candidate_bundle_manifest_sha256": "a8cfd42bc908fc93d5a2a20be527d09704b6e3f6c1217f27307c32cc2d822971", - "first_candidate_dispatch_sha256": "4790cae3c0bac5b756ba76e07103141a9cf671c941cbd59a1fc7343a9e528de1", - "meaning": "Fresh-v3 is observed regression for all subsequent unfrozen candidates. This first exposure record is never replaced." - }, - "this_evaluation_at": "2026-09-21T16:40:35.605010+00:00", - "candidate_frozen_at": "2026-09-21T16:37:26.312364+00:00", - "candidate_frozen_before_first_global_unseal": true, - "candidate_is_first_unseal_identity": true, - "confirmation_interpretation": "Independent confirmation for this already-frozen candidate, subject to documented source exposure limits.", - "historical_panel_key": "fresh", - "historical_key_is_not_a_new_freshness_claim": true - }, - "evaluation_receipt_sha256": "927c92121c2df534f2ba12523e466ecebd8bd8936f949c7792de3e747c92a47c", - "models": { - "Decision-1.0-Sol": { - "old": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 592, - "accuracy": 0.6727272727272727, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.5053115978004115, - "ece_fixed10": 0.21064225162298772, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2916406051570087 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 5, - "accuracy": 0.2, - "confidence": 0.36697886654313405 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 46, - "accuracy": 0.1956521739130435, - "confidence": 0.45570099942772246 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 58, - "accuracy": 0.3620689655172414, - "confidence": 0.5576252373499052 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 67, - "accuracy": 0.44776119402985076, - "confidence": 0.6574267938757535 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 46, - "accuracy": 0.5, - "confidence": 0.747891662317026 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 58, - "accuracy": 0.3275862068965517, - "confidence": 0.8616683949123229 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 599, - "accuracy": 0.8163606010016694, - "confidence": 0.9908293125028603 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.26384749554805875, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.26384749554805875, - "native_vs_expected_score_mean_residual": -2.7755575615628914e-17, - "native_vs_expected_score_normalized_mean_residual": -6.938893903907228e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 6.938893903907228e-18, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7291666666666666, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.8958333333333334 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 106, - "accuracy": 0.828125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.28826673836063604, - "ece_fixed10": 0.1180374270070608, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.0, - "confidence": 0.45022153854370117 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.6, - "confidence": 0.553354598546749 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.6476862470812134 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.7361337886049685 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.5, - "confidence": 0.8917180734079536 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 110, - "accuracy": 0.8818181818181818, - "confidence": 0.9897218231127872 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.828125, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 27, - "accuracy": 0.421875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.7848910679565373, - "ece_fixed10": 0.39660009279134156, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 6, - "accuracy": 0.5, - "confidence": 0.5620877347167449 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 7, - "accuracy": 0.14285714285714285, - "confidence": 0.6579369442889863 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 12, - "accuracy": 0.4166666666666667, - "confidence": 0.7535542001705245 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.2777777777777778, - "confidence": 0.851582323188389 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 21, - "accuracy": 0.6190476190476191, - "confidence": 0.9539613667088186 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.34375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.84375 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 105, - "accuracy": 0.9375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.1043970843488384, - "ece_fixed10": 0.04630639103284299, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.5, - "confidence": 0.4736528616816479 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 1.0, - "confidence": 0.5889406237848638 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6684377701710751 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.7666719528803786 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8686310934852569 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 101, - "accuracy": 0.9702970297029703, - "confidence": 0.9973975827459748 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9375, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.0002740430083565339, - "ece_fixed10": 0.003113534208550428, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 0.9968864657914496 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 96, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 2.704486750865214e-07, - "ece_fixed10": 0.00011794967110267951, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 96, - "accuracy": 1.0, - "confidence": 0.9998820503288973 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 28, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.9956448936402504, - "ece_fixed10": 0.4115817330042363, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 8, - "accuracy": 0.625, - "confidence": 0.4385291120613849 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.5590626547549981 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 15, - "accuracy": 0.6, - "confidence": 0.650477529336362 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.5, - "confidence": 0.7373985953743358 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.8605369855822738 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 30, - "accuracy": 0.26666666666666666, - "confidence": 0.9796756216583755 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.26384749554805875, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.26384749554805875, - "native_vs_expected_score_mean_residual": -2.7755575615628914e-17, - "native_vs_expected_score_normalized_mean_residual": -6.938893903907228e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 6.938893903907228e-18, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.96875 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 44, - "accuracy": 0.4583333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 1.071427121673764, - "ece_fixed10": 0.5355851163184129, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 0.0, - "confidence": 0.7897772458009645 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.0, - "confidence": 0.8746577367566413 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 93, - "accuracy": 0.4731182795698925, - "confidence": 0.9986782628736924 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4583333333333333, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 52, - "accuracy": 0.5416666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.621912564450967, - "ece_fixed10": 0.21100123150556507, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 8, - "accuracy": 0.25, - "confidence": 0.48576188711404356 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 14, - "accuracy": 0.14285714285714285, - "confidence": 0.5539485869513966 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.6111111111111112, - "confidence": 0.6654355661958973 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 14, - "accuracy": 0.6428571428571429, - "confidence": 0.7394231917592037 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 16, - "accuracy": 0.5, - "confidence": 0.856385502975958 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 26, - "accuracy": 0.7692307692307693, - "confidence": 0.9454926917896931 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5416666666666666, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9583333333333334 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 18, - "accuracy": 0.1875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 1.0953875652240594, - "ece_fixed10": 0.4649764714286558, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2916406051570087 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 5, - "accuracy": 0.2, - "confidence": 0.36697886654313405 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 20, - "accuracy": 0.05, - "confidence": 0.44912133269295584 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 16, - "accuracy": 0.1875, - "confidence": 0.5616842485995722 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 19, - "accuracy": 0.2631578947368421, - "confidence": 0.6593113730263791 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 11, - "accuracy": 0.45454545454545453, - "confidence": 0.7486772471729431 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 14, - "accuracy": 0.14285714285714285, - "confidence": 0.8732757987411874 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 10, - "accuracy": 0.1, - "confidence": 0.9553604699045805 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.125, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.625 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 52, - "accuracy": 0.8125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.2249048081225762, - "ece_fixed10": 0.07215535881816537, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 6, - "accuracy": 0.0, - "confidence": 0.4562904209936655 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.5476087771386519 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.625562107921439 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 48, - "accuracy": 1.0, - "confidence": 0.9932050138285149 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.75, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.75 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 406, - "accuracy": 0.725, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.42518428679693787, - "ece_fixed10": 0.1699993102506835, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2916406051570087 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.5, - "confidence": 0.3548010540706127 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 30, - "accuracy": 0.23333333333333334, - "confidence": 0.45872641450029555 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 39, - "accuracy": 0.48717948717948717, - "confidence": 0.5552239543779027 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 34, - "accuracy": 0.4411764705882353, - "confidence": 0.6535348995347615 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 20, - "accuracy": 0.4, - "confidence": 0.7494815883079738 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 25, - "accuracy": 0.4, - "confidence": 0.8640278165905684 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 409, - "accuracy": 0.8459657701711492, - "confidence": 0.9911773467543138 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2697671117813874, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.2697671117813874, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7357142857142858, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.9482758620689655 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 186, - "accuracy": 0.58125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.6455343920564904, - "ece_fixed10": 0.28358238584863726, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 3, - "accuracy": 0.0, - "confidence": 0.3750974081914816 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 16, - "accuracy": 0.125, - "confidence": 0.45002834616664805 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 19, - "accuracy": 0.10526315789473684, - "confidence": 0.562554186608226 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 33, - "accuracy": 0.45454545454545453, - "confidence": 0.6614366244088966 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 26, - "accuracy": 0.5769230769230769, - "confidence": 0.7466686423239893 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 33, - "accuracy": 0.2727272727272727, - "confidence": 0.8598809542469855 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 190, - "accuracy": 0.7526315789473684, - "confidence": 0.9900801229826263 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2579278793147301, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.2579278793147301, - "native_vs_expected_score_mean_residual": -5.551115123125783e-17, - "native_vs_expected_score_normalized_mean_residual": -1.3877787807814457e-17, - "native_vs_expected_score_normalized_mean_abs_residual": 1.3877787807814457e-17, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5166666666666667, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.9482758620689655 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 537, - "accuracy": 0.714095744680851, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.4397871070773507, - "ece_fixed10": 0.18339937091392294, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2916406051570087 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 5, - "accuracy": 0.2, - "confidence": 0.36697886654313405 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 38, - "accuracy": 0.10526315789473684, - "confidence": 0.4593161336101094 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 46, - "accuracy": 0.30434782608695654, - "confidence": 0.5568556832492184 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 45, - "accuracy": 0.4444444444444444, - "confidence": 0.6596638586579366 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 32, - "accuracy": 0.53125, - "confidence": 0.7464240273058822 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 37, - "accuracy": 0.35135135135135137, - "confidence": 0.8666668683993759 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 548, - "accuracy": 0.8540145985401459, - "confidence": 0.9928527387565271 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7880434782608695, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.890625 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 27, - "accuracy": 0.421875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.7848910679565373, - "ece_fixed10": 0.39660009279134156, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 6, - "accuracy": 0.5, - "confidence": 0.5620877347167449 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 7, - "accuracy": 0.14285714285714285, - "confidence": 0.6579369442889863 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 12, - "accuracy": 0.4166666666666667, - "confidence": 0.7535542001705245 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.2777777777777778, - "confidence": 0.851582323188389 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 21, - "accuracy": 0.6190476190476191, - "confidence": 0.9539613667088186 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.34375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.84375 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 28, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.9956448936402504, - "ece_fixed10": 0.4115817330042363, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 8, - "accuracy": 0.625, - "confidence": 0.4385291120613849 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.5590626547549981 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 15, - "accuracy": 0.6, - "confidence": 0.650477529336362 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.5, - "confidence": 0.7373985953743358 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.8605369855822738 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 30, - "accuracy": 0.26666666666666666, - "confidence": 0.9796756216583755 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.26384749554805875, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.26384749554805875, - "native_vs_expected_score_mean_residual": -2.7755575615628914e-17, - "native_vs_expected_score_normalized_mean_residual": -6.938893903907228e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 6.938893903907228e-18, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.96875 - } - } - }, - "family_macro_accuracy": 0.6625, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.714095744680851, - "brier": 0.4397871070773507, - "nll": 1.4093821957156258, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.421875, - "brier": 0.7848910679565373, - "nll": 1.1098509656235067, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.1, - "recall_true": 0.9583333333333334, - "balanced_accuracy": 0.5291666666666667, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.4375, - "brier": 0.9956448936402504, - "nll": 2.641703942163262, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.19456059540016718, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.26384749554805875, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.26384749554805875, - "native_vs_probability_expected_normalized_abs_residual": 6.938893903907228e-18 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 68, - "truncation_unknown_requests": 0, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 23, - "accuracy_all_requested": 0.5476190476190477, - "accuracy_accepted": 0.5476190476190477, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 9, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 80, - "accuracy_all_requested": 0.4878048780487805, - "accuracy_accepted": 0.4878048780487805, - "native_normalized_score_mae": 0.6061463205904929, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 4, - "accuracy_all_requested": 0.8, - "accuracy_accepted": 0.8, - "native_normalized_score_mae": 0.6137113922806229, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 5, - "accuracy_all_requested": 0.8333333333333334, - "accuracy_accepted": 0.8333333333333334, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 116, - "valid_questions": 220, - "accuracy_all_requested": 0.5272727272727272, - "isolation_max_probability_delta": 0.012504371317154761 - }, - "raw_T1_core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 592, - "accuracy": 0.6727272727272727, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.47438602420941745, - "ece_fixed10": 0.1762908743602762, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.2873237256111184 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 28, - "accuracy": 0.14285714285714285, - "confidence": 0.37482197208572426 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 53, - "accuracy": 0.2830188679245283, - "confidence": 0.45974826062222224 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 81, - "accuracy": 0.4444444444444444, - "confidence": 0.5471241618643258 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 55, - "accuracy": 0.45454545454545453, - "confidence": 0.6411488149636897 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 57, - "accuracy": 0.3157894736842105, - "confidence": 0.7573205049639931 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 69, - "accuracy": 0.5362318840579711, - "confidence": 0.8486467822368066 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 535, - "accuracy": 0.8523364485981308, - "confidence": 0.9898033741412835 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2713913830780261, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.2713913830780261, - "native_vs_expected_score_mean_residual": -2.7755575615628914e-17, - "native_vs_expected_score_normalized_mean_residual": -6.938893903907228e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 6.938893903907228e-18, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7291666666666666, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.8958333333333334 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 106, - "accuracy": 0.828125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.2781883720834964, - "ece_fixed10": 0.10697406107439093, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.3804080557579961 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.46452217721928046 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 7, - "accuracy": 0.7142857142857143, - "confidence": 0.5446067090889836 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 5, - "accuracy": 0.4, - "confidence": 0.6473994074810557 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.5, - "confidence": 0.7821528598999217 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 0.6363636363636364, - "confidence": 0.8427877861970415 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 99, - "accuracy": 0.9090909090909091, - "confidence": 0.9864543817005708 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.828125, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 27, - "accuracy": 0.421875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.6861289258656927, - "ece_fixed10": 0.33739840251037934, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.5566185992414401 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.18181818181818182, - "confidence": 0.6565709677608464 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 18, - "accuracy": 0.3333333333333333, - "confidence": 0.7488503986272582 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.4444444444444444, - "confidence": 0.8556962695294046 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 8, - "accuracy": 0.875, - "confidence": 0.9349762119127589 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.34375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.84375 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 105, - "accuracy": 0.9375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.10263514892187131, - "ece_fixed10": 0.04847182636085563, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.3671470565388582 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.4515374368045761 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.75, - "confidence": 0.5517381553487731 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.5, - "confidence": 0.6530999924178935 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.5, - "confidence": 0.7769065446138286 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 1.0, - "confidence": 0.8337008047122708 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 99, - "accuracy": 0.9696969696969697, - "confidence": 0.9935482514560607 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9375, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.001661962588481836, - "ece_fixed10": 0.009481499066982688, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 4, - "accuracy": 1.0, - "confidence": 0.8755262014893161 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 60, - "accuracy": 1.0, - "confidence": 0.9981846542292641 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 96, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 1.5928224131446337e-05, - "ece_fixed10": 0.0013259414472956976, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 96, - "accuracy": 1.0, - "confidence": 0.9986740585527043 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 28, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.95444914282133, - "ece_fixed10": 0.41878607823566494, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.3809384742713487 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 4, - "accuracy": 0.75, - "confidence": 0.4467318552764752 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.6470588235294118, - "confidence": 0.5471511425461935 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.6384848801505555 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 5, - "accuracy": 0.2, - "confidence": 0.7794958727886201 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.0, - "confidence": 0.8482710335935036 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 22, - "accuracy": 0.36363636363636365, - "confidence": 0.9843097711553362 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2713913830780261, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.2713913830780261, - "native_vs_expected_score_mean_residual": -2.7755575615628914e-17, - "native_vs_expected_score_normalized_mean_residual": -6.938893903907228e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 6.938893903907228e-18, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.96875 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 44, - "accuracy": 0.4583333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 1.0571802931711018, - "ece_fixed10": 0.5264062718528175, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6622324162558724 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.0, - "confidence": 0.7617138663446841 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 0.0, - "confidence": 0.8890086350252978 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 92, - "accuracy": 0.4782608695652174, - "confidence": 0.994134057759782 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4583333333333333, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 52, - "accuracy": 0.5416666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.5763060452616744, - "ece_fixed10": 0.14807547746560823, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 15, - "accuracy": 0.13333333333333333, - "confidence": 0.46014681168394467 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 18, - "accuracy": 0.5, - "confidence": 0.5479843567987072 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 21, - "accuracy": 0.6190476190476191, - "confidence": 0.6311840901493353 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 15, - "accuracy": 0.4666666666666667, - "confidence": 0.7520457332366229 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 17, - "accuracy": 0.6470588235294118, - "confidence": 0.835289845897807 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 10, - "accuracy": 1.0, - "confidence": 0.9286154032885612 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5416666666666666, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9583333333333334 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 18, - "accuracy": 0.1875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.9877375228722561, - "ece_fixed10": 0.39031380774919155, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.2873237256111184 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 17, - "accuracy": 0.0, - "confidence": 0.3718465468081864 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 19, - "accuracy": 0.21052631578947367, - "confidence": 0.4655719753231786 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 23, - "accuracy": 0.17391304347826086, - "confidence": 0.5461117198064056 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.5, - "confidence": 0.6387628504624147 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 13, - "accuracy": 0.15384615384615385, - "confidence": 0.7590961310344337 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.16666666666666666, - "confidence": 0.8371189608701117 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 4, - "accuracy": 0.0, - "confidence": 0.9402067346251132 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.125, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.625 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 52, - "accuracy": 0.8125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.21271986252997196, - "ece_fixed10": 0.0704800938179135, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.37974974523240773 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 11, - "accuracy": 0.36363636363636365, - "confidence": 0.4533233379099903 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.0, - "confidence": 0.520810910536703 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 1.0, - "confidence": 0.892982090826378 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 45, - "accuracy": 1.0, - "confidence": 0.9804181480502049 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.75, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.75 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 406, - "accuracy": 0.725, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.4028078846818651, - "ece_fixed10": 0.13966499669647112, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.2873237256111184 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 15, - "accuracy": 0.13333333333333333, - "confidence": 0.3771397925055477 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 37, - "accuracy": 0.3783783783783784, - "confidence": 0.45955789462847385 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 47, - "accuracy": 0.5106382978723404, - "confidence": 0.5467877263906958 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 23, - "accuracy": 0.34782608695652173, - "confidence": 0.643765442618408 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 27, - "accuracy": 0.3333333333333333, - "confidence": 0.7611807529410287 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 39, - "accuracy": 0.6410256410256411, - "confidence": 0.8511128030482453 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 370, - "accuracy": 0.8729729729729729, - "confidence": 0.9888524287663077 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.27515328585854965, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.27515328585854965, - "native_vs_expected_score_mean_residual": -4.163336342344337e-17, - "native_vs_expected_score_normalized_mean_residual": -1.0408340855860843e-17, - "native_vs_expected_score_normalized_mean_abs_residual": 1.0408340855860843e-17, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7357142857142858, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.9482758620689655 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 186, - "accuracy": 0.58125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.599647768382634, - "ece_fixed10": 0.24038616027193516, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 13, - "accuracy": 0.15384615384615385, - "confidence": 0.37214756390900494 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 16, - "accuracy": 0.0625, - "confidence": 0.46018848198276524 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 34, - "accuracy": 0.35294117647058826, - "confidence": 0.5475892344308143 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 32, - "accuracy": 0.53125, - "confidence": 0.6392681138368609 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 30, - "accuracy": 0.3, - "confidence": 0.7538462817846607 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 30, - "accuracy": 0.4, - "confidence": 0.8454409551819361 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 165, - "accuracy": 0.806060606060606, - "confidence": 0.9919357971033502 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.26762948029750255, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.2676294802975026, - "native_vs_expected_score_mean_residual": -1.3877787807814457e-17, - "native_vs_expected_score_normalized_mean_residual": -3.469446951953614e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 3.469446951953614e-18, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5166666666666667, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.9482758620689655 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 537, - "accuracy": 0.714095744680851, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.41550891610148655, - "ece_fixed10": 0.15323142332541886, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.2873237256111184 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 21, - "accuracy": 0.0, - "confidence": 0.3727831380238495 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 49, - "accuracy": 0.24489795918367346, - "confidence": 0.46081082432391585 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 55, - "accuracy": 0.38181818181818183, - "confidence": 0.5455621871736753 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 41, - "accuracy": 0.5365853658536586, - "confidence": 0.6372060862727306 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 34, - "accuracy": 0.3235294117647059, - "confidence": 0.7585435954033485 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 45, - "accuracy": 0.6444444444444445, - "confidence": 0.8458770871388744 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 505, - "accuracy": 0.8732673267326733, - "confidence": 0.9909112485244895 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7880434782608695, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.890625 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 27, - "accuracy": 0.421875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.6861289258656927, - "ece_fixed10": 0.33739840251037934, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.5566185992414401 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.18181818181818182, - "confidence": 0.6565709677608464 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 18, - "accuracy": 0.3333333333333333, - "confidence": 0.7488503986272582 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.4444444444444444, - "confidence": 0.8556962695294046 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 8, - "accuracy": 0.875, - "confidence": 0.9349762119127589 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.34375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.84375 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 28, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.95444914282133, - "ece_fixed10": 0.41878607823566494, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.3809384742713487 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 4, - "accuracy": 0.75, - "confidence": 0.4467318552764752 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.6470588235294118, - "confidence": 0.5471511425461935 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.6384848801505555 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 5, - "accuracy": 0.2, - "confidence": 0.7794958727886201 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.0, - "confidence": 0.8482710335935036 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 22, - "accuracy": 0.36363636363636365, - "confidence": 0.9843097711553362 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2713913830780261, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.2713913830780261, - "native_vs_expected_score_mean_residual": -2.7755575615628914e-17, - "native_vs_expected_score_normalized_mean_residual": -6.938893903907228e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 6.938893903907228e-18, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.96875 - } - } - }, - "family_macro_accuracy": 0.6625, - "reliability_contract_pass": true - }, - "raw_T1_typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.714095744680851, - "brier": 0.41550891610148655, - "nll": 1.0767177741285114, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.421875, - "brier": 0.6861289258656927, - "nll": 0.9188761046119444, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.1, - "recall_true": 0.9583333333333334, - "balanced_accuracy": 0.5291666666666667, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.4375, - "brier": 0.95444914282133, - "nll": 2.0355049570481776, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.19000176526823098, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.2713913830780261, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.2713913830780261, - "native_vs_probability_expected_normalized_abs_residual": 6.938893903907228e-18 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - } - }, - "fresh": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 431, - "accuracy": 0.48977272727272725, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.6223922877530715, - "ece_fixed10": 0.17321313998999, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 83, - "accuracy": 0.21686746987951808, - "confidence": 0.255322756801748 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 105, - "accuracy": 0.24761904761904763, - "confidence": 0.3617328854794225 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 154, - "accuracy": 0.21428571428571427, - "confidence": 0.44800143631831724 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 86, - "accuracy": 0.36046511627906974, - "confidence": 0.544861074795505 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 56, - "accuracy": 0.4107142857142857, - "confidence": 0.6466142425501442 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 44, - "accuracy": 0.5681818181818182, - "confidence": 0.7528318091133201 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 59, - "accuracy": 0.559322033898305, - "confidence": 0.8543721914644231 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 293, - "accuracy": 0.825938566552901, - "confidence": 0.9851897313845642 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3029841952119829, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.3029841952119829, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4954128440366973, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.95 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 47, - "accuracy": 0.5875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6534386529128547, - "ece_fixed10": 0.28503657731413684, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.75, - "confidence": 0.5534700286055504 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.45454545454545453, - "confidence": 0.6543047703277473 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 13, - "accuracy": 0.6153846153846154, - "confidence": 0.756148575664867 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 12, - "accuracy": 0.6666666666666666, - "confidence": 0.8524651348875686 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 36, - "accuracy": 0.5555555555555556, - "confidence": 0.9714950232798899 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.15, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 40, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8379966553497242, - "ece_fixed10": 0.40006821698017003, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.770540574464704 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 30, - "accuracy": 0.5333333333333333, - "confidence": 0.8609812417521517 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 43, - "accuracy": 0.46511627906976744, - "confidence": 0.9484240949906074 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 13, - "accuracy": 0.1625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8666126577828821, - "ece_fixed10": 0.29894820824606855, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 5, - "accuracy": 0.0, - "confidence": 0.28001110583971434 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 22, - "accuracy": 0.18181818181818182, - "confidence": 0.35875503316837176 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 30, - "accuracy": 0.16666666666666666, - "confidence": 0.4463583804450982 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 18, - "accuracy": 0.1111111111111111, - "confidence": 0.5438789730160175 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.0, - "confidence": 0.6427256029665375 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 1.0, - "confidence": 0.7427796678790695 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 108, - "accuracy": 0.9, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.15741454604446367, - "ece_fixed10": 0.07032179543907437, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.0, - "confidence": 0.5556382195527799 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.6154501831273078 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.7393184595775756 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.8527166809613788 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 104, - "accuracy": 0.9519230769230769, - "confidence": 0.9966467297490323 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 107, - "accuracy": 0.8916666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.17489673213187343, - "ece_fixed10": 0.0778917029574959, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.45414476942895937 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.5342572476073886 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.6333956125733992 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 0.0, - "confidence": 0.7219111806175175 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.5, - "confidence": 0.8101294154246075 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 110, - "accuracy": 0.9363636363636364, - "confidence": 0.993211586173689 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8916666666666667, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 21, - "accuracy": 0.2625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8290708751027251, - "ece_fixed10": 0.18028195895267932, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 25, - "accuracy": 0.2, - "confidence": 0.3736966911979181 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 42, - "accuracy": 0.2857142857142857, - "confidence": 0.44840748353146587 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 12, - "accuracy": 0.3333333333333333, - "confidence": 0.5463114336221415 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.691287924479128 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 19, - "accuracy": 0.2375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7989900557273347, - "ece_fixed10": 0.018630706763695087, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 76, - "accuracy": 0.23684210526315788, - "confidence": 0.25273645500278313 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 4, - "accuracy": 0.25, - "confidence": 0.32062149022102193 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3029841952119829, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.3029841952119829, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 23, - "accuracy": 0.2875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8246402388767562, - "ece_fixed10": 0.17319469980908542, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 29, - "accuracy": 0.3103448275862069, - "confidence": 0.35504519245542765 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 26, - "accuracy": 0.2692307692307692, - "confidence": 0.44079677306925247 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 13, - "accuracy": 0.3076923076923077, - "confidence": 0.5450286815903225 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 10, - "accuracy": 0.3, - "confidence": 0.6503860504546146 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.0, - "confidence": 0.7546579692492643 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 28, - "accuracy": 0.35, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7689374922053708, - "ece_fixed10": 0.178457020584307, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.0, - "confidence": 0.2918813525674971 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 13, - "accuracy": 0.3076923076923077, - "confidence": 0.37185034934703765 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 29, - "accuracy": 0.27586206896551724, - "confidence": 0.454258322433382 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 11, - "accuracy": 0.36363636363636365, - "confidence": 0.545439258867543 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 13, - "accuracy": 0.38461538461538464, - "confidence": 0.6435007772585114 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 0.5, - "confidence": 0.744268718095758 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 4, - "accuracy": 0.75, - "confidence": 0.8414403382150784 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 25, - "accuracy": 0.3125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.768161620061633, - "ece_fixed10": 0.22011836638109167, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 12, - "accuracy": 0.25, - "confidence": 0.36117282350702085 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 26, - "accuracy": 0.038461538461538464, - "confidence": 0.44923089429033614 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 19, - "accuracy": 0.47368421052631576, - "confidence": 0.5413410839826384 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 14, - "accuracy": 0.5714285714285714, - "confidence": 0.6434694818621299 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.75, - "confidence": 0.7636796698516521 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.2, - "confidence": 0.8493240311415567 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 216, - "accuracy": 0.4909090909090909, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.600496649307899, - "ece_fixed10": 0.15935712423808068, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 40, - "accuracy": 0.225, - "confidence": 0.2602213582582913 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 55, - "accuracy": 0.23636363636363636, - "confidence": 0.35706916840908 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 72, - "accuracy": 0.20833333333333334, - "confidence": 0.44994407725934743 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 54, - "accuracy": 0.3888888888888889, - "confidence": 0.5474486776455124 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 29, - "accuracy": 0.3793103448275862, - "confidence": 0.6523485333090463 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 30, - "accuracy": 0.5666666666666667, - "confidence": 0.7581165865603248 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 41, - "accuracy": 0.5853658536585366, - "confidence": 0.8534919726350196 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 119, - "accuracy": 0.8907563025210085, - "confidence": 0.9870293947528226 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3073117079999136, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.3073117079999136, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.518348623853211, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 215, - "accuracy": 0.48863636363636365, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.644287926198244, - "ece_fixed10": 0.1870691557418992, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 43, - "accuracy": 0.20930232558139536, - "confidence": 0.25076591823752165 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 50, - "accuracy": 0.26, - "confidence": 0.36686297425679926 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 82, - "accuracy": 0.21951219512195122, - "confidence": 0.44629570280912 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 32, - "accuracy": 0.3125, - "confidence": 0.5404944949861178 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 27, - "accuracy": 0.4444444444444444, - "confidence": 0.640455189512805 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 14, - "accuracy": 0.5714285714285714, - "confidence": 0.7415072860125961 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.5, - "confidence": 0.8563771343536198 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 174, - "accuracy": 0.7816091954022989, - "confidence": 0.9839315708051231 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2986566824240522, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.2986566824240522, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5137614678899083, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 325, - "accuracy": 0.5078125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.5694862251617342, - "ece_fixed10": 0.15511437913884718, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 7, - "accuracy": 0.0, - "confidence": 0.28340260490479513 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 101, - "accuracy": 0.24752475247524752, - "confidence": 0.3633610595490621 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 154, - "accuracy": 0.21428571428571427, - "confidence": 0.44800143631831724 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 78, - "accuracy": 0.32051282051282054, - "confidence": 0.543978105173962 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 45, - "accuracy": 0.4, - "confidence": 0.6447343357600636 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 24, - "accuracy": 0.5416666666666666, - "confidence": 0.745870170670412 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 17, - "accuracy": 0.5294117647058824, - "confidence": 0.8440553191285638 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 214, - "accuracy": 0.9439252336448598, - "confidence": 0.9948810017430146 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6176470588235294, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.95 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 87, - "accuracy": 0.54375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.7457176541312895, - "ece_fixed10": 0.34255239714715346, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.75, - "confidence": 0.5534700286055504 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.45454545454545453, - "confidence": 0.6543047703277473 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 20, - "accuracy": 0.6, - "confidence": 0.76118577524481 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 42, - "accuracy": 0.5714285714285714, - "confidence": 0.8585480683622707 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 79, - "accuracy": 0.5063291139240507, - "confidence": 0.9589374294009134 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.075, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 19, - "accuracy": 0.2375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7989900557273347, - "ece_fixed10": 0.018630706763695087, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 76, - "accuracy": 0.23684210526315788, - "confidence": 0.25273645500278313 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 4, - "accuracy": 0.25, - "confidence": 0.32062149022102193 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3029841952119829, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.3029841952119829, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.4491666666666667, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.5078125, - "brier": 0.5694862251617342, - "nll": 1.1646171015896791, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.54375, - "brier": 0.7457176541312895, - "nll": 1.2560858977530787, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.425, - "recall_true": 0.6625, - "balanced_accuracy": 0.54375, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.2375, - "brier": 0.7989900557273347, - "nll": 1.6078474985400462, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.19829673394114097, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.3029841952119829, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.3029841952119829, - "native_vs_probability_expected_normalized_abs_residual": 0.0 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 68, - "truncation_unknown_requests": 0, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 8, - "accuracy_all_requested": 0.19047619047619047, - "accuracy_accepted": 0.19047619047619047, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 106, - "accuracy_all_requested": 0.6463414634146342, - "accuracy_accepted": 0.6463414634146342, - "native_normalized_score_mae": 0.2579293511021405, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 2, - "accuracy_all_requested": 0.4, - "accuracy_accepted": 0.4, - "native_normalized_score_mae": 0.04831638270894334, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 5, - "accuracy_all_requested": 0.8333333333333334, - "accuracy_accepted": 0.8333333333333334, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 2, - "accuracy_all_requested": 0.3333333333333333, - "accuracy_accepted": 0.3333333333333333, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 1, - "accuracy_all_requested": 0.16666666666666666, - "accuracy_accepted": 0.16666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 116, - "valid_questions": 220, - "accuracy_all_requested": 0.5272727272727272, - "isolation_max_probability_delta": 0.0036232225542985885 - }, - "raw_T1_core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 431, - "accuracy": 0.48977272727272725, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.5894705875611461, - "ece_fixed10": 0.12684444494015, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 94, - "accuracy": 0.2127659574468085, - "confidence": 0.24558290407880662 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 185, - "accuracy": 0.2594594594594595, - "confidence": 0.35525741886624757 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 139, - "accuracy": 0.23741007194244604, - "confidence": 0.4398378126442102 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 64, - "accuracy": 0.4375, - "confidence": 0.5431548985574537 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 43, - "accuracy": 0.627906976744186, - "confidence": 0.6450795931587001 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 48, - "accuracy": 0.5416666666666666, - "confidence": 0.7482068263132389 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 63, - "accuracy": 0.49206349206349204, - "confidence": 0.8475803900814785 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 244, - "accuracy": 0.8934426229508197, - "confidence": 0.9871574256739893 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30206243730601, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.30206243730601, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4954128440366973, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.95 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 47, - "accuracy": 0.5875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6075338628899687, - "ece_fixed10": 0.23581712928977971, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 10, - "accuracy": 0.7, - "confidence": 0.5484593744747228 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 16, - "accuracy": 0.5625, - "confidence": 0.6371672101673301 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 13, - "accuracy": 0.46153846153846156, - "confidence": 0.7377057114572303 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 14, - "accuracy": 0.5714285714285714, - "confidence": 0.837880270373618 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 27, - "accuracy": 0.6296296296296297, - "confidence": 0.9568440996695436 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.15, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 40, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7429056360219671, - "ece_fixed10": 0.33370583431076467, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.6875695654222436 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 24, - "accuracy": 0.6666666666666666, - "confidence": 0.7582493388037667 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 36, - "accuracy": 0.3888888888888889, - "confidence": 0.8496782703171676 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 17, - "accuracy": 0.5294117647058824, - "confidence": 0.9321974226991772 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 13, - "accuracy": 0.1625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8171282387592947, - "ece_fixed10": 0.23686846903564204, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 7, - "accuracy": 0.0, - "confidence": 0.27778810367945755 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 37, - "accuracy": 0.1891891891891892, - "confidence": 0.3489180733690848 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 31, - "accuracy": 0.12903225806451613, - "confidence": 0.43752005105877184 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.25, - "confidence": 0.5386475728634614 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.6227197918367494 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 108, - "accuracy": 0.9, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.14945677829749596, - "ece_fixed10": 0.07153657551742953, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.467976593582882 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.25, - "confidence": 0.5578461731331754 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.875, - "confidence": 0.6619699759601577 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.5, - "confidence": 0.74304917176894 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.5, - "confidence": 0.8470233116505685 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 101, - "accuracy": 0.9603960396039604, - "confidence": 0.9951935076562408 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 107, - "accuracy": 0.8916666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.1629063491974198, - "ece_fixed10": 0.05820071432282314, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.3631070880189737 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.4772528691283689 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.5, - "confidence": 0.548952003653114 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6433684165297375 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 1.0, - "confidence": 0.704453557200211 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 0.7272727272727273, - "confidence": 0.8531614941075746 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 99, - "accuracy": 0.9595959595959596, - "confidence": 0.9966638455980746 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8916666666666667, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 21, - "accuracy": 0.2625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7933891373767017, - "ece_fixed10": 0.12800110332942918, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 49, - "accuracy": 0.2653061224489796, - "confidence": 0.36072985114701106 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 30, - "accuracy": 0.26666666666666666, - "confidence": 0.43340606764776385 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 0.0, - "confidence": 0.5621435307178777 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 19, - "accuracy": 0.2375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7977526914199731, - "ece_fixed10": 0.0012694457981493878, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 80, - "accuracy": 0.2375, - "confidence": 0.23876944579814938 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30206243730601, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.30206243730601, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 23, - "accuracy": 0.2875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7868633959369329, - "ece_fixed10": 0.11583875815149434, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.0, - "confidence": 0.29351164181145695 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 46, - "accuracy": 0.34782608695652173, - "confidence": 0.34698089129259246 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 19, - "accuracy": 0.21052631578947367, - "confidence": 0.44429819913236773 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 11, - "accuracy": 0.2727272727272727, - "confidence": 0.5403846768814113 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6276505693726873 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 28, - "accuracy": 0.35, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.732813525693984, - "ece_fixed10": 0.10061427823190855, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 3, - "accuracy": 0.0, - "confidence": 0.2854396544670362 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 28, - "accuracy": 0.32142857142857145, - "confidence": 0.3621890436022008 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 25, - "accuracy": 0.28, - "confidence": 0.43791050741216636 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 14, - "accuracy": 0.42857142857142855, - "confidence": 0.5390321767558247 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.625, - "confidence": 0.6329168053634893 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.5, - "confidence": 0.7469912357481687 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 25, - "accuracy": 0.3125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7372452838314122, - "ece_fixed10": 0.19195341401211785, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.2976891733878241 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 24, - "accuracy": 0.125, - "confidence": 0.3613070733773449 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 30, - "accuracy": 0.3, - "confidence": 0.44276628918344113 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 16, - "accuracy": 0.5, - "confidence": 0.540169457083491 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 1.0, - "confidence": 0.6501387122947637 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.0, - "confidence": 0.7387841417571902 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 216, - "accuracy": 0.4909090909090909, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.5699491941118726, - "ece_fixed10": 0.11062811113368827, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 48, - "accuracy": 0.22916666666666666, - "confidence": 0.2510051490480975 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 87, - "accuracy": 0.25287356321839083, - "confidence": 0.3514058388188963 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 78, - "accuracy": 0.2692307692307692, - "confidence": 0.44062753741065136 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 32, - "accuracy": 0.40625, - "confidence": 0.5426732567356176 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 29, - "accuracy": 0.5862068965517241, - "confidence": 0.6514589937539789 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 38, - "accuracy": 0.5526315789473685, - "confidence": 0.7467305631201064 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 25, - "accuracy": 0.52, - "confidence": 0.8282784795295639 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 103, - "accuracy": 0.9514563106796117, - "confidence": 0.9936532760003018 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30512565927728297, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.30512565927728297, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.518348623853211, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 215, - "accuracy": 0.48863636363636365, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.6089919810104197, - "ece_fixed10": 0.14830572562407202, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 46, - "accuracy": 0.1956521739130435, - "confidence": 0.2399249093282422 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 98, - "accuracy": 0.2653061224489796, - "confidence": 0.3586766787042023 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 61, - "accuracy": 0.19672131147540983, - "confidence": 0.43882800064777716 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 32, - "accuracy": 0.46875, - "confidence": 0.5436365403792898 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 14, - "accuracy": 0.7142857142857143, - "confidence": 0.6318651204970515 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 10, - "accuracy": 0.5, - "confidence": 0.7538166264471419 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 38, - "accuracy": 0.47368421052631576, - "confidence": 0.8602790154445803 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 141, - "accuracy": 0.851063829787234, - "confidence": 0.9824122300455482 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.29899921533473706, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.29899921533473706, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5137614678899083, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 325, - "accuracy": 0.5078125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.5419980341050874, - "ece_fixed10": 0.11254446001903412, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 14, - "accuracy": 0.07142857142857142, - "confidence": 0.2845169513968481 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 185, - "accuracy": 0.2594594594594595, - "confidence": 0.35525741886624757 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 139, - "accuracy": 0.23741007194244604, - "confidence": 0.4398378126442102 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 54, - "accuracy": 0.3888888888888889, - "confidence": 0.5421725882024039 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 24, - "accuracy": 0.7083333333333334, - "confidence": 0.6450432686200038 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 11, - "accuracy": 0.36363636363636365, - "confidence": 0.7387062984364609 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 13, - "accuracy": 0.6923076923076923, - "confidence": 0.8522171583449583 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 200, - "accuracy": 0.96, - "confidence": 0.9959213249374486 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6176470588235294, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.95 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 87, - "accuracy": 0.54375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.675219749455968, - "ece_fixed10": 0.2847614818002722, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 10, - "accuracy": 0.7, - "confidence": 0.5484593744747228 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 19, - "accuracy": 0.5263157894736842, - "confidence": 0.645125476786527 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 37, - "accuracy": 0.5945945945945946, - "confidence": 0.7510313075739025 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 50, - "accuracy": 0.44, - "confidence": 0.8463748303329737 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 44, - "accuracy": 0.5909090909090909, - "confidence": 0.947321519930993 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.075, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 19, - "accuracy": 0.2375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7977526914199731, - "ece_fixed10": 0.0012694457981493878, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 80, - "accuracy": 0.2375, - "confidence": 0.23876944579814938 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30206243730601, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.30206243730601, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.4491666666666667, - "reliability_contract_pass": true - }, - "raw_T1_typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.5078125, - "brier": 0.5419980341050874, - "nll": 1.058458921709375, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.54375, - "brier": 0.675219749455968, - "nll": 0.9984952217301725, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.425, - "recall_true": 0.6625, - "balanced_accuracy": 0.54375, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.2375, - "brier": 0.7977526914199731, - "nll": 1.6042897753425218, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.19811268178819194, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.30206243730601, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.30206243730601, - "native_vs_probability_expected_normalized_abs_residual": 0.0 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - } - } - }, - "Decision-1.0-Nox-stage4-selected": { - "old": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 715, - "accuracy": 0.8125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.2563815008904626, - "ece_fixed10": 0.0739416110507463, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2707927686448555 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 35, - "accuracy": 0.2857142857142857, - "confidence": 0.35895617117519457 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 42, - "accuracy": 0.38095238095238093, - "confidence": 0.45104707345599104 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 40, - "accuracy": 0.425, - "confidence": 0.5542498937223248 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 34, - "accuracy": 0.38235294117647056, - "confidence": 0.6552243845250667 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 44, - "accuracy": 0.6818181818181818, - "confidence": 0.7555876130865141 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 51, - "accuracy": 0.5686274509803921, - "confidence": 0.8482791417081645 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 633, - "accuracy": 0.9478672985781991, - "confidence": 0.9910500250459087 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.026444750263919684, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.026444750263919684, - "native_vs_expected_score_mean_residual": -4.336808689942018e-19, - "native_vs_expected_score_normalized_mean_residual": -1.0842021724855044e-19, - "native_vs_expected_score_normalized_mean_abs_residual": 1.0842021724855044e-19, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8379629629629629, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.9010416666666666 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 107, - "accuracy": 0.8359375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.29764301777177343, - "ece_fixed10": 0.13909436073213155, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.3607881963253021 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.4361547166682077 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.5701769496422767 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.75, - "confidence": 0.6319101799980144 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.7754910600222014 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.4, - "confidence": 0.8399881895643109 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 109, - "accuracy": 0.8807339449541285, - "confidence": 0.9940264586762672 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8359375, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 60, - "accuracy": 0.9375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.09990714935533926, - "ece_fixed10": 0.06762614415940169, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 1.0, - "confidence": 0.5526803735893346 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6723754305841407 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 5, - "accuracy": 0.8, - "confidence": 0.7247475886368111 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 10, - "accuracy": 0.9, - "confidence": 0.8620012102439383 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 45, - "accuracy": 1.0, - "confidence": 0.9703903742703215 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.90625, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9375 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 108, - "accuracy": 0.9642857142857143, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.07211104364728835, - "ece_fixed10": 0.040489447953924014, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 1.0, - "confidence": 0.6654121014756964 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 1.0, - "confidence": 0.8722821861948878 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 109, - "accuracy": 0.963302752293578, - "confidence": 0.9975956381650071 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9642857142857143, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 1.635549181010826e-05, - "ece_fixed10": 0.0008416754427679196, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 0.9991583245572321 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 96, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 4.541666105673645e-06, - "ece_fixed10": 0.0007898867724583924, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 96, - "accuracy": 1.0, - "confidence": 0.9992101132275416 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 57, - "accuracy": 0.890625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.14676055037988497, - "ece_fixed10": 0.05585647773251878, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.5, - "confidence": 0.44013240056015046 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6707324484051193 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.5, - "confidence": 0.7829707599135725 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8492298834561699 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 53, - "accuracy": 0.9811320754716981, - "confidence": 0.9893215431918578 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.026444750263919684, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.026444750263919684, - "native_vs_expected_score_mean_residual": -4.336808689942018e-19, - "native_vs_expected_score_normalized_mean_residual": -1.0842021724855044e-19, - "native_vs_expected_score_normalized_mean_abs_residual": 1.0842021724855044e-19, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.96875 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 53, - "accuracy": 0.5520833333333334, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.5968205852873751, - "ece_fixed10": 0.16203723448339385, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.4585333389145911 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 24, - "accuracy": 0.4583333333333333, - "confidence": 0.5560145204735099 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.6602055588621407 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 20, - "accuracy": 0.75, - "confidence": 0.7563879516625714 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 21, - "accuracy": 0.5714285714285714, - "confidence": 0.8409505522674198 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 13, - "accuracy": 0.6923076923076923, - "confidence": 0.9503734852449034 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4583333333333333, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.7083333333333334 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 72, - "accuracy": 0.75, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.39404822015093494, - "ece_fixed10": 0.19572449329433667, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.0, - "confidence": 0.4647363811222791 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.0, - "confidence": 0.5572288415887351 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6732768467993844 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.0, - "confidence": 0.7421913661704651 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.0, - "confidence": 0.839842494516763 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 82, - "accuracy": 0.8780487804878049, - "confidence": 0.9897346728136834 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.75, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9583333333333334 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 34, - "accuracy": 0.3541666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.6939145508445551, - "ece_fixed10": 0.16000928138391668, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2707927686448555 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 34, - "accuracy": 0.29411764705882354, - "confidence": 0.3589022880825444 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 27, - "accuracy": 0.4074074074074074, - "confidence": 0.4499339520670623 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.125, - "confidence": 0.5450225582171467 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 10, - "accuracy": 0.1, - "confidence": 0.6438871212327919 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.75, - "confidence": 0.7421553923461858 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.0, - "confidence": 0.839039385644848 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 9, - "accuracy": 0.8888888888888888, - "confidence": 0.9324061642198954 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.20833333333333334, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.6666666666666666 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.029899373167068814, - "ece_fixed10": 0.06179332037256891, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.42306804907655343 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 1.0, - "confidence": 0.5591212851022155 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 1.0, - "confidence": 0.6685578190814477 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 1.0, - "confidence": 0.7405166189178838 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 1.0, - "confidence": 0.8820063710526257 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 53, - "accuracy": 1.0, - "confidence": 0.9881356805403135 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 463, - "accuracy": 0.8267857142857142, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.23728701511466044, - "ece_fixed10": 0.08201449665489613, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2707927686448555 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 18, - "accuracy": 0.2777777777777778, - "confidence": 0.363839868243257 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 21, - "accuracy": 0.38095238095238093, - "confidence": 0.44554574079955395 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.11764705882352941, - "confidence": 0.5582174114957772 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.3888888888888889, - "confidence": 0.6549615254277411 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 22, - "accuracy": 0.7272727272727273, - "confidence": 0.7629926352145433 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 34, - "accuracy": 0.6176470588235294, - "confidence": 0.8550234414524168 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 429, - "accuracy": 0.9417249417249417, - "confidence": 0.9921121415750537 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.02638585336000586, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.02638585336000586, - "native_vs_expected_score_mean_residual": -8.673617379884035e-19, - "native_vs_expected_score_normalized_mean_residual": -2.168404344971009e-19, - "native_vs_expected_score_normalized_mean_abs_residual": 2.168404344971009e-19, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8404761904761905, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.9224137931034483 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 252, - "accuracy": 0.7875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.2897968509981164, - "ece_fixed10": 0.07431218778432906, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 17, - "accuracy": 0.29411764705882354, - "confidence": 0.35378519780901085 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 21, - "accuracy": 0.38095238095238093, - "confidence": 0.4565484061124281 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 23, - "accuracy": 0.6521739130434783, - "confidence": 0.5513173805854253 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 16, - "accuracy": 0.375, - "confidence": 0.6555201010095579 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 22, - "accuracy": 0.6363636363636364, - "confidence": 0.7481825909584849 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 17, - "accuracy": 0.47058823529411764, - "confidence": 0.8347905422196596 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 204, - "accuracy": 0.9607843137254902, - "confidence": 0.9888164564625593 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.02650364716783351, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.02650364716783351, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7777777777777778, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.9396551724137931 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 598, - "accuracy": 0.7952127659574468, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.2790279095752031, - "ece_fixed10": 0.08427107054707456, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2707927686448555 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 35, - "accuracy": 0.2857142857142857, - "confidence": 0.35895617117519457 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 40, - "accuracy": 0.375, - "confidence": 0.45159280710078303 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 38, - "accuracy": 0.39473684210526316, - "confidence": 0.5543325000451137 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 30, - "accuracy": 0.43333333333333335, - "confidence": 0.6530471105291249 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 35, - "accuracy": 0.6857142857142857, - "confidence": 0.7568638283705224 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 38, - "accuracy": 0.47368421052631576, - "confidence": 0.8445930124818025 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 535, - "accuracy": 0.9401869158878504, - "confidence": 0.9929589854630415 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8288043478260869, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 60, - "accuracy": 0.9375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.09990714935533926, - "ece_fixed10": 0.06762614415940169, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 1.0, - "confidence": 0.5526803735893346 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6723754305841407 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 5, - "accuracy": 0.8, - "confidence": 0.7247475886368111 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 10, - "accuracy": 0.9, - "confidence": 0.8620012102439383 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 45, - "accuracy": 1.0, - "confidence": 0.9703903742703215 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.90625, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9375 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 57, - "accuracy": 0.890625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.14676055037988497, - "ece_fixed10": 0.05585647773251878, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.5, - "confidence": 0.44013240056015046 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6707324484051193 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.5, - "confidence": 0.7829707599135725 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8492298834561699 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 53, - "accuracy": 0.9811320754716981, - "confidence": 0.9893215431918578 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.026444750263919684, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.026444750263919684, - "native_vs_expected_score_mean_residual": -4.336808689942018e-19, - "native_vs_expected_score_normalized_mean_residual": -1.0842021724855044e-19, - "native_vs_expected_score_normalized_mean_abs_residual": 1.0842021724855044e-19, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.96875 - } - } - }, - "family_macro_accuracy": 0.8284598214285713, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.7952127659574468, - "brier": 0.2790279095752031, - "nll": 0.5937347671420453, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.9375, - "brier": 0.09990714935533926, - "nll": 0.16940839586653852, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.9, - "recall_true": 1.0, - "balanced_accuracy": 0.95, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.890625, - "brier": 0.14676055037988497, - "nll": 0.23593972621600515, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.017881983250196833, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.026444750263919684, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.026444750263919684, - "native_vs_probability_expected_normalized_abs_residual": 1.0842021724855044e-19 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 68, - "truncation_unknown_requests": 0, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 23, - "accuracy_all_requested": 0.5476190476190477, - "accuracy_accepted": 0.5476190476190477, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 9, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 164, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": 0.0753000229766748, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 4, - "accuracy_all_requested": 0.8, - "accuracy_accepted": 0.8, - "native_normalized_score_mae": 0.14946960375474413, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 5, - "accuracy_all_requested": 0.8333333333333334, - "accuracy_accepted": 0.8333333333333334, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 200, - "valid_questions": 220, - "accuracy_all_requested": 0.9090909090909091, - "isolation_max_probability_delta": 1.261876552752102e-06 - }, - "raw_T1_core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 715, - "accuracy": 0.8125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.2552243936077084, - "ece_fixed10": 0.07151212321144008, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.27018074973609674 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 37, - "accuracy": 0.2972972972972973, - "confidence": 0.3583211405229185 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 41, - "accuracy": 0.36585365853658536, - "confidence": 0.4501311738439686 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 41, - "accuracy": 0.43902439024390244, - "confidence": 0.5528081995639954 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 35, - "accuracy": 0.4, - "confidence": 0.6541964204731542 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 42, - "accuracy": 0.6904761904761905, - "confidence": 0.752341913686174 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 53, - "accuracy": 0.5849056603773585, - "confidence": 0.8444003312578867 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 630, - "accuracy": 0.9476190476190476, - "confidence": 0.9905295849511253 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.026796004587655253, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.026796004587655253, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8379629629629629, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.9010416666666666 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 107, - "accuracy": 0.8359375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.2965606946083775, - "ece_fixed10": 0.13807017974859365, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.3578651089430552 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.4323395998046353 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 1.0, - "confidence": 0.583023895105738 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.6338000824355867 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.7685904127995836 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.4, - "confidence": 0.8320521706202278 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 109, - "accuracy": 0.8807339449541285, - "confidence": 0.99334414489451 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8359375, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 60, - "accuracy": 0.9375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.10024861220671905, - "ece_fixed10": 0.05807661575524867, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 1.0, - "confidence": 0.5511630127794737 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.5, - "confidence": 0.6818999097532907 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.7345366101371958 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 0.9090909090909091, - "confidence": 0.8594738766693791 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 44, - "accuracy": 1.0, - "confidence": 0.9694947134583345 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.90625, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9375 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 108, - "accuracy": 0.9642857142857143, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.0719969965375669, - "ece_fixed10": 0.040385270871796934, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 1.0, - "confidence": 0.6579636510067146 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 1.0, - "confidence": 0.865794913142063 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 109, - "accuracy": 0.963302752293578, - "confidence": 0.9972924087412546 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9642857142857143, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 2.161178129809161e-05, - "ece_fixed10": 0.000979963203247336, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 0.9990200367967527 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 96, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 6.3334066916379615e-06, - "ece_fixed10": 0.0009479893055344135, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 96, - "accuracy": 1.0, - "confidence": 0.9990520106944656 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 57, - "accuracy": 0.890625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.14539211186139428, - "ece_fixed10": 0.05339129836223357, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.5, - "confidence": 0.4350465765880037 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6612984917334279 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.5, - "confidence": 0.7718411901293278 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8394175352352105 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 53, - "accuracy": 0.9811320754716981, - "confidence": 0.9879041867673426 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.026796004587655253, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.026796004587655253, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.96875 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 53, - "accuracy": 0.5520833333333334, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.5937373986470937, - "ece_fixed10": 0.16073432938078536, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.4538738759173777 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 24, - "accuracy": 0.4583333333333333, - "confidence": 0.5514576969489319 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.6543023930785954 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 21, - "accuracy": 0.7619047619047619, - "confidence": 0.7506698765383812 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 20, - "accuracy": 0.55, - "confidence": 0.8345469004400765 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 13, - "accuracy": 0.6923076923076923, - "confidence": 0.9457733747170575 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4583333333333333, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.7083333333333334 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 72, - "accuracy": 0.75, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.3909571256481359, - "ece_fixed10": 0.1935941832510808, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.0, - "confidence": 0.4622017904321838 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.0, - "confidence": 0.5523818486374154 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6649697522828056 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.0, - "confidence": 0.7327586251155883 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.0, - "confidence": 0.8310021629791244 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 82, - "accuracy": 0.8780487804878049, - "confidence": 0.9884515231174967 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.75, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9583333333333334 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 34, - "accuracy": 0.3541666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.6908659061408465, - "ece_fixed10": 0.1700013025300769, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.27018074973609674 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 36, - "accuracy": 0.3055555555555556, - "confidence": 0.35833380806680365 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 26, - "accuracy": 0.38461538461538464, - "confidence": 0.4509067577221217 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.125, - "confidence": 0.5500099314132889 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 10, - "accuracy": 0.1, - "confidence": 0.6432578635979257 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 1.0, - "confidence": 0.745564124657371 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.4, - "confidence": 0.8577265157347416 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 7, - "accuracy": 0.8571428571428571, - "confidence": 0.9345361728548609 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.20833333333333334, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.6666666666666666 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.031206797334929672, - "ece_fixed10": 0.064291980565045, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.42010120972613957 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 1.0, - "confidence": 0.5522763211802342 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 1.0, - "confidence": 0.6593702888359869 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 1.0, - "confidence": 0.7304879313920132 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 1.0, - "confidence": 0.8733717129950435 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 53, - "accuracy": 1.0, - "confidence": 0.9868199532730829 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 463, - "accuracy": 0.8267857142857142, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.2362328055298427, - "ece_fixed10": 0.08093819680377065, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.27018074973609674 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 19, - "accuracy": 0.2631578947368421, - "confidence": 0.3628442858376164 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 20, - "accuracy": 0.4, - "confidence": 0.4433060316324977 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 18, - "accuracy": 0.16666666666666666, - "confidence": 0.5558003702109998 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.3333333333333333, - "confidence": 0.652385573620027 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 22, - "accuracy": 0.7727272727272727, - "confidence": 0.7596966349157493 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 36, - "accuracy": 0.6388888888888888, - "confidence": 0.8526714943301653 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 426, - "accuracy": 0.9413145539906104, - "confidence": 0.9919362736622368 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.026746416757918296, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.026746416757918296, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8404761904761905, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.9224137931034483 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 252, - "accuracy": 0.7875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.28845967274397327, - "ece_fixed10": 0.07142864763440956, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 18, - "accuracy": 0.3333333333333333, - "confidence": 0.3535467093574042 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 21, - "accuracy": 0.3333333333333333, - "confidence": 0.45663130928346474 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 23, - "accuracy": 0.6521739130434783, - "confidence": 0.5504665007967746 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 17, - "accuracy": 0.47058823529411764, - "confidence": 0.6561137877294064 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 20, - "accuracy": 0.6, - "confidence": 0.7442517203336412 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 17, - "accuracy": 0.47058823529411764, - "confidence": 0.826884927104826 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 204, - "accuracy": 0.9607843137254902, - "confidence": 0.9875920879367455 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.02684559241739221, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.02684559241739221, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7777777777777778, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.9396551724137931 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 598, - "accuracy": 0.7952127659574468, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.27776125004577673, - "ece_fixed10": 0.08195202592289716, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.27018074973609674 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 37, - "accuracy": 0.2972972972972973, - "confidence": 0.3583211405229185 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 39, - "accuracy": 0.358974358974359, - "confidence": 0.4509047429340181 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 39, - "accuracy": 0.41025641025641024, - "confidence": 0.5528925681170478 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 29, - "accuracy": 0.41379310344827586, - "confidence": 0.649885451520013 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 35, - "accuracy": 0.7142857142857143, - "confidence": 0.7516395938254402 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 39, - "accuracy": 0.48717948717948717, - "confidence": 0.8405321104512614 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 533, - "accuracy": 0.9399624765478424, - "confidence": 0.9925271092464785 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8288043478260869, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 60, - "accuracy": 0.9375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.10024861220671905, - "ece_fixed10": 0.05807661575524867, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 1.0, - "confidence": 0.5511630127794737 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.5, - "confidence": 0.6818999097532907 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.7345366101371958 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 0.9090909090909091, - "confidence": 0.8594738766693791 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 44, - "accuracy": 1.0, - "confidence": 0.9694947134583345 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.90625, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9375 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 57, - "accuracy": 0.890625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.14539211186139428, - "ece_fixed10": 0.05339129836223357, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.5, - "confidence": 0.4350465765880037 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6612984917334279 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.5, - "confidence": 0.7718411901293278 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8394175352352105 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 53, - "accuracy": 0.9811320754716981, - "confidence": 0.9879041867673426 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.026796004587655253, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.026796004587655253, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.96875 - } - } - }, - "family_macro_accuracy": 0.8284598214285713, - "reliability_contract_pass": true - }, - "raw_T1_typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.7952127659574468, - "brier": 0.27776125004577673, - "nll": 0.5847624437362341, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.9375, - "brier": 0.10024861220671905, - "nll": 0.17133698040132303, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.9, - "recall_true": 1.0, - "balanced_accuracy": 0.95, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.890625, - "brier": 0.14539211186139428, - "nll": 0.23458975132485918, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.017690913385239468, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.026796004587655253, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.026796004587655253, - "native_vs_probability_expected_normalized_abs_residual": 0.0 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - } - }, - "fresh": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 482, - "accuracy": 0.5477272727272727, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.5690726888948539, - "ece_fixed10": 0.1401451664351401, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 53, - "accuracy": 0.3018867924528302, - "confidence": 0.2622791761866259 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 104, - "accuracy": 0.27884615384615385, - "confidence": 0.3514970323952078 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 95, - "accuracy": 0.43157894736842106, - "confidence": 0.4499388687064587 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 105, - "accuracy": 0.42857142857142855, - "confidence": 0.5460500332570607 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 106, - "accuracy": 0.32075471698113206, - "confidence": 0.6471308224433712 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 82, - "accuracy": 0.5, - "confidence": 0.7473118825722479 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 68, - "accuracy": 0.47058823529411764, - "confidence": 0.8458129628136472 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 267, - "accuracy": 0.9138576779026217, - "confidence": 0.9857778366717537 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.29570965302222324, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.29570965302222324, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4724770642201835, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.9166666666666666 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 40, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6729724810339484, - "ece_fixed10": 0.33444791492404974, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.0, - "confidence": 0.5224795045113395 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 13, - "accuracy": 0.15384615384615385, - "confidence": 0.6563136246112697 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 14, - "accuracy": 0.5714285714285714, - "confidence": 0.7569601050480769 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 22, - "accuracy": 0.5, - "confidence": 0.8579169395343587 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 29, - "accuracy": 0.6551724137931034, - "confidence": 0.9554201008457182 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 35, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6695970038300371, - "ece_fixed10": 0.2987477355091148, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.6470588235294118, - "confidence": 0.5407078391687326 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 20, - "accuracy": 0.3, - "confidence": 0.6422737971777123 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 32, - "accuracy": 0.46875, - "confidence": 0.7587555418169254 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 0.2727272727272727, - "confidence": 0.81510898408198 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 34, - "accuracy": 0.425, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7656620536965113, - "ece_fixed10": 0.1724788433286279, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 1.0, - "confidence": 0.29543900941103496 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 20, - "accuracy": 0.35, - "confidence": 0.3603830888601033 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 22, - "accuracy": 0.5, - "confidence": 0.45046913638869746 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 10, - "accuracy": 0.5, - "confidence": 0.5392713561293389 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 10, - "accuracy": 0.1, - "confidence": 0.6597995759653417 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 7, - "accuracy": 0.2857142857142857, - "confidence": 0.7288856762307929 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.8395687082499034 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 2, - "accuracy": 1.0, - "confidence": 0.9390226519250505 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 110, - "accuracy": 0.9166666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.13013554049448825, - "ece_fixed10": 0.05570040018493194, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 1.0, - "confidence": 0.38961951019705804 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.0, - "confidence": 0.5462775933556349 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6498591024638636 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.75, - "confidence": 0.7741977705158665 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 4, - "accuracy": 0.75, - "confidence": 0.8394611427550467 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 107, - "accuracy": 0.9626168224299065, - "confidence": 0.9937414952969577 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9166666666666666, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 106, - "accuracy": 0.8833333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.17940777529618954, - "ece_fixed10": 0.08449366118328214, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.27017720807423273 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 3, - "accuracy": 0.0, - "confidence": 0.45895270187135884 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.5, - "confidence": 0.5372139345146069 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6961979285267498 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 1.0, - "confidence": 0.7265464446127164 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 4, - "accuracy": 0.25, - "confidence": 0.8677214462562848 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 105, - "accuracy": 0.9523809523809523, - "confidence": 0.992770127655098 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8833333333333333, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 25, - "accuracy": 0.3125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7943975966393777, - "ece_fixed10": 0.182686713709271, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 13, - "accuracy": 0.07692307692307693, - "confidence": 0.37564379976716034 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 32, - "accuracy": 0.34375, - "confidence": 0.45189783040728193 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 24, - "accuracy": 0.3333333333333333, - "confidence": 0.553458542265703 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.45454545454545453, - "confidence": 0.6352574647598818 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 23, - "accuracy": 0.2875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7801516348061177, - "ece_fixed10": 0.05163973003648243, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 48, - "accuracy": 0.2708333333333333, - "confidence": 0.2601486268454573 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 28, - "accuracy": 0.25, - "confidence": 0.3349926456578705 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 4, - "accuracy": 0.75, - "confidence": 0.4403703967299571 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.29570965302222324, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.29570965302222324, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 28, - "accuracy": 0.35, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.9130777240040153, - "ece_fixed10": 0.3497009538169479, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 7, - "accuracy": 0.42857142857142855, - "confidence": 0.46869576339092495 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 18, - "accuracy": 0.2222222222222222, - "confidence": 0.5433731275245486 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.05555555555555555, - "confidence": 0.6462110238970182 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 12, - "accuracy": 0.4166666666666667, - "confidence": 0.7259571363336926 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 13, - "accuracy": 0.38461538461538464, - "confidence": 0.8516767334269317 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 12, - "accuracy": 0.8333333333333334, - "confidence": 0.9582840054563944 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 33, - "accuracy": 0.4125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6984842676583607, - "ece_fixed10": 0.04674734792255455, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2824298143386841 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 36, - "accuracy": 0.3611111111111111, - "confidence": 0.3485960379441696 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 16, - "accuracy": 0.4375, - "confidence": 0.437161197544004 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 14, - "accuracy": 0.5, - "confidence": 0.5401671081972307 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.45454545454545453, - "confidence": 0.6382152405997821 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.5, - "confidence": 0.7093436924005085 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 48, - "accuracy": 0.6, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.5011418424890071, - "ece_fixed10": 0.10654180478677518, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 6, - "accuracy": 0.0, - "confidence": 0.35763154005315273 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 11, - "accuracy": 0.5454545454545454, - "confidence": 0.4508501596867971 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 15, - "accuracy": 0.6, - "confidence": 0.5577484121420504 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 21, - "accuracy": 0.6666666666666666, - "confidence": 0.6492506255119014 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 0.5, - "confidence": 0.736644048973393 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 7, - "accuracy": 0.7142857142857143, - "confidence": 0.8424858754750828 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 12, - "accuracy": 0.8333333333333334, - "confidence": 0.9622368914121524 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 252, - "accuracy": 0.5727272727272728, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.5484665850626557, - "ece_fixed10": 0.12678143517596172, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 34, - "accuracy": 0.35294117647058826, - "confidence": 0.25596090207883604 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 46, - "accuracy": 0.32608695652173914, - "confidence": 0.3558246300913118 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 47, - "accuracy": 0.425531914893617, - "confidence": 0.44897588087134743 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 44, - "accuracy": 0.4318181818181818, - "confidence": 0.5395860544824255 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 43, - "accuracy": 0.32558139534883723, - "confidence": 0.6507896367024352 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 57, - "accuracy": 0.5614035087719298, - "confidence": 0.7518348655739209 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 42, - "accuracy": 0.5, - "confidence": 0.8368992424064126 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 127, - "accuracy": 0.937007874015748, - "confidence": 0.9865087123522492 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2865956690428598, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.2865956690428598, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5091743119266054, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 230, - "accuracy": 0.5227272727272727, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.5896787927270519, - "ece_fixed10": 0.15895492345436738, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 19, - "accuracy": 0.21052631578947367, - "confidence": 0.2735855614321444 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 58, - "accuracy": 0.2413793103448276, - "confidence": 0.3480647997396769 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 48, - "accuracy": 0.4375, - "confidence": 0.4508817942950052 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 61, - "accuracy": 0.4262295081967213, - "confidence": 0.5507125753240107 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 63, - "accuracy": 0.31746031746031744, - "confidence": 0.6446335365205179 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 25, - "accuracy": 0.36, - "confidence": 0.7369994813284336 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 26, - "accuracy": 0.4230769230769231, - "confidence": 0.8602120496253343 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 140, - "accuracy": 0.8928571428571429, - "confidence": 0.9851148280187327 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30482363700158666, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.30482363700158666, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.48623853211009177, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 384, - "accuracy": 0.6, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.5171348072716611, - "ece_fixed10": 0.11661544737464838, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 5, - "accuracy": 0.6, - "confidence": 0.28273244986184387 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 76, - "accuracy": 0.2894736842105263, - "confidence": 0.3575775959300162 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 91, - "accuracy": 0.4175824175824176, - "confidence": 0.45035946088124995 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 86, - "accuracy": 0.3953488372093023, - "confidence": 0.5476542001988399 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 73, - "accuracy": 0.3561643835616438, - "confidence": 0.6468262207602272 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 36, - "accuracy": 0.5, - "confidence": 0.7333876545030455 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 35, - "accuracy": 0.5142857142857142, - "confidence": 0.8478545707620098 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 238, - "accuracy": 0.9453781512605042, - "confidence": 0.9894768885161025 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6058823529411764, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.9166666666666666 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 75, - "accuracy": 0.46875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.6712847424319927, - "ece_fixed10": 0.30353583760379876, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 19, - "accuracy": 0.5789473684210527, - "confidence": 0.5387890670995332 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 33, - "accuracy": 0.24242424242424243, - "confidence": 0.6478046382879016 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 46, - "accuracy": 0.5, - "confidence": 0.7582091045394498 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 33, - "accuracy": 0.42424242424242425, - "confidence": 0.8436476210502324 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 29, - "accuracy": 0.6551724137931034, - "confidence": 0.9554201008457182 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 23, - "accuracy": 0.2875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7801516348061177, - "ece_fixed10": 0.05163973003648243, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 48, - "accuracy": 0.2708333333333333, - "confidence": 0.2601486268454573 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 28, - "accuracy": 0.25, - "confidence": 0.3349926456578705 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 4, - "accuracy": 0.75, - "confidence": 0.4403703967299571 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.29570965302222324, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.29570965302222324, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.5125, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.6, - "brier": 0.5171348072716611, - "nll": 1.0541795364972422, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.46875, - "brier": 0.6712847424319927, - "nll": 0.9540232824016679, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.7625, - "recall_true": 0.175, - "balanced_accuracy": 0.46875, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.2875, - "brier": 0.7801516348061177, - "nll": 1.5687792016990254, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.19142155144448664, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.29570965302222324, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.29570965302222324, - "native_vs_probability_expected_normalized_abs_residual": 0.0 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 68, - "truncation_unknown_requests": 0, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 5, - "accuracy_all_requested": 0.11904761904761904, - "accuracy_accepted": 0.11904761904761904, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 3, - "accuracy_all_requested": 0.3333333333333333, - "accuracy_accepted": 0.3333333333333333, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 128, - "accuracy_all_requested": 0.7804878048780488, - "accuracy_accepted": 0.7804878048780488, - "native_normalized_score_mae": 0.14034636642238343, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 2, - "accuracy_all_requested": 0.4, - "accuracy_accepted": 0.4, - "native_normalized_score_mae": 0.16618976409022268, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 2, - "accuracy_all_requested": 0.3333333333333333, - "accuracy_accepted": 0.3333333333333333, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 138, - "valid_questions": 220, - "accuracy_all_requested": 0.6272727272727273, - "isolation_max_probability_delta": 0.002524121026493395 - }, - "raw_T1_core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 482, - "accuracy": 0.5477272727272727, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.5661458794370604, - "ece_fixed10": 0.13552209311179103, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 57, - "accuracy": 0.2982456140350877, - "confidence": 0.26300775036344 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 103, - "accuracy": 0.2912621359223301, - "confidence": 0.351501020428267 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 98, - "accuracy": 0.4387755102040816, - "confidence": 0.4493772373904034 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 110, - "accuracy": 0.4, - "confidence": 0.5478216026935456 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 106, - "accuracy": 0.33962264150943394, - "confidence": 0.6500621455361346 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 78, - "accuracy": 0.48717948717948717, - "confidence": 0.7512420862494339 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 63, - "accuracy": 0.49206349206349204, - "confidence": 0.844473247107483 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 265, - "accuracy": 0.9169811320754717, - "confidence": 0.9850628238580136 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2958081964921219, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.2958081964921219, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4724770642201835, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.9166666666666666 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 40, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6667285315820373, - "ece_fixed10": 0.32961626892838114, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.0, - "confidence": 0.5218274436239243 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 13, - "accuracy": 0.15384615384615385, - "confidence": 0.6521069725411077 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 16, - "accuracy": 0.5625, - "confidence": 0.7568698030202837 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 21, - "accuracy": 0.47619047619047616, - "confidence": 0.858691400344854 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 28, - "accuracy": 0.6785714285714286, - "confidence": 0.9537792760150631 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 35, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.663582561405367, - "ece_fixed10": 0.2946822233740528, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.6470588235294118, - "confidence": 0.5395407841202988 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 20, - "accuracy": 0.3, - "confidence": 0.6383861823069047 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 34, - "accuracy": 0.4411764705882353, - "confidence": 0.7553174731792548 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 9, - "accuracy": 0.3333333333333333, - "confidence": 0.8109170517485055 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 34, - "accuracy": 0.425, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7619630446873139, - "ece_fixed10": 0.15811616112417942, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 1.0, - "confidence": 0.29416592821502674 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 21, - "accuracy": 0.38095238095238093, - "confidence": 0.35901725661619477 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 22, - "accuracy": 0.45454545454545453, - "confidence": 0.4490037686915842 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 9, - "accuracy": 0.5555555555555556, - "confidence": 0.5348251779777786 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.08333333333333333, - "confidence": 0.657263511370418 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.7386142221241498 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.8359946060009326 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 2, - "accuracy": 1.0, - "confidence": 0.9331592284380767 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 110, - "accuracy": 0.9166666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.12964844890664082, - "ece_fixed10": 0.0543869897903284, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 1.0, - "confidence": 0.38075330943670616 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.0, - "confidence": 0.5385279104624456 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6457388307476394 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.75, - "confidence": 0.7661831445340055 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 4, - "accuracy": 0.75, - "confidence": 0.8311142293069116 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 107, - "accuracy": 0.9626168224299065, - "confidence": 0.9930530843624062 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9166666666666666, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 106, - "accuracy": 0.8833333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.17835211927231975, - "ece_fixed10": 0.07854765777413539, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.26189933860186926 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 3, - "accuracy": 0.0, - "confidence": 0.4532004443180007 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.5, - "confidence": 0.5352401433493199 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.5, - "confidence": 0.6910016107575014 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 1.0, - "confidence": 0.7302944457087439 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.4, - "confidence": 0.8659590080428401 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 104, - "accuracy": 0.9519230769230769, - "confidence": 0.9929637175013041 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8833333333333333, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 25, - "accuracy": 0.3125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7908539569991557, - "ece_fixed10": 0.17654718483194967, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 13, - "accuracy": 0.07692307692307693, - "confidence": 0.3726284232913511 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 35, - "accuracy": 0.37142857142857144, - "confidence": 0.4514573902771032 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 24, - "accuracy": 0.2916666666666667, - "confidence": 0.5575025980132381 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.5, - "confidence": 0.6373167839690106 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 23, - "accuracy": 0.2875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7801702822282384, - "ece_fixed10": 0.036616517024516015, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 51, - "accuracy": 0.2549019607843137, - "confidence": 0.26075484571055313 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 25, - "accuracy": 0.28, - "confidence": 0.3344809679072888 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 4, - "accuracy": 0.75, - "confidence": 0.43279999173978717 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2958081964921219, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.2958081964921219, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 28, - "accuracy": 0.35, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.9047549767981821, - "ece_fixed10": 0.34876963084332213, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 8, - "accuracy": 0.5, - "confidence": 0.4681648328574127 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 20, - "accuracy": 0.2, - "confidence": 0.5480116210468976 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 20, - "accuracy": 0.1, - "confidence": 0.6579411155506681 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 0.375, - "confidence": 0.7392836574021613 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 12, - "accuracy": 0.4166666666666667, - "confidence": 0.8473028186078908 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 12, - "accuracy": 0.8333333333333334, - "confidence": 0.9538276096551481 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 33, - "accuracy": 0.4125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6973290576896682, - "ece_fixed10": 0.05134835532609467, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.29040705292203867 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 37, - "accuracy": 0.35135135135135137, - "confidence": 0.3498614570840675 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 15, - "accuracy": 0.4666666666666667, - "confidence": 0.44088067697375066 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 16, - "accuracy": 0.4375, - "confidence": 0.5453152907091006 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.5555555555555556, - "confidence": 0.6460648130806665 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 0.0, - "confidence": 0.7071386295768006 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 48, - "accuracy": 0.6, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.500221410149261, - "ece_fixed10": 0.10984450944417443, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 6, - "accuracy": 0.0, - "confidence": 0.3555702988590701 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 11, - "accuracy": 0.5454545454545454, - "confidence": 0.44641341599703244 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.5294117647058824, - "confidence": 0.5576292961476371 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 21, - "accuracy": 0.7142857142857143, - "confidence": 0.651172811831365 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.7468224428491627 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.8388647130084678 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 12, - "accuracy": 0.8333333333333334, - "confidence": 0.9582226811917316 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 252, - "accuracy": 0.5727272727272728, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.5457552399050928, - "ece_fixed10": 0.1257046550744604, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 35, - "accuracy": 0.37142857142857144, - "confidence": 0.2556243258415061 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 47, - "accuracy": 0.3191489361702128, - "confidence": 0.35566009586331826 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 49, - "accuracy": 0.4489795918367347, - "confidence": 0.4501950393328742 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 44, - "accuracy": 0.38636363636363635, - "confidence": 0.5414348413270162 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 45, - "accuracy": 0.35555555555555557, - "confidence": 0.6541002084603419 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 56, - "accuracy": 0.5357142857142857, - "confidence": 0.754656813454397 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 38, - "accuracy": 0.5263157894736842, - "confidence": 0.8352340822797443 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 126, - "accuracy": 0.9444444444444444, - "confidence": 0.9859047549614911 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2869674562426989, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.2869674562426989, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5091743119266054, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 230, - "accuracy": 0.5227272727272727, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.586536518969028, - "ece_fixed10": 0.15463312372304597, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 22, - "accuracy": 0.18181818181818182, - "confidence": 0.27475410755742574 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 56, - "accuracy": 0.26785714285714285, - "confidence": 0.3480103678309918 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 49, - "accuracy": 0.42857142857142855, - "confidence": 0.44855943544793264 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 66, - "accuracy": 0.4090909090909091, - "confidence": 0.5520794436045652 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 61, - "accuracy": 0.32786885245901637, - "confidence": 0.647083246657621 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 22, - "accuracy": 0.36363636363636365, - "confidence": 0.7425500533640732 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 25, - "accuracy": 0.44, - "confidence": 0.8585167776456457 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 139, - "accuracy": 0.8920863309352518, - "confidence": 0.984299634512415 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3046489367415449, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.3046489367415449, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.48623853211009177, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 384, - "accuracy": 0.6, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.5146404123240027, - "ece_fixed10": 0.11433820134482509, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.2821574399129782 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 78, - "accuracy": 0.2948717948717949, - "confidence": 0.35695616546704206 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 94, - "accuracy": 0.425531914893617, - "confidence": 0.4500826520989402 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 91, - "accuracy": 0.3626373626373626, - "confidence": 0.549939868999968 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 73, - "accuracy": 0.3835616438356164, - "confidence": 0.6528968922966818 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 28, - "accuracy": 0.5, - "confidence": 0.7430775639655943 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 33, - "accuracy": 0.5454545454545454, - "confidence": 0.8445770210543315 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 237, - "accuracy": 0.9451476793248945, - "confidence": 0.9887587704386153 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6058823529411764, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.9166666666666666 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 75, - "accuracy": 0.46875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.6651555464937021, - "ece_fixed10": 0.2991035600606189, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 19, - "accuracy": 0.5789473684210527, - "confidence": 0.5376762219627856 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 33, - "accuracy": 0.24242424242424243, - "confidence": 0.6437913420961362 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 50, - "accuracy": 0.48, - "confidence": 0.7558142187283841 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 30, - "accuracy": 0.43333333333333335, - "confidence": 0.8443590957659495 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 28, - "accuracy": 0.6785714285714286, - "confidence": 0.9537792760150631 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 23, - "accuracy": 0.2875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7801702822282384, - "ece_fixed10": 0.036616517024516015, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 51, - "accuracy": 0.2549019607843137, - "confidence": 0.26075484571055313 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 25, - "accuracy": 0.28, - "confidence": 0.3344809679072888 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 4, - "accuracy": 0.75, - "confidence": 0.43279999173978717 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.2958081964921219, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.2958081964921219, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.5125, - "reliability_contract_pass": true - }, - "raw_T1_typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.6, - "brier": 0.5146404123240027, - "nll": 1.0432214479782773, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.46875, - "brier": 0.6651555464937021, - "nll": 0.9407458807123558, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.7625, - "recall_true": 0.175, - "balanced_accuracy": 0.46875, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.2875, - "brier": 0.7801702822282384, - "nll": 1.56847652545463, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.1913739757480281, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.2958081964921219, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.2958081964921219, - "native_vs_probability_expected_normalized_abs_residual": 0.0 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - } - } - }, - "jev-1.13.0": { - "old": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 674, - "accuracy": 0.7659090909090909, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.2809666873510124, - "ece_fixed10": 0.08375149219467398, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 23, - "accuracy": 0.17391304347826086, - "confidence": 0.36869565217391304 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 56, - "accuracy": 0.35714285714285715, - "confidence": 0.45089285714285715 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 83, - "accuracy": 0.2891566265060241, - "confidence": 0.5472970670561033 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 61, - "accuracy": 0.4426229508196721, - "confidence": 0.6380327868852459 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 61, - "accuracy": 0.5573770491803278, - "confidence": 0.7403278688524589 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 35, - "accuracy": 0.7142857142857143, - "confidence": 0.8444646464646465 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 561, - "accuracy": 0.9625668449197861, - "confidence": 0.9891433047048019 - } - ], - "zero_gold_probability_count": 9, - "normalized_score_mae": 0.0, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.0, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 880, - "all_variants_correct_group_fraction": 0.8148148148148148, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.8229166666666666 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 109, - "accuracy": 0.8515625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.26243125, - "ece_fixed10": 0.13453124999999996, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.5766666666666667 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.6266666666666666 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 1.0, - "confidence": 0.735 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 1.0, - "confidence": 0.852 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 115, - "accuracy": 0.8608695652173913, - "confidence": 0.9921739130434782 - } - ], - "zero_gold_probability_count": 7, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 128, - "all_variants_correct_group_fraction": 0.8515625, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.0029968750000000013, - "ece_fixed10": 0.030781249999999982, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 0.96921875 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 64, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 108, - "accuracy": 0.9642857142857143, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.054753571428571436, - "ece_fixed10": 0.03080357142857145, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.4 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.57 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 0.0, - "confidence": 0.78 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 109, - "accuracy": 0.981651376146789, - "confidence": 0.9985321100917431 - } - ], - "zero_gold_probability_count": 2, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 112, - "all_variants_correct_group_fraction": 0.9642857142857143, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.0, - "ece_fixed10": 0.0, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 1.0 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 64, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 29, - "accuracy": 0.3020833333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.7030341684947113, - "ece_fixed10": 0.3260616582491582, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 6, - "accuracy": 0.16666666666666666, - "confidence": 0.3466666666666667 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.47111111111111115 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 25, - "accuracy": 0.08, - "confidence": 0.5588000000000001 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 24, - "accuracy": 0.125, - "confidence": 0.6341666666666667 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 29, - "accuracy": 0.5517241379310345, - "confidence": 0.7348275862068965 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 1.0, - "confidence": 0.8393602693602693 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 96, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.0 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.0, - "ece_fixed10": 0.0, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 1.0 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.0, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.0, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 64, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 54, - "accuracy": 0.5625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.5663166666666667, - "ece_fixed10": 0.10677083333333337, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 11, - "accuracy": 0.18181818181818182, - "confidence": 0.3709090909090909 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 28, - "accuracy": 0.42857142857142855, - "confidence": 0.45357142857142857 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 26, - "accuracy": 0.6153846153846154, - "confidence": 0.5396153846153846 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 13, - "accuracy": 0.9230769230769231, - "confidence": 0.6453846153846154 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 12, - "accuracy": 0.6666666666666666, - "confidence": 0.7425 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.8300000000000001 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 96, - "all_variants_correct_group_fraction": 0.5, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 86, - "accuracy": 0.8958333333333334, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.12794375, - "ece_fixed10": 0.04468750000000002, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.39 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 5, - "accuracy": 0.2, - "confidence": 0.44399999999999995 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.5, - "confidence": 0.55 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 1.0, - "confidence": 0.63 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.5, - "confidence": 0.7625 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.8333333333333334, - "confidence": 0.8516666666666667 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 74, - "accuracy": 1.0, - "confidence": 0.9921621621621621 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 96, - "all_variants_correct_group_fraction": 0.8333333333333334, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 32, - "accuracy": 0.3333333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.7610530734533891, - "ece_fixed10": 0.30670664983164986, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 5, - "accuracy": 0.2, - "confidence": 0.386 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 13, - "accuracy": 0.23076923076923078, - "confidence": 0.4376923076923077 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 24, - "accuracy": 0.08333333333333333, - "confidence": 0.5385690235690236 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 19, - "accuracy": 0.42105263157894735, - "confidence": 0.6405263157894737 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 13, - "accuracy": 0.46153846153846156, - "confidence": 0.7415384615384616 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 15, - "accuracy": 0.5333333333333333, - "confidence": 0.8458787878787878 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.9142857142857144 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 96, - "all_variants_correct_group_fraction": 0.2916666666666667, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.8333333333333334 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.0020923381542699735, - "ece_fixed10": 0.02938446969696973, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 0.9706155303030303 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 64, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 446, - "accuracy": 0.7964285714285714, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.2567138843339601, - "ece_fixed10": 0.08155699855699851, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 15, - "accuracy": 0.2, - "confidence": 0.362 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 16, - "accuracy": 0.25, - "confidence": 0.458125 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 48, - "accuracy": 0.3541666666666667, - "confidence": 0.5511595117845117 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 42, - "accuracy": 0.4523809523809524, - "confidence": 0.6388095238095238 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 27, - "accuracy": 0.5925925925925926, - "confidence": 0.7366666666666667 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.7222222222222222, - "confidence": 0.8447923681257015 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 394, - "accuracy": 0.949238578680203, - "confidence": 0.9911928934010151 - } - ], - "zero_gold_probability_count": 9, - "normalized_score_mae": 0.0, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.0, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 560, - "all_variants_correct_group_fraction": 0.8428571428571429, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.8448275862068966 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 228, - "accuracy": 0.7125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.323409092630854, - "ece_fixed10": 0.09772064393939388, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 8, - "accuracy": 0.125, - "confidence": 0.38125 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 40, - "accuracy": 0.4, - "confidence": 0.44799999999999995 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 35, - "accuracy": 0.2, - "confidence": 0.5419999999999999 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 19, - "accuracy": 0.42105263157894735, - "confidence": 0.6363157894736842 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 34, - "accuracy": 0.5294117647058824, - "confidence": 0.743235294117647 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 17, - "accuracy": 0.7058823529411765, - "confidence": 0.8441176470588235 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 167, - "accuracy": 0.9940119760479041, - "confidence": 0.9843077481400835 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.0, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.0, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 320, - "all_variants_correct_group_fraction": 0.7611111111111111, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.853448275862069 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 546, - "accuracy": 0.726063829787234, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.3285357511554401, - "ece_fixed10": 0.10062674618525683, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 23, - "accuracy": 0.17391304347826086, - "confidence": 0.36869565217391304 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 56, - "accuracy": 0.35714285714285715, - "confidence": 0.45089285714285715 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 83, - "accuracy": 0.2891566265060241, - "confidence": 0.5472970670561033 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 61, - "accuracy": 0.4426229508196721, - "confidence": 0.6380327868852459 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 61, - "accuracy": 0.5573770491803278, - "confidence": 0.7403278688524589 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 35, - "accuracy": 0.7142857142857143, - "confidence": 0.8444646464646465 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 433, - "accuracy": 0.9515011547344111, - "confidence": 0.9904835887745819 - } - ], - "zero_gold_probability_count": 9, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 752, - "all_variants_correct_group_fraction": 0.782608695652174, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.734375 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.0029968750000000013, - "ece_fixed10": 0.030781249999999982, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 0.96921875 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 64, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.0, - "ece_fixed10": 0.0, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 1.0 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.0, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.0, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 64, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - } - } - }, - "family_macro_accuracy": 0.7909598214285715, - "reliability_contract_pass": false - }, - "typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.726063829787234, - "brier": 0.3285357511554401, - "nll": "infinity", - "zero_true_probability_count": 9, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 1.0, - "brier": 0.0029968750000000013, - "nll": 0.03156331132314722, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 1.0, - "recall_true": 1.0, - "balanced_accuracy": 1.0, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 1.0, - "brier": 0.0, - "nll": 0.0, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.0, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.0, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.0, - "native_vs_probability_expected_normalized_abs_residual": 0.0 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 0, - "truncation_unknown_requests": 68, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 25, - "accuracy_all_requested": 0.5952380952380952, - "accuracy_accepted": 0.5952380952380952, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 9, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 164, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": 0.00093749999999998, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 5, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": 0.0, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 6, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 4, - "accuracy_all_requested": 0.6666666666666666, - "accuracy_accepted": 0.6666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 203, - "valid_questions": 220, - "accuracy_all_requested": 0.9227272727272727, - "isolation_max_probability_delta": 0.020000000000000018 - } - }, - "fresh": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 603, - "accuracy": 0.6852272727272727, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.4129858524640343, - "ece_fixed10": 0.04721108815426997, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 48, - "accuracy": 0.3125, - "confidence": 0.26125000000000004 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 60, - "accuracy": 0.4, - "confidence": 0.3478333333333333 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 100, - "accuracy": 0.57, - "confidence": 0.4486 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 132, - "accuracy": 0.49242424242424243, - "confidence": 0.5476515151515152 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 127, - "accuracy": 0.5905511811023622, - "confidence": 0.6347244094488189 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 103, - "accuracy": 0.7184466019417476, - "confidence": 0.7401941747572816 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 81, - "accuracy": 0.8888888888888888, - "confidence": 0.8440029928918817 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 229, - "accuracy": 0.9650655021834061, - "confidence": 0.9870742358078602 - } - ], - "zero_gold_probability_count": 5, - "normalized_score_mae": 0.25353125, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.25353125, - "native_vs_expected_score_mean_residual": 0.00024999999999996135, - "native_vs_expected_score_normalized_mean_residual": 6.249999999999034e-05, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0029375000000000026, - "reported_truncations": 0, - "unknown_truncation_status": 880, - "all_variants_correct_group_fraction": 0.5458715596330275, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.9 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 61, - "accuracy": 0.7625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.34932250000000004, - "ece_fixed10": 0.10212499999999998, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 19, - "accuracy": 0.6842105263157895, - "confidence": 0.5494736842105263 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 26, - "accuracy": 0.6153846153846154, - "confidence": 0.6407692307692308 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 25, - "accuracy": 0.88, - "confidence": 0.7476 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 10, - "accuracy": 1.0, - "confidence": 0.836 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 80, - "all_variants_correct_group_fraction": 0.45, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 55, - "accuracy": 0.6875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.40536500000000003, - "ece_fixed10": 0.052999999999999985, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 31, - "accuracy": 0.5806451612903226, - "confidence": 0.5512903225806451 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 29, - "accuracy": 0.6551724137931034, - "confidence": 0.6317241379310345 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 14, - "accuracy": 0.8571428571428571, - "confidence": 0.7414285714285713 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 1.0, - "confidence": 0.8283333333333335 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 80, - "all_variants_correct_group_fraction": 0.25, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 44, - "accuracy": 0.55, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.5861500000000001, - "ece_fixed10": 0.13912499999999997, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 25, - "accuracy": 0.48, - "confidence": 0.35240000000000005 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 23, - "accuracy": 0.5217391304347826, - "confidence": 0.44739130434782604 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.35294117647058826, - "confidence": 0.5376470588235295 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 7, - "accuracy": 0.8571428571428571, - "confidence": 0.6285714285714287 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 1.0, - "confidence": 0.74 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 1.0, - "confidence": 0.826 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 1, - "accuracy": 1.0, - "confidence": 0.9 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 80, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 110, - "accuracy": 0.9166666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.11781166666666666, - "ece_fixed10": 0.048750000000000016, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.29 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.5, - "confidence": 0.5549999999999999 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.68 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.25, - "confidence": 0.74 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8766666666666666 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 109, - "accuracy": 0.9724770642201835, - "confidence": 0.9924770642201836 - } - ], - "zero_gold_probability_count": 2, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 120, - "all_variants_correct_group_fraction": 0.9166666666666666, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 106, - "accuracy": 0.8833333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.160085, - "ece_fixed10": 0.06566666666666669, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.29 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.5366666666666667 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.6316666666666667 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 6, - "accuracy": 0.5, - "confidence": 0.765 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 1.0, - "confidence": 0.8733333333333334 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 101, - "accuracy": 0.9603960396039604, - "confidence": 0.9922772277227723 - } - ], - "zero_gold_probability_count": 2, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 120, - "all_variants_correct_group_fraction": 0.8833333333333333, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 47, - "accuracy": 0.5875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.48014834991327415, - "ece_fixed10": 0.2243989898989899, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 3, - "accuracy": 0.0, - "confidence": 0.38666666666666666 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 20, - "accuracy": 0.7, - "confidence": 0.4495 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 15, - "accuracy": 0.26666666666666666, - "confidence": 0.548 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.16666666666666666, - "confidence": 0.6241666666666666 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 0.875, - "confidence": 0.725 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.8888888888888888, - "confidence": 0.8560044893378227 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 4, - "accuracy": 1.0, - "confidence": 0.9299999999999999 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 80, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 34, - "accuracy": 0.425, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7210325, - "ece_fixed10": 0.10124999999999999, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 46, - "accuracy": 0.32608695652173914, - "confidence": 0.26 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 16, - "accuracy": 0.375, - "confidence": 0.33 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 12, - "accuracy": 0.75, - "confidence": 0.43916666666666665 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.6, - "confidence": 0.548 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.65 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.25353125, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.25353125, - "native_vs_expected_score_mean_residual": 0.00024999999999996135, - "native_vs_expected_score_normalized_mean_residual": 6.249999999999034e-05, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0029375000000000026, - "reported_truncations": 0, - "unknown_truncation_status": 80, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 53, - "accuracy": 0.6625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.520445786399347, - "ece_fixed10": 0.13035101010101013, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 4, - "accuracy": 0.5, - "confidence": 0.37 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 16, - "accuracy": 0.5625, - "confidence": 0.45 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 13, - "accuracy": 0.6153846153846154, - "confidence": 0.5530769230769231 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.8181818181818182, - "confidence": 0.6354545454545455 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 14, - "accuracy": 0.5714285714285714, - "confidence": 0.7428571428571429 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 15, - "accuracy": 0.6666666666666666, - "confidence": 0.8298720538720539 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 7, - "accuracy": 1.0, - "confidence": 0.9371428571428572 - } - ], - "zero_gold_probability_count": 1, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 80, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 33, - "accuracy": 0.4125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6965927407917559, - "ece_fixed10": 0.13947601010101007, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 12, - "accuracy": 0.3333333333333333, - "confidence": 0.345 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 22, - "accuracy": 0.4090909090909091, - "confidence": 0.44727272727272727 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 12, - "accuracy": 0.25, - "confidence": 0.5525 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 19, - "accuracy": 0.42105263157894735, - "confidence": 0.633157894736842 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 11, - "accuracy": 0.6363636363636364, - "confidence": 0.7309090909090908 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8426936026936027 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 1, - "accuracy": 0.0, - "confidence": 0.95 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 80, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 60, - "accuracy": 0.75, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.3669425, - "ece_fixed10": 0.10462500000000005, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.46714285714285714 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 15, - "accuracy": 0.5333333333333333, - "confidence": 0.5413333333333333 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 15, - "accuracy": 0.8, - "confidence": 0.64 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 19, - "accuracy": 0.631578947368421, - "confidence": 0.7315789473684211 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 1.0, - "confidence": 0.8483333333333333 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 6, - "accuracy": 1.0, - "confidence": 0.9183333333333333 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 80, - "all_variants_correct_group_fraction": 0.2, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 303, - "accuracy": 0.6886363636363636, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.4073813636363637, - "ece_fixed10": 0.06643181818181818, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 27, - "accuracy": 0.3333333333333333, - "confidence": 0.26185185185185184 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 28, - "accuracy": 0.4642857142857143, - "confidence": 0.34750000000000003 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 51, - "accuracy": 0.6078431372549019, - "confidence": 0.44882352941176473 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 67, - "accuracy": 0.43283582089552236, - "confidence": 0.5470149253731343 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 53, - "accuracy": 0.5660377358490566, - "confidence": 0.6354716981132076 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 52, - "accuracy": 0.7307692307692307, - "confidence": 0.7386538461538461 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 43, - "accuracy": 0.8837209302325582, - "confidence": 0.8444186046511628 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 119, - "accuracy": 0.9663865546218487, - "confidence": 0.9873109243697479 - } - ], - "zero_gold_probability_count": 2, - "normalized_score_mae": 0.2546875, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.2548125, - "native_vs_expected_score_mean_residual": 0.0009999999999999675, - "native_vs_expected_score_normalized_mean_residual": 0.0002499999999999919, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0023749999999999995, - "reported_truncations": 0, - "unknown_truncation_status": 440, - "all_variants_correct_group_fraction": 0.5963302752293578, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 300, - "accuracy": 0.6818181818181818, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.41859034129170497, - "ece_fixed10": 0.030263085399449035, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 21, - "accuracy": 0.2857142857142857, - "confidence": 0.2604761904761905 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 32, - "accuracy": 0.34375, - "confidence": 0.348125 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 49, - "accuracy": 0.5306122448979592, - "confidence": 0.4483673469387755 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 65, - "accuracy": 0.5538461538461539, - "confidence": 0.5483076923076923 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 74, - "accuracy": 0.6081081081081081, - "confidence": 0.6341891891891892 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 51, - "accuracy": 0.7058823529411765, - "confidence": 0.7417647058823529 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 38, - "accuracy": 0.8947368421052632, - "confidence": 0.8435326953748007 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 110, - "accuracy": 0.9636363636363636, - "confidence": 0.9868181818181818 - } - ], - "zero_gold_probability_count": 3, - "normalized_score_mae": 0.252375, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.25225, - "native_vs_expected_score_mean_residual": -0.0005000000000000449, - "native_vs_expected_score_normalized_mean_residual": -0.00012500000000001122, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0035000000000000057, - "reported_truncations": 0, - "unknown_truncation_status": 440, - "all_variants_correct_group_fraction": 0.6146788990825688, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 453, - "accuracy": 0.7078125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.3833905471380471, - "ece_fixed10": 0.06496212121212121, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.0, - "confidence": 0.29 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 44, - "accuracy": 0.4090909090909091, - "confidence": 0.3543181818181818 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 88, - "accuracy": 0.5454545454545454, - "confidence": 0.4498863636363636 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 77, - "accuracy": 0.4025974025974026, - "confidence": 0.5457142857142857 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 71, - "accuracy": 0.5492957746478874, - "confidence": 0.6335211267605634 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 64, - "accuracy": 0.625, - "confidence": 0.73703125 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 65, - "accuracy": 0.8615384615384616, - "confidence": 0.8466806526806526 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 229, - "accuracy": 0.9650655021834061, - "confidence": 0.9870742358078602 - } - ], - "zero_gold_probability_count": 5, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 640, - "all_variants_correct_group_fraction": 0.6176470588235294, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.9 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 116, - "accuracy": 0.725, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.37734375, - "ece_fixed10": 0.06931250000000003, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 50, - "accuracy": 0.62, - "confidence": 0.5506 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 55, - "accuracy": 0.6363636363636364, - "confidence": 0.6359999999999999 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 39, - "accuracy": 0.8717948717948718, - "confidence": 0.7453846153846154 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 16, - "accuracy": 1.0, - "confidence": 0.833125 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 160, - "all_variants_correct_group_fraction": 0.35, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 34, - "accuracy": 0.425, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7210325, - "ece_fixed10": 0.10124999999999999, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 46, - "accuracy": 0.32608695652173914, - "confidence": 0.26 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 16, - "accuracy": 0.375, - "confidence": 0.33 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 12, - "accuracy": 0.75, - "confidence": 0.43916666666666665 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.6, - "confidence": 0.548 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.65 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.25353125, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.25353125, - "native_vs_expected_score_mean_residual": 0.00024999999999996135, - "native_vs_expected_score_normalized_mean_residual": 6.249999999999034e-05, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0029375000000000026, - "reported_truncations": 0, - "unknown_truncation_status": 80, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.6637500000000001, - "reliability_contract_pass": false - }, - "typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.7078125, - "brier": 0.3833905471380471, - "nll": "infinity", - "zero_true_probability_count": 5, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.725, - "brier": 0.37734375, - "nll": 0.5589288200828377, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.75, - "recall_true": 0.7, - "balanced_accuracy": 0.725, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.425, - "brier": 0.7210325, - "nll": 1.4384123588303563, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.1589446875, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.25353125, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.25353125, - "native_vs_probability_expected_normalized_abs_residual": 0.0029375000000000026 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 0, - "truncation_unknown_requests": 68, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 10, - "accuracy_all_requested": 0.23809523809523808, - "accuracy_accepted": 0.23809523809523808, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 8, - "accuracy_all_requested": 0.8888888888888888, - "accuracy_accepted": 0.8888888888888888, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 146, - "accuracy_all_requested": 0.8902439024390244, - "accuracy_accepted": 0.8902439024390244, - "native_normalized_score_mae": 0.12901041666666666, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 4, - "accuracy_all_requested": 0.8, - "accuracy_accepted": 0.8, - "native_normalized_score_mae": 0.01749999999999996, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 5, - "accuracy_all_requested": 0.8333333333333334, - "accuracy_accepted": 0.8333333333333334, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 2, - "accuracy_all_requested": 0.3333333333333333, - "accuracy_accepted": 0.3333333333333333, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 168, - "valid_questions": 220, - "accuracy_all_requested": 0.7636363636363637, - "isolation_max_probability_delta": 0.10999999999999999 - } - } - }, - "laya-routed": { - "old": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 522, - "accuracy": 0.5931818181818181, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.541972347789284, - "ece_fixed10": 0.09208847514926827, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 63, - "accuracy": 0.2698412698412698, - "confidence": 0.2868391254543119 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 206, - "accuracy": 0.34951456310679613, - "confidence": 0.34896115322846105 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 160, - "accuracy": 0.58125, - "confidence": 0.4439010963992985 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 90, - "accuracy": 0.6888888888888889, - "confidence": 0.5418373106788509 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 51, - "accuracy": 0.7058823529411765, - "confidence": 0.6513938451498601 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 55, - "accuracy": 0.6181818181818182, - "confidence": 0.7475488229365247 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 76, - "accuracy": 0.868421052631579, - "confidence": 0.8493798957928252 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 179, - "accuracy": 0.7932960893854749, - "confidence": 0.9793368553026283 - } - ], - "zero_gold_probability_count": 12, - "normalized_score_mae": 0.3121438866634389, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.31215078125, - "native_vs_expected_score_mean_residual": -1.756684739436659e-05, - "native_vs_expected_score_normalized_mean_residual": -4.391711848591648e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 2.2106737221068082e-05, - "reported_truncations": 112, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6365740740740741, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.5989583333333334 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 117, - "accuracy": 0.9140625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.21569237867224522, - "ece_fixed10": 0.22369205828706903, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 6, - "accuracy": 0.5, - "confidence": 0.3591376062393761 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 12, - "accuracy": 0.9166666666666666, - "confidence": 0.45390190269026903 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 20, - "accuracy": 0.8, - "confidence": 0.5355501350628014 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 20, - "accuracy": 0.95, - "confidence": 0.6583218809256688 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 30, - "accuracy": 0.9333333333333333, - "confidence": 0.7498675427261088 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 33, - "accuracy": 1.0, - "confidence": 0.8456029388089081 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 7, - "accuracy": 1.0, - "confidence": 0.9267720696127207 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9140625, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 28, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.9074458078125, - "ece_fixed10": 0.4347984375, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.6, - "confidence": 0.54516 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 5, - "accuracy": 0.8, - "confidence": 0.6519600000000001 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 9, - "accuracy": 0.1111111111111111, - "confidence": 0.7397111111111111 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.7777777777777778, - "confidence": 0.8627444444444445 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 27, - "accuracy": 0.2222222222222222, - "confidence": 0.9491074074074075 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 94, - "accuracy": 0.8392857142857143, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.28964485024757874, - "ece_fixed10": 0.14074109106946442, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.3161 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 0.0, - "confidence": 0.5675 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.668 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.5, - "confidence": 0.8079 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 107, - "accuracy": 0.8691588785046729, - "confidence": 0.9962205813063553 - } - ], - "zero_gold_probability_count": 12, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 112, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8392857142857143, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 55, - "accuracy": 0.859375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.23422780176567864, - "ece_fixed10": 0.11583625272144824, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.2813718628137186 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 6, - "accuracy": 0.5, - "confidence": 0.3517894489448945 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 4, - "accuracy": 0.75, - "confidence": 0.4277002445747524 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.8, - "confidence": 0.5618589262659893 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 5, - "accuracy": 0.8, - "confidence": 0.6500535586441356 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 1.0, - "confidence": 0.7438643099386432 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 1.0, - "confidence": 0.8299929452945294 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 33, - "accuracy": 0.9393939393939394, - "confidence": 0.9691753321983742 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5625, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.5625 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 92, - "accuracy": 0.9583333333333334, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.4026366225154631, - "ece_fixed10": 0.49693354209340834, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 1.0, - "confidence": 0.2995 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 21, - "accuracy": 0.8095238095238095, - "confidence": 0.36363692343789306 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 45, - "accuracy": 1.0, - "confidence": 0.45467554851904435 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 26, - "accuracy": 1.0, - "confidence": 0.5351415312936846 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 1.0, - "confidence": 0.6258991662589917 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 1.0, - "confidence": 0.7326267373262674 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9166666666666666, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9166666666666666 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 12, - "accuracy": 0.1875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.8746096750160117, - "ece_fixed10": 0.3393338173107288, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 12, - "accuracy": 0.0, - "confidence": 0.2861283354997834 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 11, - "accuracy": 0.0, - "confidence": 0.3433723500377235 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 17, - "accuracy": 0.11764705882352941, - "confidence": 0.45015309811047216 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 10, - "accuracy": 0.7, - "confidence": 0.5592292993314929 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.6375680343743471 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 0.0, - "confidence": 0.7581875681498756 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8227380361963803 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3121438866634389, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.31215078125, - "native_vs_expected_score_mean_residual": -1.756684739436659e-05, - "native_vs_expected_score_normalized_mean_residual": -4.391711848591648e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 2.2106737221068082e-05, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.1875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.78125 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 24, - "accuracy": 0.25, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.7333075393144273, - "ece_fixed10": 0.07850770644636876, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 39, - "accuracy": 0.3076923076923077, - "confidence": 0.2862292148762921 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 54, - "accuracy": 0.2037037037037037, - "confidence": 0.3234803304609514 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.41058045137847116 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.08333333333333333 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 36, - "accuracy": 0.375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.6888206153798193, - "ece_fixed10": 0.1861250311141357, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 35, - "accuracy": 0.5142857142857142, - "confidence": 0.3750906255656491 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 39, - "accuracy": 0.3076923076923077, - "confidence": 0.4343953303661071 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.125, - "confidence": 0.547981301880188 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 10, - "accuracy": 0.3, - "confidence": 0.6390106582435066 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 0.0, - "confidence": 0.7055 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8584333333333333 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.16666666666666666, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.375 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 23, - "accuracy": 0.23958333333333334, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.7941007588421473, - "ece_fixed10": 0.15774738390745302, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 7, - "accuracy": 0.2857142857142857, - "confidence": 0.2925530498376734 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 52, - "accuracy": 0.21153846153846154, - "confidence": 0.3510752316843677 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 28, - "accuracy": 0.25, - "confidence": 0.43794457071976717 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.25, - "confidence": 0.5299189118911891 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.6618338166183382 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.125, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.5 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 41, - "accuracy": 0.640625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.5692749481529262, - "ece_fixed10": 0.21475055577283372, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.2821352114788521 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 20, - "accuracy": 0.45, - "confidence": 0.35194211020352106 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 12, - "accuracy": 1.0, - "confidence": 0.4431618369586184 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 7, - "accuracy": 0.42857142857142855, - "confidence": 0.5460837301984087 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.5, - "confidence": 0.6650344640535947 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.0, - "confidence": 0.7413372287228723 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 12, - "accuracy": 0.75, - "confidence": 0.8591080220415802 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 5, - "accuracy": 1.0, - "confidence": 0.921920889911009 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.5 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 375, - "accuracy": 0.6696428571428571, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.4608826326996567, - "ece_fixed10": 0.1127898791294149, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 29, - "accuracy": 0.20689655172413793, - "confidence": 0.28352322517526674 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 107, - "accuracy": 0.38317757009345793, - "confidence": 0.3483243875828981 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 88, - "accuracy": 0.6022727272727273, - "confidence": 0.44592885019925443 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 55, - "accuracy": 0.7454545454545455, - "confidence": 0.544136730580331 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 33, - "accuracy": 0.7878787878787878, - "confidence": 0.6562042286122847 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 41, - "accuracy": 0.7073170731707317, - "confidence": 0.7475299826033242 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 58, - "accuracy": 0.9310344827586207, - "confidence": 0.8497513150602718 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 149, - "accuracy": 0.8389261744966443, - "confidence": 0.9843739180146119 - } - ], - "zero_gold_probability_count": 12, - "normalized_score_mae": 0.3076297982109308, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.307634375, - "native_vs_expected_score_mean_residual": -2.295544144824413e-05, - "native_vs_expected_score_normalized_mean_residual": -5.738860362061032e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 2.005086666145503e-05, - "reported_truncations": 112, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6547619047619048, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.603448275862069 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 147, - "accuracy": 0.459375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.6838793491961319, - "ece_fixed10": 0.11586251246745156, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 34, - "accuracy": 0.3235294117647059, - "confidence": 0.28966739333937985 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 99, - "accuracy": 0.31313131313131315, - "confidence": 0.3496493746837665 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 72, - "accuracy": 0.5555555555555556, - "confidence": 0.4414227306437967 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 35, - "accuracy": 0.6, - "confidence": 0.5382239365479536 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.5555555555555556, - "confidence": 0.6425748088020815 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 14, - "accuracy": 0.35714285714285715, - "confidence": 0.7476039981980399 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.6666666666666666, - "confidence": 0.8481831003754977 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 30, - "accuracy": 0.5666666666666667, - "confidence": 0.9543194438331096 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.31665797511594695, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.3166671875, - "native_vs_expected_score_mean_residual": -1.2178253340489054e-05, - "native_vs_expected_score_normalized_mean_residual": -3.0445633351222634e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 2.4162607780681133e-05, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.3333333333333333, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.6982758620689655 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 482, - "accuracy": 0.6409574468085106, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.4825586637680122, - "ece_fixed10": 0.0932369788505473, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 51, - "accuracy": 0.3333333333333333, - "confidence": 0.2870063701494951 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 195, - "accuracy": 0.36923076923076925, - "confidence": 0.349276419049477 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 143, - "accuracy": 0.6363636363636364, - "confidence": 0.44315785144062747 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 75, - "accuracy": 0.6933333333333334, - "confidence": 0.5392968662370886 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 43, - "accuracy": 0.7209302325581395, - "confidence": 0.6522926046399958 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 38, - "accuracy": 0.868421052631579, - "confidence": 0.7471653872713118 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 55, - "accuracy": 0.9090909090909091, - "confidence": 0.8464592358484652 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 152, - "accuracy": 0.8947368421052632, - "confidence": 0.9847065598629635 - } - ], - "zero_gold_probability_count": 12, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 112, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6983695652173914, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.484375 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 28, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.9074458078125, - "ece_fixed10": 0.4347984375, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.6, - "confidence": 0.54516 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 5, - "accuracy": 0.8, - "confidence": 0.6519600000000001 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 9, - "accuracy": 0.1111111111111111, - "confidence": 0.7397111111111111 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.7777777777777778, - "confidence": 0.8627444444444445 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 27, - "accuracy": 0.2222222222222222, - "confidence": 0.9491074074074075 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 12, - "accuracy": 0.1875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.8746096750160117, - "ece_fixed10": 0.3393338173107288, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 12, - "accuracy": 0.0, - "confidence": 0.2861283354997834 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 11, - "accuracy": 0.0, - "confidence": 0.3433723500377235 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 17, - "accuracy": 0.11764705882352941, - "confidence": 0.45015309811047216 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 10, - "accuracy": 0.7, - "confidence": 0.5592292993314929 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.6375680343743471 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 0.0, - "confidence": 0.7581875681498756 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8227380361963803 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3121438866634389, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.31215078125, - "native_vs_expected_score_mean_residual": -1.756684739436659e-05, - "native_vs_expected_score_normalized_mean_residual": -4.391711848591648e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 2.2106737221068082e-05, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.1875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.78125 - } - } - }, - "family_macro_accuracy": 0.5701264880952381, - "reliability_contract_pass": false - }, - "typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.6409574468085106, - "brier": 0.4825586637680122, - "nll": "infinity", - "zero_true_probability_count": 12, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.4375, - "brier": 0.9074458078125, - "nll": 1.4928367521053874, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.2, - "recall_true": 0.8333333333333334, - "balanced_accuracy": 0.5166666666666667, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.1875, - "brier": 0.8746096750160117, - "nll": 1.6546029819166304, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.20031317264074974, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.3121438866634389, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.31215078125, - "native_vs_probability_expected_normalized_abs_residual": 2.2106737221068082e-05 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 56, - "reported_truncated_requests": 39, - "reported_untruncated_requests": 17, - "truncation_unknown_requests": 12, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 30, - "contract_valid_questions": 30, - "correct_questions": 6, - "accuracy_all_requested": 0.14285714285714285, - "accuracy_accepted": 0.2, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 12 - } - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 4, - "accuracy_all_requested": 0.4444444444444444, - "accuracy_accepted": 0.4444444444444444, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 76, - "accuracy_all_requested": 0.4634146341463415, - "accuracy_accepted": 0.4634146341463415, - "native_normalized_score_mae": 0.5167053571428571, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 2, - "accuracy_all_requested": 0.4, - "accuracy_accepted": 0.4, - "native_normalized_score_mae": 0.7271666666666667, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 4, - "accuracy_all_requested": 0.6666666666666666, - "accuracy_accepted": 0.6666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 2, - "accuracy_all_requested": 0.3333333333333333, - "accuracy_accepted": 0.3333333333333333, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "255": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 88, - "valid_questions": 208, - "accuracy_all_requested": 0.4, - "isolation_max_probability_delta": 0.006400000000000017 - } - }, - "fresh": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 361, - "accuracy": 0.4102272727272727, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.6741105054850807, - "ece_fixed10": 0.1610202776068644, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 5, - "accuracy": 0.0, - "confidence": 0.17776660396925292 - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 144, - "accuracy": 0.19444444444444445, - "confidence": 0.27346359441618223 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 236, - "accuracy": 0.21610169491525424, - "confidence": 0.34033579987559104 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 74, - "accuracy": 0.24324324324324326, - "confidence": 0.4379108004603339 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 61, - "accuracy": 0.3770491803278688, - "confidence": 0.5433013780722488 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 48, - "accuracy": 0.4583333333333333, - "confidence": 0.6512191767899873 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 50, - "accuracy": 0.6, - "confidence": 0.7503762880332681 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 60, - "accuracy": 0.55, - "confidence": 0.8498566965511878 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 202, - "accuracy": 0.7722772277227723, - "confidence": 0.9742350739133602 - } - ], - "zero_gold_probability_count": 16, - "normalized_score_mae": 0.31729610675246617, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.3172984375, - "native_vs_expected_score_mean_residual": -9.298056100889852e-06, - "native_vs_expected_score_normalized_mean_residual": -2.324514025222463e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 1.8051798859790995e-05, - "reported_truncations": 40, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.3440366972477064, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.6416666666666667 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 38, - "accuracy": 0.475, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7625778925000001, - "ece_fixed10": 0.3426575, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 10, - "accuracy": 0.4, - "confidence": 0.53983 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.6614571428571429 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 9, - "accuracy": 0.5555555555555556, - "confidence": 0.7535444444444445 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 25, - "accuracy": 0.48, - "confidence": 0.862728 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 29, - "accuracy": 0.4482758620689655, - "confidence": 0.9322068965517241 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 51, - "accuracy": 0.6375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.51554342275, - "ece_fixed10": 0.14916875000000002, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 14, - "accuracy": 0.5, - "confidence": 0.5409285714285714 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.6666666666666666, - "confidence": 0.64745 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 15, - "accuracy": 0.7333333333333333, - "confidence": 0.7528066666666666 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 13, - "accuracy": 0.46153846153846156, - "confidence": 0.8483769230769231 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 26, - "accuracy": 0.7307692307692307, - "confidence": 0.9541884615384616 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.1, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 18, - "accuracy": 0.225, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7931300147179258, - "ece_fixed10": 0.12061709050628343, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 15, - "accuracy": 0.2, - "confidence": 0.290729404392894 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 54, - "accuracy": 0.24074074074074073, - "confidence": 0.340308048958562 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.42215461453854614 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.0, - "confidence": 0.5562 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 89, - "accuracy": 0.7416666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.4517366211475811, - "ece_fixed10": 0.2076123150544259, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.2, - "confidence": 0.5665711395545783 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.5, - "confidence": 0.6512827242275773 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 5, - "accuracy": 0.4, - "confidence": 0.7661998191387982 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.6, - "confidence": 0.8575371657165716 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 101, - "accuracy": 0.801980198019802, - "confidence": 0.9936317454214957 - } - ], - "zero_gold_probability_count": 16, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7416666666666667, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 88, - "accuracy": 0.7333333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.36910535066653494, - "ece_fixed10": 0.10167561630479818, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 5, - "accuracy": 0.0, - "confidence": 0.17776660396925292 - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 15, - "accuracy": 0.3333333333333333, - "confidence": 0.24627061410519904 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 10, - "accuracy": 0.6, - "confidence": 0.3620106585409066 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 11, - "accuracy": 0.6363636363636364, - "confidence": 0.4613322610612968 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.5, - "confidence": 0.5611705621133306 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.75, - "confidence": 0.6639022752400695 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 1.0, - "confidence": 0.7342900281713077 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 13, - "accuracy": 0.9230769230769231, - "confidence": 0.8349347852394443 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 46, - "accuracy": 0.9347826086956522, - "confidence": 0.969473448759298 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7333333333333333, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 21, - "accuracy": 0.2625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8923025227843047, - "ece_fixed10": 0.28641102378962274, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 17, - "accuracy": 0.23529411764705882, - "confidence": 0.3637684467834492 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 16, - "accuracy": 0.25, - "confidence": 0.4361073047568855 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 14, - "accuracy": 0.2857142857142857, - "confidence": 0.5387220622424349 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 17, - "accuracy": 0.29411764705882354, - "confidence": 0.6501657236311866 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 13, - "accuracy": 0.3076923076923077, - "confidence": 0.7491919292768424 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.0, - "confidence": 0.8055600593392672 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 16, - "accuracy": 0.2, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8277920500293308, - "ece_fixed10": 0.08094118847539025, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 62, - "accuracy": 0.20967741935483872, - "confidence": 0.2635774024995526 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 17, - "accuracy": 0.17647058823529413, - "confidence": 0.3344644778269977 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.4476 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.31729610675246617, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.3172984375, - "native_vs_expected_score_mean_residual": -9.298056100889852e-06, - "native_vs_expected_score_normalized_mean_residual": -2.324514025222463e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 1.8051798859790995e-05, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 15, - "accuracy": 0.1875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.763582398463182, - "ece_fixed10": 0.1405338660727512, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 22, - "accuracy": 0.13636363636363635, - "confidence": 0.28807092469002743 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 50, - "accuracy": 0.22, - "confidence": 0.33056720251205207 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 8, - "accuracy": 0.125, - "confidence": 0.42209860212961103 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 40, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 10, - "accuracy": 0.125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8091924288622117, - "ece_fixed10": 0.21117321521679464, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 19, - "accuracy": 0.21052631578947367, - "confidence": 0.2892911168498059 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 53, - "accuracy": 0.11320754716981132, - "confidence": 0.3383360641345681 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 8, - "accuracy": 0.0, - "confidence": 0.43318932475814326 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 15, - "accuracy": 0.1875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8198318725077589, - "ece_fixed10": 0.21223096749820972, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 11, - "accuracy": 0.0, - "confidence": 0.28616970910542433 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 35, - "accuracy": 0.22857142857142856, - "confidence": 0.3426393228101647 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 21, - "accuracy": 0.19047619047619047, - "confidence": 0.44113012096815835 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.375, - "confidence": 0.5241693745416938 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.0, - "confidence": 0.636340796579658 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 0.0, - "confidence": 0.8357835783578358 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 179, - "accuracy": 0.4068181818181818, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.6884448928436154, - "ece_fixed10": 0.19660842339511514, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 58, - "accuracy": 0.22413793103448276, - "confidence": 0.2741043543085309 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 119, - "accuracy": 0.19327731092436976, - "confidence": 0.34191800239776826 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 42, - "accuracy": 0.19047619047619047, - "confidence": 0.43101559507839404 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 32, - "accuracy": 0.34375, - "confidence": 0.5417596176222316 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 19, - "accuracy": 0.47368421052631576, - "confidence": 0.650097762608925 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 14, - "accuracy": 0.2857142857142857, - "confidence": 0.759235649692428 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 22, - "accuracy": 0.45454545454545453, - "confidence": 0.8561993558446754 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 134, - "accuracy": 0.753731343283582, - "confidence": 0.982573927519187 - } - ], - "zero_gold_probability_count": 16, - "normalized_score_mae": 0.3177614181748619, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.317766875, - "native_vs_expected_score_mean_residual": -1.0060490866409255e-05, - "native_vs_expected_score_normalized_mean_residual": -2.515122716602314e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 1.9826781244307314e-05, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.42201834862385323, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 182, - "accuracy": 0.41363636363636364, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.6597761181265459, - "ece_fixed10": 0.12543213181861373, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 5, - "accuracy": 0.0, - "confidence": 0.17776660396925292 - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 86, - "accuracy": 0.1744186046511628, - "confidence": 0.27303145402366796 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 117, - "accuracy": 0.23931623931623933, - "confidence": 0.33872655115645356 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 32, - "accuracy": 0.3125, - "confidence": 0.44696075752412995 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 29, - "accuracy": 0.41379310344827586, - "confidence": 0.5450026309826125 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 29, - "accuracy": 0.4482758620689655, - "confidence": 0.6519538964258558 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 36, - "accuracy": 0.7222222222222222, - "confidence": 0.7469309807213727 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 38, - "accuracy": 0.6052631578947368, - "confidence": 0.846184630644432 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 68, - "accuracy": 0.8088235294117647, - "confidence": 0.957802627101878 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.31683079533007047, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.31683, - "native_vs_expected_score_mean_residual": -8.535621335370447e-06, - "native_vs_expected_score_normalized_mean_residual": -2.1339053338426116e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 1.627681647527468e-05, - "reported_truncations": 40, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4541284403669725, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 256, - "accuracy": 0.4, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.6636627743820697, - "ece_fixed10": 0.15052757690001484, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 5, - "accuracy": 0.0, - "confidence": 0.17776660396925292 - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 82, - "accuracy": 0.18292682926829268, - "confidence": 0.2809385200116827 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 219, - "accuracy": 0.2191780821917808, - "confidence": 0.3407915646008243 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 73, - "accuracy": 0.2465753424657534, - "confidence": 0.43777807169951655 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 37, - "accuracy": 0.32432432432432434, - "confidence": 0.5451374070920858 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 29, - "accuracy": 0.3448275862068966, - "confidence": 0.6503076029627376 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 26, - "accuracy": 0.5384615384615384, - "confidence": 0.747877476987054 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 22, - "accuracy": 0.6818181818181818, - "confidence": 0.8361046269577851 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 147, - "accuracy": 0.8435374149659864, - "confidence": 0.9860720063299236 - } - ], - "zero_gold_probability_count": 16, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 40, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4294117647058823, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.6416666666666667 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 89, - "accuracy": 0.55625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.639060657625, - "ece_fixed10": 0.24303062499999997, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 24, - "accuracy": 0.4583333333333333, - "confidence": 0.5404708333333333 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 19, - "accuracy": 0.631578947368421, - "confidence": 0.6526105263157894 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 24, - "accuracy": 0.6666666666666666, - "confidence": 0.7530833333333332 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 38, - "accuracy": 0.47368421052631576, - "confidence": 0.8578184210526315 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 55, - "accuracy": 0.5818181818181818, - "confidence": 0.9425981818181818 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.05, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 16, - "accuracy": 0.2, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8277920500293308, - "ece_fixed10": 0.08094118847539025, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 62, - "accuracy": 0.20967741935483872, - "confidence": 0.2635774024995526 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 17, - "accuracy": 0.17647058823529413, - "confidence": 0.3344644778269977 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.4476 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.31729610675246617, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.3172984375, - "native_vs_expected_score_mean_residual": -9.298056100889852e-06, - "native_vs_expected_score_normalized_mean_residual": -2.324514025222463e-06, - "native_vs_expected_score_normalized_mean_abs_residual": 1.8051798859790995e-05, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.3775, - "reliability_contract_pass": false - }, - "typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.4, - "brier": 0.6636627743820697, - "nll": "infinity", - "zero_true_probability_count": 16, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.55625, - "brier": 0.639060657625, - "nll": 0.9543628597553081, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.225, - "recall_true": 0.8875, - "balanced_accuracy": 0.55625, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.2, - "brier": 0.8277920500293308, - "nll": 1.6751829850565383, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.21449872654729513, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.31729610675246617, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.3172984375, - "native_vs_probability_expected_normalized_abs_residual": 1.8051798859790995e-05 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 56, - "reported_truncated_requests": 39, - "reported_untruncated_requests": 17, - "truncation_unknown_requests": 12, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 30, - "contract_valid_questions": 30, - "correct_questions": 4, - "accuracy_all_requested": 0.09523809523809523, - "accuracy_accepted": 0.13333333333333333, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 12 - } - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 1, - "accuracy_all_requested": 0.1111111111111111, - "accuracy_accepted": 0.1111111111111111, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 92, - "accuracy_all_requested": 0.5609756097560976, - "accuracy_accepted": 0.5609756097560976, - "native_normalized_score_mae": 0.273746875, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 1, - "accuracy_all_requested": 0.2, - "accuracy_accepted": 0.2, - "native_normalized_score_mae": 0.03564999999999996, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 1, - "accuracy_all_requested": 0.16666666666666666, - "accuracy_accepted": 0.16666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "255": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 98, - "valid_questions": 208, - "accuracy_all_requested": 0.44545454545454544, - "isolation_max_probability_delta": 0.005299999999999971 - } - } - }, - "decider": { - "old": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 557, - "accuracy": 0.6329545454545454, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.4520407761345868, - "ece_fixed10": 0.07885360091443505, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 13, - "accuracy": 0.15384615384615385, - "confidence": 0.2878359414438979 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 96, - "accuracy": 0.2916666666666667, - "confidence": 0.35808643979185395 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 117, - "accuracy": 0.29914529914529914, - "confidence": 0.4550644212136613 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 113, - "accuracy": 0.5486725663716814, - "confidence": 0.5502992807871521 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 95, - "accuracy": 0.5684210526315789, - "confidence": 0.6512159897370967 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 74, - "accuracy": 0.6216216216216216, - "confidence": 0.7526469829550104 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 75, - "accuracy": 0.52, - "confidence": 0.8476349920573549 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 297, - "accuracy": 0.9797979797979798, - "confidence": 0.9821973978685894 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.1744074886944538, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.174375, - "native_vs_expected_score_mean_residual": 9.361716157681166e-05, - "native_vs_expected_score_normalized_mean_residual": 2.3404290394202915e-05, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0005045907105654512, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7199074074074074, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.65625 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 111, - "accuracy": 0.8671875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.20685864539976934, - "ece_fixed10": 0.06862000612412975, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 3, - "accuracy": 0.0, - "confidence": 0.4706333333333334 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.5, - "confidence": 0.5423752925292529 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.5555555555555556, - "confidence": 0.6526817099013726 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 10, - "accuracy": 1.0, - "confidence": 0.746597304269573 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 21, - "accuracy": 0.8095238095238095, - "confidence": 0.863892183867017 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 83, - "accuracy": 0.9397590361445783, - "confidence": 0.9727326745928448 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8671875, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 46, - "accuracy": 0.71875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.2886980934375, - "ece_fixed10": 0.12390156250000003, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 12, - "accuracy": 0.25, - "confidence": 0.5467833333333334 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.5, - "confidence": 0.65995 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 10, - "accuracy": 0.8, - "confidence": 0.75133 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 15, - "accuracy": 0.9333333333333333, - "confidence": 0.85722 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 15, - "accuracy": 1.0, - "confidence": 0.9452999999999999 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.59375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.75 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 110, - "accuracy": 0.9821428571428571, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.03316499374993111, - "ece_fixed10": 0.004496716518667586, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.7228674980893416 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 1.0, - "confidence": 0.8960792158431686 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 108, - "accuracy": 0.9907407407407407, - "confidence": 0.9928806386265362 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9821428571428571, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.0004934676351429451, - "ece_fixed10": 0.005615684110938213, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 1.0, - "confidence": 0.8923892389238923 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 63, - "accuracy": 1.0, - "confidence": 0.9960032853646994 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 8, - "accuracy": 0.08333333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 1.0800082229848227, - "ece_fixed10": 0.50554161486276, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 14, - "accuracy": 0.2857142857142857, - "confidence": 0.376522054708792 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 31, - "accuracy": 0.12903225806451613, - "confidence": 0.4507396519073965 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 12, - "accuracy": 0.0, - "confidence": 0.555789290237643 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.0, - "confidence": 0.6573421920307969 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 0.0, - "confidence": 0.7774846499090965 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 23, - "accuracy": 0.0, - "confidence": 0.8321596014522482 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.0 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 54, - "accuracy": 0.84375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.3337635760293656, - "ece_fixed10": 0.27780819322601946, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 4, - "accuracy": 0.5, - "confidence": 0.35614134336566344 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 15, - "accuracy": 0.4666666666666667, - "confidence": 0.4703410534046106 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 20, - "accuracy": 1.0, - "confidence": 0.5365210268973103 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 17, - "accuracy": 1.0, - "confidence": 0.649362480198213 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 1.0, - "confidence": 0.7601554199781793 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.1744074886944538, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.174375, - "native_vs_expected_score_mean_residual": 9.361716157681166e-05, - "native_vs_expected_score_normalized_mean_residual": 2.3404290394202915e-05, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0005045907105654512, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.78125, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 52, - "accuracy": 0.5416666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.580678091882917, - "ece_fixed10": 0.053511011940026776, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.29287071292870714 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 15, - "accuracy": 0.3333333333333333, - "confidence": 0.366598098296981 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 30, - "accuracy": 0.5333333333333333, - "confidence": 0.4590508943280756 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 24, - "accuracy": 0.5833333333333334, - "confidence": 0.5562243556181186 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.6111111111111112, - "confidence": 0.6434918756134187 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.7503287604157877 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 1.0, - "confidence": 0.80935 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.3333333333333333, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.5833333333333334 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 46, - "accuracy": 0.4791666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.7310978053151042, - "ece_fixed10": 0.1746020992020731, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 1.0, - "confidence": 0.3889 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 14, - "accuracy": 0.5, - "confidence": 0.4634992234937779 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 26, - "accuracy": 0.38461538461538464, - "confidence": 0.5505581221049659 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 25, - "accuracy": 0.56, - "confidence": 0.6555970160827301 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 24, - "accuracy": 0.5, - "confidence": 0.7483272531464392 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.8292333333333334 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4583333333333333, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9166666666666666 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 28, - "accuracy": 0.2916666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.7266625879529256, - "ece_fixed10": 0.16986915508050404, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 12, - "accuracy": 0.16666666666666666, - "confidence": 0.2874163771534971 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 57, - "accuracy": 0.2631578947368421, - "confidence": 0.3495558744377868 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 16, - "accuracy": 0.0625, - "confidence": 0.44484933412955585 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 11, - "accuracy": 0.9090909090909091, - "confidence": 0.562166529740029 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.125, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.5833333333333334 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 38, - "accuracy": 0.59375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.44317944268298853, - "ece_fixed10": 0.21596898359993988, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 5, - "accuracy": 0.2, - "confidence": 0.3735735526447356 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 8, - "accuracy": 0.0, - "confidence": 0.4280608708714837 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.548341431064581 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 6, - "accuracy": 0.16666666666666666, - "confidence": 0.6335507955310079 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 5, - "accuracy": 0.4, - "confidence": 0.757010800919908 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.8417248758209155 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 28, - "accuracy": 1.0, - "confidence": 0.9577499723615354 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.5 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 380, - "accuracy": 0.6785714285714286, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.3962902799041901, - "ece_fixed10": 0.09161579012759397, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 8, - "accuracy": 0.25, - "confidence": 0.2860890692798907 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 37, - "accuracy": 0.21621621621621623, - "confidence": 0.3523982919289018 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 57, - "accuracy": 0.24561403508771928, - "confidence": 0.4583402233980338 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 54, - "accuracy": 0.5555555555555556, - "confidence": 0.5490698860969027 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 64, - "accuracy": 0.46875, - "confidence": 0.6537192822473022 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 42, - "accuracy": 0.7142857142857143, - "confidence": 0.7499940033396543 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 51, - "accuracy": 0.49019607843137253, - "confidence": 0.8512756827028034 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 247, - "accuracy": 0.9757085020242915, - "confidence": 0.9827891992285552 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.17575494897140104, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.17578125, - "native_vs_expected_score_mean_residual": -0.0003528793468725658, - "native_vs_expected_score_normalized_mean_residual": -8.821983671814145e-05, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0004932722069405388, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7714285714285715, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.8189655172413793 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 177, - "accuracy": 0.553125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.5496041445377812, - "ece_fixed10": 0.09001380874173501, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 5, - "accuracy": 0.0, - "confidence": 0.29063093690630937 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 59, - "accuracy": 0.3389830508474576, - "confidence": 0.36165358336692566 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 60, - "accuracy": 0.35, - "confidence": 0.4519524091385074 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 59, - "accuracy": 0.5423728813559322, - "confidence": 0.5514244894867025 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 31, - "accuracy": 0.7741935483870968, - "confidence": 0.646047901974092 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 32, - "accuracy": 0.5, - "confidence": 0.7561290187001652 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 24, - "accuracy": 0.5833333333333334, - "confidence": 0.839898524435777 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 50, - "accuracy": 1.0, - "confidence": 0.9792738991503588 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.17306002841750653, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.17296875, - "native_vs_expected_score_mean_residual": 0.0005401136700261891, - "native_vs_expected_score_normalized_mean_residual": 0.00013502841750654727, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0005159092141903635, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5777777777777777, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.7758620689655172 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 457, - "accuracy": 0.6077127659574468, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.4760084257347833, - "ece_fixed10": 0.11174037442690131, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 13, - "accuracy": 0.15384615384615385, - "confidence": 0.2878359414438979 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 92, - "accuracy": 0.2826086956521739, - "confidence": 0.3581710092016884 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 102, - "accuracy": 0.27450980392156865, - "confidence": 0.4528178576561688 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 81, - "accuracy": 0.48148148148148145, - "confidence": 0.5542221998889133 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 66, - "accuracy": 0.4696969696969697, - "confidence": 0.6501054069947663 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 56, - "accuracy": 0.5357142857142857, - "confidence": 0.751809524622238 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 60, - "accuracy": 0.4166666666666667, - "confidence": 0.8452387400716936 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 282, - "accuracy": 0.9787234042553191, - "confidence": 0.984160025414791 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7255434782608695, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.578125 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 46, - "accuracy": 0.71875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.2886980934375, - "ece_fixed10": 0.12390156250000003, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 12, - "accuracy": 0.25, - "confidence": 0.5467833333333334 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.5, - "confidence": 0.65995 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 10, - "accuracy": 0.8, - "confidence": 0.75133 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 15, - "accuracy": 0.9333333333333333, - "confidence": 0.85722 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 15, - "accuracy": 1.0, - "confidence": 0.9452999999999999 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.59375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.75 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 54, - "accuracy": 0.84375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.3337635760293656, - "ece_fixed10": 0.27780819322601946, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 4, - "accuracy": 0.5, - "confidence": 0.35614134336566344 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 15, - "accuracy": 0.4666666666666667, - "confidence": 0.4703410534046106 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 20, - "accuracy": 1.0, - "confidence": 0.5365210268973103 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 17, - "accuracy": 1.0, - "confidence": 0.649362480198213 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 1.0, - "confidence": 0.7601554199781793 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.1744074886944538, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.174375, - "native_vs_expected_score_mean_residual": 9.361716157681166e-05, - "native_vs_expected_score_normalized_mean_residual": 2.3404290394202915e-05, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0005045907105654512, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.78125, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.875 - } - } - }, - "family_macro_accuracy": 0.6401413690476191, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.6077127659574468, - "brier": 0.4760084257347833, - "nll": 0.8560895165680907, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.71875, - "brier": 0.2886980934375, - "nll": 0.4329852497065267, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.55, - "recall_true": 1.0, - "balanced_accuracy": 0.775, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.84375, - "brier": 0.3337635760293656, - "nll": 0.6027084825031839, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.07299414938303234, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.1744074886944538, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.174375, - "native_vs_probability_expected_normalized_abs_residual": 0.0005045907105654512 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 68, - "truncation_unknown_requests": 0, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 18, - "accuracy_all_requested": 0.42857142857142855, - "accuracy_accepted": 0.42857142857142855, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 9, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 80, - "accuracy_all_requested": 0.4878048780487805, - "accuracy_accepted": 0.4878048780487805, - "native_normalized_score_mae": 0.42892857142857144, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 4, - "accuracy_all_requested": 0.8, - "accuracy_accepted": 0.8, - "native_normalized_score_mae": 0.34, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 5, - "accuracy_all_requested": 0.8333333333333334, - "accuracy_accepted": 0.8333333333333334, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 2, - "accuracy_all_requested": 0.3333333333333333, - "accuracy_accepted": 0.3333333333333333, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 1, - "accuracy_all_requested": 0.16666666666666666, - "accuracy_accepted": 0.16666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 1, - "accuracy_all_requested": 0.16666666666666666, - "accuracy_accepted": 0.16666666666666666, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 111, - "valid_questions": 220, - "accuracy_all_requested": 0.5045454545454545, - "isolation_max_probability_delta": 0.020799999999999985 - } - }, - "fresh": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 444, - "accuracy": 0.5045454545454545, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.5570073599453393, - "ece_fixed10": 0.07443340245185769, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 113, - "accuracy": 0.23893805309734514, - "confidence": 0.2742519193080649 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 207, - "accuracy": 0.2898550724637681, - "confidence": 0.34900076133466945 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 97, - "accuracy": 0.30927835051546393, - "confidence": 0.44528515293987214 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 137, - "accuracy": 0.49635036496350365, - "confidence": 0.5440101928542644 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 60, - "accuracy": 0.4166666666666667, - "confidence": 0.6401449564064996 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 34, - "accuracy": 0.5588235294117647, - "confidence": 0.7495632941391928 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 26, - "accuracy": 0.5769230769230769, - "confidence": 0.8410491565692684 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 206, - "accuracy": 0.970873786407767, - "confidence": 0.9843911940166093 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30254762315699846, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.30253125000000003, - "native_vs_expected_score_mean_residual": 0.0004805141190689866, - "native_vs_expected_score_normalized_mean_residual": 0.00012012852976724664, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0006340132177241678, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4908256880733945, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.9416666666666667 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 45, - "accuracy": 0.5625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.52729967975, - "ece_fixed10": 0.14442375000000005, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.5294117647058824, - "confidence": 0.5408176470588236 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 19, - "accuracy": 0.42105263157894735, - "confidence": 0.6469736842105264 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 24, - "accuracy": 0.5833333333333334, - "confidence": 0.75055 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 16, - "accuracy": 0.6875, - "confidence": 0.837875 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 4, - "accuracy": 0.75, - "confidence": 0.912075 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.1, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 41, - "accuracy": 0.5125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.506689249, - "ece_fixed10": 0.050554999999999996, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 63, - "accuracy": 0.49206349206349204, - "confidence": 0.544368253968254 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 17, - "accuracy": 0.5882352941176471, - "confidence": 0.6323058823529412 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 19, - "accuracy": 0.2375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7603848411671006, - "ece_fixed10": 0.09655607828858574, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 18, - "accuracy": 0.2777777777777778, - "confidence": 0.2834224342870021 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 55, - "accuracy": 0.21818181818181817, - "confidence": 0.3383732025977865 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 7, - "accuracy": 0.2857142857142857, - "confidence": 0.43033661472036616 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 108, - "accuracy": 0.9, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.14011371882556184, - "ece_fixed10": 0.05479117360047251, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.5, - "confidence": 0.38935 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.42955704429557046 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.25, - "confidence": 0.5602034531377553 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6447644764476448 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.5, - "confidence": 0.7443818156989171 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.4, - "confidence": 0.844218352126776 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 103, - "accuracy": 0.9805825242718447, - "confidence": 0.9873689375138258 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 106, - "accuracy": 0.8833333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.16493426704488157, - "ece_fixed10": 0.04847207022023625, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 4, - "accuracy": 0.5, - "confidence": 0.4620973722372237 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.2, - "confidence": 0.5754221073736986 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.6896103089691031 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.75, - "confidence": 0.7604061844842216 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.4, - "confidence": 0.8480372620334198 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 99, - "accuracy": 0.9696969696969697, - "confidence": 0.9842150040757319 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8833333333333333, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 19, - "accuracy": 0.2375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8518319270189384, - "ece_fixed10": 0.25204326296304513, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 22, - "accuracy": 0.2727272727272727, - "confidence": 0.3520317540052721 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 21, - "accuracy": 0.09523809523809523, - "confidence": 0.4565743529392674 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 22, - "accuracy": 0.4090909090909091, - "confidence": 0.5507323756748692 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 13, - "accuracy": 0.15384615384615385, - "confidence": 0.6355222132581452 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.0, - "confidence": 0.7263999999999999 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 18, - "accuracy": 0.225, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.813013988237628, - "ece_fixed10": 0.04541527584334862, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 78, - "accuracy": 0.23076923076923078, - "confidence": 0.2696105393265114 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.30179999999999996 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30254762315699846, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.30253125000000003, - "native_vs_expected_score_mean_residual": 0.0004805141190689866, - "native_vs_expected_score_normalized_mean_residual": 0.00012012852976724664, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0006340132177241678, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 21, - "accuracy": 0.2625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8102507752368616, - "ece_fixed10": 0.1409205892091684, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 44, - "accuracy": 0.29545454545454547, - "confidence": 0.3646309584363323 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 32, - "accuracy": 0.21875, - "confidence": 0.4397294831036698 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.5184006133946728 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6033396660333967 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 34, - "accuracy": 0.425, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6978794815139786, - "ece_fixed10": 0.1263082413822574, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.0, - "confidence": 0.28354999999999997 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 26, - "accuracy": 0.23076923076923078, - "confidence": 0.3634336147337977 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 23, - "accuracy": 0.391304347826087, - "confidence": 0.45874216310607385 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 23, - "accuracy": 0.6956521739130435, - "confidence": 0.5326546941992861 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 6, - "accuracy": 0.5, - "confidence": 0.6313789237742893 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.1, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 33, - "accuracy": 0.4125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7021590386685603, - "ece_fixed10": 0.08234577387217024, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 15, - "accuracy": 0.26666666666666666, - "confidence": 0.28614273314516064 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 56, - "accuracy": 0.39285714285714285, - "confidence": 0.3395106555536601 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.7777777777777778, - "confidence": 0.41020915293320265 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 220, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.5527869842270272, - "ece_fixed10": 0.07535481952233898, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 54, - "accuracy": 0.2777777777777778, - "confidence": 0.2817668779706812 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 120, - "accuracy": 0.25, - "confidence": 0.3500957839390162 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 44, - "accuracy": 0.36363636363636365, - "confidence": 0.4452007421910528 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 70, - "accuracy": 0.4857142857142857, - "confidence": 0.5388917626289017 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 16, - "accuracy": 0.3125, - "confidence": 0.6375149674342122 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 15, - "accuracy": 0.5333333333333333, - "confidence": 0.7457618175197112 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 14, - "accuracy": 0.5714285714285714, - "confidence": 0.8488565543309914 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 107, - "accuracy": 0.9719626168224299, - "confidence": 0.9845542108777948 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.29988745169355646, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.299875, - "native_vs_expected_score_mean_residual": 0.0001402615253802919, - "native_vs_expected_score_normalized_mean_residual": 3.5065381345072975e-05, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0006563565430441371, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5321100917431193, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 224, - "accuracy": 0.509090909090909, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.5612277356636516, - "ece_fixed10": 0.07351198538137638, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 59, - "accuracy": 0.2033898305084746, - "confidence": 0.26737382154906025 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 87, - "accuracy": 0.3448275862068966, - "confidence": 0.347490385328674 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 53, - "accuracy": 0.2641509433962264, - "confidence": 0.4453552297879485 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 67, - "accuracy": 0.5074626865671642, - "confidence": 0.5493578065225541 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 44, - "accuracy": 0.45454545454545453, - "confidence": 0.6411013160327859 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 19, - "accuracy": 0.5789473684210527, - "confidence": 0.7525644598914151 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 12, - "accuracy": 0.5833333333333334, - "confidence": 0.8319405258472582 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 99, - "accuracy": 0.9696969696969697, - "confidence": 0.9842150040757319 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30520779462044045, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.3051875, - "native_vs_expected_score_mean_residual": 0.0008207667127576812, - "native_vs_expected_score_normalized_mean_residual": 0.0002051916781894203, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0006116698924041985, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5275229357798165, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 340, - "accuracy": 0.53125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.5350097553013881, - "ece_fixed10": 0.07229667514088574, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 35, - "accuracy": 0.2571428571428571, - "confidence": 0.2845955661240985 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 205, - "accuracy": 0.2926829268292683, - "confidence": 0.3494612565672028 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 97, - "accuracy": 0.30927835051546393, - "confidence": 0.44528515293987214 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 57, - "accuracy": 0.49122807017543857, - "confidence": 0.5445666038777935 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 24, - "accuracy": 0.2916666666666667, - "confidence": 0.6402915576829156 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 10, - "accuracy": 0.5, - "confidence": 0.7471952000732555 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 10, - "accuracy": 0.4, - "confidence": 0.8461278070800979 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 202, - "accuracy": 0.9752475247524752, - "confidence": 0.9858231978585223 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6176470588235294, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.9416666666666667 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 86, - "accuracy": 0.5375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.516994464375, - "ece_fixed10": 0.097489375, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 80, - "accuracy": 0.5, - "confidence": 0.54361375 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 36, - "accuracy": 0.5, - "confidence": 0.6400472222222222 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 24, - "accuracy": 0.5833333333333334, - "confidence": 0.75055 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 16, - "accuracy": 0.6875, - "confidence": 0.837875 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 4, - "accuracy": 0.75, - "confidence": 0.912075 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.05, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 18, - "accuracy": 0.225, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.813013988237628, - "ece_fixed10": 0.04541527584334862, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 78, - "accuracy": 0.23076923076923078, - "confidence": 0.2696105393265114 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.30179999999999996 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30254762315699846, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.30253125000000003, - "native_vs_expected_score_mean_residual": 0.0004805141190689866, - "native_vs_expected_score_normalized_mean_residual": 0.00012012852976724664, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0006340132177241678, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.4658333333333333, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.53125, - "brier": 0.5350097553013881, - "nll": 1.0203942121826428, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.5375, - "brier": 0.516994464375, - "nll": 0.714786410834577, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.575, - "recall_true": 0.5, - "balanced_accuracy": 0.5375, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.225, - "brier": 0.813013988237628, - "nll": 1.6466804789065077, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.20152128376829256, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.30254762315699846, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.30253125000000003, - "native_vs_probability_expected_normalized_abs_residual": 0.0006340132177241678 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 68, - "truncation_unknown_requests": 0, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 4, - "accuracy_all_requested": 0.09523809523809523, - "accuracy_accepted": 0.09523809523809523, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 2, - "accuracy_all_requested": 0.2222222222222222, - "accuracy_accepted": 0.2222222222222222, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 86, - "accuracy_all_requested": 0.524390243902439, - "accuracy_accepted": 0.524390243902439, - "native_normalized_score_mae": 0.36557291666666664, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 4, - "accuracy_all_requested": 0.8, - "accuracy_accepted": 0.8, - "native_normalized_score_mae": 0.13, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 4, - "accuracy_all_requested": 0.6666666666666666, - "accuracy_accepted": 0.6666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 96, - "valid_questions": 220, - "accuracy_all_requested": 0.43636363636363634, - "isolation_max_probability_delta": 0.013000000000000012 - } - } - }, - "base-2b": { - "old": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 500, - "accuracy": 0.5681818181818182, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.5768518209070048, - "ece_fixed10": 0.2026392868028203, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 10, - "accuracy": 0.4, - "confidence": 0.2896300923775639 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 49, - "accuracy": 0.32653061224489793, - "confidence": 0.36176176402715016 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 79, - "accuracy": 0.26582278481012656, - "confidence": 0.4515540928045029 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 83, - "accuracy": 0.4457831325301205, - "confidence": 0.5538946626071113 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 80, - "accuracy": 0.3, - "confidence": 0.651600192919048 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 138, - "accuracy": 0.2753623188405797, - "confidence": 0.7627499337950342 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 117, - "accuracy": 0.5213675213675214, - "confidence": 0.8516555091928629 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 324, - "accuracy": 0.9228395061728395, - "confidence": 0.9778250483709188 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.103275410034405, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.10327540668049551, - "native_vs_expected_score_mean_residual": 2.571971649892313e-08, - "native_vs_expected_score_normalized_mean_residual": 6.429929124730782e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 3.0280404299427854e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.625, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.4895833333333333 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 103, - "accuracy": 0.8046875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.30272308590154934, - "ece_fixed10": 0.13434160798282319, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.0, - "confidence": 0.47522017203803746 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.5746833541128021 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.25, - "confidence": 0.6562359617637399 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.25, - "confidence": 0.7365675427632561 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 4, - "accuracy": 0.5, - "confidence": 0.8709501222152677 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 107, - "accuracy": 0.8878504672897196, - "confidence": 0.9922195092707161 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8046875, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 24, - "accuracy": 0.375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.8024335381077476, - "ece_fixed10": 0.42220145277678967, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.6513549089431763 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 51, - "accuracy": 0.3137254901960784, - "confidence": 0.7756630626379275 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 12, - "accuracy": 0.5833333333333334, - "confidence": 0.8427859743436178 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 104, - "accuracy": 0.9285714285714286, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.10722459870949627, - "ece_fixed10": 0.053708577573857476, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.44178535660632334 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.5, - "confidence": 0.5649608281452168 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.6423012398154174 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.0, - "confidence": 0.7579985683695122 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 1.0, - "confidence": 0.8483063181600814 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 97, - "accuracy": 0.979381443298969, - "confidence": 0.9901973328807759 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9285714285714286, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 62, - "accuracy": 0.96875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.08978995158855806, - "ece_fixed10": 0.07838535744862543, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.5, - "confidence": 0.35270214493854546 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 3, - "accuracy": 1.0, - "confidence": 0.42897341263101163 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 1.0, - "confidence": 0.5802746952391644 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 1.0, - "confidence": 0.7516588319922766 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 7, - "accuracy": 0.8571428571428571, - "confidence": 0.8571914254270874 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 46, - "accuracy": 1.0, - "confidence": 0.9818899158383967 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 9, - "accuracy": 0.09375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 1.056400567536298, - "ece_fixed10": 0.48612752923496405, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 4, - "accuracy": 0.5, - "confidence": 0.29650773642401923 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 26, - "accuracy": 0.15384615384615385, - "confidence": 0.36324824388582877 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 20, - "accuracy": 0.15, - "confidence": 0.4521390085994783 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.0, - "confidence": 0.5448212123467716 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.0, - "confidence": 0.6551088429974189 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 15, - "accuracy": 0.0, - "confidence": 0.7468211648313328 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 13, - "accuracy": 0.0, - "confidence": 0.8497315940045133 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 3, - "accuracy": 0.0, - "confidence": 0.910909640781386 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.0 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 52, - "accuracy": 0.8125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.2361466873636782, - "ece_fixed10": 0.17061250568165753, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 4, - "accuracy": 0.5, - "confidence": 0.28092096034442054 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.3649985963778772 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.42737326503589673 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.0, - "confidence": 0.5334895984838766 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.0, - "confidence": 0.6297378709379815 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 0.0, - "confidence": 0.7041150279036534 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 1.0, - "confidence": 0.8636305362923078 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 35, - "accuracy": 1.0, - "confidence": 0.9424173511763662 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.103275410034405, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.10327540668049551, - "native_vs_expected_score_mean_residual": 2.571971649892313e-08, - "native_vs_expected_score_normalized_mean_residual": 6.429929124730782e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 3.0280404299427854e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.78125, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9375 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 47, - "accuracy": 0.4895833333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.7350433643975048, - "ece_fixed10": 0.2651467590248772, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.29692274110821826 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 1.0, - "confidence": 0.35873429439875537 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 13, - "accuracy": 0.3076923076923077, - "confidence": 0.4603141344511844 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 11, - "accuracy": 0.36363636363636365, - "confidence": 0.5531705524073626 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 14, - "accuracy": 0.42857142857142855, - "confidence": 0.6579994942916609 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 15, - "accuracy": 0.2, - "confidence": 0.7616150698637135 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 20, - "accuracy": 0.55, - "confidence": 0.8628444049992717 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 20, - "accuracy": 0.85, - "confidence": 0.9456283910853364 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.125, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.125 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 35, - "accuracy": 0.3645833333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.82971294777414, - "ece_fixed10": 0.2727446020739942, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.3790053909952391 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 20, - "accuracy": 0.3, - "confidence": 0.4579237847518359 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 23, - "accuracy": 0.43478260869565216, - "confidence": 0.5570073123980198 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 15, - "accuracy": 0.3333333333333333, - "confidence": 0.6587323726893473 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 19, - "accuracy": 0.3684210526315789, - "confidence": 0.7577618120377959 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 12, - "accuracy": 0.4166666666666667, - "confidence": 0.84387124094373 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 4, - "accuracy": 0.25, - "confidence": 0.9179767088858656 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.08333333333333333, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.25 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 24, - "accuracy": 0.25, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 1.028429930217442, - "ece_fixed10": 0.41349586297171964, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2896633955936612 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.36034894690684355 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.4337235304545297 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 20, - "accuracy": 0.3, - "confidence": 0.5526630145986594 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 22, - "accuracy": 0.22727272727272727, - "confidence": 0.6455567447861381 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 18, - "accuracy": 0.2222222222222222, - "confidence": 0.7568799011393468 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 21, - "accuracy": 0.23809523809523808, - "confidence": 0.8442343699914515 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 2, - "accuracy": 0.5, - "confidence": 0.9065560836415889 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.0 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 40, - "accuracy": 0.625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.5358729259785378, - "ece_fixed10": 0.14642755633021687, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 7, - "accuracy": 0.42857142857142855, - "confidence": 0.3501349465215281 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 10, - "accuracy": 0.2, - "confidence": 0.4477402738961949 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.875, - "confidence": 0.5217453530124536 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.625, - "confidence": 0.6486299731585321 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 9, - "accuracy": 0.5555555555555556, - "confidence": 0.7630134847898982 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 12, - "accuracy": 0.75, - "confidence": 0.8454899536154773 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 10, - "accuracy": 0.9, - "confidence": 0.9316827724159189 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4375, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.5625 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 350, - "accuracy": 0.625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.48320891087299356, - "ece_fixed10": 0.17946984285816178, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.2850449963465936 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 23, - "accuracy": 0.17391304347826086, - "confidence": 0.3624359691899322 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 46, - "accuracy": 0.2826086956521739, - "confidence": 0.4525511210348905 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 50, - "accuracy": 0.4, - "confidence": 0.5579141355125624 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 38, - "accuracy": 0.21052631578947367, - "confidence": 0.650834336753684 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 73, - "accuracy": 0.3013698630136986, - "confidence": 0.7639968651332872 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 61, - "accuracy": 0.5737704918032787, - "confidence": 0.855409641163366 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 263, - "accuracy": 0.935361216730038, - "confidence": 0.98281685278218 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.10692898568770887, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.10692897137505497, - "native_vs_expected_score_mean_residual": 5.732414400799746e-08, - "native_vs_expected_score_normalized_mean_residual": 1.4331036001999364e-08, - "native_vs_expected_score_normalized_mean_abs_residual": 3.0030024978852654e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.669047619047619, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.4827586206896552 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 150, - "accuracy": 0.46875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.7407269134665244, - "ece_fixed10": 0.25949644404371097, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 4, - "accuracy": 0.5, - "confidence": 0.29650773642401923 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 26, - "accuracy": 0.46153846153846156, - "confidence": 0.3611653517677661 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 33, - "accuracy": 0.24242424242424243, - "confidence": 0.45016429587729595 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 33, - "accuracy": 0.5151515151515151, - "confidence": 0.5478045521443066 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 42, - "accuracy": 0.38095238095238093, - "confidence": 0.6522931104019966 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 65, - "accuracy": 0.24615384615384617, - "confidence": 0.7613495339843809 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 56, - "accuracy": 0.4642857142857143, - "confidence": 0.8475661868678507 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 61, - "accuracy": 0.8688524590163934, - "confidence": 0.9563030064010551 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.09962183438110114, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.09962184198593604, - "native_vs_expected_score_mean_residual": -5.8847110101512e-09, - "native_vs_expected_score_normalized_mean_residual": -1.4711777525378e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 3.0530783620003055e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.42777777777777776, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.5172413793103449 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 424, - "accuracy": 0.5638297872340425, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.5866495584680758, - "ece_fixed10": 0.2017695474600319, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.2954361803996594 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 43, - "accuracy": 0.27906976744186046, - "confidence": 0.3613101130014673 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 78, - "accuracy": 0.2692307692307692, - "confidence": 0.45186410341692096 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 81, - "accuracy": 0.4567901234567901, - "confidence": 0.5543984913508948 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 75, - "accuracy": 0.30666666666666664, - "confidence": 0.6527694538777166 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 86, - "accuracy": 0.2558139534883721, - "confidence": 0.7557739493171717 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 94, - "accuracy": 0.4574468085106383, - "confidence": 0.8513864572790018 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 289, - "accuracy": 0.9134948096885813, - "confidence": 0.9821131777889441 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6331521739130435, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.25 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 24, - "accuracy": 0.375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.8024335381077476, - "ece_fixed10": 0.42220145277678967, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.6513549089431763 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 51, - "accuracy": 0.3137254901960784, - "confidence": 0.7756630626379275 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 12, - "accuracy": 0.5833333333333334, - "confidence": 0.8427859743436178 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.375, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 52, - "accuracy": 0.8125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.2361466873636782, - "ece_fixed10": 0.17061250568165753, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 4, - "accuracy": 0.5, - "confidence": 0.28092096034442054 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.3649985963778772 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.42737326503589673 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.0, - "confidence": 0.5334895984838766 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.0, - "confidence": 0.6297378709379815 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 0.0, - "confidence": 0.7041150279036534 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 1.0, - "confidence": 0.8636305362923078 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 35, - "accuracy": 1.0, - "confidence": 0.9424173511763662 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.103275410034405, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.10327540668049551, - "native_vs_expected_score_mean_residual": 2.571971649892313e-08, - "native_vs_expected_score_normalized_mean_residual": 6.429929124730782e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 3.0280404299427854e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.78125, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9375 - } - } - }, - "family_macro_accuracy": 0.5712425595238095, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.5638297872340425, - "brier": 0.5866495584680758, - "nll": 1.1525129603560054, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.375, - "brier": 0.8024335381077476, - "nll": 1.0482543011815757, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.0, - "recall_true": 1.0, - "balanced_accuracy": 0.5, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.8125, - "brier": 0.2361466873636782, - "nll": 0.4554143784168391, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.0560214998368148, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.103275410034405, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.10327540668049551, - "native_vs_probability_expected_normalized_abs_residual": 3.0280404299427854e-08 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 44, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 44, - "truncation_unknown_requests": 24, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 18, - "contract_valid_questions": 18, - "correct_questions": 10, - "accuracy_all_requested": 0.23809523809523808, - "accuracy_accepted": 0.5555555555555556, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 24 - } - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 9, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 64, - "accuracy_all_requested": 0.3902439024390244, - "accuracy_accepted": 0.3902439024390244, - "native_normalized_score_mae": 0.42433279513248373, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 3, - "accuracy_all_requested": 0.6, - "accuracy_accepted": 0.6, - "native_normalized_score_mae": 0.5912923862536749, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 5, - "accuracy_all_requested": 0.8333333333333334, - "accuracy_accepted": 0.8333333333333334, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 4, - "accuracy_all_requested": 0.6666666666666666, - "accuracy_accepted": 0.6666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 1, - "accuracy_all_requested": 0.16666666666666666, - "accuracy_accepted": 0.16666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "64": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "128": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "255": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 86, - "valid_questions": 196, - "accuracy_all_requested": 0.39090909090909093, - "isolation_max_probability_delta": 0.0 - } - }, - "fresh": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 370, - "accuracy": 0.42045454545454547, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.7131401721155806, - "ece_fixed10": 0.20155998603191633, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 1, - "accuracy": 0.0, - "confidence": 0.19998528876980515 - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 7, - "accuracy": 0.14285714285714285, - "confidence": 0.277494439076451 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 45, - "accuracy": 0.3111111111111111, - "confidence": 0.3604697947188639 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 181, - "accuracy": 0.2430939226519337, - "confidence": 0.4593789679669198 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 285, - "accuracy": 0.34035087719298246, - "confidence": 0.5391371953006456 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 106, - "accuracy": 0.39622641509433965, - "confidence": 0.6501476717045056 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 59, - "accuracy": 0.4915254237288136, - "confidence": 0.7424703107499199 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 90, - "accuracy": 0.5666666666666667, - "confidence": 0.8499739834170628 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 106, - "accuracy": 0.8679245283018868, - "confidence": 0.9715891196063917 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3125684816392898, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.31256848259363323, - "native_vs_expected_score_mean_residual": 4.91213872910734e-09, - "native_vs_expected_score_normalized_mean_residual": 1.228034682276835e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 1.403788882636814e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.3302752293577982, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.7 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 37, - "accuracy": 0.4625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.5053272452460555, - "ece_fixed10": 0.08540617898106581, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 77, - "accuracy": 0.45454545454545453, - "confidence": 0.5415567840848651 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.622459352016449 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 40, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6681559322600907, - "ece_fixed10": 0.271347726136446, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.5926666259765625 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 27, - "accuracy": 0.5555555555555556, - "confidence": 0.6553181763048526 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 12, - "accuracy": 0.3333333333333333, - "confidence": 0.7183037201563517 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 40, - "accuracy": 0.5, - "confidence": 0.849681232869625 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 17, - "accuracy": 0.2125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.9502399902211524, - "ece_fixed10": 0.34056563518179933, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 28, - "accuracy": 0.21428571428571427, - "confidence": 0.467252758867365 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 31, - "accuracy": 0.16129032258064516, - "confidence": 0.5425235403350285 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 14, - "accuracy": 0.21428571428571427, - "confidence": 0.6581545127476934 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 6, - "accuracy": 0.5, - "confidence": 0.7210360224764091 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 0.0, - "confidence": 0.803564502545681 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 85, - "accuracy": 0.7083333333333334, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.3898917407864388, - "ece_fixed10": 0.06744351177024703, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 3, - "accuracy": 0.0, - "confidence": 0.2808071114307083 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 12, - "accuracy": 0.16666666666666666, - "confidence": 0.3522191428122105 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 12, - "accuracy": 0.5, - "confidence": 0.45139684363239163 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 16, - "accuracy": 0.625, - "confidence": 0.5431096585222035 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.625, - "confidence": 0.639049680914457 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 11, - "accuracy": 0.8181818181818182, - "confidence": 0.7611687095633329 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 16, - "accuracy": 0.875, - "confidence": 0.8506196667882987 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 42, - "accuracy": 0.9285714285714286, - "confidence": 0.9762157154959222 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7083333333333334, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 89, - "accuracy": 0.7416666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.34374235132350583, - "ece_fixed10": 0.03763027425547704, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 1, - "accuracy": 0.0, - "confidence": 0.19998528876980515 - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 3, - "accuracy": 0.0, - "confidence": 0.27068390222114386 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 8, - "accuracy": 0.375, - "confidence": 0.3492501363915494 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 12, - "accuracy": 0.4166666666666667, - "confidence": 0.44373169176567395 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 9, - "accuracy": 0.5555555555555556, - "confidence": 0.5505915751403035 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.6666666666666666, - "confidence": 0.6639880779737576 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 9, - "accuracy": 0.7777777777777778, - "confidence": 0.7493475535738284 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 13, - "accuracy": 0.8461538461538461, - "confidence": 0.8482895153232632 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 56, - "accuracy": 0.9285714285714286, - "confidence": 0.9753642476099447 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7416666666666667, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 22, - "accuracy": 0.275, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8620738003787114, - "ece_fixed10": 0.2676285395531104, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 11, - "accuracy": 0.5454545454545454, - "confidence": 0.373983864978012 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 32, - "accuracy": 0.25, - "confidence": 0.4515128367339108 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 28, - "accuracy": 0.25, - "confidence": 0.5460034536313526 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.125, - "confidence": 0.6355890614433023 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 0.0, - "confidence": 0.7028857102975251 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 16, - "accuracy": 0.2, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.95210702105606, - "ece_fixed10": 0.3170794129084339, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 19, - "accuracy": 0.15789473684210525, - "confidence": 0.47567572207584946 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 60, - "accuracy": 0.21666666666666667, - "confidence": 0.5286724203257959 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6081690936858222 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3125684816392898, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.31256848259363323, - "native_vs_expected_score_mean_residual": 4.91213872910734e-09, - "native_vs_expected_score_normalized_mean_residual": 1.228034682276835e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 1.403788882636814e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 22, - "accuracy": 0.275, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 1.0729411103683213, - "ece_fixed10": 0.42029023663635107, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.4682453990307693 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.29411764705882354, - "confidence": 0.5388575922534251 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 15, - "accuracy": 0.2, - "confidence": 0.6491615865495685 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 15, - "accuracy": 0.26666666666666666, - "confidence": 0.7583260100309652 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 16, - "accuracy": 0.3125, - "confidence": 0.8605705350826894 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 8, - "accuracy": 0.125, - "confidence": 0.9208735951614871 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 23, - "accuracy": 0.2875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.820557778989014, - "ece_fixed10": 0.2202510551902553, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 1.0, - "confidence": 0.2879880325796003 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 13, - "accuracy": 0.23076923076923078, - "confidence": 0.362888643149522 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 30, - "accuracy": 0.2, - "confidence": 0.4555873774029548 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 26, - "accuracy": 0.3076923076923077, - "confidence": 0.5402949676648278 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 10, - "accuracy": 0.5, - "confidence": 0.6475229605482073 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 19, - "accuracy": 0.2375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.9126878765870636, - "ece_fixed10": 0.30146817486402083, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.3691350817680359 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 39, - "accuracy": 0.15384615384615385, - "confidence": 0.46038190052968275 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 20, - "accuracy": 0.4, - "confidence": 0.5340781524174794 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.18181818181818182, - "confidence": 0.6496996425356557 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 5, - "accuracy": 0.4, - "confidence": 0.7330255818774957 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 4, - "accuracy": 0.25, - "confidence": 0.8250094402666813 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 186, - "accuracy": 0.42272727272727273, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.7014073191387005, - "ece_fixed10": 0.17726061240570337, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 4, - "accuracy": 0.25, - "confidence": 0.2826023417179313 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 31, - "accuracy": 0.3225806451612903, - "confidence": 0.36202080985421853 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 98, - "accuracy": 0.25510204081632654, - "confidence": 0.4589980848817875 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 138, - "accuracy": 0.34782608695652173, - "confidence": 0.5379414974782126 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 61, - "accuracy": 0.4426229508196721, - "confidence": 0.6509687541462338 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 42, - "accuracy": 0.5, - "confidence": 0.7422923991532395 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 20, - "accuracy": 0.7, - "confidence": 0.843083363300026 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 46, - "accuracy": 0.8695652173913043, - "confidence": 0.9712355213563397 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3180997988455819, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.31809979926329107, - "native_vs_expected_score_mean_residual": -1.2935113530287979e-08, - "native_vs_expected_score_normalized_mean_residual": -3.2337783825719947e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 1.5635727675100685e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.38990825688073394, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 184, - "accuracy": 0.41818181818181815, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.7248730250924607, - "ece_fixed10": 0.22585935965812934, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 1, - "accuracy": 0.0, - "confidence": 0.19998528876980515 - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 3, - "accuracy": 0.0, - "confidence": 0.27068390222114386 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 14, - "accuracy": 0.2857142857142857, - "confidence": 0.35703540406200723 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 83, - "accuracy": 0.2289156626506024, - "confidence": 0.45982868534454596 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 147, - "accuracy": 0.3333333333333333, - "confidence": 0.5402596871339501 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 45, - "accuracy": 0.3333333333333333, - "confidence": 0.649034648839052 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 17, - "accuracy": 0.47058823529411764, - "confidence": 0.7429098570476009 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 70, - "accuracy": 0.5285714285714286, - "confidence": 0.8519427320219304 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 60, - "accuracy": 0.8666666666666667, - "confidence": 0.9718602115980984 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30703716443299756, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.3070371659239754, - "native_vs_expected_score_mean_residual": 2.275939098850266e-08, - "native_vs_expected_score_normalized_mean_residual": 5.689847747125665e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 1.2440049977635593e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.40825688073394495, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 277, - "accuracy": 0.4328125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.7148689618386475, - "ece_fixed10": 0.19460317640931074, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 1, - "accuracy": 0.0, - "confidence": 0.19998528876980515 - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 7, - "accuracy": 0.14285714285714285, - "confidence": 0.277494439076451 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 45, - "accuracy": 0.3111111111111111, - "confidence": 0.3604697947188639 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 162, - "accuracy": 0.25308641975308643, - "confidence": 0.4574676202627861 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 147, - "accuracy": 0.32653061224489793, - "confidence": 0.5417769825892862 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 75, - "accuracy": 0.3333333333333333, - "confidence": 0.6499535372094856 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 47, - "accuracy": 0.5319148936170213, - "confidence": 0.7486405040929586 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 50, - "accuracy": 0.62, - "confidence": 0.8502081838550127 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 106, - "accuracy": 0.8679245283018868, - "confidence": 0.9715891196063917 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4235294117647059, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.7 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 77, - "accuracy": 0.48125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.5867415887530731, - "ece_fixed10": 0.17162751108407973, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 78, - "accuracy": 0.46153846153846156, - "confidence": 0.542212038468092 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 30, - "accuracy": 0.5666666666666667, - "confidence": 0.6520322938760121 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 12, - "accuracy": 0.3333333333333333, - "confidence": 0.7183037201563517 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 40, - "accuracy": 0.5, - "confidence": 0.849681232869625 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 16, - "accuracy": 0.2, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.95210702105606, - "ece_fixed10": 0.3170794129084339, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 19, - "accuracy": 0.15789473684210525, - "confidence": 0.47567572207584946 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 60, - "accuracy": 0.21666666666666667, - "confidence": 0.5286724203257959 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6081690936858222 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3125684816392898, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.31256848259363323, - "native_vs_expected_score_mean_residual": 4.91213872910734e-09, - "native_vs_expected_score_normalized_mean_residual": 1.228034682276835e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 1.403788882636814e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.39, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.4328125, - "brier": 0.7148689618386475, - "nll": 1.5125692348302717, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.48125, - "brier": 0.5867415887530731, - "nll": 0.8074560956855128, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.0, - "recall_true": 0.9625, - "balanced_accuracy": 0.48125, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.2, - "brier": 0.95210702105606, - "nll": 2.144207760094438, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.22731912785976321, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.3125684816392898, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.31256848259363323, - "native_vs_probability_expected_normalized_abs_residual": 1.403788882636814e-08 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 44, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 44, - "truncation_unknown_requests": 24, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 18, - "contract_valid_questions": 18, - "correct_questions": 4, - "accuracy_all_requested": 0.09523809523809523, - "accuracy_accepted": 0.2222222222222222, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 24 - } - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 2, - "accuracy_all_requested": 0.2222222222222222, - "accuracy_accepted": 0.2222222222222222, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 60, - "accuracy_all_requested": 0.36585365853658536, - "accuracy_accepted": 0.36585365853658536, - "native_normalized_score_mae": 0.45097056970310706, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 5, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": 0.11979871056973934, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 1, - "accuracy_all_requested": 0.16666666666666666, - "accuracy_accepted": 0.16666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "64": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "128": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "255": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 71, - "valid_questions": 196, - "accuracy_all_requested": 0.32272727272727275, - "isolation_max_probability_delta": 0.033267274498939514 - } - } - }, - "base-4b": { - "old": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 613, - "accuracy": 0.696590909090909, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.4310028636947534, - "ece_fixed10": 0.12689413363782995, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.365892778908834 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 59, - "accuracy": 0.3389830508474576, - "confidence": 0.4647860507803505 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 87, - "accuracy": 0.5287356321839081, - "confidence": 0.5541594431241464 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 97, - "accuracy": 0.422680412371134, - "confidence": 0.6535381234539136 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 106, - "accuracy": 0.5188679245283019, - "confidence": 0.748862284808273 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 74, - "accuracy": 0.5675675675675675, - "confidence": 0.8519432126286925 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 448, - "accuracy": 0.9084821428571429, - "confidence": 0.9819713306786599 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.048436180626773864, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.048436173722564035, - "native_vs_expected_score_mean_residual": 2.3944358527178133e-08, - "native_vs_expected_score_normalized_mean_residual": 5.986089631794533e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 2.906006874459771e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6921296296296297, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.5104166666666666 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 108, - "accuracy": 0.84375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.2342417146166414, - "ece_fixed10": 0.10329694445253162, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.3331189230036495 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 0.0, - "confidence": 0.4774424373211804 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.542445819289184 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.0, - "confidence": 0.6302549123634461 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 6, - "accuracy": 0.5, - "confidence": 0.7539497916610597 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 4, - "accuracy": 0.75, - "confidence": 0.8527144822019697 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 109, - "accuracy": 0.9174311926605505, - "confidence": 0.9884038636172571 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.84375, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 40, - "accuracy": 0.625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.4583870923415588, - "ece_fixed10": 0.1820569833507761, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.575727664762073 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.5, - "confidence": 0.648589680592219 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 11, - "accuracy": 0.36363636363636365, - "confidence": 0.7426146106286482 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.16666666666666666, - "confidence": 0.8576921708881855 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 26, - "accuracy": 0.9615384615384616, - "confidence": 0.9758503093169286 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.46875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.6875 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 109, - "accuracy": 0.9732142857142857, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.05274966636037035, - "ece_fixed10": 0.02456981423315471, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.5031886262446713 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 0.0, - "confidence": 0.8884582793815097 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 110, - "accuracy": 0.9818181818181818, - "confidence": 0.9942413594634226 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9732142857142857, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.0044542244611689066, - "ece_fixed10": 0.01439576842329148, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.5393320317013822 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 63, - "accuracy": 1.0, - "confidence": 0.9926879172890153 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 70, - "accuracy": 0.7291666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.5036952682523081, - "ece_fixed10": 0.22992641439149022, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.49607636415787715 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 11, - "accuracy": 1.0, - "confidence": 0.5574286243097496 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 16, - "accuracy": 0.6875, - "confidence": 0.6756403992418057 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 22, - "accuracy": 0.8636363636363636, - "confidence": 0.758396398912988 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 17, - "accuracy": 0.7058823529411765, - "confidence": 0.8606520713778083 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 29, - "accuracy": 0.5517241379310345, - "confidence": 0.9505037573682784 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4166666666666667, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.4166666666666667 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 58, - "accuracy": 0.90625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.1309433975394751, - "ece_fixed10": 0.06284843798673961, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 1.0, - "confidence": 0.47597716854436145 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.5, - "confidence": 0.5372950983257349 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.6820185306503338 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 1.0, - "confidence": 0.7511516287201252 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 1.0, - "confidence": 0.8137494633696232 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 48, - "accuracy": 1.0, - "confidence": 0.9802939435313948 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.048436180626773864, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.048436173722564035, - "native_vs_expected_score_mean_residual": 2.3944358527178133e-08, - "native_vs_expected_score_normalized_mean_residual": 5.986089631794533e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 2.906006874459771e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9375 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 49, - "accuracy": 0.5104166666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.6927366676648784, - "ece_fixed10": 0.21294870469794336, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.3437981547010289 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 6, - "accuracy": 0.16666666666666666, - "confidence": 0.44974637370800724 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 15, - "accuracy": 0.4666666666666667, - "confidence": 0.5471388335535708 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.5, - "confidence": 0.6474989102771106 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 19, - "accuracy": 0.3684210526315789, - "confidence": 0.7568199990726887 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 22, - "accuracy": 0.5909090909090909, - "confidence": 0.8480075385286749 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 14, - "accuracy": 0.8571428571428571, - "confidence": 0.9399423129320695 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.16666666666666666, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.16666666666666666 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 49, - "accuracy": 0.5104166666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.6348621799548558, - "ece_fixed10": 0.17008874989123873, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.5, - "confidence": 0.3824469873212201 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 26, - "accuracy": 0.34615384615384615, - "confidence": 0.46616753591997157 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 14, - "accuracy": 0.6428571428571429, - "confidence": 0.5592327091356926 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 17, - "accuracy": 0.4117647058823529, - "confidence": 0.6506453309332758 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 20, - "accuracy": 0.5, - "confidence": 0.7476268579139838 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 0.6363636363636364, - "confidence": 0.853853298746921 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 6, - "accuracy": 1.0, - "confidence": 0.9339296853632834 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.16666666666666666, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.25 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 27, - "accuracy": 0.28125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.9311168155017566, - "ece_fixed10": 0.3208738073425578, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 4, - "accuracy": 0.25, - "confidence": 0.37685645078283964 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 22, - "accuracy": 0.3181818181818182, - "confidence": 0.4636648747265642 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 23, - "accuracy": 0.2608695652173913, - "confidence": 0.5558794044239661 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 21, - "accuracy": 0.19047619047619047, - "confidence": 0.6523059228827675 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 21, - "accuracy": 0.3333333333333333, - "confidence": 0.7337675265977411 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.4, - "confidence": 0.8406127433855757 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.041666666666666664 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 39, - "accuracy": 0.609375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.6280929190360265, - "ece_fixed10": 0.27727676750302865, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.5, - "confidence": 0.5544110330804386 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 7, - "accuracy": 0.42857142857142855, - "confidence": 0.6355256713744237 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 5, - "accuracy": 0.6, - "confidence": 0.7517366980112288 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 7, - "accuracy": 0.42857142857142855, - "confidence": 0.8431254247767892 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 43, - "accuracy": 0.6744186046511628, - "confidence": 0.9657592997887982 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.3125, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.5625 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 434, - "accuracy": 0.775, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.3217840838473388, - "ece_fixed10": 0.08316366021945415, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 4, - "accuracy": 0.25, - "confidence": 0.3721035309075323 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 26, - "accuracy": 0.2692307692307692, - "confidence": 0.46203103254432476 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 45, - "accuracy": 0.5111111111111111, - "confidence": 0.5488461398618623 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 53, - "accuracy": 0.49056603773584906, - "confidence": 0.6534536384864872 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 53, - "accuracy": 0.5849056603773585, - "confidence": 0.7472654107022569 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 38, - "accuracy": 0.7105263157894737, - "confidence": 0.8562330574892958 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 341, - "accuracy": 0.9354838709677419, - "confidence": 0.9841565591429069 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.04299290817328186, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.042992904612333405, - "native_vs_expected_score_mean_residual": 3.842089410510674e-09, - "native_vs_expected_score_normalized_mean_residual": 9.605223526276685e-10, - "native_vs_expected_score_normalized_mean_abs_residual": 2.6064077843918065e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7833333333333333, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.6637931034482759 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 179, - "accuracy": 0.559375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.6221357284277288, - "ece_fixed10": 0.20342246211998768, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 5, - "accuracy": 0.2, - "confidence": 0.36092417730987536 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 33, - "accuracy": 0.3939393939393939, - "confidence": 0.46695667120873435 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 42, - "accuracy": 0.5476190476190477, - "confidence": 0.5598522680480225 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 44, - "accuracy": 0.3409090909090909, - "confidence": 0.6536398894374045 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 53, - "accuracy": 0.4528301886792453, - "confidence": 0.750459158914289 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 36, - "accuracy": 0.4166666666666667, - "confidence": 0.8474150430536113 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 107, - "accuracy": 0.822429906542056, - "confidence": 0.975007191367368 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.05387945308026588, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.053879442832794666, - "native_vs_expected_score_mean_residual": 4.404662764384559e-08, - "native_vs_expected_score_normalized_mean_residual": 1.1011656910961398e-08, - "native_vs_expected_score_normalized_mean_abs_residual": 3.205605964527736e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.4722222222222222, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.4827586206896552 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 515, - "accuracy": 0.6848404255319149, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.45420926688696384, - "ece_fixed10": 0.13477215151712532, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.365892778908834 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 57, - "accuracy": 0.3157894736842105, - "confidence": 0.46439337998161323 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 70, - "accuracy": 0.5428571428571428, - "confidence": 0.5533137397476601 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 82, - "accuracy": 0.4146341463414634, - "confidence": 0.6532203197069755 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 93, - "accuracy": 0.5268817204301075, - "confidence": 0.749552023820662 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 67, - "accuracy": 0.5970149253731343, - "confidence": 0.8519984365048434 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 374, - "accuracy": 0.893048128342246, - "confidence": 0.9826121358617447 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.6956521739130435, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.359375 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 40, - "accuracy": 0.625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.4583870923415588, - "ece_fixed10": 0.1820569833507761, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.575727664762073 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.5, - "confidence": 0.648589680592219 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 11, - "accuracy": 0.36363636363636365, - "confidence": 0.7426146106286482 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.16666666666666666, - "confidence": 0.8576921708881855 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 26, - "accuracy": 0.9615384615384616, - "confidence": 0.9758503093169286 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.46875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.6875 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 58, - "accuracy": 0.90625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.1309433975394751, - "ece_fixed10": 0.06284843798673961, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 2, - "accuracy": 1.0, - "confidence": 0.47597716854436145 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.5, - "confidence": 0.5372950983257349 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.6820185306503338 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 1.0, - "confidence": 0.7511516287201252 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 1.0, - "confidence": 0.8137494633696232 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 48, - "accuracy": 1.0, - "confidence": 0.9802939435313948 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.048436180626773864, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.048436173722564035, - "native_vs_expected_score_mean_residual": 2.3944358527178133e-08, - "native_vs_expected_score_normalized_mean_residual": 5.986089631794533e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 2.906006874459771e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9375 - } - } - }, - "family_macro_accuracy": 0.6988839285714286, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.6848404255319149, - "brier": 0.45420926688696384, - "nll": 0.8927424485088513, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.625, - "brier": 0.4583870923415588, - "nll": 0.6215398616594311, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.4, - "recall_true": 1.0, - "balanced_accuracy": 0.7, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.90625, - "brier": 0.1309433975394751, - "nll": 0.23971378960303502, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.021382184843620543, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.048436180626773864, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.048436173722564035, - "native_vs_probability_expected_normalized_abs_residual": 2.906006874459771e-08 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 44, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 44, - "truncation_unknown_requests": 24, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 18, - "contract_valid_questions": 18, - "correct_questions": 11, - "accuracy_all_requested": 0.2619047619047619, - "accuracy_accepted": 0.6111111111111112, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 24 - } - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 9, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 88, - "accuracy_all_requested": 0.5365853658536586, - "accuracy_accepted": 0.5365853658536586, - "native_normalized_score_mae": 0.2929119200977896, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 4, - "accuracy_all_requested": 0.8, - "accuracy_accepted": 0.8, - "native_normalized_score_mae": 0.6345627953608831, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 5, - "accuracy_all_requested": 0.8333333333333334, - "accuracy_accepted": 0.8333333333333334, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "64": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "128": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "255": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 112, - "valid_questions": 196, - "accuracy_all_requested": 0.509090909090909, - "isolation_max_probability_delta": 0.0 - } - }, - "fresh": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 418, - "accuracy": 0.475, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.6402890146128556, - "ece_fixed10": 0.1739532918881776, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 31, - "accuracy": 0.22580645161290322, - "confidence": 0.27976998062086345 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 82, - "accuracy": 0.3170731707317073, - "confidence": 0.36276918991906937 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 173, - "accuracy": 0.2138728323699422, - "confidence": 0.4450651432909496 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 150, - "accuracy": 0.3466666666666667, - "confidence": 0.5608208440007151 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 122, - "accuracy": 0.3770491803278688, - "confidence": 0.638608648398192 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 77, - "accuracy": 0.5454545454545454, - "confidence": 0.7662667946197042 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 45, - "accuracy": 0.4666666666666667, - "confidence": 0.8281357625326449 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 200, - "accuracy": 0.935, - "confidence": 0.986803249476393 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3154027037954301, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.3154027041979134, - "native_vs_expected_score_mean_residual": -3.2023384222146234e-09, - "native_vs_expected_score_normalized_mean_residual": -8.005846055536558e-10, - "native_vs_expected_score_normalized_mean_abs_residual": 1.7496862074473096e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.45871559633027525, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 40, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.5194999377059087, - "ece_fixed10": 0.11421643644571303, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 43, - "accuracy": 0.46511627906976744, - "confidence": 0.5834375952565393 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 36, - "accuracy": 0.5277777777777778, - "confidence": 0.632091204325358 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 1.0, - "confidence": 0.7057850360870361 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 40, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6853863901211932, - "ece_fixed10": 0.3001618586480618, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 49, - "accuracy": 0.5306122448979592, - "confidence": 0.78478919243326 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 31, - "accuracy": 0.45161290322580644, - "confidence": 0.8244605891166195 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 19, - "accuracy": 0.2375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8477309390010331, - "ece_fixed10": 0.21720037730498049, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 20, - "accuracy": 0.25, - "confidence": 0.37356698170788083 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 42, - "accuracy": 0.23809523809523808, - "confidence": 0.43824218010983995 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 10, - "accuracy": 0.1, - "confidence": 0.5392214012570051 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.375, - "confidence": 0.6382881216321863 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 110, - "accuracy": 0.9166666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.1694904332404825, - "ece_fixed10": 0.08044444744794255, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 5, - "accuracy": 0.8, - "confidence": 0.32374713880996486 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.47709706068839935 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.5677687913342871 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.6967713013337351 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 1.0, - "confidence": 0.7499077794352096 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8585500354523005 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 101, - "accuracy": 0.9405940594059405, - "confidence": 0.9854199926394916 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9166666666666666, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 104, - "accuracy": 0.8666666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.22579310300825411, - "ece_fixed10": 0.09705926319656139, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 1.0, - "confidence": 0.3678000130038011 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.4689939019880286 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.502805145768911 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 7, - "accuracy": 0.2857142857142857, - "confidence": 0.6438311193053671 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.75, - "confidence": 0.7326826143005916 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 6, - "accuracy": 0.6666666666666666, - "confidence": 0.8345341394584503 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 99, - "accuracy": 0.9292929292929293, - "confidence": 0.988214450895858 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8666666666666667, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 22, - "accuracy": 0.275, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.913679931200533, - "ece_fixed10": 0.2631312723834899, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 11, - "accuracy": 0.36363636363636365, - "confidence": 0.3682292623579022 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 31, - "accuracy": 0.25806451612903225, - "confidence": 0.44076523484739305 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 12, - "accuracy": 0.3333333333333333, - "confidence": 0.5558144017140504 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.08333333333333333, - "confidence": 0.6497594806626936 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.7493772141895492 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.2, - "confidence": 0.8249952216492424 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 17, - "accuracy": 0.2125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.859496975751117, - "ece_fixed10": 0.15698497440359274, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 31, - "accuracy": 0.22580645161290322, - "confidence": 0.27976998062086345 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 17, - "accuracy": 0.17647058823529413, - "confidence": 0.34365725315486456 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 31, - "accuracy": 0.1935483870967742, - "confidence": 0.4379436170609851 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.5324968794825837 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3154027037954301, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.3154027041979134, - "native_vs_expected_score_mean_residual": -3.2023384222146234e-09, - "native_vs_expected_score_normalized_mean_residual": -8.005846055536558e-10, - "native_vs_expected_score_normalized_mean_abs_residual": 1.7496862074473096e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 20, - "accuracy": 0.25, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8930902687292035, - "ece_fixed10": 0.2805215893105245, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 12, - "accuracy": 0.4166666666666667, - "confidence": 0.3620775583206594 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 28, - "accuracy": 0.10714285714285714, - "confidence": 0.44626034747300913 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 19, - "accuracy": 0.2631578947368421, - "confidence": 0.543351478076717 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 15, - "accuracy": 0.4, - "confidence": 0.6420993048885709 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 6, - "accuracy": 0.16666666666666666, - "confidence": 0.7227000764432386 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 25, - "accuracy": 0.3125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8565948390870451, - "ece_fixed10": 0.22337046331866284, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 7, - "accuracy": 0.14285714285714285, - "confidence": 0.39199727332611317 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 23, - "accuracy": 0.21739130434782608, - "confidence": 0.45635655811643516 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 31, - "accuracy": 0.3225806451612903, - "confidence": 0.5592555922966547 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 16, - "accuracy": 0.4375, - "confidence": 0.6350884697968401 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.7103721458621637 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 21, - "accuracy": 0.2625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8747745747722723, - "ece_fixed10": 0.2940912171108159, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 8, - "accuracy": 0.25, - "confidence": 0.36747392705604004 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 14, - "accuracy": 0.21428571428571427, - "confidence": 0.46131058325612706 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 30, - "accuracy": 0.26666666666666666, - "confidence": 0.5524704442343048 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 27, - "accuracy": 0.25925925925925924, - "confidence": 0.6390762511595336 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 1.0, - "confidence": 0.7000143215053832 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 211, - "accuracy": 0.47954545454545455, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.6243438529234981, - "ece_fixed10": 0.1603833377990506, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 52, - "accuracy": 0.28846153846153844, - "confidence": 0.36641264293690795 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 107, - "accuracy": 0.22429906542056074, - "confidence": 0.4411199101327656 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 88, - "accuracy": 0.375, - "confidence": 0.5597935830953831 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 40, - "accuracy": 0.375, - "confidence": 0.6369634374832602 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 37, - "accuracy": 0.5675675675675675, - "confidence": 0.7722026858810082 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 15, - "accuracy": 0.5333333333333333, - "confidence": 0.8317392912498025 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 101, - "accuracy": 0.9405940594059405, - "confidence": 0.9854199926394916 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3087484111873172, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.3087484152056277, - "native_vs_expected_score_mean_residual": 1.7507504879166192e-08, - "native_vs_expected_score_normalized_mean_residual": 4.376876219791548e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 1.6801123556287044e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5045871559633027, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 207, - "accuracy": 0.47045454545454546, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.6562341763022129, - "ece_fixed10": 0.18891589988346608, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 31, - "accuracy": 0.22580645161290322, - "confidence": 0.27976998062086345 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 30, - "accuracy": 0.36666666666666664, - "confidence": 0.3564538713548159 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 66, - "accuracy": 0.19696969696969696, - "confidence": 0.4514612031080054 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 62, - "accuracy": 0.3064516129032258, - "confidence": 0.5622788917373155 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 82, - "accuracy": 0.3780487804878049, - "confidence": 0.6394111903079148 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 40, - "accuracy": 0.525, - "confidence": 0.760776095202998 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 30, - "accuracy": 0.43333333333333335, - "confidence": 0.826333998174066 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 99, - "accuracy": 0.9292929292929293, - "confidence": 0.988214450895858 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.322056996403543, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.32205699319019915, - "native_vs_expected_score_mean_residual": -2.391218172359544e-08, - "native_vs_expected_score_normalized_mean_residual": -5.97804543089886e-09, - "native_vs_expected_score_normalized_mean_abs_residual": 1.8192600592659147e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.47706422018348627, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 321, - "accuracy": 0.5015625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.622349482145399, - "ece_fixed10": 0.1701457366729181, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 65, - "accuracy": 0.35384615384615387, - "confidence": 0.36776769645739993 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 142, - "accuracy": 0.21830985915492956, - "confidence": 0.4466198426791812 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 106, - "accuracy": 0.29245283018867924, - "confidence": 0.5519133313640896 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 86, - "accuracy": 0.313953488372093, - "confidence": 0.6413368808007736 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 27, - "accuracy": 0.5555555555555556, - "confidence": 0.7348921377926094 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 14, - "accuracy": 0.5, - "confidence": 0.8362736465252726 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 200, - "accuracy": 0.935, - "confidence": 0.986803249476393 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5882352941176471, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 80, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.6024431639135509, - "ece_fixed10": 0.20351146049797533, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 43, - "accuracy": 0.46511627906976744, - "confidence": 0.5834375952565393 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 36, - "accuracy": 0.5277777777777778, - "confidence": 0.632091204325358 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 50, - "accuracy": 0.54, - "confidence": 0.7832091093063355 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 31, - "accuracy": 0.45161290322580644, - "confidence": 0.8244605891166195 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 17, - "accuracy": 0.2125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.859496975751117, - "ece_fixed10": 0.15698497440359274, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 31, - "accuracy": 0.22580645161290322, - "confidence": 0.27976998062086345 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 17, - "accuracy": 0.17647058823529413, - "confidence": 0.34365725315486456 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 31, - "accuracy": 0.1935483870967742, - "confidence": 0.4379436170609851 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.5324968794825837 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3154027037954301, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.3154027041979134, - "native_vs_expected_score_mean_residual": -3.2023384222146234e-09, - "native_vs_expected_score_normalized_mean_residual": -8.005846055536558e-10, - "native_vs_expected_score_normalized_mean_abs_residual": 1.7496862074473096e-08, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.4333333333333333, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.5015625, - "brier": 0.622349482145399, - "nll": 1.2835724369942125, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.5, - "brier": 0.6024431639135509, - "nll": 0.8198428138728368, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.0, - "recall_true": 1.0, - "balanced_accuracy": 0.5, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.2125, - "brier": 0.859496975751117, - "nll": 1.8285665978728267, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.21582302767156475, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.3154027037954301, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.3154027041979134, - "native_vs_probability_expected_normalized_abs_residual": 1.7496862074473096e-08 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 44, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 44, - "truncation_unknown_requests": 24, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 18, - "contract_valid_questions": 18, - "correct_questions": 4, - "accuracy_all_requested": 0.09523809523809523, - "accuracy_accepted": 0.2222222222222222, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 24 - } - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 2, - "accuracy_all_requested": 0.2222222222222222, - "accuracy_accepted": 0.2222222222222222, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 96, - "accuracy_all_requested": 0.5853658536585366, - "accuracy_accepted": 0.5853658536585366, - "native_normalized_score_mae": 0.20985999035959443, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 2, - "accuracy_all_requested": 0.4, - "accuracy_accepted": 0.4, - "native_normalized_score_mae": 0.11019603256136179, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 4, - "accuracy_all_requested": 0.6666666666666666, - "accuracy_accepted": 0.6666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "64": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "128": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - }, - "255": { - "questions": 6, - "accepted_questions": 0, - "contract_valid_questions": 0, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": null, - "native_normalized_score_mae": null, - "errors": { - "request_rejected_or_missing": 6 - } - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 104, - "valid_questions": 196, - "accuracy_all_requested": 0.4727272727272727, - "isolation_max_probability_delta": 0.027673304080963135 - } - } - } - }, - "statistics": { - "models": { - "Decision-1.0-Sol": { - "old": { - "point": 0.6625, - "exact_fraction": "53/80", - "ci95": [ - 0.6200502232142857, - 0.7049860491071429 - ] - }, - "fresh": { - "point": 0.44916666666666666, - "exact_fraction": "539/1200", - "ci95": [ - 0.4241666666666667, - 0.475 - ] - }, - "joint": { - "point": 0.5558333333333333, - "exact_fraction": "667/1200", - "ci95": [ - 0.5310824962797619, - 0.5805065104166667 - ] - } - }, - "Decision-1.0-Nox-stage4-selected": { - "old": { - "point": 0.8284598214285714, - "exact_fraction": "7423/8960", - "ci95": [ - 0.7940838913690476, - 0.8607905505952381 - ] - }, - "fresh": { - "point": 0.5125, - "exact_fraction": "41/80", - "ci95": [ - 0.48708333333333337, - 0.5387500000000001 - ] - }, - "joint": { - "point": 0.6704799107142857, - "exact_fraction": "2403/3584", - "ci95": [ - 0.648708240327381, - 0.6915596168154762 - ] - } - }, - "jev-1.13.0": { - "old": { - "point": 0.7909598214285715, - "exact_fraction": "7087/8960", - "ci95": [ - 0.7619410342261904, - 0.8200892857142857 - ] - }, - "fresh": { - "point": 0.66375, - "exact_fraction": "531/800", - "ci95": [ - 0.6320833333333333, - 0.6941666666666667 - ] - }, - "joint": { - "point": 0.7273549107142857, - "exact_fraction": "65171/89600", - "ci95": [ - 0.7058666294642857, - 0.747957775297619 - ] - } - }, - "laya-routed": { - "old": { - "point": 0.5701264880952381, - "exact_fraction": "3065/5376", - "ci95": [ - 0.5316220238095237, - 0.6088169642857142 - ] - }, - "fresh": { - "point": 0.3775, - "exact_fraction": "151/400", - "ci95": [ - 0.355, - 0.39999999999999997 - ] - }, - "joint": { - "point": 0.47381324404761904, - "exact_fraction": "127361/268800", - "ci95": [ - 0.451599609375, - 0.4963211495535715 - ] - } - }, - "decider": { - "old": { - "point": 0.6401413690476191, - "exact_fraction": "17207/26880", - "ci95": [ - 0.5991443452380952, - 0.6812509300595239 - ] - }, - "fresh": { - "point": 0.4658333333333333, - "exact_fraction": "559/1200", - "ci95": [ - 0.4375, - 0.495 - ] - }, - "joint": { - "point": 0.5529873511904762, - "exact_fraction": "148643/268800", - "ci95": [ - 0.5279351748511906, - 0.5784305245535714 - ] - } - }, - "base-2b": { - "old": { - "point": 0.5712425595238095, - "exact_fraction": "3071/5376", - "ci95": [ - 0.5355654761904762, - 0.6078896949404763 - ] - }, - "fresh": { - "point": 0.39, - "exact_fraction": "39/100", - "ci95": [ - 0.37333333333333335, - 0.4058333333333334 - ] - }, - "joint": { - "point": 0.4806212797619048, - "exact_fraction": "129191/268800", - "ci95": [ - 0.46069847470238096, - 0.5003390066964286 - ] - } - }, - "base-4b": { - "old": { - "point": 0.6988839285714286, - "exact_fraction": "3131/4480", - "ci95": [ - 0.6621270461309524, - 0.7343043154761904 - ] - }, - "fresh": { - "point": 0.43333333333333335, - "exact_fraction": "13/30", - "ci95": [ - 0.4183333333333333, - 0.4479166666666667 - ] - }, - "joint": { - "point": 0.566108630952381, - "exact_fraction": "15217/26880", - "ci95": [ - 0.546599609375, - 0.5855662202380953 - ] - } - } - }, - "released_reference": { - "Decision-1.0-Nox": { - "old": { - "point": 0.7931919642857143, - "exact_fraction": "7107/8960", - "ci95": [ - 0.7568061755952382, - 0.8291294642857142 - ] - }, - "fresh": { - "point": 0.46125, - "exact_fraction": "369/800", - "ci95": [ - 0.43166666666666664, - 0.49000000000000005 - ] - }, - "joint": { - "point": 0.6272209821428572, - "exact_fraction": "56199/89600", - "ci95": [ - 0.6035586867559524, - 0.6507930617559524 - ] - } - } - }, - "method": { - "bootstrap_resamples": 10000, - "seed": 202609215301, - "weights": { - "observed_core_family_macro": 0.5, - "fresh_family_macro": 0.5 - }, - "fresh_families": 10, - "clusters": "MASSIVE source utterance IDs once per menu stratum shared EN/ZH. Synthetic entire counterfactual menu/rubric blocks. Old original within-family group_id. All model comparisons share draws; old/fresh independently resampled.", - "exact_point_arithmetic": "fractions.Fraction, no rounding eligibility" - }, - "paired_update_comparison": { - "panels": { - "old": { - "delta": 0.03526785714285714, - "exact_fraction": "79/2240", - "paired_ci95": [ - 0.0037183779761905196, - 0.0658854166666667 - ] - }, - "fresh": { - "delta": 0.05125, - "exact_fraction": "41/800", - "paired_ci95": [ - 0.022083333333333344, - 0.08083333333333337 - ] - }, - "joint": { - "delta": 0.04325892857142857, - "exact_fraction": "969/22400", - "paired_ci95": [ - 0.021729631696428615, - 0.06456110491071432 - ] - } - }, - "positive_joint_point": true, - "fresh_point_gain": true, - "fresh_paired_ci95_lower_positive": true, - "eligibility_note": "Positive exact joint point is the quality update condition; candidate100%valid/full-input checks are separate required conditions. Neither point gain nor interval alone proves universal superiority." - } - }, - "released_reference_metrics": { - "old": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 686, - "accuracy": 0.7795454545454545, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.38197586576363796, - "ece_fixed10": 0.1799057621571309, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.3961949618083316 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.4640871259421855 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 17, - "accuracy": 0.23529411764705882, - "confidence": 0.5404332791017599 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 20, - "accuracy": 0.35, - "confidence": 0.6556441848182006 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 22, - "accuracy": 0.22727272727272727, - "confidence": 0.7393310900586078 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 39, - "accuracy": 0.358974358974359, - "confidence": 0.8489406679946168 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 771, - "accuracy": 0.8456549935149157, - "confidence": 0.9956857031997477 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.03023279190211094, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.030232791902110947, - "native_vs_expected_score_mean_residual": -3.469448275442594e-17, - "native_vs_expected_score_normalized_mean_residual": -8.673620688606486e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 1.5612514592513714e-17, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8101851851851852, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 105, - "accuracy": 0.8203125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.29825263563136184, - "ece_fixed10": 0.15093597286326702, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.0, - "confidence": 0.5319159512038862 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6450092196070495 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 5, - "accuracy": 0.6, - "confidence": 0.7474386684360821 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 118, - "accuracy": 0.864406779661017, - "confidence": 0.9983893064397896 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8203125, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 58, - "accuracy": 0.90625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.13202739973587088, - "ece_fixed10": 0.06574701897512272, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.5180861795518941 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.25, - "confidence": 0.6690595293418109 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8478037189508436 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 56, - "accuracy": 0.9642857142857143, - "confidence": 0.9911829664239281 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8125, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.8125 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 107, - "accuracy": 0.9553571428571429, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.07584595357260272, - "ece_fixed10": 0.038785515636364926, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.4900017073607324 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 111, - "accuracy": 0.963963963963964, - "confidence": 0.9986844688640734 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9553571428571429, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 1.2554789328800606e-09, - "ece_fixed10": 4.436076588953952e-06, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 0.999995563923411 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 96, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 3.4517435270668484e-14, - "ece_fixed10": 6.582595402093716e-08, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 96, - "accuracy": 1.0, - "confidence": 0.999999934174046 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 56, - "accuracy": 0.875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.2320648985477132, - "ece_fixed10": 0.11952358943929525, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.0, - "confidence": 0.8395234604717723 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 62, - "accuracy": 0.9032258064516129, - "confidence": 0.9995235935995379 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.03023279190211094, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.030232791902110947, - "native_vs_expected_score_mean_residual": -3.469448275442594e-17, - "native_vs_expected_score_normalized_mean_residual": -8.673620688606486e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 1.5612514592513714e-17, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 40, - "accuracy": 0.4166666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 1.005582551479196, - "ece_fixed10": 0.47396052043047876, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.42794579528748794 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 7, - "accuracy": 0.2857142857142857, - "confidence": 0.5567994877185918 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 5, - "accuracy": 0.6, - "confidence": 0.6478223443220029 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 7, - "accuracy": 0.14285714285714285, - "confidence": 0.7240551699699642 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 12, - "accuracy": 0.5, - "confidence": 0.8467033391730664 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 0.4375, - "confidence": 0.979792652664559 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.3333333333333333, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.7083333333333334 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 70, - "accuracy": 0.7291666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.5346262256601714, - "ece_fixed10": 0.26746488303447746, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 96, - "accuracy": 0.7291666666666666, - "confidence": 0.9966315497011441 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7083333333333334, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9583333333333334 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 34, - "accuracy": 0.3541666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 1.0999519100829234, - "ece_fixed10": 0.496495353057829, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.3961949618083316 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.46554809011877846 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.2, - "confidence": 0.5288038692664676 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.3333333333333333, - "confidence": 0.6507873374846909 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 7, - "accuracy": 0.14285714285714285, - "confidence": 0.7316838424444192 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.3333333333333333, - "confidence": 0.8524397403558017 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 48, - "accuracy": 0.3958333333333333, - "confidence": 0.9825665206990927 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.2916666666666667, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.75 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 56, - "accuracy": 0.875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.1985991338626924, - "ece_fixed10": 0.10044431245400727, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6954386004174882 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.0, - "confidence": 0.7793058507360925 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 4, - "accuracy": 0.0, - "confidence": 0.8454681443781865 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 56, - "accuracy": 1.0, - "confidence": 0.9997641559478937 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.75, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.75 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 458, - "accuracy": 0.8178571428571428, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.3176364372395694, - "ece_fixed10": 0.14940440517433312, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.3961949618083316 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.4577656569576309 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 8, - "accuracy": 0.25, - "confidence": 0.5371407482648365 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.3333333333333333, - "confidence": 0.6526098946025211 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 12, - "accuracy": 0.4166666666666667, - "confidence": 0.7357347739051966 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 13, - "accuracy": 0.38461538461538464, - "confidence": 0.8538651196678484 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 510, - "accuracy": 0.8647058823529412, - "confidence": 0.9961329493179968 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.03019358362298899, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.03019358362298899, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 1.3877787807814457e-17, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8357142857142857, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.9224137931034483 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 228, - "accuracy": 0.7125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.4945698656807579, - "ece_fixed10": 0.2368444481786901, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.4767300639112948 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.5433599731790253 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 11, - "accuracy": 0.36363636363636365, - "confidence": 0.6581267859037567 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 10, - "accuracy": 0.0, - "confidence": 0.7436466694427013 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 26, - "accuracy": 0.34615384615384615, - "confidence": 0.8464784421580008 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 261, - "accuracy": 0.8084291187739464, - "confidence": 0.9948117740031691 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.030272000181232894, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.030272000181232904, - "native_vs_expected_score_mean_residual": -6.938896550885189e-17, - "native_vs_expected_score_normalized_mean_residual": -1.734724137721297e-17, - "native_vs_expected_score_normalized_mean_abs_residual": 1.734724137721297e-17, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.9137931034482759 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 572, - "accuracy": 0.7606382978723404, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.4160064558269309, - "ece_fixed10": 0.19604199388384114, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.3961949618083316 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.4444444444444444, - "confidence": 0.4640871259421855 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 16, - "accuracy": 0.1875, - "confidence": 0.5418299728236265 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 16, - "accuracy": 0.375, - "confidence": 0.6522903486872982 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 22, - "accuracy": 0.22727272727272727, - "confidence": 0.7393310900586078 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 34, - "accuracy": 0.35294117647058826, - "confidence": 0.8495949404115875 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 653, - "accuracy": 0.8300153139356815, - "confidence": 0.9957074551976939 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8043478260869565, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.859375 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 58, - "accuracy": 0.90625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.13202739973587088, - "ece_fixed10": 0.06574701897512272, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.5180861795518941 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.25, - "confidence": 0.6690595293418109 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.8478037189508436 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 56, - "accuracy": 0.9642857142857143, - "confidence": 0.9911829664239281 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8125, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.8125 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 56, - "accuracy": 0.875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.2320648985477132, - "ece_fixed10": 0.11952358943929525, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.0, - "confidence": 0.8395234604717723 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 62, - "accuracy": 0.9032258064516129, - "confidence": 0.9995235935995379 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.03023279190211094, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.030232791902110947, - "native_vs_expected_score_mean_residual": -3.469448275442594e-17, - "native_vs_expected_score_normalized_mean_residual": -8.673620688606486e-18, - "native_vs_expected_score_normalized_mean_abs_residual": 1.5612514592513714e-17, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - } - } - }, - "family_macro_accuracy": 0.7931919642857144, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.7606382978723404, - "brier": 0.4160064558269309, - "nll": 2.0020913793476565, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.90625, - "brier": 0.13202739973587088, - "nll": 0.20782075375364445, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.85, - "recall_true": 1.0, - "balanced_accuracy": 0.925, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.875, - "brier": 0.2320648985477132, - "nll": 0.9362552270884734, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.029087145398789996, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.03023279190211094, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.030232791902110947, - "native_vs_probability_expected_normalized_abs_residual": 1.5612514592513714e-17 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 68, - "truncation_unknown_requests": 0, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 23, - "accuracy_all_requested": 0.5476190476190477, - "accuracy_accepted": 0.5476190476190477, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 9, - "accuracy_all_requested": 1.0, - "accuracy_accepted": 1.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 126, - "accuracy_all_requested": 0.7682926829268293, - "accuracy_accepted": 0.7682926829268293, - "native_normalized_score_mae": 0.2678790319519212, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 4, - "accuracy_all_requested": 0.8, - "accuracy_accepted": 0.8, - "native_normalized_score_mae": 0.013801039323420644, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 5, - "accuracy_all_requested": 0.8333333333333334, - "accuracy_accepted": 0.8333333333333334, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 3, - "accuracy_all_requested": 0.5, - "accuracy_accepted": 0.5, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 162, - "valid_questions": 220, - "accuracy_all_requested": 0.7363636363636363, - "isolation_max_probability_delta": 7.065123629468317e-07 - }, - "raw_T1_core": { - "overall": { - "n": 880, - "semantic_groups": 432, - "correct": 686, - "accuracy": 0.7795454545454545, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.34924920509664126, - "ece_fixed10": 0.14978427413321835, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 8, - "accuracy": 0.375, - "confidence": 0.3789598810039931 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 12, - "accuracy": 0.25, - "confidence": 0.4508167774176413 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 40, - "accuracy": 0.275, - "confidence": 0.5533991419434927 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 38, - "accuracy": 0.2631578947368421, - "confidence": 0.645767040213949 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 32, - "accuracy": 0.625, - "confidence": 0.7525072363839229 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 42, - "accuracy": 0.5476190476190477, - "confidence": 0.8426957582241604 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 708, - "accuracy": 0.8700564971751412, - "confidence": 0.9932487489330769 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.029855060917887352, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.029855060917887352, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8101851851851852, - "consistency_eligible_groups": 192, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.875 - }, - "slices": { - "family": { - "ag_news": { - "n": 128, - "semantic_groups": 128, - "correct": 105, - "accuracy": 0.8203125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 128, - "brier": 0.29015262122947405, - "ece_fixed10": 0.15030071597302647, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.4183163594196057 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.0, - "confidence": 0.528539760126942 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.75, - "confidence": 0.6486613821715659 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 0.0, - "confidence": 0.78389012296573 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 1.0, - "confidence": 0.853665204867415 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 114, - "accuracy": 0.868421052631579, - "confidence": 0.9960458553054022 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8203125, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 128, - "semantic_prediction_consistency_group_fraction": null - }, - "boolean_constraints": { - "n": 64, - "semantic_groups": 32, - "correct": 58, - "accuracy": 0.90625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.11687687193874806, - "ece_fixed10": 0.05513261817576882, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.5, - "confidence": 0.5682763729114986 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6493828308981164 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 1.0, - "confidence": 0.7415196299552917 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 8, - "accuracy": 0.75, - "confidence": 0.8367337812857574 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 47, - "accuracy": 1.0, - "confidence": 0.9896318074598555 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8125, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.8125 - }, - "dbpedia_14": { - "n": 112, - "semantic_groups": 112, - "correct": 107, - "accuracy": 0.9553571428571429, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 112, - "brier": 0.07384239019154944, - "ece_fixed10": 0.04127978724945051, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.4288516204400607 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 1.0, - "confidence": 0.769678790462397 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 1, - "accuracy": 1.0, - "confidence": 0.8493065967769572 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 109, - "accuracy": 0.963302752293578, - "confidence": 0.9982887150342913 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9553571428571429, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 112, - "semantic_prediction_consistency_group_fraction": null - }, - "natural_intents": { - "n": 64, - "semantic_groups": 16, - "correct": 64, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 5.541605461058211e-06, - "ece_fixed10": 0.00038602894824601997, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 64, - "accuracy": 1.0, - "confidence": 0.999613971051754 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "option_carrier": { - "n": 96, - "semantic_groups": 24, - "correct": 96, - "accuracy": 1.0, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 5.721580194020575e-08, - "ece_fixed10": 0.0001063835418321224, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 96, - "accuracy": 1.0, - "confidence": 0.9998936164581679 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 1.0, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "ordinal_rubric": { - "n": 64, - "semantic_groups": 32, - "correct": 56, - "accuracy": 0.875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.21418641288906323, - "ece_fixed10": 0.10735882766272516, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6642419825327354 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 62, - "accuracy": 0.9032258064516129, - "confidence": 0.9926206613765958 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.029855060917887352, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.029855060917887352, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - }, - "relational_composition": { - "n": 96, - "semantic_groups": 24, - "correct": 40, - "accuracy": 0.4166666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.9367582549712957, - "ece_fixed10": 0.4098851186377697, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 1, - "accuracy": 0.0, - "confidence": 0.38266130863104914 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.4906760384643468 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 13, - "accuracy": 0.3076923076923077, - "confidence": 0.5544107572360889 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.25, - "confidence": 0.6356539568956644 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 17, - "accuracy": 0.7058823529411765, - "confidence": 0.7420487249749115 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 14, - "accuracy": 0.5, - "confidence": 0.842142555237115 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 42, - "accuracy": 0.3333333333333333, - "confidence": 0.9704664409983718 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.3333333333333333, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.7083333333333334 - }, - "scoped_evidence": { - "n": 96, - "semantic_groups": 24, - "correct": 70, - "accuracy": 0.7291666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.5162315690889163, - "ece_fixed10": 0.25462995936692034, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.0, - "confidence": 0.7843199527707216 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.6, - "confidence": 0.860865284169783 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 89, - "accuracy": 0.7528089887640449, - "confidence": 0.9951855030655504 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7083333333333334, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.9583333333333334 - }, - "state_tracking": { - "n": 96, - "semantic_groups": 24, - "correct": 34, - "accuracy": 0.3541666666666667, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 96, - "brier": 0.9540808854885041, - "ece_fixed10": 0.3758089136189465, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 7, - "accuracy": 0.42857142857142855, - "confidence": 0.3784311056286994 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.4524397011875203 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 14, - "accuracy": 0.35714285714285715, - "confidence": 0.5517447466039547 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.2777777777777778, - "confidence": 0.647157587611922 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 0.5, - "confidence": 0.7648294413595741 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 0.2727272727272727, - "confidence": 0.8358842899619042 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 29, - "accuracy": 0.41379310344827586, - "confidence": 0.9644169098691234 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.2916666666666667, - "consistency_eligible_groups": 24, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.75 - }, - "unknown_rejection": { - "n": 64, - "semantic_groups": 16, - "correct": 56, - "accuracy": 0.875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.15097216820460807, - "ece_fixed10": 0.08333766238282168, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.0, - "confidence": 0.5720987722336208 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.0, - "confidence": 0.6457960351005461 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 56, - "accuracy": 1.0, - "confidence": 0.99174944351493 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.75, - "consistency_eligible_groups": 16, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.75 - } - }, - "language": { - "en": { - "n": 560, - "semantic_groups": 420, - "correct": 458, - "accuracy": 0.8178571428571428, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 560, - "brier": 0.296595765891396, - "ece_fixed10": 0.12609645897725402, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.3746286956125433 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 6, - "accuracy": 0.16666666666666666, - "confidence": 0.4551360573073951 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 18, - "accuracy": 0.2777777777777778, - "confidence": 0.5469138774373282 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 16, - "accuracy": 0.375, - "confidence": 0.6449739747942691 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 16, - "accuracy": 0.75, - "confidence": 0.7555742083311783 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 24, - "accuracy": 0.5833333333333334, - "confidence": 0.8392953216553245 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 474, - "accuracy": 0.8818565400843882, - "confidence": 0.994175358789316 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.029745834644878795, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.029745834644878795, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8357142857142857, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 304, - "semantic_prediction_consistency_group_fraction": 0.9224137931034483 - }, - "zh": { - "n": 320, - "semantic_groups": 180, - "correct": 228, - "accuracy": 0.7125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 320, - "brier": 0.44139272370582044, - "ece_fixed10": 0.19258853269142662, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.5, - "confidence": 0.3919534371783424 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.44649749752788753 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 22, - "accuracy": 0.2727272727272727, - "confidence": 0.5587052674485364 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 22, - "accuracy": 0.18181818181818182, - "confidence": 0.6463438150646253 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 16, - "accuracy": 0.5, - "confidence": 0.7494402644366676 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 18, - "accuracy": 0.5, - "confidence": 0.8472296736492749 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 234, - "accuracy": 0.8461538461538461, - "confidence": 0.9913717699935157 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.02996428719089591, - "native_score_metric_coverage": 32, - "native_score_normalized_mae": 0.02996428719089591, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.7, - "consistency_eligible_groups": 116, - "consistency_excluded_groups": 64, - "semantic_prediction_consistency_group_fraction": 0.9137931034482759 - } - }, - "task_type": { - "choice": { - "n": 752, - "semantic_groups": 368, - "correct": 572, - "accuracy": 0.7606382978723404, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 752, - "brier": 0.3805202795958303, - "ece_fixed10": 0.16480877795559878, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 8, - "accuracy": 0.375, - "confidence": 0.3789598810039931 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 12, - "accuracy": 0.25, - "confidence": 0.4508167774176413 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 36, - "accuracy": 0.25, - "confidence": 0.551746116280381 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 34, - "accuracy": 0.29411764705882354, - "confidence": 0.6444675853314223 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 29, - "accuracy": 0.5862068965517241, - "confidence": 0.7536438853248157 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 34, - "accuracy": 0.5, - "confidence": 0.844098576327314 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 599, - "accuracy": 0.8564273789649416, - "confidence": 0.9935975597473393 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8043478260869565, - "consistency_eligible_groups": 128, - "consistency_excluded_groups": 240, - "semantic_prediction_consistency_group_fraction": 0.859375 - }, - "noul": { - "n": 64, - "semantic_groups": 32, - "correct": 58, - "accuracy": 0.90625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.11687687193874806, - "ece_fixed10": 0.05513261817576882, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.5, - "confidence": 0.5682763729114986 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6493828308981164 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 1.0, - "confidence": 0.7415196299552917 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 8, - "accuracy": 0.75, - "confidence": 0.8367337812857574 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 47, - "accuracy": 1.0, - "confidence": 0.9896318074598555 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8125, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 0.8125 - }, - "score": { - "n": 64, - "semantic_groups": 32, - "correct": 56, - "accuracy": 0.875, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 64, - "brier": 0.21418641288906323, - "ece_fixed10": 0.10735882766272516, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6642419825327354 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 62, - "accuracy": 0.9032258064516129, - "confidence": 0.9926206613765958 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.029855060917887352, - "native_score_metric_coverage": 64, - "native_score_normalized_mae": 0.029855060917887352, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.875, - "consistency_eligible_groups": 32, - "consistency_excluded_groups": 0, - "semantic_prediction_consistency_group_fraction": 1.0 - } - } - }, - "family_macro_accuracy": 0.7931919642857144, - "reliability_contract_pass": true - }, - "raw_T1_typed": { - "by_task_type": { - "choice": { - "n": 752, - "valid_probability_coverage": 752, - "missing_or_invalid": 0, - "accuracy": 0.7606382978723404, - "brier": 0.3805202795958303, - "nll": 1.134342708648393, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.90625, - "brier": 0.11687687193874806, - "nll": 0.17671388441281916, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 40, - "1": 24 - }, - "recall_false": 0.85, - "recall_true": 1.0, - "balanced_accuracy": 0.925, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 64, - "valid_probability_coverage": 64, - "missing_or_invalid": 0, - "accuracy": 0.875, - "brier": 0.21418641288906323, - "nll": 0.5163372930908202, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.02709231741266556, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.029855060917887352, - "native_expected_value_coverage": 64, - "native_expected_value_normalized_mae": 0.029855060917887352, - "native_vs_probability_expected_normalized_abs_residual": 0.0 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - } - }, - "fresh": { - "core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 441, - "accuracy": 0.5011363636363636, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.7388050786689678, - "ece_fixed10": 0.29359708391636685, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.2786213110516698 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 54, - "accuracy": 0.2222222222222222, - "confidence": 0.35798449586197995 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 67, - "accuracy": 0.2537313432835821, - "confidence": 0.4545989150311905 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 105, - "accuracy": 0.3523809523809524, - "confidence": 0.5453419392005783 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 78, - "accuracy": 0.32051282051282054, - "confidence": 0.6424311133694965 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 66, - "accuracy": 0.4393939393939394, - "confidence": 0.7477015333462058 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 64, - "accuracy": 0.421875, - "confidence": 0.8492773194144988 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 443, - "accuracy": 0.6613995485327314, - "confidence": 0.9872221841843392 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.33540512705924624, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.33540512705924624, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.47706422018348627, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.925 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 41, - "accuracy": 0.5125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8805893680894341, - "ece_fixed10": 0.43310371196819547, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.5, - "confidence": 0.5383665486812819 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.6399883558050903 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.5, - "confidence": 0.7446048539147114 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 7, - "accuracy": 0.5714285714285714, - "confidence": 0.8487777640786572 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 66, - "accuracy": 0.5151515151515151, - "confidence": 0.9881961323681482 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 28, - "accuracy": 0.35, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 1.0511189652013402, - "ece_fixed10": 0.48816312903522757, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 10, - "accuracy": 0.5, - "confidence": 0.5474020932109125 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 5, - "accuracy": 0.6, - "confidence": 0.6500892395879854 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 12, - "accuracy": 0.4166666666666667, - "confidence": 0.7505033439245764 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 15, - "accuracy": 0.5333333333333333, - "confidence": 0.8578323480413631 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 38, - "accuracy": 0.18421052631578946, - "confidence": 0.9593436275014157 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 16, - "accuracy": 0.2, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 1.0082875679310133, - "ece_fixed10": 0.41649334317546005, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.36344537669335963 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.45788118142888007 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 18, - "accuracy": 0.16666666666666666, - "confidence": 0.5384545991925728 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 21, - "accuracy": 0.09523809523809523, - "confidence": 0.6372387595003688 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 10, - "accuracy": 0.4, - "confidence": 0.7508557272667347 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 9, - "accuracy": 0.3333333333333333, - "confidence": 0.8534133968893274 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 4, - "accuracy": 0.0, - "confidence": 0.9160134628228233 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 110, - "accuracy": 0.9166666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.16002341454539376, - "ece_fixed10": 0.08291932240219652, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 1.0, - "confidence": 0.6729006210440398 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.8544363567477417 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 115, - "accuracy": 0.9304347826086956, - "confidence": 0.9976766161748559 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9166666666666666, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 106, - "accuracy": 0.8833333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.19374992564313637, - "ece_fixed10": 0.10242363498464385, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.45510023273375416 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 0.0, - "confidence": 0.5878685913081293 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 3, - "accuracy": 0.0, - "confidence": 0.6583998123609399 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 1, - "accuracy": 1.0, - "confidence": 0.728755592428129 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 1.0, - "confidence": 0.85319127988061 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 112, - "accuracy": 0.9196428571428571, - "confidence": 0.9973911257966241 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8833333333333333, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 29, - "accuracy": 0.3625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8184311286506973, - "ece_fixed10": 0.21764745455185117, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 6, - "accuracy": 0.16666666666666666, - "confidence": 0.3615332128007198 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 18, - "accuracy": 0.16666666666666666, - "confidence": 0.46421170659121597 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 26, - "accuracy": 0.5, - "confidence": 0.5489769227841929 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.3333333333333333, - "confidence": 0.631826344611855 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.742552039932256 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 5, - "accuracy": 0.4, - "confidence": 0.8477312526248522 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 4, - "accuracy": 0.5, - "confidence": 0.9435499475946132 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 20, - "accuracy": 0.25, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.9067128857383592, - "ece_fixed10": 0.2110961151628111, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.2786213110516698 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 35, - "accuracy": 0.2571428571428571, - "confidence": 0.3538826635523328 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 17, - "accuracy": 0.17647058823529413, - "confidence": 0.4371497199024658 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 13, - "accuracy": 0.23076923076923078, - "confidence": 0.5434482019324861 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.25, - "confidence": 0.6682398017969211 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.7468760516028238 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.5, - "confidence": 0.8435362707896914 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.33540512705924624, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.33540512705924624, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 20, - "accuracy": 0.25, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 1.4487255153980025, - "ece_fixed10": 0.7272779405682706, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.0, - "confidence": 0.7499384493226128 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.0, - "confidence": 0.8644580601900228 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 76, - "accuracy": 0.2631578947368421, - "confidence": 0.986229502979426 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 35, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7773862408056179, - "ece_fixed10": 0.22535539964992451, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.3655339269743879 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 16, - "accuracy": 0.4375, - "confidence": 0.45794191115518595 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 14, - "accuracy": 0.42857142857142855, - "confidence": 0.5452510951845959 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 12, - "accuracy": 0.5, - "confidence": 0.6484462934869737 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 17, - "accuracy": 0.4117647058823529, - "confidence": 0.7502643068466619 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 11, - "accuracy": 0.2727272727272727, - "confidence": 0.8290959484496617 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 8, - "accuracy": 0.75, - "confidence": 0.9601092544743316 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 36, - "accuracy": 0.45, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7049441832613865, - "ece_fixed10": 0.26289290803109966, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 2, - "accuracy": 0.0, - "confidence": 0.38699701561096983 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.4612783176715516 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 21, - "accuracy": 0.2857142857142857, - "confidence": 0.5456360123675678 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 10, - "accuracy": 0.4, - "confidence": 0.6409012016742133 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 13, - "accuracy": 0.5384615384615384, - "confidence": 0.7424966120290447 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 8, - "accuracy": 0.375, - "confidence": 0.8524632891648147 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 20, - "accuracy": 0.7, - "confidence": 0.9575119079539783 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 221, - "accuracy": 0.5022727272727273, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.719720813888149, - "ece_fixed10": 0.29564620078269743, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.0, - "confidence": 0.27469785469492036 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 20, - "accuracy": 0.15, - "confidence": 0.3673486273196197 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 34, - "accuracy": 0.23529411764705882, - "confidence": 0.45274346147285865 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 58, - "accuracy": 0.27586206896551724, - "confidence": 0.5439937018911372 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 34, - "accuracy": 0.35294117647058826, - "confidence": 0.6483806264666461 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 36, - "accuracy": 0.5, - "confidence": 0.7451363656166522 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 36, - "accuracy": 0.3611111111111111, - "confidence": 0.850469925528925 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 220, - "accuracy": 0.6863636363636364, - "confidence": 0.9852558177624308 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.361054411545659, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.361054411545659, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5091743119266054, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 220, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.7578893434497866, - "ece_fixed10": 0.2940452202745356, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 1.0, - "confidence": 0.2864682237651687 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 34, - "accuracy": 0.2647058823529412, - "confidence": 0.352476183239839 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 33, - "accuracy": 0.2727272727272727, - "confidence": 0.45651059445492637 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 47, - "accuracy": 0.44680851063829785, - "confidence": 0.5470057214122293 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 44, - "accuracy": 0.29545454545454547, - "confidence": 0.6378337623398811 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 30, - "accuracy": 0.36666666666666664, - "confidence": 0.7507797346216701 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 28, - "accuracy": 0.5, - "confidence": 0.847743968695951 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 223, - "accuracy": 0.6367713004484304, - "confidence": 0.9891620972463114 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.30975584257283345, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.30975584257283345, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5137614678899083, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 352, - "accuracy": 0.55, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.6610543307911891, - "ece_fixed10": 0.2621506208642252, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 19, - "accuracy": 0.15789473684210525, - "confidence": 0.3655405027481721 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 50, - "accuracy": 0.28, - "confidence": 0.46053164137495695 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 80, - "accuracy": 0.35, - "confidence": 0.545566537018334 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 66, - "accuracy": 0.30303030303030304, - "confidence": 0.6403978237920886 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 46, - "accuracy": 0.45652173913043476, - "confidence": 0.7472129360067022 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 40, - "accuracy": 0.35, - "confidence": 0.8464436582944372 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 339, - "accuracy": 0.7433628318584071, - "confidence": 0.9901575959065211 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.611764705882353, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.925 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 69, - "accuracy": 0.43125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.9658541666453871, - "ece_fixed10": 0.4606334205017115, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 12, - "accuracy": 0.5, - "confidence": 0.5458961691226408 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 8, - "accuracy": 0.5, - "confidence": 0.6463014081693997 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 14, - "accuracy": 0.42857142857142855, - "confidence": 0.7496607024945956 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 22, - "accuracy": 0.5454545454545454, - "confidence": 0.8549513440532295 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 104, - "accuracy": 0.3942307692307692, - "confidence": 0.9776538709745343 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 20, - "accuracy": 0.25, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.9067128857383592, - "ece_fixed10": 0.2110961151628111, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.2786213110516698 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 35, - "accuracy": 0.2571428571428571, - "confidence": 0.3538826635523328 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 17, - "accuracy": 0.17647058823529413, - "confidence": 0.4371497199024658 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 13, - "accuracy": 0.23076923076923078, - "confidence": 0.5434482019324861 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 4, - "accuracy": 0.25, - "confidence": 0.6682398017969211 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.7468760516028238 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 0.5, - "confidence": 0.8435362707896914 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.33540512705924624, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.33540512705924624, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.46125, - "reliability_contract_pass": true - }, - "typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.55, - "brier": 0.6610543307911891, - "nll": 1.843491345208554, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.43125, - "brier": 0.9658541666453871, - "nll": 2.133991826696492, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.7875, - "recall_true": 0.075, - "balanced_accuracy": 0.43124999999999997, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.25, - "brier": 0.9067128857383592, - "nll": 2.0455607363272255, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.2478722595148728, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.33540512705924624, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.33540512705924624, - "native_vs_probability_expected_normalized_abs_residual": 0.0 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - }, - "native": { - "requests": 68, - "accepted_requests": 68, - "reported_truncated_requests": 0, - "reported_untruncated_requests": 68, - "truncation_unknown_requests": 0, - "families": { - "large_k": { - "questions": 42, - "accepted_questions": 42, - "contract_valid_questions": 42, - "correct_questions": 6, - "accuracy_all_requested": 0.14285714285714285, - "accuracy_accepted": 0.14285714285714285, - "native_normalized_score_mae": null, - "errors": {} - }, - "length": { - "questions": 9, - "accepted_questions": 9, - "contract_valid_questions": 9, - "correct_questions": 3, - "accuracy_all_requested": 0.3333333333333333, - "accuracy_accepted": 0.3333333333333333, - "native_normalized_score_mae": null, - "errors": {} - }, - "multiquestion": { - "questions": 164, - "accepted_questions": 164, - "contract_valid_questions": 164, - "correct_questions": 96, - "accuracy_all_requested": 0.5853658536585366, - "accuracy_accepted": 0.5853658536585366, - "native_normalized_score_mae": 0.21452193730597016, - "errors": {} - }, - "structured": { - "questions": 5, - "accepted_questions": 5, - "contract_valid_questions": 5, - "correct_questions": 3, - "accuracy_all_requested": 0.6, - "accuracy_accepted": 0.6, - "native_normalized_score_mae": 0.000979818335837379, - "errors": {} - } - }, - "large_k": { - "2": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 4, - "accuracy_all_requested": 0.6666666666666666, - "accuracy_accepted": 0.6666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "8": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 1, - "accuracy_all_requested": 0.16666666666666666, - "accuracy_accepted": 0.16666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "26": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "27": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "64": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 1, - "accuracy_all_requested": 0.16666666666666666, - "accuracy_accepted": 0.16666666666666666, - "native_normalized_score_mae": null, - "errors": {} - }, - "128": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - }, - "255": { - "questions": 6, - "accepted_questions": 6, - "contract_valid_questions": 6, - "correct_questions": 0, - "accuracy_all_requested": 0.0, - "accuracy_accepted": 0.0, - "native_normalized_score_mae": null, - "errors": {} - } - }, - "method": "Separate interface/capability probe, not part of core880. No inference of attention mask from order/packing effects. API display rounding retained; no fake normalization of all-zero distributions. Accuracy over all requested and accepted are both shown. Question-isolation deltas need comparison with repeat noise, especially hosted APIs.", - "status": "private_confirmation_do_not_use_for_iterative_training_selection", - "questions": 220, - "correct_questions": 108, - "valid_questions": 220, - "accuracy_all_requested": 0.4909090909090909, - "isolation_max_probability_delta": 0.0016605969439759116 - }, - "raw_T1_core": { - "overall": { - "n": 880, - "semantic_groups": 218, - "correct": 441, - "accuracy": 0.5011363636363636, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 880, - "brier": 0.651653604575396, - "ece_fixed10": 0.2065773010082514, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 32, - "accuracy": 0.25, - "confidence": 0.2785886717383138 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 105, - "accuracy": 0.23809523809523808, - "confidence": 0.3548937732491704 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 124, - "accuracy": 0.3387096774193548, - "confidence": 0.44266943470519643 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 102, - "accuracy": 0.4411764705882353, - "confidence": 0.542543144854638 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 61, - "accuracy": 0.2786885245901639, - "confidence": 0.6497851876502001 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 57, - "accuracy": 0.5087719298245614, - "confidence": 0.7563743689356955 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 67, - "accuracy": 0.208955223880597, - "confidence": 0.8588796199278543 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 332, - "accuracy": 0.786144578313253, - "confidence": 0.9821800015382283 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.31869644502251177, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.31869644502251177, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.47706422018348627, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 98, - "semantic_prediction_consistency_group_fraction": 0.925 - }, - "slices": { - "family": { - "canonical_record_identity": { - "n": 80, - "semantic_groups": 20, - "correct": 41, - "accuracy": 0.5125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7752525412301469, - "ece_fixed10": 0.37319676061618046, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 4, - "accuracy": 0.5, - "confidence": 0.5409895374319105 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 6, - "accuracy": 0.3333333333333333, - "confidence": 0.6563282079252918 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.75, - "confidence": 0.7421665523018188 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 20, - "accuracy": 0.3, - "confidence": 0.8642938249186141 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 46, - "accuracy": 0.6086956521739131, - "confidence": 0.9660131122358705 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "capacitated_assignment": { - "n": 80, - "semantic_groups": 20, - "correct": 28, - "accuracy": 0.35, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.868463355467086, - "ece_fixed10": 0.4025769997674017, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 14, - "accuracy": 0.5714285714285714, - "confidence": 0.53990157809654 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 18, - "accuracy": 0.3333333333333333, - "confidence": 0.6543809822302284 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 21, - "accuracy": 0.47619047619047616, - "confidence": 0.7646390950013868 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 17, - "accuracy": 0.11764705882352941, - "confidence": 0.8583419504432053 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 10, - "accuracy": 0.2, - "confidence": 0.9336690242035972 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 20, - "semantic_prediction_consistency_group_fraction": null - }, - "constraint_assignment": { - "n": 80, - "semantic_groups": 10, - "correct": 16, - "accuracy": 0.2, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8491132149725859, - "ece_fixed10": 0.284200380500303, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.5, - "confidence": 0.2973425344059627 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 20, - "accuracy": 0.2, - "confidence": 0.3560114611010223 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 28, - "accuracy": 0.10714285714285714, - "confidence": 0.4479232981370042 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 18, - "accuracy": 0.3333333333333333, - "confidence": 0.5358566612795327 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.6496293605247315 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.0, - "confidence": 0.7255165970751442 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_en": { - "n": 120, - "semantic_groups": 120, - "correct": 110, - "accuracy": 0.9166666666666666, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.14693524558535592, - "ece_fixed10": 0.0710880933483507, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 1.0, - "confidence": 0.4923306152212746 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 1, - "accuracy": 1.0, - "confidence": 0.5888841790751713 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.6131500875913756 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 4, - "accuracy": 0.5, - "confidence": 0.7540222685430414 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.8615053521417181 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 110, - "accuracy": 0.9545454545454546, - "confidence": 0.9945275525264531 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.9166666666666666, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "massive_zh": { - "n": 120, - "semantic_groups": 120, - "correct": 106, - "accuracy": 0.8833333333333333, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 120, - "brier": 0.1799344384179658, - "ece_fixed10": 0.07930415721715395, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 1, - "accuracy": 0.0, - "confidence": 0.2696392396346406 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 1, - "accuracy": 0.0, - "confidence": 0.45889463455991647 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 5, - "accuracy": 0.4, - "confidence": 0.573211766326071 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 1.0, - "confidence": 0.6890833645066142 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 2, - "accuracy": 0.5, - "confidence": 0.7543345841014786 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 3, - "accuracy": 0.3333333333333333, - "confidence": 0.8578152437679298 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 107, - "accuracy": 0.9439252336448598, - "confidence": 0.9955969591143311 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.8833333333333333, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 120, - "semantic_prediction_consistency_group_fraction": null - }, - "multiset_reconciliation": { - "n": 80, - "semantic_groups": 10, - "correct": 29, - "accuracy": 0.3625, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.7498638140825309, - "ece_fixed10": 0.09619182058368667, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 2, - "accuracy": 0.0, - "confidence": 0.29166235869314033 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 23, - "accuracy": 0.21739130434782608, - "confidence": 0.36725494190807667 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 38, - "accuracy": 0.42105263157894735, - "confidence": 0.44141914785961983 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 9, - "accuracy": 0.5555555555555556, - "confidence": 0.5302814787643088 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 5, - "accuracy": 0.2, - "confidence": 0.6636009805040217 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.781919351038668 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "ordinal_service_loss": { - "n": 80, - "semantic_groups": 8, - "correct": 20, - "accuracy": 0.25, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8364493392336845, - "ece_fixed10": 0.09533978419229992, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 27, - "accuracy": 0.25925925925925924, - "confidence": 0.2765625359182665 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 37, - "accuracy": 0.24324324324324326, - "confidence": 0.33753269527483903 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.1111111111111111, - "confidence": 0.43762848995239595 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 6, - "accuracy": 0.5, - "confidence": 0.5203051357986799 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.61079731605811 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.31869644502251177, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.31869644502251177, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - }, - "paraconsistent_rule_closure": { - "n": 80, - "semantic_groups": 10, - "correct": 20, - "accuracy": 0.25, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 1.2880490099910342, - "ece_fixed10": 0.6536380757428393, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 2, - "accuracy": 0.0, - "confidence": 0.580986906577504 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 2, - "accuracy": 0.0, - "confidence": 0.6637393249544824 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 9, - "accuracy": 0.1111111111111111, - "confidence": 0.7585614748634553 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 14, - "accuracy": 0.0, - "confidence": 0.8676932016285311 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 53, - "accuracy": 0.3584905660377358, - "confidence": 0.9589968962225026 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "temporal_exclusion": { - "n": 80, - "semantic_groups": 10, - "correct": 35, - "accuracy": 0.4375, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6869951479713968, - "ece_fixed10": 0.1268309449327932, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 20, - "accuracy": 0.35, - "confidence": 0.37342950544472286 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 19, - "accuracy": 0.5263157894736842, - "confidence": 0.4414450142812487 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 25, - "accuracy": 0.4, - "confidence": 0.5470867274312142 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.2222222222222222, - "confidence": 0.6289269449470292 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 3, - "accuracy": 0.6666666666666666, - "confidence": 0.7565413100748869 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 2, - "accuracy": 1.0, - "confidence": 0.8266827505096428 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 2, - "accuracy": 1.0, - "confidence": 0.9442141812181338 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - }, - "transaction_recovery": { - "n": 80, - "semantic_groups": 10, - "correct": 36, - "accuracy": 0.45, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.6236987013759078, - "ece_fixed10": 0.18464066019021938, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 5, - "accuracy": 0.0, - "confidence": 0.3478906942386364 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 28, - "accuracy": 0.39285714285714285, - "confidence": 0.43921046450879314 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 18, - "accuracy": 0.4444444444444444, - "confidence": 0.5434973053194739 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 9, - "accuracy": 0.3333333333333333, - "confidence": 0.6505053832372583 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 8, - "accuracy": 1.0, - "confidence": 0.7429385667982417 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 8, - "accuracy": 0.25, - "confidence": 0.838526595963975 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 4, - "accuracy": 1.0, - "confidence": 0.9170744601083907 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 10, - "semantic_prediction_consistency_group_fraction": null - } - }, - "language": { - "en": { - "n": 440, - "semantic_groups": 218, - "correct": 221, - "accuracy": 0.5022727272727273, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.6266078920510654, - "ece_fixed10": 0.2054996233651954, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 7, - "accuracy": 0.0, - "confidence": 0.2719072129290506 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 57, - "accuracy": 0.21052631578947367, - "confidence": 0.35445134168927017 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 61, - "accuracy": 0.3114754098360656, - "confidence": 0.43837811061122633 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 50, - "accuracy": 0.5, - "confidence": 0.5407776162305715 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 37, - "accuracy": 0.16216216216216217, - "confidence": 0.6464748392996447 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 36, - "accuracy": 0.5, - "confidence": 0.7586326549306528 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 33, - "accuracy": 0.24242424242424243, - "confidence": 0.8608714166546161 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 159, - "accuracy": 0.8364779874213837, - "confidence": 0.9804635872571582 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.330161225444139, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.330161225444139, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5091743119266054, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - }, - "zh": { - "n": 440, - "semantic_groups": 218, - "correct": 220, - "accuracy": 0.5, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 440, - "brier": 0.6766993170997264, - "ece_fixed10": 0.21214821953711333, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 25, - "accuracy": 0.32, - "confidence": 0.2804594802049075 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 48, - "accuracy": 0.2708333333333333, - "confidence": 0.355419160726552 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 63, - "accuracy": 0.36507936507936506, - "confidence": 0.44682452628824687 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 52, - "accuracy": 0.38461538461538464, - "confidence": 0.5442407685316248 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 24, - "accuracy": 0.4583333333333333, - "confidence": 0.6548886413573064 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 21, - "accuracy": 0.5238095238095238, - "confidence": 0.7525030215157691 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 34, - "accuracy": 0.17647058823529413, - "confidence": 0.856946405457762 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 173, - "accuracy": 0.7398843930635838, - "confidence": 0.9837575152416392 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.3072316646008846, - "native_score_metric_coverage": 40, - "native_score_normalized_mae": 0.3072316646008846, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.5137614678899083, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 218, - "semantic_prediction_consistency_group_fraction": null - } - }, - "task_type": { - "choice": { - "n": 640, - "semantic_groups": 170, - "correct": 352, - "accuracy": 0.55, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 640, - "brier": 0.5860030517998048, - "ece_fixed10": 0.17663181986886395, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 5, - "accuracy": 0.2, - "confidence": 0.28952980516656934 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 68, - "accuracy": 0.23529411764705882, - "confidence": 0.3643402421469683 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 115, - "accuracy": 0.3565217391304348, - "confidence": 0.4430639434249808 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 78, - "accuracy": 0.41025641025641024, - "confidence": 0.544807560478228 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 36, - "accuracy": 0.25, - "confidence": 0.6474797834141176 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 32, - "accuracy": 0.5, - "confidence": 0.7527266195343203 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 30, - "accuracy": 0.2, - "confidence": 0.8555748293086489 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 276, - "accuracy": 0.8369565217391305, - "confidence": 0.9866321271949483 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.611764705882353, - "consistency_eligible_groups": 120, - "consistency_excluded_groups": 50, - "semantic_prediction_consistency_group_fraction": 0.925 - }, - "noul": { - "n": 160, - "semantic_groups": 40, - "correct": 69, - "accuracy": 0.43125, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 160, - "brier": 0.8218579483486165, - "ece_fixed10": 0.38544573093528656, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 18, - "accuracy": 0.5555555555555556, - "confidence": 0.5401433468377335 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 24, - "accuracy": 0.3333333333333333, - "confidence": 0.6548677886539943 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 25, - "accuracy": 0.52, - "confidence": 0.761043488169456 - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 37, - "accuracy": 0.21621621621621623, - "confidence": 0.8615591798893723 - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 56, - "accuracy": 0.5357142857142857, - "confidence": 0.9602373822301074 - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": null, - "native_score_metric_coverage": 0, - "native_score_normalized_mae": null, - "native_vs_expected_score_mean_residual": null, - "native_vs_expected_score_normalized_mean_residual": null, - "native_vs_expected_score_normalized_mean_abs_residual": null, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 40, - "semantic_prediction_consistency_group_fraction": null - }, - "score": { - "n": 80, - "semantic_groups": 8, - "correct": 20, - "accuracy": 0.25, - "valid_rate": 1.0, - "native_contract_rate": 1.0, - "probability_coverage": 80, - "brier": 0.8364493392336845, - "ece_fixed10": 0.09533978419229992, - "calibration_bins": [ - { - "lower": 0.0, - "upper": 0.1, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.1, - "upper": 0.2, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.2, - "upper": 0.3, - "n": 27, - "accuracy": 0.25925925925925924, - "confidence": 0.2765625359182665 - }, - { - "lower": 0.3, - "upper": 0.4, - "n": 37, - "accuracy": 0.24324324324324326, - "confidence": 0.33753269527483903 - }, - { - "lower": 0.4, - "upper": 0.5, - "n": 9, - "accuracy": 0.1111111111111111, - "confidence": 0.43762848995239595 - }, - { - "lower": 0.5, - "upper": 0.6, - "n": 6, - "accuracy": 0.5, - "confidence": 0.5203051357986799 - }, - { - "lower": 0.6, - "upper": 0.7, - "n": 1, - "accuracy": 0.0, - "confidence": 0.61079731605811 - }, - { - "lower": 0.7, - "upper": 0.8, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.8, - "upper": 0.9, - "n": 0, - "accuracy": null, - "confidence": null - }, - { - "lower": 0.9, - "upper": 1.0, - "n": 0, - "accuracy": null, - "confidence": null - } - ], - "zero_gold_probability_count": 0, - "normalized_score_mae": 0.31869644502251177, - "native_score_metric_coverage": 80, - "native_score_normalized_mae": 0.31869644502251177, - "native_vs_expected_score_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_residual": 0.0, - "native_vs_expected_score_normalized_mean_abs_residual": 0.0, - "reported_truncations": 0, - "unknown_truncation_status": 0, - "all_variants_correct_group_fraction": 0.0, - "consistency_eligible_groups": 0, - "consistency_excluded_groups": 8, - "semantic_prediction_consistency_group_fraction": null - } - } - }, - "family_macro_accuracy": 0.46125, - "reliability_contract_pass": true - }, - "raw_T1_typed": { - "by_task_type": { - "choice": { - "n": 640, - "valid_probability_coverage": 640, - "missing_or_invalid": 0, - "accuracy": 0.55, - "brier": 0.5860030517998048, - "nll": 1.2402040701257664, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null - }, - "noul": { - "n": 160, - "valid_probability_coverage": 160, - "missing_or_invalid": 0, - "accuracy": 0.43125, - "brier": 0.8218579483486165, - "nll": 1.2938835243494435, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "class_support": { - "0": 80, - "1": 80 - }, - "recall_false": 0.7875, - "recall_true": 0.075, - "balanced_accuracy": 0.43124999999999997, - "recall_denominator": "All gold rows, including invalid/missing predictions as wrong" - }, - "score": { - "n": 80, - "valid_probability_coverage": 80, - "missing_or_invalid": 0, - "accuracy": 0.25, - "brier": 0.8364493392336845, - "nll": 1.7371047897738634, - "zero_true_probability_count": 0, - "nll_policy": "Natural log; no epsilon clipping; infinity is explicit JSON string; proper scores conditional on valid coverage", - "soft_target_coverage": 0, - "soft_target_nll": null, - "normalized_rps": 0.21888460314506486, - "soft_target_normalized_rps": null, - "rps_definition": "sum over K-1 cumulative boundaries, divided by K-1", - "probability_expected_value_normalized_mae": 0.31869644502251177, - "native_expected_value_coverage": 80, - "native_expected_value_normalized_mae": 0.31869644502251177, - "native_vs_probability_expected_normalized_abs_residual": 0.0 - } - }, - "supplement_only": true, - "original_accuracy_brier_gate_definitions_changed": false, - "hard_target_note": "Existing core gold is hard objective labels; never relabel official model probabilities as business ground truth" - } - } - }, - "update_readiness": { - "allowed": true, - "update": { - "policy_sha256": "359de2b6b81be91a66c6c538b3655bed06f4728357328cd318b1a765e64565cd", - "validity": { - "valid_finite_outputs": true, - "accepted_in_scope": true, - "no_silent_truncation": true - }, - "candidate": { - "evaluation_model_name": "Decision-1.0-Nox-stage4-selected", - "old": { - "numerator": 7423, - "denominator": 8960 - }, - "fresh": { - "numerator": 41, - "denominator": 80 - }, - "joint": { - "numerator": 2403, - "denominator": 3584 - }, - "bundle_manifest_sha256": "a8cfd42bc908fc93d5a2a20be527d09704b6e3f6c1217f27307c32cc2d822971" - }, - "released_reference": { - "evaluation_model_name": "Decision-1.0-Nox", - "old": { - "numerator": 7107, - "denominator": 8960 - }, - "fresh": { - "numerator": 369, - "denominator": 800 - }, - "joint": { - "numerator": 56199, - "denominator": 89600 - }, - "repo_id": "llm-semantic-router/Decision-1.0-Nox", - "model_revision": "43d60043795a1ddd840c0edd5bbdc5a399c9715b", - "bundle_manifest_sha256": "adf5d88146d4f4a7fdcf0b265aa0c804a6116537c1e92bd620e40dfdf6803550", - "release_manifest_sha256": "dc5ca2bcd8caa6199f5c9bbb71d6bdff604a77f1b568a651c81c6fdb190fb042" - }, - "joint_delta": { - "numerator": 969, - "denominator": 22400 - }, - "evaluation_receipt_sha256": "927c92121c2df534f2ba12523e466ecebd8bd8936f949c7792de3e747c92a47c" - } - } -} diff --git a/release-manifest.json b/release-manifest.json index 15267d88a5ae5c74b21139feaa435602e2123792..5294b93c699b9402f001f0157ab2ddd270ae4984 100644 --- a/release-manifest.json +++ b/release-manifest.json @@ -1,18 +1,18 @@ { "format": "decision-public-release-v1", "status": "assembled-not-published", - "bundle_manifest_sha256": "a8cfd42bc908fc93d5a2a20be527d09704b6e3f6c1217f27307c32cc2d822971", - "readiness_sha256": "8faa9b038f05857991bb6f51afb3a4671484f09e6b4f320e1a7b725f39704154", - "model_card_sha256": "18c0e7aa8967fc6ca502e2a8bca36022c005bd94902a26b494743864708698fc", + "bundle_manifest_sha256": "7762761b5da5d51d33e06d322c037f7cfdee54f910e6e83d3e8c7ce02ae42b2e", + "readiness_sha256": "82bc316e1f85083b32315e894f9bf3cf2f731f78376b33bbdf206492fd41c7f1", + "model_card_sha256": "9a9f62761cffdef286d9916169b27f40d540e38085834882c4923f29c6c6ca2d", "repo_id": "llm-semantic-router/Decision-1.0-Nox", - "assembly_script_sha256": "0ca32612da99c16b9801149d69f375b157e51b6e426b0a74efa3efb6bcb9f792", + "assembly_script_sha256": "7831ef5952b8256e585d4efa91cac4ce87738f4eb93c84b1cf1e7250def1f430", "original_bundle_manifest_preserved": true, "files_exclude_this_manifest": true, "files": [ { "file": "ATTRIBUTIONS.md", - "bytes": 4182, - "sha256": "76ff86ae499bdabb46ef75f974dfbc48eec794b8651bd9671780b55a1fc108e3" + "bytes": 6606, + "sha256": "e9ed3c41423a6d15edbf82e14b80897fe5ef6cd454bcd9957d91b31460461718" }, { "file": "Dockerfile.runtime", @@ -21,18 +21,23 @@ }, { "file": "EVALUATION.md", - "bytes": 9158, - "sha256": "02e362cc09c844ce7fb2ba1df25438098befbd35cc4bf95895687623199f959c" + "bytes": 6091, + "sha256": "2402026626d5f2ef76f68eb9480baf98da0cf454b1410f195d2a7bda05525513" }, { "file": "LICENSE", "bytes": 11544, "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a" }, + { + "file": "NORMALIZATION_RUNTIME.md", + "bytes": 756, + "sha256": "cf8e6ce1f07687a68b6adeb98e6704b8f292cb6adc7b10e4b84e8e70aa5472e5" + }, { "file": "QUESTION-SCALING.md", - "bytes": 3763, - "sha256": "7dde01b31f9292116e715b501f78bf7577061f979ece9e95f7e3739ff2ead141" + "bytes": 3527, + "sha256": "9f8c8bf81b64da7d3309f4d9f79cdbcef60b315f8ed4fa77cc98d1d912d0d29a" }, { "file": "QWEN-LICENSE", @@ -41,28 +46,28 @@ }, { "file": "README.md", - "bytes": 5019, - "sha256": "18c0e7aa8967fc6ca502e2a8bca36022c005bd94902a26b494743864708698fc" + "bytes": 3796, + "sha256": "9a9f62761cffdef286d9916169b27f40d540e38085834882c4923f29c6c6ca2d" }, { "file": "RUNTIME.md", - "bytes": 5389, - "sha256": "734f699d337745858a8cd456a0bf2f30e97fb1551619ffcdd2a2d48d09e35578" + "bytes": 5205, + "sha256": "be35b2f8a9c977f38be9b5eb892fdc809db5167c327575c9311355aae3e781c7" }, { - "file": "USAGE.md", - "bytes": 5435, - "sha256": "46ec1078d2bb8cb54a1850809a101f07a77bb54c18b92276cb89d40245dbbc8f" + "file": "RUNTIME_BINDING.json", + "bytes": 2621, + "sha256": "65a7772ed57e811faefe6b3b3868e174c5e4a3d7f3aa0138a3838464bb2612fe" }, { - "file": "assets/architecture-atlas.pdf", - "bytes": 306242, - "sha256": "d1582ac115aa6a40129b6221e15a197e33d20847e03d69396f02fdab82771376" + "file": "SOURCE_BUNDLE_MANIFEST.json", + "bytes": 5637, + "sha256": "6d402d6d55e734851cb0d6417d6f3b416b25ae807c3a0d6dcf12e40acb6ac546" }, { - "file": "assets/architecture.pdf", - "bytes": 183111, - "sha256": "ee87f997a0aa4343c2ec66038b7526361e0a3c0ae02b383f8928aac9dd9bd74c" + "file": "USAGE.md", + "bytes": 5435, + "sha256": "46ec1078d2bb8cb54a1850809a101f07a77bb54c18b92276cb89d40245dbbc8f" }, { "file": "assets/architecture.png", @@ -75,75 +80,155 @@ "sha256": "215d9af6f94cc24847fc1d23a0b2a287b36afdc1e10ea4f5c82895b2fe103002" }, { - "file": "assets/decision-capabilities.pdf", - "bytes": 51584, - "sha256": "1db9c9fcad30a0721692eb2143149001686f37e1ab9639395aa40886185f0e1f" + "file": "assets/decision-expanded-old_core-600px.png", + "bytes": 140155, + "sha256": "519923f36d62df99c6769731416d70413c9f5ec3991ba43a75de0714767a12cc" }, { - "file": "assets/decision-capabilities.png", - "bytes": 451771, - "sha256": "bf584ab8924fe6f1e4623a5d402cac6fbfde96f12eaefa8c6b650ede9d95f21b" + "file": "assets/decision-expanded-old_core.pdf", + "bytes": 44043, + "sha256": "29e7f84737b7bfe2d0a444db76ce9d0b867fb206cb4253192cd19594bba0f9c0" }, { - "file": "assets/decision-capabilities.svg", - "bytes": 114811, - "sha256": "5b5c47c96d9f48000f2593a6735aba8fba6391470f15949381088036a1c78a26" + "file": "assets/decision-expanded-old_core.png", + "bytes": 261350, + "sha256": "c59e464a07320a13a182ec841fef0a826eed777d03264260d3a3d6bcf99a2819" }, { - "file": "assets/decision-family-header.png", - "bytes": 2962868, - "sha256": "213511289ce8df038d938ac470e803c427ed57f0f85cc397dd4d79964b866541" + "file": "assets/decision-expanded-old_core.svg", + "bytes": 72452, + "sha256": "593cada5637013c20daf90c8c6d8656554666c134e0123c5099d703f6a750be6" + }, + { + "file": "assets/decision-expanded-overview-600px.png", + "bytes": 83667, + "sha256": "f1e659e6ba773c872d6f6e64ca783acface0a29c7db57f0e2d06758bbcf1a397" + }, + { + "file": "assets/decision-expanded-overview.pdf", + "bytes": 40642, + "sha256": "b0427a480be62113f7c14f60c882673eb0c822b4b23b2444411f939a1fab638d" + }, + { + "file": "assets/decision-expanded-overview.png", + "bytes": 167930, + "sha256": "fa9972beb8e9081bb5a8c59e487e69f31567470fd4af6b0d6d36bdb82d60cdf1" + }, + { + "file": "assets/decision-expanded-overview.svg", + "bytes": 44991, + "sha256": "c8c4dff1759965471229ccd5abb782e4050099adc328c28a18fb451694683f5a" + }, + { + "file": "assets/decision-expanded-ranking-600px.png", + "bytes": 75990, + "sha256": "80d65d7fbe04b86fa6ca45c3659b18b968f767ff1102ca5b23de2da2fbd91a59" + }, + { + "file": "assets/decision-expanded-ranking.pdf", + "bytes": 37284, + "sha256": "38ef0ccf62b82ec3da5076241646f42c21f2387c864e186165a3a9c1b2b61e37" + }, + { + "file": "assets/decision-expanded-ranking.png", + "bytes": 171628, + "sha256": "038b995974df964403278b2d121bf9b888c52387aeed4e1726ddf5cbbb18f0b3" + }, + { + "file": "assets/decision-expanded-ranking.svg", + "bytes": 36214, + "sha256": "236c5add9d316a3ecbd328beb5c171c71d4b1501a84dee5a83581e808c5f6fa3" + }, + { + "file": "assets/decision-expanded-v3_core-600px.png", + "bytes": 143107, + "sha256": "ba4b96800e5f24d1029911d75fae19d7cf9dfd0f2ded82c05fef3b4997259e8a" + }, + { + "file": "assets/decision-expanded-v3_core.pdf", + "bytes": 44142, + "sha256": "f253e527c21c26544ab23a024ad31e28cab1336b6fdc14d7b8e42b533e02a13d" + }, + { + "file": "assets/decision-expanded-v3_core.png", + "bytes": 269511, + "sha256": "8c9b3b703cacfca5eed2099665cfcd4fab210c2b0dc6e7540e8c6a9b84b2096f" + }, + { + "file": "assets/decision-expanded-v3_core.svg", + "bytes": 72477, + "sha256": "27fc95d04dc6fd6dbaa1a353fe30588a0c1832650350509cbb5c88b6a60b7476" }, { - "file": "assets/decision-mark.png", - "bytes": 1828679, - "sha256": "d83cf7878c3839f3085feb8c02334217ad2fef0a615b805e1726b292552c46e8" + "file": "assets/decision-expanded-v4-600px.png", + "bytes": 66445, + "sha256": "b22c8b55bc594fccfeda1b33132cfd94932fb9dd101749699e7c91899a3f1166" }, { - "file": "assets/decision-quality.pdf", - "bytes": 37117, - "sha256": "5064d2b565520513dd3da59e2678203125727afc700278643b08bae6f7b6c48e" + "file": "assets/decision-expanded-v4.pdf", + "bytes": 38130, + "sha256": "005cac2146b983187002fab407d611f9b506f9cc53fec9ece370c6cf3a638eec" }, { - "file": "assets/decision-quality.png", - "bytes": 171749, - "sha256": "991ec98a7ea95250b9040fbddba09513e1ae740682c94f8d0f72580c9373c07e" + "file": "assets/decision-expanded-v4.png", + "bytes": 135301, + "sha256": "3b392a7f29d9cf0d87920658d2d57ddd2d5fc500f9658b916a518a9570d2fd08" }, { - "file": "assets/decision-quality.svg", - "bytes": 36911, - "sha256": "7a13c7cb33316a26b95ad91ee6ed131362fb76f409be381f483e233674cfaeae" + "file": "assets/decision-expanded-v4.svg", + "bytes": 39927, + "sha256": "1210b42e075012c39001ef33324ff2391681e86592212878d26086ebe3a496f2" + }, + { + "file": "assets/decision-expanded-v5-600px.png", + "bytes": 79064, + "sha256": "ef46e061eb5411360b846e4df64eafca18ce4561d5fb7eaf5dafd21681e7a193" + }, + { + "file": "assets/decision-expanded-v5.pdf", + "bytes": 40093, + "sha256": "9988a0ba3538f0ddf1c83430dd0763963dc4b6eba1e80969b4c81311389e2018" + }, + { + "file": "assets/decision-expanded-v5.png", + "bytes": 161808, + "sha256": "aef547fd323bf044b1f81edf89f07e1e0175f9a52b9742b14eda6890e7fa0f4d" + }, + { + "file": "assets/decision-expanded-v5.svg", + "bytes": 44951, + "sha256": "27bd11dbac7c6bdcc35dff7b9c7cc2c3ed984454121045694e3555dfc73f7b1a" + }, + { + "file": "assets/decision-family-header.png", + "bytes": 2962868, + "sha256": "213511289ce8df038d938ac470e803c427ed57f0f85cc397dd4d79964b866541" + }, + { + "file": "assets/decision-question-scaling-600px.png", + "bytes": 44739, + "sha256": "6d39aae1592415df0413bba6f35a6dfa41af8c32caffa4804717bc3001cca774" }, { "file": "assets/decision-question-scaling.pdf", - "bytes": 25845, - "sha256": "e1e6606982eed8280db78072e30f7cf3ca27ceaabdd69d1253371f6c9d6fafc0" + "bytes": 32062, + "sha256": "887da4cd137ed378dae40c44f79953e27b07e771cfbd5fa0fe788b1bae665517" }, { "file": "assets/decision-question-scaling.png", - "bytes": 54575, - "sha256": "83c68fe79fd316c1640e56e85e5e1424bc6b145a7baf83ea174c11237118cb55" + "bytes": 105103, + "sha256": "9f032e63eb2e5e30b2c42f5db70d68ef2039222afdb7495f8f3a3dfb0286747e" }, { "file": "assets/decision-question-scaling.svg", - "bytes": 19837, - "sha256": "93de52c74cfed97fe660bd79619a9e41eeeeb03d0a9f082b4033d8cb919deeb0" - }, - { - "file": "assets/readout.pdf", - "bytes": 123008, - "sha256": "d489b0d1814e37495a8612c12fabd3bf4a6e73d11c7e356bf1ce0cedded8a745" + "bytes": 26992, + "sha256": "cd038389b040662e57b2129dee8da8aef9d0c4c7918ce82c923fe11a7439bcac" }, { "file": "assets/readout.png", "bytes": 285481, "sha256": "fb6b3d1cf149a1a3ccb584eb1c213df73c07bcd328732b4238928e2a2a4af085" }, - { - "file": "assets/readout.svg", - "bytes": 8067, - "sha256": "701084bf7b0858a3adb247ae446df737b77d4b75004341dfd5f956b933f3d253" - }, { "file": "backbone/config.json", "bytes": 1978, @@ -152,17 +237,17 @@ { "file": "backbone/model-00001-of-00003.safetensors", "bytes": 3991295368, - "sha256": "8fe3a641db8854b36595b094abc8f8ff1e94ce9328cdbb64c36b72c92ca2a9d6" + "sha256": "5f9c4bc396605b551b5983a1699de8e96755fafdb6afdb228a5cc4d9b438a5da" }, { "file": "backbone/model-00002-of-00003.safetensors", "bytes": 3979828128, - "sha256": "e646f2549824a57e46e80c29260ae972d56104d37794af596a6e2330666cc73f" + "sha256": "656cc757ba03f87cedc0e4c88237db1ae8215686f9b6477334a34dc0a32136db" }, { "file": "backbone/model-00003-of-00003.safetensors", "bytes": 440425856, - "sha256": "893a513659cc7e917261336fcb81ea08e7e145f7409d6e7feaa13554a4faf52e" + "sha256": "0a30a07b6efbaa37d016008848c1fa6dadcb46ae451a0e5e4b927ac9b5771804" }, { "file": "backbone/model.safetensors.index.json", @@ -171,8 +256,8 @@ }, { "file": "bundle-manifest.json", - "bytes": 5649, - "sha256": "a8cfd42bc908fc93d5a2a20be527d09704b6e3f6c1217f27307c32cc2d822971" + "bytes": 7635, + "sha256": "7762761b5da5d51d33e06d322c037f7cfdee54f910e6e83d3e8c7ce02ae42b2e" }, { "file": "chat_template.jinja", @@ -181,8 +266,8 @@ }, { "file": "code/decision_api.py", - "bytes": 6724, - "sha256": "147b2fec32cbbbbb1b92cf2a19bb887f9d945e4974e141ca7ea93b8c542f5b21" + "bytes": 8251, + "sha256": "1b068eccdffd3c3b67bfa52f8f526e6b482d92551c927668ad28a794767f8a40" }, { "file": "code/decision_model.py", @@ -194,6 +279,16 @@ "bytes": 3164, "sha256": "02352e8385ab47157b6459910da54d962e5da4bb4940d571faedc86bc5da9aee" }, + { + "file": "code/profile_guard.py", + "bytes": 6184, + "sha256": "061d6ba3edf032038b074b061879a1ff81cfd2928fc0131c66b15563f7822509" + }, + { + "file": "code/runtime_profile.py", + "bytes": 2857, + "sha256": "3e26ed65f2cd706def209761421cb8fc825281e5c1ed8f168350365ad803dc96" + }, { "file": "decision_config.json", "bytes": 753, @@ -202,43 +297,63 @@ { "file": "decision_head.safetensors", "bytes": 10529624, - "sha256": "f4f1df3f8c3bf507300bb810e098fb8854f182c715e1dd9cc57d44361f949fde" + "sha256": "8b8e342445035e503b4963e16c5f7a4fef95c08368e27eace312145fc3660772" + }, + { + "file": "metrics/comparator-coverage.json", + "bytes": 21192, + "sha256": "7612486e073438ad11f91122af39e9ffc89e774157f7f14aebe58184ab9b4632" + }, + { + "file": "metrics/expanded-quality.json", + "bytes": 58851, + "sha256": "d8f37dcb8fdb274a4d3c11bd4d059c4d52f018087e33ca6b340ca8d64b2b6cce" }, { - "file": "metrics/natural-confirmation.json", - "bytes": 24161, - "sha256": "7863d103cc47442beb5aaf925c8be5d9abfe1cdeb815ae05d0b2ff537b012ff5" + "file": "metrics/figure-evidence.json", + "bytes": 16646, + "sha256": "00703ecd5f2ac1ecfd4fcdd8a42712f78c943ab4bb8dab7d18689d44aecce87d" + }, + { + "file": "metrics/materials-provenance.json", + "bytes": 736, + "sha256": "a8277d5a8cc7f06d90ff97f55219eb3ce9de8ed285c82416020b83da031fb7fa" }, { "file": "metrics/question-scaling.json", - "bytes": 9629, - "sha256": "4708635c6f26cd0209dafb4e45af6921ed22091e1996bbeb5861c98214ff5886" + "bytes": 17711, + "sha256": "25fc1e00f836156e1c58d933ace5bfc24774ebae33eebd64f632f8febd92d21c" }, { "file": "model-card-example.json", - "bytes": 2804, - "sha256": "956e790b58bf37bf4c86a0e2a305a3f596a6bc6a936fde69c9f8fc1ff3737e81" + "bytes": 3772, + "sha256": "847cc6ebe4def51d7172345cc42706b864c0078fd227c23f05973faa7fba8e73" }, { "file": "pyproject.toml", "bytes": 436, - "sha256": "6a557fbe472027af103ca2cfc07e977caab3e257de930aa57cf9fea8cf90f8a0" - }, - { - "file": "quality-metrics.json", - "bytes": 1375528, - "sha256": "68adfdde02ec23a9947a1df8d44d3837bc66a23681a364bca8986f2644120b8b" + "sha256": "135a9516e87ca2fb41ef54a974e29132256b6c2d528a1cbdea1001e28f306946" }, { "file": "runtime-build-provenance.json", - "bytes": 3033, - "sha256": "8a22aed74b1210c44c918c1dc3f8ac9778b55ebcc70a9a805eb2f2de76a24870" + "bytes": 2894, + "sha256": "61d064b0db35d0ebd04a038e376896d2aab8b69e6525a8ec1f317f73b5284345" }, { "file": "runtime-fla-requirements.lock", "bytes": 654, "sha256": "35e1fedff9ca49092a7ada277bbd794abe7e8474e14a678a7e4bcec6406d0eb5" }, + { + "file": "runtime-profile/l2norm_fwd_kernel.json", + "bytes": 26330, + "sha256": "d7ed7c9962a48efdfa76ed695c8bdc0f56afe1397c202c936eb78315e87a26df" + }, + { + "file": "runtime-profile/profile.json", + "bytes": 35040, + "sha256": "be32858d15233e0a3fbee0e4257fb02be0b3439deee4eb9c3f61151df7b73850" + }, { "file": "runtime-provenance.json", "bytes": 5805, @@ -246,8 +361,8 @@ }, { "file": "runtime.json", - "bytes": 378, - "sha256": "c5d3521358b2817f4e56ea8150c5c612139b4e2b2c5bb09f512d9a7c0b5298b9" + "bytes": 1112, + "sha256": "78a2c21f8138beeffb3e6b1c11e03cb48972a78ca5ef3d1f58c50651031c34b6" }, { "file": "src/decision/__init__.py", @@ -256,18 +371,18 @@ }, { "file": "src/decision/example.py", - "bytes": 3381, - "sha256": "d52c20b602bd1049aecda55244e71c27d8ce44bd1aa8b1941db62ae41d31ba12" + "bytes": 3581, + "sha256": "a54dec885f92c2d38d07ef2333dff51965be52bc119f772cc74200ed314e69b6" }, { "file": "src/decision/model.py", - "bytes": 8790, - "sha256": "ab428234d508a1245d3723c5a5bb0c796685abebd0447f77c7fbed9113954b21" + "bytes": 9415, + "sha256": "ba240d7493fc29203fe036966f0ab911200cd4a9252b50b05423409977639ee0" }, { "file": "temperature.json", - "bytes": 13765, - "sha256": "09a7559b6b85cc8e7e7f3dff176b9b2433f165019ceb6e9215bf467d28bb8256" + "bytes": 6095, + "sha256": "1d652e75f49b5c232d1e742b189c1c826ad52ced1a931a124e29e65e7d7fb569" }, { "file": "tokenizer.json", @@ -285,29 +400,46 @@ "Dockerfile.runtime": "copy", "EVALUATION.md": "copy", "LICENSE": "copy", + "NORMALIZATION_RUNTIME.md": "copy", "QUESTION-SCALING.md": "copy", "QWEN-LICENSE": "copy", "README.md": "copy", "RUNTIME.md": "copy", + "RUNTIME_BINDING.json": "copy", + "SOURCE_BUNDLE_MANIFEST.json": "copy", "USAGE.md": "copy", - "assets/architecture-atlas.pdf": "copy", - "assets/architecture.pdf": "copy", "assets/architecture.png": "copy", "assets/architecture.svg": "copy", - "assets/decision-capabilities.pdf": "copy", - "assets/decision-capabilities.png": "copy", - "assets/decision-capabilities.svg": "copy", + "assets/decision-expanded-old_core-600px.png": "copy", + "assets/decision-expanded-old_core.pdf": "copy", + "assets/decision-expanded-old_core.png": "copy", + "assets/decision-expanded-old_core.svg": "copy", + "assets/decision-expanded-overview-600px.png": "copy", + "assets/decision-expanded-overview.pdf": "copy", + "assets/decision-expanded-overview.png": "copy", + "assets/decision-expanded-overview.svg": "copy", + "assets/decision-expanded-ranking-600px.png": "copy", + "assets/decision-expanded-ranking.pdf": "copy", + "assets/decision-expanded-ranking.png": "copy", + "assets/decision-expanded-ranking.svg": "copy", + "assets/decision-expanded-v3_core-600px.png": "copy", + "assets/decision-expanded-v3_core.pdf": "copy", + "assets/decision-expanded-v3_core.png": "copy", + "assets/decision-expanded-v3_core.svg": "copy", + "assets/decision-expanded-v4-600px.png": "copy", + "assets/decision-expanded-v4.pdf": "copy", + "assets/decision-expanded-v4.png": "copy", + "assets/decision-expanded-v4.svg": "copy", + "assets/decision-expanded-v5-600px.png": "copy", + "assets/decision-expanded-v5.pdf": "copy", + "assets/decision-expanded-v5.png": "copy", + "assets/decision-expanded-v5.svg": "copy", "assets/decision-family-header.png": "copy", - "assets/decision-mark.png": "copy", - "assets/decision-quality.pdf": "copy", - "assets/decision-quality.png": "copy", - "assets/decision-quality.svg": "copy", + "assets/decision-question-scaling-600px.png": "copy", "assets/decision-question-scaling.pdf": "copy", "assets/decision-question-scaling.png": "copy", "assets/decision-question-scaling.svg": "copy", - "assets/readout.pdf": "copy", "assets/readout.png": "copy", - "assets/readout.svg": "copy", "backbone/config.json": "copy", "backbone/model-00001-of-00003.safetensors": "hardlink", "backbone/model-00002-of-00003.safetensors": "hardlink", @@ -318,15 +450,21 @@ "code/decision_api.py": "copy", "code/decision_model.py": "copy", "code/predict.py": "copy", + "code/profile_guard.py": "copy", + "code/runtime_profile.py": "copy", "decision_config.json": "copy", "decision_head.safetensors": "hardlink", - "metrics/natural-confirmation.json": "copy", + "metrics/comparator-coverage.json": "copy", + "metrics/expanded-quality.json": "copy", + "metrics/figure-evidence.json": "copy", + "metrics/materials-provenance.json": "copy", "metrics/question-scaling.json": "copy", "model-card-example.json": "copy", "pyproject.toml": "copy", - "quality-metrics.json": "copy", "runtime-build-provenance.json": "copy", "runtime-fla-requirements.lock": "copy", + "runtime-profile/l2norm_fwd_kernel.json": "copy", + "runtime-profile/profile.json": "copy", "runtime-provenance.json": "copy", "runtime.json": "copy", "src/decision/__init__.py": "copy", diff --git a/runtime-build-provenance.json b/runtime-build-provenance.json index c02560e1c21a4261c231b62c8a708c5cf3c71e2d..11431cf3821a0f44e76123d06cedfa9945d16f8c 100644 --- a/runtime-build-provenance.json +++ b/runtime-build-provenance.json @@ -25,7 +25,7 @@ "build_exit_code": 0, "gpu_validation": { "status": "passed", - "scope": "Real packaged three-question example and explicit complete-input overflow rejection for each bundle. Exact response equality against qualified runtime on this request; not a replacement for full benchmark rerun. These entries identify the initial v1.0 weights; they are not new measurements of a later weight update.", + "scope": "Real packaged three-question example and explicit complete-input overflow rejection for each bundle. Exact response equality against qualified runtime on this request; not a replacement for full benchmark rerun.", "models": { "Decision-1.0-Sol": { "bundle_manifest_sha256": "076558961011e924bf17084f940401150245bae6e135d4b1b0a6ee16be01f13c", @@ -51,8 +51,7 @@ "overflow_rejected": true, "board_product_name_verified": false } - }, - "validation_release": "v1.0" + } }, "built_utc": "2026-09-21T12:57:54.147566+00:00", "files": { diff --git a/runtime-profile/l2norm_fwd_kernel.json b/runtime-profile/l2norm_fwd_kernel.json new file mode 100644 index 0000000000000000000000000000000000000000..97e3762d25463be34cfdc28177f74d5eeb5430df --- /dev/null +++ b/runtime-profile/l2norm_fwd_kernel.json @@ -0,0 +1,1286 @@ +{ + "kernel_name": "l2norm_fwd_kernel", + "triton_version": "3.7.1", + "autotune_entries": { + "878a9638751a7375a0fb15d3a7767659": { + "autotune_key": [ + 128, + 1, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "eca743e0790be43b6867d648e9430f67": { + "autotune_key": [ + 128, + 2, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 64 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "7e2d02ce45e4f072828b4c73e16fe0af": { + "autotune_key": [ + 128, + 3, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "424d41ef00a3161badd7f960c34e6ece": { + "autotune_key": [ + 128, + 4, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "6bc2b57d9035425140554c225e667aad": { + "autotune_key": [ + 128, + 5, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "91b9289c43a2d1b21042009d44c57b8a": { + "autotune_key": [ + 128, + 6, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "f44702dfc86a96df1de10439f5c5575a": { + "autotune_key": [ + 128, + 7, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "87c993ac4858bff6042f319bec1647f7": { + "autotune_key": [ + 128, + 8, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "b35c6bfa2cd85845ba58306ee63e28b1": { + "autotune_key": [ + 128, + 9, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 8 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "3527d130e8f6f1415c6d24df9821e638": { + "autotune_key": [ + 128, + 10, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "fe2e43a7c21c6a4814d2636f3004d7dc": { + "autotune_key": [ + 128, + 11, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "c39c159634377c459eccde8a272f74fe": { + "autotune_key": [ + 128, + 12, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "a10453829fb545fd2e01acfb4a809d17": { + "autotune_key": [ + 128, + 13, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "bd8734e29b9704051fe467ede8e8cc03": { + "autotune_key": [ + 128, + 14, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "d5ffd2f70a70e13ac03901bd22a150a7": { + "autotune_key": [ + 128, + 15, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 8 + }, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "d522f744092358110c9ad2cc5f54b501": { + "autotune_key": [ + 128, + 16, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 8 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "6003bee20af205e16410f1983eadd67c": { + "autotune_key": [ + 128, + 17, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "89d2cb2e8a818ce3ecb80bd340ba42b6": { + "autotune_key": [ + 128, + 18, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "157376310735556f015b8ac2dc6e9e2e": { + "autotune_key": [ + 128, + 19, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "e19965e082b861f637bf4bab6abe025b": { + "autotune_key": [ + 128, + 20, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "49feda728414835e54048468abd808ab": { + "autotune_key": [ + 128, + 21, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "6b419788330d3468ef22dc741ec9d7f7": { + "autotune_key": [ + 128, + 22, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "8827081d0dda52c9376b3a1d97a82a8e": { + "autotune_key": [ + 128, + 23, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "49e16fa1a715a8913c54c4ff9ce61aa9": { + "autotune_key": [ + 128, + 24, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "7bf76d12f90b95bbddff57fbc8fe3c39": { + "autotune_key": [ + 128, + 25, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "3f2cd7a3a429540e08f0e4dd8ac9afc5": { + "autotune_key": [ + 128, + 26, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "837afa436bfab6256a535e9360403f34": { + "autotune_key": [ + 128, + 27, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "d4e411f29afbb396dcfe1455427e5178": { + "autotune_key": [ + 128, + 28, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "1b96f1802f4f63f5c872de3d1972ad8a": { + "autotune_key": [ + 128, + 29, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "aec76b66508d5e884ffcc743e8f55a92": { + "autotune_key": [ + 128, + 30, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "439d493be8fe1159bd65368245a4aa60": { + "autotune_key": [ + 128, + 31, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "801422dba59f6c5e144328aabdd135a3": { + "autotune_key": [ + 128, + 32, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "b64090a85e13119b2b9d6da44b47fb1e": { + "autotune_key": [ + 128, + 33, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "591ced9f0b071d7d5a741d78aa40e3af": { + "autotune_key": [ + 128, + 34, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "553860b3206072a8cf6ed8c5ae85afe2": { + "autotune_key": [ + 128, + 35, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "26ff6db9448cafd3c7f492ae07031b32": { + "autotune_key": [ + 128, + 36, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "fb66f24d6d8a81ecf6a0a0e4e1f593a7": { + "autotune_key": [ + 128, + 37, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "3337731ee71bf642f56a748ea1aab501": { + "autotune_key": [ + 128, + 38, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "fc613db432ecc2c79054a5ebb0b0f104": { + "autotune_key": [ + 128, + 39, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "b4b4ed9dccb4dd3c339c85409aaeda5d": { + "autotune_key": [ + 128, + 40, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "68c25491b32ce640bfa9532aad1e131c": { + "autotune_key": [ + 128, + 41, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "a2c9d6cf96c29ebed795632c6c98417b": { + "autotune_key": [ + 128, + 42, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "d64c1e9286cf2fca5455c05aa0e69bef": { + "autotune_key": [ + 128, + 43, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "0f46c3a503133bba9c70842a005f3daf": { + "autotune_key": [ + 128, + 44, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "dbaaf0399e2aae7951b56d9edfd4451b": { + "autotune_key": [ + 128, + 45, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "259a20948e5dee0f8046f0223f897c95": { + "autotune_key": [ + 128, + 46, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "ab079566edd7fa22e0127e2484b882ab": { + "autotune_key": [ + 128, + 47, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "4f7d00ab9063f086ce36b50b35a81f98": { + "autotune_key": [ + 128, + 48, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "02e6e8517e527840af546cdb42334ac4": { + "autotune_key": [ + 128, + 49, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "d70afc9c1fe4a386a1ca8777ab49d932": { + "autotune_key": [ + 128, + 50, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "57f93c08735118a0b07e796268d8e680": { + "autotune_key": [ + 128, + 51, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "8050c4b753748cf6d1f947264733bfbb": { + "autotune_key": [ + 128, + 52, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "f67948a8216d01d8be93e4427f4b6d88": { + "autotune_key": [ + 128, + 53, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "7c6a5d20e4c3ff7e6971db3725e4bb6f": { + "autotune_key": [ + 128, + 54, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "5a150e6b0fe064390c1274548b7c0ecd": { + "autotune_key": [ + 128, + 55, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "0071d83e2db8c9148c0f03ee34b8d71e": { + "autotune_key": [ + 128, + 56, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "d5d40be2025ed11198418892288d5652": { + "autotune_key": [ + 128, + 57, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "de06293f9b9bc51247734636346f23a8": { + "autotune_key": [ + 128, + 58, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "0bc19354dd96c84cf9339134d2a17081": { + "autotune_key": [ + 128, + 59, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "6ca809c36b8b4cda5731f4d5d0f5fd17": { + "autotune_key": [ + 128, + 60, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "6a5149de5c37304e620a00cbad6bdec7": { + "autotune_key": [ + 128, + 61, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "eadca55e5be25288c8e751b793ad159e": { + "autotune_key": [ + 128, + 62, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "246bcd09bb9fab2a10dccb3785664a3e": { + "autotune_key": [ + 128, + 63, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + "2ce7d874a814694b43550e5d3bf554bb": { + "autotune_key": [ + 128, + 64, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + } + } +} diff --git a/runtime-profile/profile.json b/runtime-profile/profile.json new file mode 100644 index 0000000000000000000000000000000000000000..82a7507e7d1069570b80d7189a4c877a59c43b03 --- /dev/null +++ b/runtime-profile/profile.json @@ -0,0 +1,1599 @@ +{ + "format": "decision-fla-l2norm-profile-v1", + "model_family": "Qwen/Qwen3.5-4B", + "status": "CPU-frozen-candidate-Nox-GPU-validation-not-performed", + "target": "Decision Nox v1.1 published underlying CE1563; prospective fixed inference contract for all loss-pilot candidates", + "model_changes": false, + "cache_mode": "strict", + "numerical_tolerance_unchanged": 1e-06, + "supported": { + "head_dimension": 128, + "configured_key_heads": 16, + "normalized_qk_heads_after_repeat": 32, + "value_heads": 32, + "body_dtype": "bfloat16", + "output_dtype": "bfloat16", + "rstd_dtype": "float32", + "batch_size_min": 1, + "batch_size_max": 8, + "padded_tokens_max": 16384, + "NB_min": 1, + "NB_max": 64, + "NB_formula": "ceil(B*padded_tokens*32/65536)", + "unknown_key": "raise before kernel/autotune" + }, + "runtime": { + "torch": "2.12.0+git6bbd260", + "hip": "7.2.53211", + "triton": "3.7.1", + "fla": "0.5.2", + "gpu_arch": "gfx942" + }, + "files": [ + { + "file": "l2norm_fwd_kernel.json", + "sha256": "d7ed7c9962a48efdfa76ed695c8bdc0f56afe1397c202c936eb78315e87a26df" + } + ], + "fla_source_sha256": { + "modules/l2norm.py": "30f7feebdbfa87b8e90c143ec62a7fa0781bfd736c856c0ec69452f0c778be3f", + "ops/utils/cache.py": "a64b09ffd3f51aed547ad72559dee94cb9aeafd2a5f02c6c8e60a6e020147a9c" + }, + "source_evidence": [ + { + "key": [ + 128, + 1, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "e6f711e04262c8fa01dfad28e01cdc65f9fa37f27f759a5065f72dd0f1209fd3", + "chosen_config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 7, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "08eae71633625102aaece6014e381b19b92cc6fa89e0610974f41f0bf17fcfe2", + "chosen_config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 8, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "aa490c4857c426e2a17edc7e05a87fe72c8ab1f6485d5d480212811e5d2ef0cd", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 21, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "b7da8ec1b352d1dc859bf110533f249948af0f893883300cc5ff15dfcbc62a63", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 24, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "29687aa12cb9a100995a67ab840c5ee3fc59cabc601a8cf9c54356658007063d", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 9, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "715030923db674463c19081ec50a00a8e300545dac6e6b9a2842c915944e5c15", + "chosen_config": { + "kwargs": { + "BT": 8 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 23, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "f76b7b0b67064fdc5d36b8e99df2c70c3fae53327a1050cb90f48aebf81c5065", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 36, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "c8878c0b77b8dae60045aeb2d1855b7c7c964877671e2f34656bb7bd2277a75f", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 16, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "c552a842bed537d40c8d2fa9812beae11367c3cce8bcbb79b4bf31ebb03dfdd9", + "chosen_config": { + "kwargs": { + "BT": 8 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 5, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "7bed2aa7816fef4a6bfce2136d7850949f7db0c4006583949c9df6497c6aa760", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 15, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "46dfa6ffab7a8dd3ea5eb94e1f23c1a47414f6fce90319aba8939d7c84f1a5f3", + "chosen_config": { + "kwargs": { + "BT": 8 + }, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 2, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "3535c1cba64c085418f8bb0704091a195f723279acb06cd55be1de44b52400c3", + "chosen_config": { + "kwargs": { + "BT": 64 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 26, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "ef35040761c5bcaddf580c15f650ef0d9091d6d145bce3121cffdaa0fe5d1d9c", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 12, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "e87ee6127a33dee7d4f28413a8571ff338b8a80719943b2311d45d6890cd736d", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 22, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "a747c7f1d616e064dc67504413e45584c424af4da793f32819c252e64e4fcfec", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 19, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "edeccb6df1178ec72808f8d419ce375d2f94e6874ae178a168851261fd8276be", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 17, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "9708ef5c4c817c23253941c4019af9b31053b274457402b2b0897505ef864ee4", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 10, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "2b7e23b9b8211c063cecb17ce75715da538145a8dd15b9d76ec55aeebc2e1aba", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 13, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "717b02c96ce67de3dc50b8582d5dae96d7b5f64d3e9978d066ff485093657639", + "chosen_config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 11, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "0bfee1a5fbc6eb91071fd6a5cb9b48da2bbd19168494bfe50f39c6adc89f938b", + "chosen_config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 35, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "f00fe37e0e1a00fe180898b120afdbb58d0d4a06712b4aa67bd6cd942745e4f1", + "chosen_config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 20, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "4094205b5f109780da3315a48dfeca840bdf59b0087f4f1894e3603182ddd04d", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 3, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "4cc1b13319b9111b61fdb4374403099c728dad955a94f007f384870a1a6e5d17", + "chosen_config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 25, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "73738f6ec2d4f97b22865f6f2f3326e64188f7ee33cbca9427f012d876ad6d53", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 4, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "d0e13a488637bc8fccadf05b70c668743968d1299f071b7d694225447ca1f8d6", + "chosen_config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 18, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "4f2af84cb027b6c0dbda7377716829423960f4fbdf51e8755abeddffb8f16f55", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 6, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "7890d73b0d592703953109555cabc535f6682e5d4d0d2c66fed6ef91e9294fab", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "key": [ + 128, + 14, + "torch.bfloat16", + "torch.bfloat16", + "torch.float32" + ], + "source_autotune_sha256": "ddd1a082f05e91a47cdba6e7fc062570c1f5c9f57fc30ca68bd9b0b3c2914a21", + "chosen_config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + } + ], + "per_NB_policy": [ + { + "NB": 1, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 2, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 64 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 3, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 4, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 5, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 6, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 7, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 8, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 9, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 8 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 10, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 11, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 12, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 13, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 14, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 15, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 8 + }, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 16, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 8 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 17, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 18, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 19, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 20, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 21, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 22, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 23, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 24, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 25, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 26, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 27, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 28, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 29, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 30, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 31, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 32, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 33, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 34, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 35, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 16 + }, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 36, + "basis": "Nox-original-production-chosen", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 37, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 38, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 39, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 40, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 41, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 42, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 43, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 44, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 45, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 46, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 47, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 48, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 49, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 50, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 51, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 52, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 53, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 54, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 55, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 56, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 57, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 58, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 59, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 60, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 61, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 62, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 63, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + }, + { + "NB": 64, + "basis": "prospective-fixed-Nox-original-NB36-setting", + "config": { + "kwargs": { + "BT": 32 + }, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3, + "maxnreg": null, + "pre_hook": null, + "ir_override": null + } + } + ], + "unseen_NB_claim": "Prospective fixed launch setting; no assertion it reproduces a previously undefined autotune result or optimizes speed", + "other_kernels": "Unmodified FLA/Transformers dispatch; only l2norm forward has fixed launch configs. Remaining kernels can still autotune.", + "compiled_binaries_included": false, + "private_paths_included": false, + "builder_sha256": "0113251d806e0fbef6f8b94f5dbb7bdca632f94adb90bd8c60ece1d9f84ba1e6" +} diff --git a/runtime.json b/runtime.json index 864d1c0a1bf0cb6187eae906a627fb21d1b5783c..71f409f7a67f754b73d9ddbd7ed689a6fd47dce8 100644 --- a/runtime.json +++ b/runtime.json @@ -9,5 +9,17 @@ "gdn_new_implementation": true, "tokenizers": "0.23.2", "safetensors": "0.8.0", - "gated_delta": "fla.ops.gated_delta_rule.chunk" + "gated_delta": "fla.ops.gated_delta_rule.chunk", + "normalization_profile": { + "kind": "decision-fla-l2norm-profile-v1", + "profile_file": "runtime-profile/profile.json", + "profile_sha256": "be32858d15233e0a3fbee0e4257fb02be0b3439deee4eb9c3f61151df7b73850", + "guard_file": "code/profile_guard.py", + "guard_sha256": "061d6ba3edf032038b074b061879a1ff81cfd2928fc0131c66b15563f7822509", + "loader_file": "code/runtime_profile.py", + "loader_sha256": "3e26ed65f2cd706def209761421cb8fc825281e5c1ed8f168350365ad803dc96", + "validated_arch": "gfx942", + "activation": "automatic before FLA import by public from_pretrained and bundle DecisionEngine", + "process_scope": "one profile per fresh Python process; same-profile loads allowed; mixed profiles rejected" + } } diff --git a/src/decision/example.py b/src/decision/example.py index 377ea54992916d96e53b264187ca4fdc00f16692..7bcc62c41334fd27f6b8b67628b674f14d912bad 100644 --- a/src/decision/example.py +++ b/src/decision/example.py @@ -52,6 +52,9 @@ def main(): 'bundle_manifest_sha256': hashlib.sha256((model.bundle_path / 'bundle-manifest.json').read_bytes()).hexdigest(), 'runtime': model.runtime, 'revision': args.revision, 'device': args.device, 'model_name': response['model'], 'example_source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} + if model.runtime.get('normalization_profile') is not None: + import sys + record['normalization_telemetry'] = dict(sys.modules['_decision_process_normalization_profile_v1'].telemetry) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(record, ensure_ascii=False, indent=2) + '\n') print(json.dumps(response, ensure_ascii=False, indent=2)) diff --git a/src/decision/model.py b/src/decision/model.py index c9f0d9c65f32b6527bc6029286e3e5d40d4ae634..395a9f77fa6d7f2c550b34fec732e77d957fe51d 100644 --- a/src/decision/model.py +++ b/src/decision/model.py @@ -131,10 +131,21 @@ class DecisionModel: isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or v <= 0 for v in temperatures.values()): raise ValueError('Bundle must contain finite positive temperatures for all three types') - runtime = _runtime_report(json.loads((path / 'runtime.json').read_text()), device, allow_unvalidated_runtime) + api = _load_api(path) + expected_runtime = json.loads((path / 'runtime.json').read_text()) + if expected_runtime.get('normalization_profile') is not None: + if not hasattr(api, 'prepare_runtime_profile'): + raise RuntimeError('Profiled bundle omits its automatic runtime entrypoint') + profile = api.prepare_runtime_profile(path, device=device) + else: + import sys + if '_decision_process_normalization_profile_v1' in sys.modules: + raise RuntimeError('Use separate processes for profiled and unprofiled models') + profile = None + runtime = _runtime_report(expected_runtime, device, allow_unvalidated_runtime) + runtime['normalization_profile'] = profile names = {'Qwen/Qwen3.5-2B': 'Decision-1.0-Sol', 'Qwen/Qwen3.5-4B': 'Decision-1.0-Nox'} name = model_name or config.get('model_name') or names.get(config.get('base_model'), path.name) - api = _load_api(path) engine = api.DecisionEngine(path, path / 'code', device=device, max_length=maximum, batch_size=manifest['production_batch_size'], temperatures=temperatures, model_name=name) diff --git a/temperature.json b/temperature.json index ee41a31c3909f4f45bfd61c2a32af2239d0a494c..e372141cd6297443a02772ba5d741f67c559e48e 100644 --- a/temperature.json +++ b/temperature.json @@ -1,420 +1,226 @@ { - "version": "stage4-nine-family-macro-nll-beta-bisection-v1", - "objective": "equal-family macro NLL", - "input_sha256": { - "cal": "5ba3bd7527da831d77cec8c6a2f260a71e10a57742e356510c719f8f30d61a70", - "predictions": "95ba29cb751506b796518940227d12fa01d4c0dda676c0e9ec85164e9b781c79", - "inference-summary": "9d62ece2bc5e80e919758cf6fbf2311446d8d208aec4632480d00e37e89c828b", - "inference-binding": "ff97f67a33660f5d7df4724a6030a9b3c8c67eb8def3305cc9d909a5cb600da1", - "selection": "3e0074ae9afdb94898c2235f3aa319b7f7b85b1984520456767314250893392e", - "model-manifest": "b64287f05a1c9b0b78239ffe6f352ec42cbb42ae256c8ee4c869b8332c0046cf", - "plan": "e5086ac34dbf190bd1f1ce6f0a426a37710160878b67120aab9bdbfed49e8348" - }, - "temperature": 0.9709503214723758, + "arm": "natural_treatment", + "selected_step": 375, + "temperature": 1.0155830584882959, "temperatures": { - "choice": 0.9709503214723758, - "noul": 0.9709503214723758, - "score": 0.9709503214723758 - }, - "ranking_unchanged": true, - "ranking_claim_scope": "Positive global scaling preserves mathematical logit ordering; finite-precision inference parity is audited separately", - "source_sha256": "80e008d3a0eab5604de4b15bc897ddfa26c6761430753478ad5972a6cf08e5df", - "family_rows": { - "stage4_dense_table": 100, - "stage4_scope": 100, - "stage4_arithmetic": 100, - "stage4_ordinal": 100, - "stage4_natural_nli": 200, - "stage4_automaton": 100, - "stage4_boolean": 100, - "stage4_relations": 100, - "stage4_registers": 100 + "choice": 1.0155830584882959, + "noul": 1.0155830584882959, + "score": 1.0155830584882959 }, - "selected_checkpoint": "checkpoint-001563", - "selected_arm": "ce", "fit": { - "temperature": 0.9709503214723758, - "inverse_temperature": 1.0299188103502273, + "temperature": 1.0155830584882959, + "inverse_temperature": 0.9846560472252349, "status": "relative-bracket-converged", "boundary_hit": false, "iterations": 38, "beta_final_bracket": [ - 1.0299188103139385, - 1.029918810386516 + 0.9846560471889461, + 0.9846560472615238 ], - "objective_at_T1": 0.5182615704773357, - "objective_at_selected_T": 0.5181319586326276, - "derivative_at_selected_beta": -1.7113407527500099e-12, + "objective_at_T1": 0.4262873409965549, + "objective_at_selected_T": 0.42624728463092443, + "derivative_at_selected_beta": 7.788978184145204e-12, "initial_boundary_derivatives": { - "beta_0.05": -3.606176602570555, - "beta_20": 0.19328400800320916 + "beta_0.05": -3.0366540479066213, + "beta_20": 0.19310407449992384 }, - "derivative_at_beta1": -0.008773648842477322, + "derivative_at_beta1": 0.005185775194699754, "initial_boundary_objectives": [ - 1.2650764843253368, - 3.9131497587262802 + 1.0957737013940982, + 3.8927751504820978 ] }, "raw": { - "temperature": 1.0, - "family_macro_nll": 0.5182615704773357, - "family_macro_brier": 0.27057614160720794, - "family_macro_native_accuracy": 0.7905555555555556, + "rows": 1600, + "accuracy_fraction": [ + 1007, + 1200 + ], + "accuracy": 0.8391666666666666, + "brier": 0.22605177485769778, + "nll": 0.4262873409965549, + "score_rps": 0.033206021482489016, "by_family": { - "stage4_dense_table": { - "nll": 0.16094672166341653, - "brier": 0.1073572972530966, - "native_accuracy": 0.92, - "rows": 100 - }, - "stage4_scope": { - "nll": 0.0982711110451638, - "brier": 0.0617345895654639, - "native_accuracy": 0.93, - "rows": 100 - }, - "stage4_arithmetic": { - "nll": 0.6106335837480057, - "brier": 0.23938910654455806, - "native_accuracy": 0.83, - "rows": 100 - }, - "stage4_ordinal": { - "nll": 0.3632582610112115, - "brier": 0.17830054368071632, - "native_accuracy": 0.89, - "rows": 100 - }, - "stage4_natural_nli": { - "nll": 0.35248374668561444, - "brier": 0.19717130281738396, - "native_accuracy": 0.875, - "rows": 200 - }, - "stage4_automaton": { - "nll": 1.1690857706226174, - "brier": 0.5880574268878196, - "native_accuracy": 0.53, - "rows": 100 - }, - "stage4_boolean": { - "nll": 0.54536841290903, - "brier": 0.3833176245261982, - "native_accuracy": 0.67, - "rows": 100 - }, - "stage4_relations": { - "nll": 0.8852863110648258, - "brier": 0.40836836379878055, - "native_accuracy": 0.65, - "rows": 100 - }, - "stage4_registers": { - "nll": 0.4790202155461354, - "brier": 0.2714890193908542, - "native_accuracy": 0.82, - "rows": 100 - } + "old/stage4_dense_table": [ + 92, + 100 + ], + "old/stage4_scope": [ + 94, + 100 + ], + "old/stage4_arithmetic": [ + 80, + 100 + ], + "old/stage4_ordinal": [ + 90, + 100 + ], + "old/stage4_natural_nli": [ + 180, + 200 + ], + "old/stage4_automaton": [ + 54, + 100 + ], + "old/stage4_boolean": [ + 69, + 100 + ], + "old/stage4_relations": [ + 68, + 100 + ], + "old/stage4_registers": [ + 77, + 100 + ], + "natural/natural_squad2_answerability": [ + 186, + 200 + ], + "natural/natural_cosmos_qa": [ + 166, + 200 + ], + "natural/natural_snli": [ + 179, + 200 + ] }, - "by_task_type": { - "score": { - "family_macro_nll": 0.3382261351389755, - "family_macro_brier": 0.18157542641640761, - "family_macro_native_accuracy": 0.8825000000000001, - "by_family": { - "stage4_dense_table": { - "nll": 0.3131940092667395, - "brier": 0.18485030915209893, - "native_accuracy": 0.875, - "rows": 16 - }, - "stage4_ordinal": { - "nll": 0.3632582610112115, - "brier": 0.17830054368071632, - "native_accuracy": 0.89, - "rows": 100 - } - } - }, - "choice": { - "family_macro_nll": 0.6096406758151754, - "family_macro_brier": 0.2868032221621859, - "family_macro_native_accuracy": 0.7675923883066741, - "by_family": { - "stage4_scope": { - "nll": 0.06959359168878981, - "brier": 0.04584186867853064, - "native_accuracy": 0.9324324324324325, - "rows": 74 - }, - "stage4_arithmetic": { - "nll": 0.6106335837480057, - "brier": 0.23938910654455806, - "native_accuracy": 0.83, - "rows": 100 - }, - "stage4_natural_nli": { - "nll": 0.35248374668561444, - "brier": 0.19717130281738396, - "native_accuracy": 0.875, - "rows": 200 - }, - "stage4_automaton": { - "nll": 1.4505220413815811, - "brier": 0.7028690104146533, - "native_accuracy": 0.4142857142857143, - "rows": 70 - }, - "stage4_relations": { - "nll": 1.2922358158773557, - "brier": 0.5663319236393812, - "native_accuracy": 0.48333333333333334, - "rows": 60 - }, - "stage4_registers": { - "nll": 0.4383348594249947, - "brier": 0.22289138346494564, - "native_accuracy": 0.8666666666666667, - "rows": 75 - }, - "stage4_dense_table": { - "nll": 0.05368109189988708, - "brier": 0.03312795957584854, - "native_accuracy": 0.9714285714285714, - "rows": 35 - } - } - }, - "noul": { - "family_macro_nll": 0.38357521019656776, - "family_macro_brier": 0.2557047396808256, - "family_macro_native_accuracy": 0.8118393511250654, - "by_family": { - "stage4_scope": { - "nll": 0.17989174305945901, - "brier": 0.10696771824365855, - "native_accuracy": 0.9230769230769231, - "rows": 26 - }, - "stage4_boolean": { - "nll": 0.54536841290903, - "brier": 0.3833176245261982, - "native_accuracy": 0.67, - "rows": 100 - }, - "stage4_relations": { - "nll": 0.2748620538460306, - "brier": 0.17142302403787968, - "native_accuracy": 0.9, - "rows": 40 - }, - "stage4_dense_table": { - "nll": 0.18785162860362803, - "brier": 0.13507441211676285, - "native_accuracy": 0.8979591836734694, - "rows": 49 - }, - "stage4_registers": { - "nll": 0.6010762839095574, - "brier": 0.4172819271685799, - "native_accuracy": 0.68, - "rows": 25 - }, - "stage4_automaton": { - "nll": 0.5124011388517018, - "brier": 0.3201637319918742, - "native_accuracy": 0.8, - "rows": 30 - } - } - } - }, - "proper_score_convention": "multiclass sum-squared Brier, including two entries for noul; NLL in nats; equal family weight within each reported task type" + "temperature": 1.0, + "probability_source": "production-T1-normalized" }, "calibrated": { - "temperature": 0.9709503214723758, - "family_macro_nll": 0.5181319586326276, - "family_macro_brier": 0.2705833688475143, - "family_macro_native_accuracy": 0.7905555555555556, + "rows": 1600, + "accuracy_fraction": [ + 1007, + 1200 + ], + "accuracy": 0.8391666666666666, + "brier": 0.2261092775549145, + "nll": 0.42624728463092443, + "score_rps": 0.03323698212819523, "by_family": { - "stage4_dense_table": { - "nll": 0.1610103183315075, - "brier": 0.10804051074170876, - "native_accuracy": 0.92, - "rows": 100 - }, - "stage4_scope": { - "nll": 0.09805690139180127, - "brier": 0.061829107241029704, - "native_accuracy": 0.93, - "rows": 100 - }, - "stage4_arithmetic": { - "nll": 0.6062748672216433, - "brier": 0.23861886616240668, - "native_accuracy": 0.83, - "rows": 100 - }, - "stage4_ordinal": { - "nll": 0.3643149926000012, - "brier": 0.17812236254633418, - "native_accuracy": 0.89, - "rows": 100 - }, - "stage4_natural_nli": { - "nll": 0.3504183765509638, - "brier": 0.19618757047681687, - "native_accuracy": 0.875, - "rows": 200 - }, - "stage4_automaton": { - "nll": 1.1681029691377338, - "brier": 0.5877681870621635, - "native_accuracy": 0.53, - "rows": 100 - }, - "stage4_boolean": { - "nll": 0.5466382119984629, - "brier": 0.3844002532174035, - "native_accuracy": 0.67, - "rows": 100 - }, - "stage4_relations": { - "nll": 0.8877925230975072, - "brier": 0.4088977034700872, - "native_accuracy": 0.65, - "rows": 100 - }, - "stage4_registers": { - "nll": 0.48057846736402743, - "brier": 0.2713857587096781, - "native_accuracy": 0.82, - "rows": 100 - } + "old/stage4_dense_table": [ + 92, + 100 + ], + "old/stage4_scope": [ + 94, + 100 + ], + "old/stage4_arithmetic": [ + 80, + 100 + ], + "old/stage4_ordinal": [ + 90, + 100 + ], + "old/stage4_natural_nli": [ + 180, + 200 + ], + "old/stage4_automaton": [ + 54, + 100 + ], + "old/stage4_boolean": [ + 69, + 100 + ], + "old/stage4_relations": [ + 68, + 100 + ], + "old/stage4_registers": [ + 77, + 100 + ], + "natural/natural_squad2_answerability": [ + 186, + 200 + ], + "natural/natural_cosmos_qa": [ + 166, + 200 + ], + "natural/natural_snli": [ + 179, + 200 + ] }, - "by_task_type": { - "score": { - "family_macro_nll": 0.33816889193659005, - "family_macro_brier": 0.1816663865227025, - "family_macro_native_accuracy": 0.8825000000000001, - "by_family": { - "stage4_dense_table": { - "nll": 0.3120227912731789, - "brier": 0.18521041049907083, - "native_accuracy": 0.875, - "rows": 16 - }, - "stage4_ordinal": { - "nll": 0.3643149926000012, - "brier": 0.17812236254633418, - "native_accuracy": 0.89, - "rows": 100 - } - } - }, - "choice": { - "family_macro_nll": 0.6089491937173043, - "family_macro_brier": 0.28658434195063703, - "family_macro_native_accuracy": 0.7675923883066741, - "by_family": { - "stage4_scope": { - "nll": 0.06926534778158984, - "brier": 0.04597518229028828, - "native_accuracy": 0.9324324324324325, - "rows": 74 - }, - "stage4_arithmetic": { - "nll": 0.6062748672216433, - "brier": 0.23861886616240668, - "native_accuracy": 0.83, - "rows": 100 - }, - "stage4_natural_nli": { - "nll": 0.3504183765509638, - "brier": 0.19618757047681687, - "native_accuracy": 0.875, - "rows": 200 - }, - "stage4_automaton": { - "nll": 1.4480794691161745, - "brier": 0.7024622954514393, - "native_accuracy": 0.4142857142857143, - "rows": 70 - }, - "stage4_relations": { - "nll": 1.2956497188092622, - "brier": 0.5670181081222113, - "native_accuracy": 0.48333333333333334, - "rows": 60 - }, - "stage4_registers": { - "nll": 0.4398741273136369, - "brier": 0.22260045014222438, - "native_accuracy": 0.8666666666666667, - "rows": 75 - }, - "stage4_dense_table": { - "nll": 0.0530824492278591, - "brier": 0.0332279210090724, - "native_accuracy": 0.9714285714285714, - "rows": 35 - } - } - }, - "noul": { - "family_macro_nll": 0.3848257414711845, - "family_macro_brier": 0.25620653197998017, - "family_macro_native_accuracy": 0.8118393511250654, - "by_family": { - "stage4_scope": { - "nll": 0.1800020924362492, - "brier": 0.10695181671621683, - "native_accuracy": 0.9230769230769231, - "rows": 26 - }, - "stage4_boolean": { - "nll": 0.5466382119984629, - "brier": 0.3844002532174035, - "native_accuracy": 0.67, - "rows": 100 - }, - "stage4_relations": { - "nll": 0.2760067295298746, - "brier": 0.171717096491901, - "native_accuracy": 0.9, - "rows": 40 - }, - "stage4_dense_table": { - "nll": 0.18879145815928203, - "brier": 0.13627974022180014, - "native_accuracy": 0.8979591836734694, - "rows": 49 - }, - "stage4_registers": { - "nll": 0.602691487515199, - "brier": 0.41774168441203935, - "native_accuracy": 0.68, - "rows": 25 - }, - "stage4_automaton": { - "nll": 0.5148244691880391, - "brier": 0.32014860082052005, - "native_accuracy": 0.8, - "rows": 30 - } - } - } + "temperature": 1.0155830584882959, + "probability_source": "binary64-logits-diagnostic" + }, + "inputs": { + "old": { + "data_sha256": "5ba3bd7527da831d77cec8c6a2f260a71e10a57742e356510c719f8f30d61a70", + "predictions_sha256": "07786988e0631422be3ec70e769abcc3b03c63785890aa51a6a147b1db43c54a", + "summary_sha256": "7237145e9368e083602d9efba7fe1c412084908db7212c673dbb656dbf20c2d9" }, - "proper_score_convention": "multiclass sum-squared Brier, including two entries for noul; NLL in nats; equal family weight within each reported task type" + "natural": { + "data_sha256": "cc75c902fe9b51c4654d6ceaeb590fe135bdef6377d45bb774e391a66bb1fa25", + "predictions_sha256": "84212958359e8a716203d394b10a4f1f6bb2aaa00082ef09ad5ee0cec442cba7", + "summary_sha256": "8a772b0ec26315adc15d4fc9d2365f9456c01906f7c496e34f55f84202b21bfc" + } }, + "protocol_sha256": "441f06467d11f3374f9bda7a3abdfc23fb2274fb3db122dfae97a97715375e1b", + "weighted_math_sha256": "19c8dd94e9f078810be943bdcbf946b842a20ba77aac359a7faade5f62b19709", + "model_manifest_sha256": "a98e10438585d0891fabd8ba8447cc49385ac349334143d38dd28df397843424", + "winner_lock_sha256": "c369b72c875eafd3a9c78128a91a94f7e2c22762303419d882b1746309386d89", + "both_winners_lock_sha256": "a9e0dd8cbffae656b20cb8bebe664319462edc5f5be1d5b07bee7e5550b0d106", "heldout_used": false, "SELECT_used_for_temperature": false, - "generalization_improvement_claim": false, - "raw_predictions_sha256": "95ba29cb751506b796518940227d12fa01d4c0dda676c0e9ec85164e9b781c79", - "dev_sha256": "5ba3bd7527da831d77cec8c6a2f260a71e10a57742e356510c719f8f30d61a70", - "cal_sha256": "5ba3bd7527da831d77cec8c6a2f260a71e10a57742e356510c719f8f30d61a70", - "calibration_dataset_role": "CAL; dev_sha256 is a legacy portable-bundle schema alias, not SELECT", - "source_calibration_artifact_sha256": "8b16a9b87b3a727efcadcff77b0e5d480c34fb02133b52271ed30c4e520b9000", - "export_metadata_bridge_sha256": "e4cd2ceaa9bd9697f5c47f5a4504526612e6f23fd87a095fe9c92fd0fae9ae31", - "temperature_refit_performed_by_bridge": false + "qualification_established": false, + "dev_sha256": "45b4cd46acc4be53b95c7ed8f333b3533972296a4d0557c7b8c48c9cab6ced61", + "raw_predictions_sha256": "cf0b1a17008bddc2e3e18289866993770c41d4a5af5f6f4219508bb34fcabefa", + "ranking_unchanged": true, + "ranking_unchanged_scope": "Positive scalar preserves mathematical within-row logit order; native float32 outputs still require the separate full public reload proof.", + "calibration_objective": "0.5mean9(old family mean NLL)+0.5mean3(natural family mean NLL)", + "calibration_weights": { + "old_panel": [ + 1, + 2 + ], + "natural_panel": [ + 1, + 2 + ], + "each_old_family": [ + 1, + 18 + ], + "each_natural_family": [ + 1, + 6 + ] + }, + "calibration_role": "original-two-panel-CAL", + "uniform_1600_fit": false, + "temperature_refitted": false, + "bridge": { + "schema": "natural-CAL-export-v1", + "family": "Nox", + "arm": "natural_treatment", + "rows": 1600, + "panel_order": [ + "old", + "natural" + ], + "panel_rows": { + "old": 1000, + "natural": 600 + }, + "original_temperature_sha256": "7ee6178ac029d830efea71305957eb4b8b788cce2e9e44d1717ffcb523ce732b", + "producer_complete_sha256": "3341c16d87acc08f36cf3ae18b3813413e01a05898e58db97ae6b332fad842d0", + "producer_freeze_sha256": "5ad8ecbd1151f303216d698ed90444c8f318265879896802c4a8e4c4e9b7d654", + "original_protocol_sha256": "441f06467d11f3374f9bda7a3abdfc23fb2274fb3db122dfae97a97715375e1b", + "ranking_sha256": "e888c82008dd82106ad160e187eac0c358c33b8d88face0f925d8702a204f11d", + "sequential_B8_boundary_preserved": true, + "normalization_profile_sha256": "be32858d15233e0a3fbee0e4257fb02be0b3439deee4eb9c3f61151df7b73850" + } }