diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..3b6d864a7f00177d252f3c1401a37f4e14e7ad86 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +demo/browser-demo.gif filter=lfs diff=lfs merge=lfs -text +demo/browser-demo.mp4 filter=lfs diff=lfs merge=lfs -text +demo/browser-preview.png filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..b8b198ce3ebaa22c9f59588a80f9f1875d85b34a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Cua AI, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NOTICES.md b/NOTICES.md new file mode 100644 index 0000000000000000000000000000000000000000..1637728ee5e11b2edf4b85b38de9fedb1622fca1 --- /dev/null +++ b/NOTICES.md @@ -0,0 +1,18 @@ +# Distribution notices + +This repository distributes FP16 Core ML derivatives of the MIT-licensed +CUA-S1-FORMS checkpoint published by Cua at +https://huggingface.co/cua-ai/cua-s1-forms/tree/f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71. +The Cua MIT license is included as LICENSE. + +The model implementation includes byte-collation and attention primitives adapted +from Minimal Labs' MIT-licensed jevlike project. Its copyright and license are +preserved in UPSTREAM-THIRD-PARTY-NOTICES.md. That file is an unmodified notice +from the source distribution; its statement that the source package does not +bundle model weights describes that upstream package. This derived distribution +does include the converted weights listed above. + +The upstream demo dataset, training datasets, source safetensors checkpoint, +Python environment, and third-party dependencies are not bundled. The local +preprocessing helper is from the FluidInference Mobius conversion toolkit. +Model/data revisions and source download hashes are recorded in assets.lock.json. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..295edaaf44e8eefdf1f61932267e2b93a353bab5 --- /dev/null +++ b/README.md @@ -0,0 +1,446 @@ +--- +license: mit +library_name: coreml +pipeline_tag: text-classification +base_model: cua-ai/cua-s1-forms +base_model_relation: quantized +datasets: +- cua-ai/cua-s1-forms +tags: +- coreml +- apple-silicon +- computer-use +- classification +- cua-s1 +- fp16 +--- + +# CUA-S1-FORMS — Core ML + +FP16 Core ML conversion of [Cua's CUA-S1-FORMS](https://huggingface.co/cua-ai/cua-s1-forms), +a **706,048-parameter** specialist that selects among supplied form actions. +The portable package is **1,511,163 bytes (1.51 MB)**. No text generation, KV cache, +or external tokenizer is required. + +The model uses small Transformer encoders over UTF-8 bytes and an attention +readout. It is a classifier, not an autoregressive LLM. The inherited checkpoint +configuration contains an unused `hf_model` value; this `tinyx` checkpoint does +not load Qwen weights. + +## Files + +| File | Purpose | +| --- | --- | +| `cua_s1_forms_fp16_options32.mlpackage/` | Portable model; compile locally or add to Xcode | +| `cua_s1_forms_fp16_options32.mlmodelc/` | Compiled bundle for FluidAudio's model loader | +| `preprocessing.py` | Upstream-compatible byte encoding and input validation | +| `conversion.json` | Architecture, conversion versions, and portable package hashes | +| `assets.lock.json` | Pinned upstream source, model, and demo hashes | +| `synthetic-test.lock.json` | Full published synthetic test revision, count, size, and SHA-256 | +| `checksums.json` | SHA-256 of each distributed file except this checksum file | +| `reports/` | Per-row parity, original Cua metrics, and compute-placement report | + +The model targets **iOS 17/macOS 14 or newer**. Runtime validation used an Apple +silicon Mac. Use the portable package for local compilation on other supported +systems; iPhone performance and compatibility of the precompiled bundle across +older OS versions have not been measured. + +## Python usage + +Install `coremltools==9.0`, `numpy==1.26.4`, and `huggingface_hub` on macOS. +Download this repository, then run from its directory: + +```python +import coremltools as ct +from preprocessing import InputLimits, prepare_inputs + +model = ct.models.MLModel( + "cua_s1_forms_fp16_options32.mlpackage", + compute_units=ct.ComputeUnit.CPU_AND_NE, +) +options = ["fill E-mail: person@example.com", "check", "click", "skip"] +inputs = prepare_inputs( + 'TASK fill the form from the document, then submit\n' + 'FORM Contact details\nELEMENT Edit "Email address" value=""', + options, + InputLimits(), +) +raw_probabilities = model.predict(inputs)["probabilities"][0, :len(options)] +print(options[int(raw_probabilities.argmax())], raw_probabilities) +``` + +Download the model artifacts: + +```bash +hf download FluidInference/cua-s1-forms-coreml --revision main --local-dir ./cua-coreml +cd cua-coreml +``` + +## Swift usage + +The [FluidAudio integration](https://github.com/FluidInference/FluidAudio/tree/main) +provides `CuaS1FormsManager`: + +```swift +import FluidAudio +import Foundation + +let manager = try await CuaS1FormsManager.load( + from: URL(fileURLWithPath: "/models/cua_s1_forms_fp16_options32.mlpackage")) +let decision = try await manager.score( + context: "TASK fill the form from the document, then submit\nFORM Contact details\nELEMENT Edit \"Email address\" value=\"\"", + options: ["fill E-mail: person@example.com", "check", "click", "skip"]) +print(decision.selectedOption, decision.probabilities) +``` + +`decision.probabilities` uses stable Swift softmax; `decision.rawProbabilities` +preserves the package output shown in the Python example. See the +[full runtime validation](#swift-probability-fix). + +`try await CuaS1FormsManager.load()` downloads +and caches the compiled artifact automatically. + +## Tensor interface + +| Name | Type | Shape | +| --- | --- | --- | +| `context_ids` | int32 | `[1, 224]` | +| `option_ids` | int32 | `[1, 32, 96]` | +| `option_mask` | int32 | `[1, 32]` | +| `logits` | float32 output | `[1, 32]` | +| `probabilities` | float32 output | `[1, 32]` | + +Encode UTF-8 bytes plus one, pad with zero, and truncate by bytes at 224 for +context and 96 per option. Supply a nonempty context and 2–32 nonempty options. +Set option-mask entries to one for supplied options and zero for padding. +Padded logits are `-10000`; padded probabilities are zero. Inputs above 32 +options must be rejected or use a separately exported larger-capacity model. +The example helper rejects overflow rather than dropping choices. + +## Conversion verification + +On the complete pinned **196-row upstream demo**, PyTorch and Core ML both +selected **196/196 labeled options correctly**, and matched each other's selected +option on every row. The unmodified upstream Cua evaluator reports 36 fill, +4 check, 6 click, and 150 skip decisions, with zero wrong actions, wrong targets, +or unsafe actions on these saved predictions. It counts `skip` as abstention, +so its 23.47% coverage corresponds to 46 actionable decisions. + +| Check | Core ML `ALL` | Core ML `CPU_AND_NE` | +| --- | ---: | ---: | +| Selected options matching PyTorch | 196 / 196 | 196 / 196 | +| Maximum absolute probability error | 0.003099 | 0.002336 | +| Warm model-call median | 1.85 ms | 0.90 ms | +| Warm model-call p95 | 2.49 ms | 0.94 ms | + +Measured September 19, 2026 on Apple M5 Pro, 24 GB, macOS 27.0, using +Python 3.11.11, PyTorch 2.7.0, and coremltools 9.0. Timing is exploratory and +includes Python call overhead; model loading, encoding, document extraction, +UI observation, and action execution are excluded. It is not an optimized +PyTorch/MPS speed comparison. + +The conversion gates require 100% selected-option agreement, no accuracy loss, +maximum absolute probability error ≤ 0.005, finite outputs, normalized live +probabilities, and zero probability for padding. The FP32 export adapter differs +from the unmodified PyTorch reference by at most 0.00000113. Six reversed-option +checks also pass on each Core ML configuration. The checked-in placement report +counts 149 Neural Engine operations, 24 CPU operations, and zero GPU operations; +operation counts are not a measurement of time spent on each processor. + +These results verify conversion on three demo forms and three PDFs. They do not +establish generalization, live GUI completion rates, or production safety. +The original demo is not redistributed here; its exact revision and SHA-256 are +in the asset lock. No training was performed. The separate full synthetic test below evaluates +the unchanged artifacts and exposes numerical failures outside this demo. + +## Full published synthetic test + +Evaluated the complete published synthetic [`test.jsonl`](https://huggingface.co/datasets/cua-ai/cua-s1-forms/blob/8273f34778b99ac2e12d9f6e7d57dad99ae20845/test.jsonl): +**24,370 decisions across 1,040 episode seeds**, with no excluded rows or truncated +inputs. The dataset revision is `8273f34778b99ac2e12d9f6e7d57dad99ae20845`; its +SHA-256 is `d63a7e0db195d4d20154a40b2f8dd09ce3bb65487a158c638da5c609d4475e7c`. +The upstream [model card](https://huggingface.co/cua-ai/cua-s1-forms/blob/f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71/README.md) reports 99.95% on an approximately 15,000-row +synthetic test. Our result rounds to that accuracy, but the released file contains +24,370 rows; this does not reconstruct the card's unspecified smaller manifest. + +| Model / backend | Correct decisions | Top-1 accuracy | Median call | p95 call | +| --- | ---: | ---: | ---: | ---: | +| Upstream PyTorch / CPU | 24,359 / 24,370 | 99.9549% | 1.787 ms | 3.104 ms | +| Original FP16 Core ML / CPU + ANE | 24,359 / 24,370 | 99.9549% | 1.003 ms | 1.133 ms | +| ANE-gather FP16 Core ML / CPU + ANE | 24,359 / 24,370 | 99.9549% | 1.052 ms | 1.177 ms | + +Both Core ML exports select the same option as PyTorch on **all 24,370 rows**. +All three share the same 11 errors: choosing `fill` when the label is `skip`. +The original Cua evaluator counts these as wrong/unsafe actions; this offline +benchmark executes no actions. All 9,802 fill, 816 check, and 1,040 click labels +are correct; skip accuracy is 12,701/12,712. Higher ANE placement is approximately +**4.8% slower** by median here, so the original remains the default. + +**Strict numerical conversion parity fails for both exports.** Each has 11 rows +above the original 0.005 absolute probability-error limit, with a maximum error +of **0.0204874**. One further row (zero-based index 19270) has a live-probability +sum of **0.99893665**, which failed the original Swift manager's 0.001 +normalization guard despite a correct argmax. The [Swift probability fix](#swift-probability-fix) +now handles that output. These raw conversion reports retain the original scores, +failed gates, tolerances, and model weights; the runtime fix does not establish +raw numerical parity. The earlier 196-row demo passed its numerical gates. + +Measured September 19, 2026 on **Apple M5 Pro, 24 GB, macOS 27.0 (26A428)**, +Python 3.11.11, PyTorch 2.7.0, coremltools 9.0. Batch size 1, three warmup rows per +model, one timed pass over the whole split; both Core ML models remain loaded +and alternate AB/BA order by row. PyTorch uses two CPU threads and one inter-op +thread, with the Transformer fast path disabled. Timers cover PyTorch +forward + softmax or synchronous Core ML prediction, excluding encoding, +validation, loading, UI, and network. These compare deployment backends, not +algorithms on equal hardware, and differ from the separate Swift timings at the end of this card. + +Upstream describes this synthetic split as disjoint from training/validation by +form signature; those signatures were not independently re-audited here. No +training, validation inference, test-based tuning, or hosted Jev/API comparison +was performed. This measures supplied-option classification, not unseen real-world +GUI completion or document extraction. + +[Full report](reports/synthetic-test.json), [complete compressed per-row trace](reports/synthetic-test-decisions.jsonl.gz), +and [test manifest](synthetic-test.lock.json) retain the exact protocol, hashes, +paired decisions, action metrics, and all failed numerical checks. Reproduce with +`benchmark-synthetic.py --require-parity` in the [Mobius toolkit](https://github.com/FluidInference/mobius/tree/main/models/computer-use/cua-s1-forms/coreml#full-published-synthetic-test). +That command exits 1 for the recorded numerical failures; it does not normalize +scores or weaken the original gates. Treat these artifacts as under review +until the numerical failures are resolved and validated. + +## ANE profile + +A separate September 19 profile uses the same portable-package hashes on the M5 +Pro, with real demo rows 0, 68, and 130 (27, 21, and 19 options). Each policy +runs two warmup passes and ten timed passes, totaling 30 timed predictions. All +120 timed predictions select the correct labels. + +| Policy | CPU ops | GPU ops | ANE ops | Warm p50 | Warm p95 | +| --- | ---: | ---: | ---: | ---: | ---: | +| `CPU_ONLY` | 173 | 0 | 0 | 1.527 ms | 1.602 ms | +| `CPU_AND_GPU` | 0 | 173 | 0 | 0.929 ms | 2.380 ms | +| `CPU_AND_NE` | 24 | 0 | 149 | 0.929 ms | 0.973 ms | +| `ALL` | 0 | 173 | 0 | 0.912 ms | 1.229 ms | + +`CPU_AND_NE` assigns 86.1% of operations to ANE; `ALL` chooses the GPU on this +Mac. CPU fallbacks cover integer/mask preparation and embedding gathers. These +are public `MLComputePlan` preferred-device assignments, not measurements of +utilization, energy, or time spent on each device. No Instruments runtime trace +was captured. + +ANE model loading took 566.8 ms, followed by a 1.65 ms first prediction, with +system caches retained. These are not first-install cold-start numbers. Warm +timing includes Python model-call overhead and excludes encoding, Swift/UI work, +and animation. This three-row timing manifest differs from the full conversion +parity run above; no weights or graph were changed. + +See [reports/ane-profile.json](reports/ane-profile.json) for all operation +assignments, individual timings, hashes, and the protocol; +[reports/ane-fallback.json](reports/ane-fallback.json) records rejection reasons. +Reproduce with `uv run --frozen python profile-coreml.py` in the +[Mobius conversion directory](https://github.com/FluidInference/mobius/tree/main/models/computer-use/cua-s1-forms/coreml). + +## Optional higher-ANE variant + +The `ane-gather/` directory contains an alternative portable package and compiled +bundle with the **same int32 inputs and float32 outputs** and all trained weights. +The variant uses shared float16 mask inputs and unsigned 16-bit embedding indices +to eliminate negative-index correction and place the gathers on ANE. Valid byte +IDs 0–256 remain exact. It is **1,509,491 bytes** as a portable package. + +On this M5 Pro, the scheduler plan is **162 ANE operations and 3 CPU input casts +(98.2% ANE)** for both `CPU_AND_NE` and `ALL`. The default model has 149 ANE and +24 CPU operations (86.1%) under `CPU_AND_NE`. Counts are not runtime or energy +shares, and host byte encoding still runs outside the model. + +The optional variant passes **196/196 decisions** against upstream on `ALL` and +`CPU_AND_NE`, with maximum probability error **0.002336** under the unchanged +0.005 tolerance. **28 Python regression tests** pass, including all byte-ID +boundaries, full option capacity, truncation, and reordered choices. The Swift +manager independently passes all 196 reference decisions, compiled-cache loading, +and concurrent/reordered requests; the native demo passes its three-form checks. + +A matched same-process ABBA comparison uses three real inputs and 60 timed calls +per model after warmup: + +| Artifact | CPU ops | ANE ops | Warm p50 | Warm p95 | +| --- | ---: | ---: | ---: | ---: | +| Root/default | 24 | 149 | 0.915 ms | 0.968 ms | +| `ane-gather/` | 3 | 162 | 0.970 ms | 0.988 ms | + +Higher ANE placement is about **6% slower** in this local comparison, so the +root/default artifact remains unchanged. No energy or CPU-time saving is claimed. +The original input names, dtypes, shapes, and byte encoding still apply. Load +`ane-gather/cua_s1_forms_fp16_options32.mlpackage` with the existing Python or +Swift APIs, or pass its local path to the Swift demo's `--model` argument. + +Reports: [parity](reports/ane-gather-verification.json), +[Swift validation](reports/swift-ane-validation.json), +[compute plans](reports/ane-gather-profile.json), +[fallbacks](reports/ane-gather-fallback.json), and +[matched comparison](reports/ane-comparison.json). Reproduce with +`uv run --frozen python convert-coreml.py --optimization ane-gather --output-dir build/ane-gather` +in the [Mobius conversion directory](https://github.com/FluidInference/mobius/tree/main/models/computer-use/cua-s1-forms/coreml#optional-higher-ane-variant). + +## INT8 weight trial + +A matched run over all **24,370 synthetic decisions** on the same M5 Pro: + +| Export | Package size | Accuracy | Median | p95 | +| --- | ---: | ---: | ---: | ---: | +| Original FP16 | 1.51 MB | 99.9549% | 0.990 ms | 1.102 ms | +| INT8 weights, FP16 compute | 0.81 MB | 99.9549% | 0.990 ms | 1.104 ms | + +**46.2% smaller**, with every selected option unchanged and effectively identical +latency. Numerical parity still fails: 64 rows exceed the 0.005 probability-error +limit (maximum 0.067738), versus 11 for FP16. No INT8 probability-sum violations +were observed. The original remains the default. + +The experimental `int8-weights/cua_s1_forms_int8_options32.mlpackage` uses INT8 +weights with FP16 computation. Compile it locally or load it with the existing +Swift `load(from:)` API. Inputs and outputs match the original model. This +variant was measured on M5 Pro; iPhone behavior has not been measured. + +[Full report](reports/int8-synthetic-test.json) · +[Complete per-row trace](reports/int8-synthetic-test-decisions.jsonl.gz) · +[Quantization manifest](int8-weights/conversion.json) · +[Reproduction](https://github.com/FluidInference/mobius/tree/main/models/computer-use/cua-s1-forms/coreml#int8-weight-trial) + +The matched benchmark uses CPU+ANE, batch 1, three warmups/model, and one complete +pass with alternating Core ML order; encoding/loading/UI are excluded. All models +get the same 24,359 decisions correct. Public compute plans assign 149 ANE and +32 CPU operations, with 19 unassigned constant-dequantization operations. This +compression does not establish an INT8 activation path or a speed/energy gain. +No calibration, training, or test-based tuning was performed. Numerical parity +remains failed; the default FP16 artifacts are unchanged. + +## INT4 weight trial + +A matched run over all **24,370 synthetic decisions** on M5 Pro, CPU+ANE: + +| Export | Package size | Accuracy | Median | p95 | +| --- | ---: | ---: | ---: | ---: | +| FP16 control, iOS 18 target | 1.51 MB | 99.9549% | 0.982 ms | 1.081 ms | +| INT4 weights, FP16 compute | 0.45 MB | 99.9302% | 0.982 ms | 1.082 ms | + +**70.1% smaller**, with essentially unchanged latency. INT4 makes **17 errors +versus 11** for FP16: 14 choices change, introducing 10 errors and correcting four. +Numerical parity fails: 356 rows exceed the 0.005 probability-error limit +(maximum 0.754359); no INT4 probability-sum violations were observed. + +Packed INT4 requires **iOS 18/macOS 15**. Both exports use the same decomposed +attention graph and FP16 computation. Batch-1 timing excludes encoding/loading/UI. +INT8 preserves all choices at 0.81 MB; the original FP16 remains the default. + +The experimental INT4 package is +`int4-weights/cua_s1_forms_int4_options32.mlpackage`. Its matched FP16 control is +`int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage`. Nineteen weight tensors +use per-channel symmetric INT4 with a 2,048-element threshold; no calibration or +retraining. This is weight compression with FP16 computation. The graph is +retargeted from the verified original to preserve decomposed attention. + +[Full report](reports/int4-synthetic-test.json) · +[Complete per-row trace](reports/int4-synthetic-test-decisions.jsonl.gz) · +[Quantization manifest](int4-weights/conversion.json) · +[FP16 control manifest](int4-source-fp16/conversion.json) · +[Reproduction](https://github.com/FluidInference/mobius/tree/main/models/computer-use/cua-s1-forms/coreml#int4-weight-trial) + +The demo retains 196/196 choices but fails numerical parity. The full benchmark +above exposes the accuracy loss. Public compute-plan placement is 149 ANE / 32 CPU +operations, plus 19 unassigned constant dequantizations; no speed/energy claim. + +## Application responsibilities + +The application must extract document entities, describe UI elements, build +candidate actions, and validate and order the selected actions. Submission and +other effects require application authorization. Scores are not calibrated +confidence guarantees. Text outside the byte limits is truncated, and arbitrary +new forms and languages require their own evaluation. + +## Source, reproduction, and license + +The [Mobius conversion toolkit](https://github.com/FluidInference/mobius/tree/main/models/computer-use/cua-s1-forms/coreml) +contains the conversion code, lockfile, original reference implementation and +evaluator, tests, and full reproduction instructions. Adaptations are limited +to export-compatible masking, a floating-point clamp constant, finite padded +logits, and disabling the fused PyTorch Transformer fast path during tracing. +All trained layers and checkpoint tensors are retained; internal compute and +weights are converted to FP16. + +- Model: [`cua-ai/cua-s1-forms` at `f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71`](https://huggingface.co/cua-ai/cua-s1-forms/tree/f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71). +- Demo: [`cua-ai/cua-s1-forms` at `8273f34778b99ac2e12d9f6e7d57dad99ae20845`](https://huggingface.co/datasets/cua-ai/cua-s1-forms/tree/8273f34778b99ac2e12d9f6e7d57dad99ae20845). +- Code: [`trycua/cua` at `83f142c4290a0f7d9ed545ae8532858c6e4f8145`](https://github.com/trycua/cua/tree/83f142c4290a0f7d9ed545ae8532858c6e4f8145/libs/cua-s1). + +The pinned model and dataset cards declare MIT. See [LICENSE](LICENSE), +[NOTICES.md](NOTICES.md), and the preserved +[upstream third-party notices](UPSTREAM-THIRD-PARTY-NOTICES.md). + +## Swift probability fix + +The Swift manager computes a stable softmax from live logits using Double arithmetic +and returns Float `probabilities`. The model's original softmax output remains +available as `rawProbabilities`, including FP16 rounding errors. + +All **73,110 real Swift API calls completed** on the pinned synthetic split +(Apple M5 Pro, 24 GB, macOS 27.0; Swift 6.2.3, release build, CPU+ANE): + +| Variant | Calls completed | Correct decisions | Changed choices after fix | Probability-sum failures after fix | +| --- | ---: | ---: | ---: | ---: | +| FP16 | 24,370 | 24,359 (99.9549%) | 0 | 0 | +| INT8 | 24,370 | 24,359 (99.9549%) | 0 | 0 | +| INT4 | 24,370 | 24,353 (99.9302%) | 0 | 0 | + +The recorded FP16 sum failure is fixed. Stable probabilities agree with an +independent float64 softmax within **0.000000030**. Raw conversion-parity failures +and INT4's accuracy loss remain. This validation changes no model artifacts and +makes no new latency claim. Earlier complete Swift timings predate this fix. + +[Runtime report](reports/swift-runtime/report.json) · [Full runtime trace](reports/swift-runtime/decisions.jsonl.gz) +The report pins the dataset, packages, saved reference traces, Swift sources, +and validation harness by SHA-256. + +Reproduce with the [Mobius Swift validation harness](https://github.com/FluidInference/mobius/tree/main/models/computer-use/cua-s1-forms/coreml#swift-probability-fix). + +## Live browser proof and expanded Swift benchmark + +[![Actual CUA-driven browser forms](demo/browser-demo.gif)](demo/browser-demo.mp4) + +The [native Swift browser demo](https://github.com/FluidInference/FluidAudio/tree/7f9eb92b0af8594c4e048a9e57f697340aacfa67/Examples/CuaS1FormsDemo) +loads both variants into independent WKWebViews. It reads actual DOM labels, +roles and state, asks the model for a choice, applies compatible fill/check +actions, dispatches events, and independently verifies the resulting DOM. +Source values are user-entered or supplied by the original public examples. +HTML contains controls, not source values or expected choices. The +[recording](https://github.com/FluidInference/FluidAudio/blob/7f9eb92b0af8594c4e048a9e57f697340aacfa67/Examples/CuaS1FormsDemo/browser-demo.mp4) +shows patient, job and insurance forms: **100/100 original decisions** across +both models, with event-count, stale-observation and explicit-click checks. +Full actual contexts/candidates/actions are in [browser-validation.json](reports/browser-validation.json). +This is bounded local browser automation, not arbitrary desktop control or PDF extraction. + +The expanded **release Swift** comparison uses all 50 initial controls, both +models resident, one warmup pass per model and ABBA with two full passes per +block (200 timed calls/model). All 400 choices match upstream labels. On the +M5 Pro / 24 GB / macOS 27.0 (26A428), original median/p95 is **0.912/0.933 ms**; +ANE gather is **0.961/0.984 ms**, about 5.4% slower by median. This timer includes +Swift encoding + Core ML + output decoding and excludes browser/rendering/animation. +See [swift-variant-comparison.json](reports/swift-variant-comparison.json) for +raw samples, exact model hashes, per-form statistics, and load/first-call costs. +The earlier compute-plan counts still apply to these unchanged artifacts; +no utilization, energy saving or held-out accuracy claim is made. + +Reproduce from FluidAudio commit `7f9eb92b0af8594c4e048a9e57f697340aacfa67` (the example is +retained in history and is not part of the current library PR): + +```bash +git worktree add --detach /tmp/cua-s1-browser-repro 7f9eb92b0af8594c4e048a9e57f697340aacfa67 +cd /tmp/cua-s1-browser-repro +Examples/CuaS1FormsDemo/run.sh --browser +swift run --package-path Examples/CuaS1FormsDemo -c release CuaS1FormsDemo \ + --benchmark --report /absolute/path/to/variant-comparison.json \ + --hardware "Describe the measured Mac" +``` + +Both packages are fetched by pinned revisions and verified hashes, or supplied +with `--model /path/to/original.mlpackage --ane-model /path/to/ane-gather.mlpackage`. +The [demo README](https://github.com/FluidInference/FluidAudio/tree/7f9eb92b0af8594c4e048a9e57f697340aacfa67/Examples/CuaS1FormsDemo#matched-swift-benchmark) +also documents real-browser recording and its separate validation trace. diff --git a/UPSTREAM-THIRD-PARTY-NOTICES.md b/UPSTREAM-THIRD-PARTY-NOTICES.md new file mode 100644 index 0000000000000000000000000000000000000000..2ed49a7cd010a557576c2d072656dd21f5ae0894 --- /dev/null +++ b/UPSTREAM-THIRD-PARTY-NOTICES.md @@ -0,0 +1,46 @@ +# Third-party notices + +This Cua-S1 component does not bundle third-party model weights, datasets, +binaries, or media. + +## jevlike + +Parts of `python/src/cua_s1/model.py` are adapted from the MIT-licensed +[`jevlike`](https://github.com/vinnylarouge/jevlike) project at commit +[`94f5fd1b0b11d52bbdfdf4e0ee6aa96b568f8452`](https://github.com/vinnylarouge/jevlike/commit/94f5fd1b0b11d52bbdfdf4e0ee6aa96b568f8452). +The adapted material is limited to the byte-collation and small attention-model +primitives in that file. + +Copyright (c) 2026 Minimal Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +The Python package declares Hatchling as a build-system requirement. Hatchling +is build tooling and is not bundled in the resulting package; it is distributed +under its own terms. + +The Python package declares its runtime dependencies in `python/pyproject.toml`. +The component-local `python/uv.lock` records the tested development resolution. +Those dependencies remain distributed under their own licenses and are not +copied into this repository. + +Future checkpoint or software distributions must update this file with the +applicable notices for all included third-party code, models, datasets, assets, +and other materials. A reference to Cua-S1 or `cua-s1-form-v0` does not grant +rights to third-party material. diff --git a/ane-gather/conversion.json b/ane-gather/conversion.json new file mode 100644 index 0000000000000000000000000000000000000000..96540444fac247c28b6c46cf441ebb9f9ca96f1f --- /dev/null +++ b/ane-gather/conversion.json @@ -0,0 +1,36 @@ +{ + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "optimization": "ane-gather", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.6771400420111604, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "de18e313c3b625e35d008ed8b6b24108edc6df7bf2fae9533af03519eca11b63", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "38f812a04eb2322080634ba788c61df362336168466ca67e5549058d346ac793" + } +} diff --git a/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/analytics/coremldata.bin b/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/analytics/coremldata.bin new file mode 100644 index 0000000000000000000000000000000000000000..2dc6e233b6c439880a1b59c554da63cc0bcf96a1 --- /dev/null +++ b/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/analytics/coremldata.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:526dccb87bdc036df1e8add0b50dd6540db534808fc2f8caa97cd897b4fc1fa3 +size 243 diff --git a/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/coremldata.bin b/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/coremldata.bin new file mode 100644 index 0000000000000000000000000000000000000000..717f3b686649634bfcef35792f7a9cc80a025d66 --- /dev/null +++ b/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/coremldata.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f2a8ed9fe8abcc39c7ee0575e2562d589d459683b92708fa9788b533be3038b +size 876 diff --git a/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/model.mil b/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/model.mil new file mode 100644 index 0000000000000000000000000000000000000000..028d76a5d8e89d73f0d117c0750ab1763ef79f3d --- /dev/null +++ b/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/model.mil @@ -0,0 +1,404 @@ +program(1.0) +[buildInfo = dict, tensor>({{"coremlc-component-MIL", "3600.16.1"}, {"coremlc-version", "3600.25.2"}, {"coremltools-component-milinternal", ""}, {"coremltools-version", "9.0"}})] +{ + func main(tensor context_ids, tensor option_ids, tensor option_mask) { + tensor mask_options_to_fp16_dtype_0 = const()[name = tensor("mask_options_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor var_29_promoted_to_fp16 = const()[name = tensor("op_29_promoted_to_fp16"), val = tensor(0x0p+0)]; + tensor option_mask_to_fp16 = cast(dtype = mask_options_to_fp16_dtype_0, x = option_mask)[name = tensor("cast_68")]; + tensor option_mask_cast_fp16 = not_equal(x = option_mask_to_fp16, y = var_29_promoted_to_fp16)[name = tensor("option_mask_cast_fp16")]; + tensor mask_context_ids_to_fp16_dtype_0 = const()[name = tensor("mask_context_ids_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor var_31_promoted_to_fp16 = const()[name = tensor("op_31_promoted_to_fp16"), val = tensor(0x0p+0)]; + tensor context_ids_to_fp16 = cast(dtype = mask_context_ids_to_fp16_dtype_0, x = context_ids)[name = tensor("cast_67")]; + tensor context_mask_cast_fp16 = not_equal(x = context_ids_to_fp16, y = var_31_promoted_to_fp16)[name = tensor("context_mask_cast_fp16")]; + tensor var_42_begin_0 = const()[name = tensor("op_42_begin_0"), val = tensor([0, 0])]; + tensor var_42_end_0 = const()[name = tensor("op_42_end_0"), val = tensor([1, 1])]; + tensor var_42_end_mask_0 = const()[name = tensor("op_42_end_mask_0"), val = tensor([true, false])]; + tensor var_42_cast_fp16 = slice_by_index(begin = var_42_begin_0, end = var_42_end_0, end_mask = var_42_end_mask_0, x = context_ids_to_fp16)[name = tensor("op_42_cast_fp16")]; + tensor fill_like_0_value_0_to_fp16 = const()[name = tensor("fill_like_0_value_0_to_fp16"), val = tensor(0x1p+0)]; + tensor fill_like_0_cast_fp16 = fill_like(ref_tensor = var_42_cast_fp16, value = fill_like_0_value_0_to_fp16)[name = tensor("fill_like_0_cast_fp16")]; + tensor var_58_begin_0 = const()[name = tensor("op_58_begin_0"), val = tensor([0, 1])]; + tensor var_58_end_0 = const()[name = tensor("op_58_end_0"), val = tensor([1, 224])]; + tensor var_58_end_mask_0 = const()[name = tensor("op_58_end_mask_0"), val = tensor([true, true])]; + tensor var_58_cast_fp16 = slice_by_index(begin = var_58_begin_0, end = var_58_end_0, end_mask = var_58_end_mask_0, x = context_ids_to_fp16)[name = tensor("op_58_cast_fp16")]; + tensor var_60 = const()[name = tensor("op_60"), val = tensor(1)]; + tensor safe_context_ids_interleave_0 = const()[name = tensor("safe_context_ids_interleave_0"), val = tensor(false)]; + tensor safe_context_ids_cast_fp16 = concat(axis = var_60, interleave = safe_context_ids_interleave_0, values = (fill_like_0_cast_fp16, var_58_cast_fp16))[name = tensor("safe_context_ids_cast_fp16")]; + tensor model_embedding_weight_to_fp16 = const()[name = tensor("model_embedding_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(64)))]; + tensor cast_0_dtype_0 = const()[name = tensor("cast_0_dtype_0"), val = tensor("uint16")]; + tensor cast_0 = cast(dtype = cast_0_dtype_0, x = context_ids_to_fp16)[name = tensor("cast_0")]; + tensor gather_0_axis_0 = const()[name = tensor("gather_0_axis_0"), val = tensor(0)]; + tensor gather_0_batch_dims_0 = const()[name = tensor("gather_0_batch_dims_0"), val = tensor(0)]; + tensor gather_0_validate_indices_0 = const()[name = tensor("gather_0_validate_indices_0"), val = tensor(false)]; + tensor gather_0 = gather(axis = gather_0_axis_0, batch_dims = gather_0_batch_dims_0, indices = cast_0, validate_indices = gather_0_validate_indices_0, x = model_embedding_weight_to_fp16)[name = tensor("gather_0")]; + tensor model_position_weight_to_fp16 = const()[name = tensor("model_position_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(65920)))]; + tensor src_1_cast_fp16 = add(x = gather_0, y = model_position_weight_to_fp16)[name = tensor("src_1_cast_fp16")]; + tensor var_81_promoted_to_fp16 = const()[name = tensor("op_81_promoted_to_fp16"), val = tensor(0x0p+0)]; + tensor mask_1_cast_fp16 = equal(x = safe_context_ids_cast_fp16, y = var_81_promoted_to_fp16)[name = tensor("mask_1_cast_fp16")]; + tensor var_97_to_fp16 = const()[name = tensor("op_97_to_fp16"), val = tensor(-inf)]; + tensor var_105_to_fp16 = const()[name = tensor("op_105_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(123328)))]; + tensor key_padding_mask_1_cast_fp16 = select(a = var_97_to_fp16, b = var_105_to_fp16, cond = mask_1_cast_fp16)[name = tensor("key_padding_mask_1_cast_fp16")]; + tensor query_1_axes_0 = const()[name = tensor("query_1_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_0_norm1_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_norm1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(123840)))]; + tensor model_encoder_layers_0_norm1_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_norm1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(124160)))]; + tensor var_84_to_fp16 = const()[name = tensor("op_84_to_fp16"), val = tensor(0x1.5p-17)]; + tensor query_1_cast_fp16 = layer_norm(axes = query_1_axes_0, beta = model_encoder_layers_0_norm1_bias_to_fp16, epsilon = var_84_to_fp16, gamma = model_encoder_layers_0_norm1_weight_to_fp16, x = src_1_cast_fp16)[name = tensor("query_1_cast_fp16")]; + tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; + tensor model_encoder_layers_0_self_attn_in_proj_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_self_attn_in_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(124480)))]; + tensor model_encoder_layers_0_self_attn_in_proj_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_self_attn_in_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(222848)))]; + tensor query_3_cast_fp16 = transpose(perm = query_3_perm_0, x = query_1_cast_fp16)[name = tensor("transpose_24")]; + tensor linear_0_cast_fp16 = linear(bias = model_encoder_layers_0_self_attn_in_proj_bias_to_fp16, weight = model_encoder_layers_0_self_attn_in_proj_weight_to_fp16, x = query_3_cast_fp16)[name = tensor("linear_0_cast_fp16")]; + tensor concat_0 = const()[name = tensor("concat_0"), val = tensor([224, 1, 3, 128])]; + tensor var_139_cast_fp16 = reshape(shape = concat_0, x = linear_0_cast_fp16)[name = tensor("op_139_cast_fp16")]; + tensor var_140_axes_0 = const()[name = tensor("op_140_axes_0"), val = tensor([0])]; + tensor var_140_cast_fp16 = expand_dims(axes = var_140_axes_0, x = var_139_cast_fp16)[name = tensor("op_140_cast_fp16")]; + tensor var_141_perm_0 = const()[name = tensor("op_141_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_142_axes_0 = const()[name = tensor("op_142_axes_0"), val = tensor([-2])]; + tensor var_141_cast_fp16 = transpose(perm = var_141_perm_0, x = var_140_cast_fp16)[name = tensor("transpose_23")]; + tensor var_142_cast_fp16 = squeeze(axes = var_142_axes_0, x = var_141_cast_fp16)[name = tensor("op_142_cast_fp16")]; + tensor q_1_begin_0 = const()[name = tensor("q_1_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor q_1_end_0 = const()[name = tensor("q_1_end_0"), val = tensor([1, 224, 1, 128])]; + tensor q_1_end_mask_0 = const()[name = tensor("q_1_end_mask_0"), val = tensor([false, true, true, true])]; + tensor q_1_squeeze_mask_0 = const()[name = tensor("q_1_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor q_1_cast_fp16 = slice_by_index(begin = q_1_begin_0, end = q_1_end_0, end_mask = q_1_end_mask_0, squeeze_mask = q_1_squeeze_mask_0, x = var_142_cast_fp16)[name = tensor("q_1_cast_fp16")]; + tensor k_1_begin_0 = const()[name = tensor("k_1_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor k_1_end_0 = const()[name = tensor("k_1_end_0"), val = tensor([2, 224, 1, 128])]; + tensor k_1_end_mask_0 = const()[name = tensor("k_1_end_mask_0"), val = tensor([false, true, true, true])]; + tensor k_1_squeeze_mask_0 = const()[name = tensor("k_1_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor k_1_cast_fp16 = slice_by_index(begin = k_1_begin_0, end = k_1_end_0, end_mask = k_1_end_mask_0, squeeze_mask = k_1_squeeze_mask_0, x = var_142_cast_fp16)[name = tensor("k_1_cast_fp16")]; + tensor v_1_begin_0 = const()[name = tensor("v_1_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor v_1_end_0 = const()[name = tensor("v_1_end_0"), val = tensor([3, 224, 1, 128])]; + tensor v_1_end_mask_0 = const()[name = tensor("v_1_end_mask_0"), val = tensor([false, true, true, true])]; + tensor v_1_squeeze_mask_0 = const()[name = tensor("v_1_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor v_1_cast_fp16 = slice_by_index(begin = v_1_begin_0, end = v_1_end_0, end_mask = v_1_end_mask_0, squeeze_mask = v_1_squeeze_mask_0, x = var_142_cast_fp16)[name = tensor("v_1_cast_fp16")]; + tensor var_150 = const()[name = tensor("op_150"), val = tensor([224, 4, 32])]; + tensor var_151_cast_fp16 = reshape(shape = var_150, x = q_1_cast_fp16)[name = tensor("op_151_cast_fp16")]; + tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([1, 0, 2])]; + tensor var_157 = const()[name = tensor("op_157"), val = tensor([224, 4, 32])]; + tensor var_158_cast_fp16 = reshape(shape = var_157, x = k_1_cast_fp16)[name = tensor("op_158_cast_fp16")]; + tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([1, 0, 2])]; + tensor var_164 = const()[name = tensor("op_164"), val = tensor([224, 4, 32])]; + tensor var_165_cast_fp16 = reshape(shape = var_164, x = v_1_cast_fp16)[name = tensor("op_165_cast_fp16")]; + tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([1, 0, 2])]; + tensor var_168 = const()[name = tensor("op_168"), val = tensor([1, 1, 1, 224])]; + tensor var_169_cast_fp16 = reshape(shape = var_168, x = key_padding_mask_1_cast_fp16)[name = tensor("op_169_cast_fp16")]; + tensor var_171_reps_0 = const()[name = tensor("op_171_reps_0"), val = tensor([1, 4, 1, 1])]; + tensor var_171_cast_fp16 = tile(reps = var_171_reps_0, x = var_169_cast_fp16)[name = tensor("op_171_cast_fp16")]; + tensor var_179 = const()[name = tensor("op_179"), val = tensor([1, 4, 224, 32])]; + tensor q_3_cast_fp16 = transpose(perm = q_3_perm_0, x = var_151_cast_fp16)[name = tensor("transpose_22")]; + tensor q_5_cast_fp16 = reshape(shape = var_179, x = q_3_cast_fp16)[name = tensor("q_5_cast_fp16")]; + tensor var_181 = const()[name = tensor("op_181"), val = tensor([1, 4, 224, 32])]; + tensor k_3_cast_fp16 = transpose(perm = k_3_perm_0, x = var_158_cast_fp16)[name = tensor("transpose_21")]; + tensor k_5_cast_fp16 = reshape(shape = var_181, x = k_3_cast_fp16)[name = tensor("k_5_cast_fp16")]; + tensor var_183 = const()[name = tensor("op_183"), val = tensor([1, 4, 224, 32])]; + tensor v_3_cast_fp16 = transpose(perm = v_3_perm_0, x = var_165_cast_fp16)[name = tensor("transpose_20")]; + tensor v_5_cast_fp16 = reshape(shape = var_183, x = v_3_cast_fp16)[name = tensor("v_5_cast_fp16")]; + tensor mul_1_y_0_to_fp16 = const()[name = tensor("mul_1_y_0_to_fp16"), val = tensor(0x1.6ap-3)]; + tensor mul_1_cast_fp16 = mul(x = q_5_cast_fp16, y = mul_1_y_0_to_fp16)[name = tensor("mul_1_cast_fp16")]; + tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; + tensor matmul_0_transpose_x_0 = const()[name = tensor("matmul_0_transpose_x_0"), val = tensor(false)]; + tensor matmul_0_cast_fp16 = matmul(transpose_x = matmul_0_transpose_x_0, transpose_y = matmul_0_transpose_y_0, x = mul_1_cast_fp16, y = k_5_cast_fp16)[name = tensor("matmul_0_cast_fp16")]; + tensor add_0_cast_fp16 = add(x = matmul_0_cast_fp16, y = var_171_cast_fp16)[name = tensor("add_0_cast_fp16")]; + tensor softmax_0_axis_0 = const()[name = tensor("softmax_0_axis_0"), val = tensor(-1)]; + tensor softmax_0_cast_fp16 = softmax(axis = softmax_0_axis_0, x = add_0_cast_fp16)[name = tensor("softmax_0_cast_fp16")]; + tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; + tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; + tensor attn_output_1_cast_fp16 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0_cast_fp16, y = v_5_cast_fp16)[name = tensor("attn_output_1_cast_fp16")]; + tensor var_186 = const()[name = tensor("op_186"), val = tensor([2, 0, 1, 3])]; + tensor var_191 = const()[name = tensor("op_191"), val = tensor([224, 128])]; + tensor var_187_cast_fp16 = transpose(perm = var_186, x = attn_output_1_cast_fp16)[name = tensor("transpose_19")]; + tensor attn_output_3_cast_fp16 = reshape(shape = var_191, x = var_187_cast_fp16)[name = tensor("attn_output_3_cast_fp16")]; + tensor model_encoder_layers_0_self_attn_out_proj_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_self_attn_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(223680)))]; + tensor model_encoder_layers_0_self_attn_out_proj_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_self_attn_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(256512)))]; + tensor linear_1_cast_fp16 = linear(bias = model_encoder_layers_0_self_attn_out_proj_bias_to_fp16, weight = model_encoder_layers_0_self_attn_out_proj_weight_to_fp16, x = attn_output_3_cast_fp16)[name = tensor("linear_1_cast_fp16")]; + tensor var_195 = const()[name = tensor("op_195"), val = tensor([224, 1, 128])]; + tensor attn_output_7_cast_fp16 = reshape(shape = var_195, x = linear_1_cast_fp16)[name = tensor("attn_output_7_cast_fp16")]; + tensor input_3_perm_0 = const()[name = tensor("input_3_perm_0"), val = tensor([1, 0, 2])]; + tensor input_3_cast_fp16 = transpose(perm = input_3_perm_0, x = attn_output_7_cast_fp16)[name = tensor("transpose_18")]; + tensor input_5_cast_fp16 = add(x = src_1_cast_fp16, y = input_3_cast_fp16)[name = tensor("input_5_cast_fp16")]; + tensor input_7_axes_0 = const()[name = tensor("input_7_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_0_norm2_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_norm2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(256832)))]; + tensor model_encoder_layers_0_norm2_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_norm2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(257152)))]; + tensor input_7_cast_fp16 = layer_norm(axes = input_7_axes_0, beta = model_encoder_layers_0_norm2_bias_to_fp16, epsilon = var_84_to_fp16, gamma = model_encoder_layers_0_norm2_weight_to_fp16, x = input_5_cast_fp16)[name = tensor("input_7_cast_fp16")]; + tensor model_encoder_layers_0_linear1_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_linear1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(257472)))]; + tensor model_encoder_layers_0_linear1_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_linear1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(388608)))]; + tensor linear_2_cast_fp16 = linear(bias = model_encoder_layers_0_linear1_bias_to_fp16, weight = model_encoder_layers_0_linear1_weight_to_fp16, x = input_7_cast_fp16)[name = tensor("linear_2_cast_fp16")]; + tensor input_11_cast_fp16 = relu(x = linear_2_cast_fp16)[name = tensor("input_11_cast_fp16")]; + tensor model_encoder_layers_0_linear2_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_linear2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(389696)))]; + tensor model_encoder_layers_0_linear2_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_linear2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(520832)))]; + tensor linear_3_cast_fp16 = linear(bias = model_encoder_layers_0_linear2_bias_to_fp16, weight = model_encoder_layers_0_linear2_weight_to_fp16, x = input_11_cast_fp16)[name = tensor("linear_3_cast_fp16")]; + tensor input_17_cast_fp16 = add(x = input_5_cast_fp16, y = linear_3_cast_fp16)[name = tensor("input_17_cast_fp16")]; + tensor query_5_axes_0 = const()[name = tensor("query_5_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_1_norm1_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_norm1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(521152)))]; + tensor model_encoder_layers_1_norm1_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_norm1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(521472)))]; + tensor query_5_cast_fp16 = layer_norm(axes = query_5_axes_0, beta = model_encoder_layers_1_norm1_bias_to_fp16, epsilon = var_84_to_fp16, gamma = model_encoder_layers_1_norm1_weight_to_fp16, x = input_17_cast_fp16)[name = tensor("query_5_cast_fp16")]; + tensor query_7_perm_0 = const()[name = tensor("query_7_perm_0"), val = tensor([1, 0, 2])]; + tensor model_encoder_layers_1_self_attn_in_proj_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_self_attn_in_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(521792)))]; + tensor model_encoder_layers_1_self_attn_in_proj_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_self_attn_in_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(620160)))]; + tensor query_7_cast_fp16 = transpose(perm = query_7_perm_0, x = query_5_cast_fp16)[name = tensor("transpose_17")]; + tensor linear_4_cast_fp16 = linear(bias = model_encoder_layers_1_self_attn_in_proj_bias_to_fp16, weight = model_encoder_layers_1_self_attn_in_proj_weight_to_fp16, x = query_7_cast_fp16)[name = tensor("linear_4_cast_fp16")]; + tensor concat_2 = const()[name = tensor("concat_2"), val = tensor([224, 1, 3, 128])]; + tensor var_246_cast_fp16 = reshape(shape = concat_2, x = linear_4_cast_fp16)[name = tensor("op_246_cast_fp16")]; + tensor var_247_axes_0 = const()[name = tensor("op_247_axes_0"), val = tensor([0])]; + tensor var_247_cast_fp16 = expand_dims(axes = var_247_axes_0, x = var_246_cast_fp16)[name = tensor("op_247_cast_fp16")]; + tensor var_248_perm_0 = const()[name = tensor("op_248_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_249_axes_0 = const()[name = tensor("op_249_axes_0"), val = tensor([-2])]; + tensor var_248_cast_fp16 = transpose(perm = var_248_perm_0, x = var_247_cast_fp16)[name = tensor("transpose_16")]; + tensor var_249_cast_fp16 = squeeze(axes = var_249_axes_0, x = var_248_cast_fp16)[name = tensor("op_249_cast_fp16")]; + tensor q_7_begin_0 = const()[name = tensor("q_7_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor q_7_end_0 = const()[name = tensor("q_7_end_0"), val = tensor([1, 224, 1, 128])]; + tensor q_7_end_mask_0 = const()[name = tensor("q_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor q_7_squeeze_mask_0 = const()[name = tensor("q_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor q_7_cast_fp16 = slice_by_index(begin = q_7_begin_0, end = q_7_end_0, end_mask = q_7_end_mask_0, squeeze_mask = q_7_squeeze_mask_0, x = var_249_cast_fp16)[name = tensor("q_7_cast_fp16")]; + tensor k_7_begin_0 = const()[name = tensor("k_7_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor k_7_end_0 = const()[name = tensor("k_7_end_0"), val = tensor([2, 224, 1, 128])]; + tensor k_7_end_mask_0 = const()[name = tensor("k_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor k_7_squeeze_mask_0 = const()[name = tensor("k_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor k_7_cast_fp16 = slice_by_index(begin = k_7_begin_0, end = k_7_end_0, end_mask = k_7_end_mask_0, squeeze_mask = k_7_squeeze_mask_0, x = var_249_cast_fp16)[name = tensor("k_7_cast_fp16")]; + tensor v_7_begin_0 = const()[name = tensor("v_7_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor v_7_end_0 = const()[name = tensor("v_7_end_0"), val = tensor([3, 224, 1, 128])]; + tensor v_7_end_mask_0 = const()[name = tensor("v_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor v_7_squeeze_mask_0 = const()[name = tensor("v_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor v_7_cast_fp16 = slice_by_index(begin = v_7_begin_0, end = v_7_end_0, end_mask = v_7_end_mask_0, squeeze_mask = v_7_squeeze_mask_0, x = var_249_cast_fp16)[name = tensor("v_7_cast_fp16")]; + tensor var_257 = const()[name = tensor("op_257"), val = tensor([224, 4, 32])]; + tensor var_258_cast_fp16 = reshape(shape = var_257, x = q_7_cast_fp16)[name = tensor("op_258_cast_fp16")]; + tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([1, 0, 2])]; + tensor var_264 = const()[name = tensor("op_264"), val = tensor([224, 4, 32])]; + tensor var_265_cast_fp16 = reshape(shape = var_264, x = k_7_cast_fp16)[name = tensor("op_265_cast_fp16")]; + tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([1, 0, 2])]; + tensor var_271 = const()[name = tensor("op_271"), val = tensor([224, 4, 32])]; + tensor var_272_cast_fp16 = reshape(shape = var_271, x = v_7_cast_fp16)[name = tensor("op_272_cast_fp16")]; + tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([1, 0, 2])]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor([1, 4, 224, 32])]; + tensor q_9_cast_fp16 = transpose(perm = q_9_perm_0, x = var_258_cast_fp16)[name = tensor("transpose_15")]; + tensor q_11_cast_fp16 = reshape(shape = var_286, x = q_9_cast_fp16)[name = tensor("q_11_cast_fp16")]; + tensor var_288 = const()[name = tensor("op_288"), val = tensor([1, 4, 224, 32])]; + tensor k_9_cast_fp16 = transpose(perm = k_9_perm_0, x = var_265_cast_fp16)[name = tensor("transpose_14")]; + tensor k_11_cast_fp16 = reshape(shape = var_288, x = k_9_cast_fp16)[name = tensor("k_11_cast_fp16")]; + tensor var_290 = const()[name = tensor("op_290"), val = tensor([1, 4, 224, 32])]; + tensor v_9_cast_fp16 = transpose(perm = v_9_perm_0, x = var_272_cast_fp16)[name = tensor("transpose_13")]; + tensor v_11_cast_fp16 = reshape(shape = var_290, x = v_9_cast_fp16)[name = tensor("v_11_cast_fp16")]; + tensor mul_3_y_0_to_fp16 = const()[name = tensor("mul_3_y_0_to_fp16"), val = tensor(0x1.6ap-3)]; + tensor mul_3_cast_fp16 = mul(x = q_11_cast_fp16, y = mul_3_y_0_to_fp16)[name = tensor("mul_3_cast_fp16")]; + tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; + tensor matmul_1_transpose_x_0 = const()[name = tensor("matmul_1_transpose_x_0"), val = tensor(false)]; + tensor matmul_1_cast_fp16 = matmul(transpose_x = matmul_1_transpose_x_0, transpose_y = matmul_1_transpose_y_0, x = mul_3_cast_fp16, y = k_11_cast_fp16)[name = tensor("matmul_1_cast_fp16")]; + tensor add_1_cast_fp16 = add(x = matmul_1_cast_fp16, y = var_171_cast_fp16)[name = tensor("add_1_cast_fp16")]; + tensor softmax_1_axis_0 = const()[name = tensor("softmax_1_axis_0"), val = tensor(-1)]; + tensor softmax_1_cast_fp16 = softmax(axis = softmax_1_axis_0, x = add_1_cast_fp16)[name = tensor("softmax_1_cast_fp16")]; + tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; + tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; + tensor attn_output_9_cast_fp16 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1_cast_fp16, y = v_11_cast_fp16)[name = tensor("attn_output_9_cast_fp16")]; + tensor var_293 = const()[name = tensor("op_293"), val = tensor([2, 0, 1, 3])]; + tensor var_298 = const()[name = tensor("op_298"), val = tensor([224, 128])]; + tensor var_294_cast_fp16 = transpose(perm = var_293, x = attn_output_9_cast_fp16)[name = tensor("transpose_12")]; + tensor attn_output_11_cast_fp16 = reshape(shape = var_298, x = var_294_cast_fp16)[name = tensor("attn_output_11_cast_fp16")]; + tensor model_encoder_layers_1_self_attn_out_proj_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_self_attn_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(620992)))]; + tensor model_encoder_layers_1_self_attn_out_proj_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_self_attn_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(653824)))]; + tensor linear_5_cast_fp16 = linear(bias = model_encoder_layers_1_self_attn_out_proj_bias_to_fp16, weight = model_encoder_layers_1_self_attn_out_proj_weight_to_fp16, x = attn_output_11_cast_fp16)[name = tensor("linear_5_cast_fp16")]; + tensor var_302 = const()[name = tensor("op_302"), val = tensor([224, 1, 128])]; + tensor attn_output_15_cast_fp16 = reshape(shape = var_302, x = linear_5_cast_fp16)[name = tensor("attn_output_15_cast_fp16")]; + tensor input_19_perm_0 = const()[name = tensor("input_19_perm_0"), val = tensor([1, 0, 2])]; + tensor input_19_cast_fp16 = transpose(perm = input_19_perm_0, x = attn_output_15_cast_fp16)[name = tensor("transpose_11")]; + tensor input_21_cast_fp16 = add(x = input_17_cast_fp16, y = input_19_cast_fp16)[name = tensor("input_21_cast_fp16")]; + tensor input_23_axes_0 = const()[name = tensor("input_23_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_1_norm2_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_norm2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(654144)))]; + tensor model_encoder_layers_1_norm2_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_norm2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(654464)))]; + tensor input_23_cast_fp16 = layer_norm(axes = input_23_axes_0, beta = model_encoder_layers_1_norm2_bias_to_fp16, epsilon = var_84_to_fp16, gamma = model_encoder_layers_1_norm2_weight_to_fp16, x = input_21_cast_fp16)[name = tensor("input_23_cast_fp16")]; + tensor model_encoder_layers_1_linear1_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_linear1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(654784)))]; + tensor model_encoder_layers_1_linear1_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_linear1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(785920)))]; + tensor linear_6_cast_fp16 = linear(bias = model_encoder_layers_1_linear1_bias_to_fp16, weight = model_encoder_layers_1_linear1_weight_to_fp16, x = input_23_cast_fp16)[name = tensor("linear_6_cast_fp16")]; + tensor input_27_cast_fp16 = relu(x = linear_6_cast_fp16)[name = tensor("input_27_cast_fp16")]; + tensor model_encoder_layers_1_linear2_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_linear2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(787008)))]; + tensor model_encoder_layers_1_linear2_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_linear2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(918144)))]; + tensor linear_7_cast_fp16 = linear(bias = model_encoder_layers_1_linear2_bias_to_fp16, weight = model_encoder_layers_1_linear2_weight_to_fp16, x = input_27_cast_fp16)[name = tensor("linear_7_cast_fp16")]; + tensor context_cast_fp16 = add(x = input_21_cast_fp16, y = linear_7_cast_fp16)[name = tensor("context_cast_fp16")]; + tensor var_340 = const()[name = tensor("op_340"), val = tensor([32, 96])]; + tensor mask_option_ids_to_fp16_dtype_0 = const()[name = tensor("mask_option_ids_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor option_ids_to_fp16 = cast(dtype = mask_option_ids_to_fp16_dtype_0, x = option_ids)[name = tensor("cast_63")]; + tensor mask_flat_ids_cast_fp16 = reshape(shape = var_340, x = option_ids_to_fp16)[name = tensor("mask_flat_ids_cast_fp16")]; + tensor var_342_promoted_to_fp16 = const()[name = tensor("op_342_promoted_to_fp16"), val = tensor(0x0p+0)]; + tensor flat_mask_cast_fp16 = not_equal(x = mask_flat_ids_cast_fp16, y = var_342_promoted_to_fp16)[name = tensor("flat_mask_cast_fp16")]; + tensor var_353_begin_0 = const()[name = tensor("op_353_begin_0"), val = tensor([0, 0])]; + tensor var_353_end_0 = const()[name = tensor("op_353_end_0"), val = tensor([32, 1])]; + tensor var_353_end_mask_0 = const()[name = tensor("op_353_end_mask_0"), val = tensor([true, false])]; + tensor var_353_cast_fp16 = slice_by_index(begin = var_353_begin_0, end = var_353_end_0, end_mask = var_353_end_mask_0, x = mask_flat_ids_cast_fp16)[name = tensor("op_353_cast_fp16")]; + tensor fill_like_1_value_0_to_fp16 = const()[name = tensor("fill_like_1_value_0_to_fp16"), val = tensor(0x1p+0)]; + tensor fill_like_1_cast_fp16 = fill_like(ref_tensor = var_353_cast_fp16, value = fill_like_1_value_0_to_fp16)[name = tensor("fill_like_1_cast_fp16")]; + tensor var_369_begin_0 = const()[name = tensor("op_369_begin_0"), val = tensor([0, 1])]; + tensor var_369_end_0 = const()[name = tensor("op_369_end_0"), val = tensor([32, 96])]; + tensor var_369_end_mask_0 = const()[name = tensor("op_369_end_mask_0"), val = tensor([true, true])]; + tensor var_369_cast_fp16 = slice_by_index(begin = var_369_begin_0, end = var_369_end_0, end_mask = var_369_end_mask_0, x = mask_flat_ids_cast_fp16)[name = tensor("op_369_cast_fp16")]; + tensor var_371 = const()[name = tensor("op_371"), val = tensor(1)]; + tensor safe_ids_interleave_0 = const()[name = tensor("safe_ids_interleave_0"), val = tensor(false)]; + tensor safe_ids_cast_fp16 = concat(axis = var_371, interleave = safe_ids_interleave_0, values = (fill_like_1_cast_fp16, var_369_cast_fp16))[name = tensor("safe_ids_cast_fp16")]; + tensor reshape_0_shape_0 = const()[name = tensor("reshape_0_shape_0"), val = tensor([32, 96])]; + tensor reshape_0 = reshape(shape = reshape_0_shape_0, x = option_ids_to_fp16)[name = tensor("reshape_0")]; + tensor cast_1_dtype_0 = const()[name = tensor("cast_1_dtype_0"), val = tensor("uint16")]; + tensor cast_1 = cast(dtype = cast_1_dtype_0, x = reshape_0)[name = tensor("cast_1")]; + tensor gather_1_axis_0 = const()[name = tensor("gather_1_axis_0"), val = tensor(0)]; + tensor gather_1_batch_dims_0 = const()[name = tensor("gather_1_batch_dims_0"), val = tensor(0)]; + tensor gather_1_validate_indices_0 = const()[name = tensor("gather_1_validate_indices_0"), val = tensor(false)]; + tensor gather_1 = gather(axis = gather_1_axis_0, batch_dims = gather_1_batch_dims_0, indices = cast_1, validate_indices = gather_1_validate_indices_0, x = model_embedding_weight_to_fp16)[name = tensor("gather_1")]; + tensor var_389_to_fp16 = const()[name = tensor("op_389_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(918464)))]; + tensor src_cast_fp16 = add(x = gather_1, y = var_389_to_fp16)[name = tensor("src_cast_fp16")]; + tensor var_392_promoted_to_fp16 = const()[name = tensor("op_392_promoted_to_fp16"), val = tensor(0x0p+0)]; + tensor mask_cast_fp16 = equal(x = safe_ids_cast_fp16, y = var_392_promoted_to_fp16)[name = tensor("mask_cast_fp16")]; + tensor var_408_to_fp16 = const()[name = tensor("op_408_to_fp16"), val = tensor(-inf)]; + tensor var_414_to_fp16 = const()[name = tensor("op_414_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(943104)))]; + tensor key_padding_mask_7_cast_fp16 = select(a = var_408_to_fp16, b = var_414_to_fp16, cond = mask_cast_fp16)[name = tensor("key_padding_mask_7_cast_fp16")]; + tensor query_9_axes_0 = const()[name = tensor("query_9_axes_0"), val = tensor([-1])]; + tensor model_option_encoder_layers_0_norm1_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_norm1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(949312)))]; + tensor model_option_encoder_layers_0_norm1_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_norm1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(949632)))]; + tensor var_395_to_fp16 = const()[name = tensor("op_395_to_fp16"), val = tensor(0x1.5p-17)]; + tensor query_9_cast_fp16 = layer_norm(axes = query_9_axes_0, beta = model_option_encoder_layers_0_norm1_bias_to_fp16, epsilon = var_395_to_fp16, gamma = model_option_encoder_layers_0_norm1_weight_to_fp16, x = src_cast_fp16)[name = tensor("query_9_cast_fp16")]; + tensor query_11_perm_0 = const()[name = tensor("query_11_perm_0"), val = tensor([1, 0, 2])]; + tensor model_option_encoder_layers_0_self_attn_in_proj_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_self_attn_in_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(949952)))]; + tensor model_option_encoder_layers_0_self_attn_in_proj_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_self_attn_in_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1048320)))]; + tensor query_11_cast_fp16 = transpose(perm = query_11_perm_0, x = query_9_cast_fp16)[name = tensor("transpose_10")]; + tensor linear_8_cast_fp16 = linear(bias = model_option_encoder_layers_0_self_attn_in_proj_bias_to_fp16, weight = model_option_encoder_layers_0_self_attn_in_proj_weight_to_fp16, x = query_11_cast_fp16)[name = tensor("linear_8_cast_fp16")]; + tensor concat_4 = const()[name = tensor("concat_4"), val = tensor([96, 32, 3, 128])]; + tensor var_448_cast_fp16 = reshape(shape = concat_4, x = linear_8_cast_fp16)[name = tensor("op_448_cast_fp16")]; + tensor var_449_axes_0 = const()[name = tensor("op_449_axes_0"), val = tensor([0])]; + tensor var_449_cast_fp16 = expand_dims(axes = var_449_axes_0, x = var_448_cast_fp16)[name = tensor("op_449_cast_fp16")]; + tensor var_450_perm_0 = const()[name = tensor("op_450_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_451_axes_0 = const()[name = tensor("op_451_axes_0"), val = tensor([-2])]; + tensor var_450_cast_fp16 = transpose(perm = var_450_perm_0, x = var_449_cast_fp16)[name = tensor("transpose_9")]; + tensor var_451_cast_fp16 = squeeze(axes = var_451_axes_0, x = var_450_cast_fp16)[name = tensor("op_451_cast_fp16")]; + tensor q_13_begin_0 = const()[name = tensor("q_13_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor q_13_end_0 = const()[name = tensor("q_13_end_0"), val = tensor([1, 96, 32, 128])]; + tensor q_13_end_mask_0 = const()[name = tensor("q_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor q_13_squeeze_mask_0 = const()[name = tensor("q_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor q_13_cast_fp16 = slice_by_index(begin = q_13_begin_0, end = q_13_end_0, end_mask = q_13_end_mask_0, squeeze_mask = q_13_squeeze_mask_0, x = var_451_cast_fp16)[name = tensor("q_13_cast_fp16")]; + tensor k_13_begin_0 = const()[name = tensor("k_13_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor k_13_end_0 = const()[name = tensor("k_13_end_0"), val = tensor([2, 96, 32, 128])]; + tensor k_13_end_mask_0 = const()[name = tensor("k_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor k_13_squeeze_mask_0 = const()[name = tensor("k_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor k_13_cast_fp16 = slice_by_index(begin = k_13_begin_0, end = k_13_end_0, end_mask = k_13_end_mask_0, squeeze_mask = k_13_squeeze_mask_0, x = var_451_cast_fp16)[name = tensor("k_13_cast_fp16")]; + tensor v_13_begin_0 = const()[name = tensor("v_13_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor v_13_end_0 = const()[name = tensor("v_13_end_0"), val = tensor([3, 96, 32, 128])]; + tensor v_13_end_mask_0 = const()[name = tensor("v_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor v_13_squeeze_mask_0 = const()[name = tensor("v_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor v_13_cast_fp16 = slice_by_index(begin = v_13_begin_0, end = v_13_end_0, end_mask = v_13_end_mask_0, squeeze_mask = v_13_squeeze_mask_0, x = var_451_cast_fp16)[name = tensor("v_13_cast_fp16")]; + tensor var_459 = const()[name = tensor("op_459"), val = tensor([96, 128, 32])]; + tensor var_460_cast_fp16 = reshape(shape = var_459, x = q_13_cast_fp16)[name = tensor("op_460_cast_fp16")]; + tensor q_15_perm_0 = const()[name = tensor("q_15_perm_0"), val = tensor([1, 0, 2])]; + tensor var_466 = const()[name = tensor("op_466"), val = tensor([96, 128, 32])]; + tensor var_467_cast_fp16 = reshape(shape = var_466, x = k_13_cast_fp16)[name = tensor("op_467_cast_fp16")]; + tensor k_15_perm_0 = const()[name = tensor("k_15_perm_0"), val = tensor([1, 0, 2])]; + tensor var_473 = const()[name = tensor("op_473"), val = tensor([96, 128, 32])]; + tensor var_474_cast_fp16 = reshape(shape = var_473, x = v_13_cast_fp16)[name = tensor("op_474_cast_fp16")]; + tensor v_15_perm_0 = const()[name = tensor("v_15_perm_0"), val = tensor([1, 0, 2])]; + tensor var_477 = const()[name = tensor("op_477"), val = tensor([32, 1, 1, 96])]; + tensor var_478_cast_fp16 = reshape(shape = var_477, x = key_padding_mask_7_cast_fp16)[name = tensor("op_478_cast_fp16")]; + tensor var_480_reps_0 = const()[name = tensor("op_480_reps_0"), val = tensor([1, 4, 1, 1])]; + tensor var_480_cast_fp16 = tile(reps = var_480_reps_0, x = var_478_cast_fp16)[name = tensor("op_480_cast_fp16")]; + tensor var_488 = const()[name = tensor("op_488"), val = tensor([32, 4, 96, 32])]; + tensor q_15_cast_fp16 = transpose(perm = q_15_perm_0, x = var_460_cast_fp16)[name = tensor("transpose_8")]; + tensor q_cast_fp16 = reshape(shape = var_488, x = q_15_cast_fp16)[name = tensor("q_cast_fp16")]; + tensor var_490 = const()[name = tensor("op_490"), val = tensor([32, 4, 96, 32])]; + tensor k_15_cast_fp16 = transpose(perm = k_15_perm_0, x = var_467_cast_fp16)[name = tensor("transpose_7")]; + tensor k_cast_fp16 = reshape(shape = var_490, x = k_15_cast_fp16)[name = tensor("k_cast_fp16")]; + tensor var_492 = const()[name = tensor("op_492"), val = tensor([32, 4, 96, 32])]; + tensor v_15_cast_fp16 = transpose(perm = v_15_perm_0, x = var_474_cast_fp16)[name = tensor("transpose_6")]; + tensor v_cast_fp16 = reshape(shape = var_492, x = v_15_cast_fp16)[name = tensor("v_cast_fp16")]; + tensor mul_5_y_0_to_fp16 = const()[name = tensor("mul_5_y_0_to_fp16"), val = tensor(0x1.6ap-3)]; + tensor mul_5_cast_fp16 = mul(x = q_cast_fp16, y = mul_5_y_0_to_fp16)[name = tensor("mul_5_cast_fp16")]; + tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(true)]; + tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; + tensor matmul_2_cast_fp16 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = mul_5_cast_fp16, y = k_cast_fp16)[name = tensor("matmul_2_cast_fp16")]; + tensor add_2_cast_fp16 = add(x = matmul_2_cast_fp16, y = var_480_cast_fp16)[name = tensor("add_2_cast_fp16")]; + tensor softmax_2_axis_0 = const()[name = tensor("softmax_2_axis_0"), val = tensor(-1)]; + tensor softmax_2_cast_fp16 = softmax(axis = softmax_2_axis_0, x = add_2_cast_fp16)[name = tensor("softmax_2_cast_fp16")]; + tensor attn_output_17_transpose_x_0 = const()[name = tensor("attn_output_17_transpose_x_0"), val = tensor(false)]; + tensor attn_output_17_transpose_y_0 = const()[name = tensor("attn_output_17_transpose_y_0"), val = tensor(false)]; + tensor attn_output_17_cast_fp16 = matmul(transpose_x = attn_output_17_transpose_x_0, transpose_y = attn_output_17_transpose_y_0, x = softmax_2_cast_fp16, y = v_cast_fp16)[name = tensor("attn_output_17_cast_fp16")]; + tensor var_495 = const()[name = tensor("op_495"), val = tensor([2, 0, 1, 3])]; + tensor var_500 = const()[name = tensor("op_500"), val = tensor([3072, 128])]; + tensor var_496_cast_fp16 = transpose(perm = var_495, x = attn_output_17_cast_fp16)[name = tensor("transpose_5")]; + tensor attn_output_19_cast_fp16 = reshape(shape = var_500, x = var_496_cast_fp16)[name = tensor("attn_output_19_cast_fp16")]; + tensor model_option_encoder_layers_0_self_attn_out_proj_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_self_attn_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1049152)))]; + tensor model_option_encoder_layers_0_self_attn_out_proj_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_self_attn_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1081984)))]; + tensor linear_9_cast_fp16 = linear(bias = model_option_encoder_layers_0_self_attn_out_proj_bias_to_fp16, weight = model_option_encoder_layers_0_self_attn_out_proj_weight_to_fp16, x = attn_output_19_cast_fp16)[name = tensor("linear_9_cast_fp16")]; + tensor var_504 = const()[name = tensor("op_504"), val = tensor([96, 32, 128])]; + tensor attn_output_cast_fp16 = reshape(shape = var_504, x = linear_9_cast_fp16)[name = tensor("attn_output_cast_fp16")]; + tensor input_35_perm_0 = const()[name = tensor("input_35_perm_0"), val = tensor([1, 0, 2])]; + tensor input_35_cast_fp16 = transpose(perm = input_35_perm_0, x = attn_output_cast_fp16)[name = tensor("transpose_4")]; + tensor input_37_cast_fp16 = add(x = src_cast_fp16, y = input_35_cast_fp16)[name = tensor("input_37_cast_fp16")]; + tensor input_39_axes_0 = const()[name = tensor("input_39_axes_0"), val = tensor([-1])]; + tensor model_option_encoder_layers_0_norm2_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_norm2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1082304)))]; + tensor model_option_encoder_layers_0_norm2_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_norm2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1082624)))]; + tensor input_39_cast_fp16 = layer_norm(axes = input_39_axes_0, beta = model_option_encoder_layers_0_norm2_bias_to_fp16, epsilon = var_395_to_fp16, gamma = model_option_encoder_layers_0_norm2_weight_to_fp16, x = input_37_cast_fp16)[name = tensor("input_39_cast_fp16")]; + tensor model_option_encoder_layers_0_linear1_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_linear1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1082944)))]; + tensor model_option_encoder_layers_0_linear1_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_linear1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1214080)))]; + tensor linear_10_cast_fp16 = linear(bias = model_option_encoder_layers_0_linear1_bias_to_fp16, weight = model_option_encoder_layers_0_linear1_weight_to_fp16, x = input_39_cast_fp16)[name = tensor("linear_10_cast_fp16")]; + tensor input_43_cast_fp16 = relu(x = linear_10_cast_fp16)[name = tensor("input_43_cast_fp16")]; + tensor model_option_encoder_layers_0_linear2_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_linear2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1215168)))]; + tensor model_option_encoder_layers_0_linear2_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_linear2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1346304)))]; + tensor linear_11_cast_fp16 = linear(bias = model_option_encoder_layers_0_linear2_bias_to_fp16, weight = model_option_encoder_layers_0_linear2_weight_to_fp16, x = input_43_cast_fp16)[name = tensor("linear_11_cast_fp16")]; + tensor hidden_cast_fp16 = add(x = input_37_cast_fp16, y = linear_11_cast_fp16)[name = tensor("hidden_cast_fp16")]; + tensor var_524_axes_0 = const()[name = tensor("op_524_axes_0"), val = tensor([-1])]; + tensor var_524 = expand_dims(axes = var_524_axes_0, x = flat_mask_cast_fp16)[name = tensor("op_524")]; + tensor weights_to_fp16_dtype_0 = const()[name = tensor("weights_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor var_524_to_fp16 = cast(dtype = weights_to_fp16_dtype_0, x = var_524)[name = tensor("cast_59")]; + tensor var_530_cast_fp16 = mul(x = hidden_cast_fp16, y = var_524_to_fp16)[name = tensor("op_530_cast_fp16")]; + tensor var_535_axes_0 = const()[name = tensor("op_535_axes_0"), val = tensor([1])]; + tensor var_535_keep_dims_0 = const()[name = tensor("op_535_keep_dims_0"), val = tensor(false)]; + tensor var_535_cast_fp16 = reduce_sum(axes = var_535_axes_0, keep_dims = var_535_keep_dims_0, x = var_530_cast_fp16)[name = tensor("op_535_cast_fp16")]; + tensor var_540_axes_0 = const()[name = tensor("op_540_axes_0"), val = tensor([1])]; + tensor var_540_keep_dims_0 = const()[name = tensor("op_540_keep_dims_0"), val = tensor(false)]; + tensor var_540_cast_fp16 = reduce_sum(axes = var_540_axes_0, keep_dims = var_540_keep_dims_0, x = var_524_to_fp16)[name = tensor("op_540_cast_fp16")]; + tensor var_541_to_fp16 = const()[name = tensor("op_541_to_fp16"), val = tensor(0x1p+0)]; + tensor var_542_cast_fp16 = maximum(x = var_540_cast_fp16, y = var_541_to_fp16)[name = tensor("op_542_cast_fp16")]; + tensor pooled_cast_fp16 = real_div(x = var_535_cast_fp16, y = var_542_cast_fp16)[name = tensor("pooled_cast_fp16")]; + tensor var_545 = const()[name = tensor("op_545"), val = tensor([1, 32, -1])]; + tensor options_cast_fp16 = reshape(shape = var_545, x = pooled_cast_fp16)[name = tensor("options_cast_fp16")]; + tensor var_548 = const()[name = tensor("op_548"), val = tensor(-1)]; + tensor input_axes_0 = const()[name = tensor("input_axes_0"), val = tensor([-1])]; + tensor model_head_context_norm_weight_to_fp16 = const()[name = tensor("model_head_context_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1346624)))]; + tensor model_head_context_norm_bias_to_fp16 = const()[name = tensor("model_head_context_norm_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1346944)))]; + tensor var_556_to_fp16 = const()[name = tensor("op_556_to_fp16"), val = tensor(0x1.5p-17)]; + tensor input_cast_fp16 = layer_norm(axes = input_axes_0, beta = model_head_context_norm_bias_to_fp16, epsilon = var_556_to_fp16, gamma = model_head_context_norm_weight_to_fp16, x = context_cast_fp16)[name = tensor("input_cast_fp16")]; + tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; + tensor model_head_option_norm_weight_to_fp16 = const()[name = tensor("model_head_option_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1347264)))]; + tensor model_head_option_norm_bias_to_fp16 = const()[name = tensor("model_head_option_norm_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1347584)))]; + tensor input_53_cast_fp16 = layer_norm(axes = input_53_axes_0, beta = model_head_option_norm_bias_to_fp16, epsilon = var_556_to_fp16, gamma = model_head_option_norm_weight_to_fp16, x = options_cast_fp16)[name = tensor("input_53_cast_fp16")]; + tensor model_head_query_weight_to_fp16 = const()[name = tensor("model_head_query_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1347904)))]; + tensor linear_12_bias_0_to_fp16 = const()[name = tensor("linear_12_bias_0_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1380736)))]; + tensor linear_12_cast_fp16 = linear(bias = linear_12_bias_0_to_fp16, weight = model_head_query_weight_to_fp16, x = input_53_cast_fp16)[name = tensor("linear_12_cast_fp16")]; + tensor model_head_key_weight_to_fp16 = const()[name = tensor("model_head_key_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1381056)))]; + tensor linear_13_cast_fp16 = linear(bias = linear_12_bias_0_to_fp16, weight = model_head_key_weight_to_fp16, x = input_cast_fp16)[name = tensor("linear_13_cast_fp16")]; + tensor model_head_value_weight_to_fp16 = const()[name = tensor("model_head_value_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1413888)))]; + tensor linear_14_cast_fp16 = linear(bias = linear_12_bias_0_to_fp16, weight = model_head_value_weight_to_fp16, x = input_cast_fp16)[name = tensor("linear_14_cast_fp16")]; + tensor matmul_3_transpose_x_1 = const()[name = tensor("matmul_3_transpose_x_1"), val = tensor(false)]; + tensor matmul_3_transpose_y_1 = const()[name = tensor("matmul_3_transpose_y_1"), val = tensor(true)]; + tensor matmul_3_cast_fp16 = matmul(transpose_x = matmul_3_transpose_x_1, transpose_y = matmul_3_transpose_y_1, x = linear_12_cast_fp16, y = linear_13_cast_fp16)[name = tensor("matmul_3_cast_fp16")]; + tensor _inversed_scores_1_y_0_to_fp16 = const()[name = tensor("_inversed_scores_1_y_0_to_fp16"), val = tensor(0x1.6ap-4)]; + tensor _inversed_scores_1_cast_fp16 = mul(x = matmul_3_cast_fp16, y = _inversed_scores_1_y_0_to_fp16)[name = tensor("_inversed_scores_1_cast_fp16")]; + tensor var_587_axes_0 = const()[name = tensor("op_587_axes_0"), val = tensor([1])]; + tensor var_587 = expand_dims(axes = var_587_axes_0, x = context_mask_cast_fp16)[name = tensor("op_587")]; + tensor var_589 = logical_not(x = var_587)[name = tensor("op_589")]; + tensor var_549_to_fp16 = const()[name = tensor("op_549_to_fp16"), val = tensor(-inf)]; + tensor scores_cast_fp16 = select(a = var_549_to_fp16, b = _inversed_scores_1_cast_fp16, cond = var_589)[name = tensor("scores_cast_fp16")]; + tensor var_591_cast_fp16 = softmax(axis = var_548, x = scores_cast_fp16)[name = tensor("op_591_cast_fp16")]; + tensor matmul_4_transpose_x_0 = const()[name = tensor("matmul_4_transpose_x_0"), val = tensor(false)]; + tensor matmul_4_transpose_y_0 = const()[name = tensor("matmul_4_transpose_y_0"), val = tensor(false)]; + tensor matmul_4_cast_fp16 = matmul(transpose_x = matmul_4_transpose_x_0, transpose_y = matmul_4_transpose_y_0, x = var_591_cast_fp16, y = linear_14_cast_fp16)[name = tensor("matmul_4_cast_fp16")]; + tensor var_594_cast_fp16 = mul(x = linear_12_cast_fp16, y = matmul_4_cast_fp16)[name = tensor("op_594_cast_fp16")]; + tensor var_596_axes_0 = const()[name = tensor("op_596_axes_0"), val = tensor([-1])]; + tensor var_596_keep_dims_0 = const()[name = tensor("op_596_keep_dims_0"), val = tensor(false)]; + tensor var_596_cast_fp16 = reduce_sum(axes = var_596_axes_0, keep_dims = var_596_keep_dims_0, x = var_594_cast_fp16)[name = tensor("op_596_cast_fp16")]; + tensor _inversed_logits_1_y_0_to_fp16 = const()[name = tensor("_inversed_logits_1_y_0_to_fp16"), val = tensor(0x1.6ap-4)]; + tensor _inversed_logits_1_cast_fp16 = mul(x = var_596_cast_fp16, y = _inversed_logits_1_y_0_to_fp16)[name = tensor("_inversed_logits_1_cast_fp16")]; + tensor var_599 = logical_not(x = option_mask_cast_fp16)[name = tensor("op_599")]; + tensor logits_3_cast_fp16 = select(a = var_549_to_fp16, b = _inversed_logits_1_cast_fp16, cond = var_599)[name = tensor("logits_3_cast_fp16")]; + tensor var_607_value_0_to_fp16 = const()[name = tensor("op_607_value_0_to_fp16"), val = tensor(-0x1.388p+13)]; + tensor var_607_cast_fp16 = fill_like(ref_tensor = logits_3_cast_fp16, value = var_607_value_0_to_fp16)[name = tensor("op_607_cast_fp16")]; + tensor logits_cast_fp16 = select(a = logits_3_cast_fp16, b = var_607_cast_fp16, cond = option_mask_cast_fp16)[name = tensor("logits_cast_fp16")]; + tensor logits_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("logits_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; + tensor var_609 = const()[name = tensor("op_609"), val = tensor(-1)]; + tensor var_611_cast_fp16 = softmax(axis = var_609, x = logits_cast_fp16)[name = tensor("op_611_cast_fp16")]; + tensor var_611_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("op_611_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; + tensor probabilities = cast(dtype = var_611_cast_fp16_to_fp32_dtype_0, x = var_611_cast_fp16)[name = tensor("cast_57")]; + tensor logits = cast(dtype = logits_cast_fp16_to_fp32_dtype_0, x = logits_cast_fp16)[name = tensor("cast_58")]; + } -> (logits, probabilities); +} \ No newline at end of file diff --git a/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/weights/weight.bin b/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/weights/weight.bin new file mode 100644 index 0000000000000000000000000000000000000000..bff6eee099e8b8b4e659c073c6b48112ba92dfb6 --- /dev/null +++ b/ane-gather/cua_s1_forms_fp16_options32.mlmodelc/weights/weight.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895 +size 1446720 diff --git a/ane-gather/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel b/ane-gather/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel new file mode 100644 index 0000000000000000000000000000000000000000..e270b2ec3e1081128d03cdd64d573f5ac942521d --- /dev/null +++ b/ane-gather/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de18e313c3b625e35d008ed8b6b24108edc6df7bf2fae9533af03519eca11b63 +size 62154 diff --git a/ane-gather/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin b/ane-gather/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin new file mode 100644 index 0000000000000000000000000000000000000000..bff6eee099e8b8b4e659c073c6b48112ba92dfb6 --- /dev/null +++ b/ane-gather/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895 +size 1446720 diff --git a/ane-gather/cua_s1_forms_fp16_options32.mlpackage/Manifest.json b/ane-gather/cua_s1_forms_fp16_options32.mlpackage/Manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d942eb1ac325c1abded21b89415fb283012a0034 --- /dev/null +++ b/ane-gather/cua_s1_forms_fp16_options32.mlpackage/Manifest.json @@ -0,0 +1,18 @@ +{ + "fileFormatVersion": "1.0.0", + "itemInfoEntries": { + "1CB24175-461C-4E33-BE2C-F580A319AEC8": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" + }, + "869A4EBA-1FCF-41FC-8DD9-70ECEF96023D": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" + } + }, + "rootModelIdentifier": "869A4EBA-1FCF-41FC-8DD9-70ECEF96023D" +} diff --git a/assets.lock.json b/assets.lock.json new file mode 100644 index 0000000000000000000000000000000000000000..401091ad1933cd7b0b26ce39daf573eabd563b83 --- /dev/null +++ b/assets.lock.json @@ -0,0 +1,54 @@ +{ + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "evaluation_file": "artifacts/demo.jsonl", + "evaluation_rows": 196, + "files": [ + { + "path": "vendor/cua_s1/__init__.py", + "url": "https://raw.githubusercontent.com/trycua/cua/83f142c4290a0f7d9ed545ae8532858c6e4f8145/libs/cua-s1/python/src/cua_s1/__init__.py", + "sha256": "68ca526b67bdb95401ddaa2ccc09b0717879d16a9102e2014f4f9018df292ddf" + }, + { + "path": "vendor/cua_s1/model.py", + "url": "https://raw.githubusercontent.com/trycua/cua/83f142c4290a0f7d9ed545ae8532858c6e4f8145/libs/cua-s1/python/src/cua_s1/model.py", + "sha256": "7538cb4c9730d70a0b325050f370f981a9e109a1b182b8c66be3ba0fa2474344" + }, + { + "path": "vendor/cua_s1/checkpoint.py", + "url": "https://raw.githubusercontent.com/trycua/cua/83f142c4290a0f7d9ed545ae8532858c6e4f8145/libs/cua-s1/python/src/cua_s1/checkpoint.py", + "sha256": "6d96ab6e946c88af341947d656ff244a8de5846edfd1f6a410466a97d89b375f" + }, + { + "path": "vendor/CUA-LICENSE", + "url": "https://raw.githubusercontent.com/trycua/cua/83f142c4290a0f7d9ed545ae8532858c6e4f8145/libs/cua-s1/python/LICENSE", + "sha256": "c0779290c1d4783169aa3dbfb55feb505e563ef8a004bbf55298ceffcfbda8d9" + }, + { + "path": "artifacts/cua-s1-forms.safetensors", + "url": "https://huggingface.co/cua-ai/cua-s1-forms/resolve/f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71/cua-s1-forms.safetensors", + "sha256": "05954c1caf51c2fb6c13ea4acbfc88a2e7653dea192252bb51dc89e76a356ddc" + }, + { + "path": "artifacts/cua-s1-forms.json", + "url": "https://huggingface.co/cua-ai/cua-s1-forms/resolve/f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71/cua-s1-forms.json", + "sha256": "62d31e2f9a001a8e9b6f8534c5194d07ebdd3f9d62ef1ac281906622992650ca" + }, + { + "path": "artifacts/demo.jsonl", + "url": "https://huggingface.co/datasets/cua-ai/cua-s1-forms/resolve/8273f34778b99ac2e12d9f6e7d57dad99ae20845/demo.jsonl", + "sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca" + }, + { + "path": "artifacts/model-card.md", + "url": "https://huggingface.co/cua-ai/cua-s1-forms/resolve/f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71/README.md", + "sha256": "a52b7930d12d59a9f70fba537f02c1683eb4be2b451ec0fd5f407620c5670a93" + }, + { + "path": "artifacts/dataset-card.md", + "url": "https://huggingface.co/datasets/cua-ai/cua-s1-forms/resolve/8273f34778b99ac2e12d9f6e7d57dad99ae20845/README.md", + "sha256": "e5768e2a4b8fee2622ca73fb522bdbb0caca567a49c93266ef0f0fdef5b0af11" + } + ] +} diff --git a/checksums.json b/checksums.json new file mode 100644 index 0000000000000000000000000000000000000000..9ed1f305666a4866817fc8b0a4b24a88e3246e68 --- /dev/null +++ b/checksums.json @@ -0,0 +1,66 @@ +{ + "LICENSE": "c0779290c1d4783169aa3dbfb55feb505e563ef8a004bbf55298ceffcfbda8d9", + "NOTICES.md": "027c72741eaa695e60d7b6cebd3666372c81d443a96b673b72a897cf432ddfd0", + "README.md": "377a7149ce6de7064b3a7d26de00940dd92f1ce7f7935296a735435942cae618", + "UPSTREAM-THIRD-PARTY-NOTICES.md": "4091e69b45c8cc97e30a066fbbd56148dbef66ab716432c048d2333c9c464213", + "ane-gather/conversion.json": "4d569a4020f1e9711da3f7fff0d11acef1812c4652414cf64728c77bc2bae4c4", + "ane-gather/cua_s1_forms_fp16_options32.mlmodelc/analytics/coremldata.bin": "526dccb87bdc036df1e8add0b50dd6540db534808fc2f8caa97cd897b4fc1fa3", + "ane-gather/cua_s1_forms_fp16_options32.mlmodelc/coremldata.bin": "0f2a8ed9fe8abcc39c7ee0575e2562d589d459683b92708fa9788b533be3038b", + "ane-gather/cua_s1_forms_fp16_options32.mlmodelc/model.mil": "578b5ef1e50c98d0577459a3a4c020f4fd83d7e465213d1263ddf1a021fef077", + "ane-gather/cua_s1_forms_fp16_options32.mlmodelc/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "ane-gather/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel": "de18e313c3b625e35d008ed8b6b24108edc6df7bf2fae9533af03519eca11b63", + "ane-gather/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "ane-gather/cua_s1_forms_fp16_options32.mlpackage/Manifest.json": "38f812a04eb2322080634ba788c61df362336168466ca67e5549058d346ac793", + "assets.lock.json": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "conversion.json": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "cua_s1_forms_fp16_options32.mlmodelc/analytics/coremldata.bin": "a7a11d7772fb6879778e6223d0f3b16a6b2a44df20a7991bf9bcc24042d05542", + "cua_s1_forms_fp16_options32.mlmodelc/coremldata.bin": "2ddf9edc046a1e66b3d3a0cd57a1d9e25c75df5f8d937249dc02d9806617d884", + "cua_s1_forms_fp16_options32.mlmodelc/model.mil": "e93676bd4f553ffcf9403ae5049185cb55acdcb63908e39952578db0845e0a12", + "cua_s1_forms_fp16_options32.mlmodelc/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "cua_s1_forms_fp16_options32.mlpackage/Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc", + "demo/browser-demo.gif": "6cf597927e7bcb5617719453eecb86c7c06ab9a91fcaf46de272b42abe12d4fc", + "demo/browser-demo.mp4": "5fbb3925c4824d15e6c24c504db91a96d318f4e751a20a070fdedf64e587d1a0", + "demo/browser-preview.png": "e624ec2a797d0b3b050e66055d6a9fb09a1f6a9ace67b8eb071ce5f2f9e282b1", + "int4-source-fp16/conversion.json": "01105dc8bef7a30fc3430e150caa2a143dc0ba23a295b6280de3b8ff3b40af55", + "int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel": "6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627", + "int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Manifest.json": "e5b1c7dc66b2014b3985c180a75548a709788c6e314d9d2ae14e4e6d441c3c68", + "int4-weights/conversion.json": "4675febbdef9cc15d46bf8aed17d7c7aaa0e91b3168a525c31b81ad2cfdb6875", + "int4-weights/cua_s1_forms_int4_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel": "cadad6bfa6875383a7ee3de68d0d22871148aa21cf117e7fe44c2c971287c4e5", + "int4-weights/cua_s1_forms_int4_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin": "0f766fcb9dbd612ed4f6c7a40117a5993a4789b3ae94afba0df37193c831f63c", + "int4-weights/cua_s1_forms_int4_options32.mlpackage/Manifest.json": "fb91c8b5ec3168111567de663bb3140d2790c6e7c48f32e0df3f5135af0673af", + "int8-weights/conversion.json": "b8a51ceeb65fe8020b2a186fae2ebf586443956111acfbac1a3f208a062c1c80", + "int8-weights/cua_s1_forms_int8_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel": "a7d9747308b8b754a46f3e70c8465b821f90e12376b3a71dc909f7364ae73c3a", + "int8-weights/cua_s1_forms_int8_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin": "f600c4434b7f7e3e789de8f1a4aa54a55f84df51fa9b788e18f5c97b68e4840f", + "int8-weights/cua_s1_forms_int8_options32.mlpackage/Manifest.json": "2206ec01787a85c12f1376fa5a832c5ac912f06172c6ff06b26ae8415470d246", + "preprocessing.py": "18c74ede43f95f91cd638c7e9631923a353e3a24d3b522ddb8b18b480f9c2846", + "reports/ane-comparison.json": "14b29f87704c6e8bd5f9bd866ea825be3972fcea6cf3c4760e38dc8774f937b6", + "reports/ane-fallback.json": "2f52b0e11f7cfbea6a6be6942de8816d8fed0630ac384a28551d405648fcb4e0", + "reports/ane-gather-fallback.json": "1e9bf8a336ae6d63ff2f07e32fed21b5e5ee4491a8a217b15deb3e1e3ac32988", + "reports/ane-gather-profile.json": "023c16760ca124176e399f1c692741f5220bc4cc391cd59bffffa291e2c332a6", + "reports/ane-gather-verification.json": "fa51fe0537dcd9cc8d1d75fc0be4747d887aa2e3af199ec08edd8f9400ce3318", + "reports/ane-profile.json": "8f654717397a294acc793861676a1480c160bb3f7ab5be6ff67ebb6a4473b832", + "reports/browser-validation.json": "299641c7b9f6cc2464cfcd984b4ef4445385e4710bc15abae7cee8ba262331b6", + "reports/int4-demo-verification.json": "7465aecddc5c6806a0c1882db3a587eb7ea0f8de7978f60bcbc7e39ffe2cfa32", + "reports/int4-fallback.json": "7e14ab1bb6b8c2ede589773af3d39aea1a7349d199a27d597f1cd47a40217b07", + "reports/int4-profile.json": "71f6b3ae5c9340a77936e368529201acef0d1e3b987b7436b1d35a829b6e350e", + "reports/int4-source-demo-verification.json": "f0c026f462dde1a8f86851d55ec35ab33ab8e68e5af3e4f1eb4b9c52f3dd08b1", + "reports/int4-synthetic-test-decisions.jsonl.gz": "6922913041f95ac8872958b92312cb5b869b9ce88116c4684b36ea7753284674", + "reports/int4-synthetic-test.json": "9d6a691aeb2ddc352ab7c12327472c92814991c201b67c7da8db39a76c972b32", + "reports/int8-demo-verification.json": "5602c8c254c0ae4c696e771291b25d1196ccc0d63f48eaaccdde0245803a5368", + "reports/int8-fallback.json": "9bb86a529a8e7e353efa12449a5ea930817dc7b1afdacec6ae24f3c318bc5835", + "reports/int8-profile.json": "0b962528dc7e5bb1d53abeb4b549f93b93f549a1e1935c3d77ef2606e83bf6a9", + "reports/int8-synthetic-test-decisions.jsonl.gz": "69720b9e11430ecff7ea0718f14bf16792de6f39f114faba250cfe855e4e4156", + "reports/int8-synthetic-test.json": "952ebe281d0aef0c05667d3c0dc5f9f175be709639a542028940059393078b5d", + "reports/swift-ane-validation.json": "ed323e6d6619fb7698d3546599d6450b745bc8cc6100c02e13f7ad4d59859a3a", + "reports/swift-runtime/decisions.jsonl.gz": "5d7ae5df86489b38e8f97167ab58c1d6bf5fb74c171a57b00d818b05d6c772b5", + "reports/swift-runtime/report.json": "81c30c4943550356da71ea03b1ff87d6d9a66ec3bbc17c20de5ff802a6a5f817", + "reports/swift-variant-comparison.json": "593053d7011a1c6d6881b130c3408099d0fcc09c714c4064d4f853f0dc63b651", + "reports/synthetic-test-decisions.jsonl.gz": "b6f6edc9de34fb37b10847a1f9568b530ce2b8e1bd241f51bf2e637f489d6554", + "reports/synthetic-test.json": "2ae85cfeb270b4f31be4f1d9082f4c73b3b9b084d6f1ddb661712a87d46b0f59", + "reports/upstream-metrics.json": "7d6207e430504dc0baec96d8180cb03f8aadc5a6e016b53c4cf03a80baf169ab", + "reports/verification.json": "9a5d22ec4b447f0710c12e03c69d7ed802158bb94e9a3c6b059731730ced9f30", + "synthetic-test.lock.json": "e582b951ecc07809a0b06a2bc0790827b90a86ce57c70d5b6177ef04f0b4f1ca" +} diff --git a/conversion.json b/conversion.json new file mode 100644 index 0000000000000000000000000000000000000000..f228889207832108ba6958f467f1029f466d41e1 --- /dev/null +++ b/conversion.json @@ -0,0 +1,35 @@ +{ + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + } +} diff --git a/cua_s1_forms_fp16_options32.mlmodelc/analytics/coremldata.bin b/cua_s1_forms_fp16_options32.mlmodelc/analytics/coremldata.bin new file mode 100644 index 0000000000000000000000000000000000000000..c869b55bfd7c1839ccc06896566a173948ece9d2 --- /dev/null +++ b/cua_s1_forms_fp16_options32.mlmodelc/analytics/coremldata.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7a11d7772fb6879778e6223d0f3b16a6b2a44df20a7991bf9bcc24042d05542 +size 243 diff --git a/cua_s1_forms_fp16_options32.mlmodelc/coremldata.bin b/cua_s1_forms_fp16_options32.mlmodelc/coremldata.bin new file mode 100644 index 0000000000000000000000000000000000000000..2649756bf9693e24486db4913da208381c30ba64 --- /dev/null +++ b/cua_s1_forms_fp16_options32.mlmodelc/coremldata.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ddf9edc046a1e66b3d3a0cd57a1d9e25c75df5f8d937249dc02d9806617d884 +size 909 diff --git a/cua_s1_forms_fp16_options32.mlmodelc/model.mil b/cua_s1_forms_fp16_options32.mlmodelc/model.mil new file mode 100644 index 0000000000000000000000000000000000000000..3e8758754fd6dba157cad04cefd6ecc3b7b82d77 --- /dev/null +++ b/cua_s1_forms_fp16_options32.mlmodelc/model.mil @@ -0,0 +1,418 @@ +program(1.0) +[buildInfo = dict, tensor>({{"coremlc-component-MIL", "3600.16.1"}, {"coremlc-version", "3600.25.2"}, {"coremltools-component-torch", "2.7.0"}, {"coremltools-source-dialect", "TorchScript"}, {"coremltools-version", "9.0"}})] +{ + func main(tensor context_ids, tensor option_ids, tensor option_mask) { + tensor var_14 = const()[name = tensor("op_14"), val = tensor(0)]; + tensor option_mask_1 = not_equal(x = option_mask, y = var_14)[name = tensor("option_mask")]; + tensor var_16 = const()[name = tensor("op_16"), val = tensor(0)]; + tensor context_mask = not_equal(x = context_ids, y = var_16)[name = tensor("context_mask")]; + tensor var_27_begin_0 = const()[name = tensor("op_27_begin_0"), val = tensor([0, 0])]; + tensor var_27_end_0 = const()[name = tensor("op_27_end_0"), val = tensor([1, 1])]; + tensor var_27_end_mask_0 = const()[name = tensor("op_27_end_mask_0"), val = tensor([true, false])]; + tensor var_27 = slice_by_index(begin = var_27_begin_0, end = var_27_end_0, end_mask = var_27_end_mask_0, x = context_ids)[name = tensor("op_27")]; + tensor fill_like_0_value_0_to_fp16 = const()[name = tensor("fill_like_0_value_0_to_fp16"), val = tensor(0x1p+0)]; + tensor fill_like_0_cast_fp16 = fill_like(ref_tensor = var_27, value = fill_like_0_value_0_to_fp16)[name = tensor("fill_like_0_cast_fp16")]; + tensor var_43_begin_0 = const()[name = tensor("op_43_begin_0"), val = tensor([0, 1])]; + tensor var_43_end_0 = const()[name = tensor("op_43_end_0"), val = tensor([1, 224])]; + tensor var_43_end_mask_0 = const()[name = tensor("op_43_end_mask_0"), val = tensor([true, true])]; + tensor var_43 = slice_by_index(begin = var_43_begin_0, end = var_43_end_0, end_mask = var_43_end_mask_0, x = context_ids)[name = tensor("op_43")]; + tensor var_45 = const()[name = tensor("op_45"), val = tensor(1)]; + tensor safe_context_ids_interleave_0 = const()[name = tensor("safe_context_ids_interleave_0"), val = tensor(false)]; + tensor var_43_promoted_to_fp16_dtype_0 = const()[name = tensor("op_43_promoted_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor var_43_to_fp16 = cast(dtype = var_43_promoted_to_fp16_dtype_0, x = var_43)[name = tensor("cast_66")]; + tensor safe_context_ids_cast_fp16 = concat(axis = var_45, interleave = safe_context_ids_interleave_0, values = (fill_like_0_cast_fp16, var_43_to_fp16))[name = tensor("safe_context_ids_cast_fp16")]; + tensor var_59_batch_dims_0 = const()[name = tensor("op_59_batch_dims_0"), val = tensor(0)]; + tensor var_59_validate_indices_0 = const()[name = tensor("op_59_validate_indices_0"), val = tensor(false)]; + tensor model_embedding_weight_to_fp16 = const()[name = tensor("model_embedding_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(64)))]; + tensor context_ids_to_int16_dtype_0 = const()[name = tensor("context_ids_to_int16_dtype_0"), val = tensor("int16")]; + tensor cast_54_dtype_0 = const()[name = tensor("cast_54_dtype_0"), val = tensor("int32")]; + tensor greater_equal_0_y_0 = const()[name = tensor("greater_equal_0_y_0"), val = tensor(0)]; + tensor context_ids_to_int16 = cast(dtype = context_ids_to_int16_dtype_0, x = context_ids)[name = tensor("cast_65")]; + tensor cast_54 = cast(dtype = cast_54_dtype_0, x = context_ids_to_int16)[name = tensor("cast_64")]; + tensor greater_equal_0 = greater_equal(x = cast_54, y = greater_equal_0_y_0)[name = tensor("greater_equal_0")]; + tensor slice_by_index_18 = const()[name = tensor("slice_by_index_18"), val = tensor(257)]; + tensor add_3 = add(x = cast_54, y = slice_by_index_18)[name = tensor("add_3")]; + tensor select_0 = select(a = cast_54, b = add_3, cond = greater_equal_0)[name = tensor("select_0")]; + tensor var_59_cast_fp16_cast_uint16_axis_0 = const()[name = tensor("op_59_cast_fp16_cast_uint16_axis_0"), val = tensor(0)]; + tensor select_0_to_int16_dtype_0 = const()[name = tensor("select_0_to_int16_dtype_0"), val = tensor("int16")]; + tensor select_0_to_int16 = cast(dtype = select_0_to_int16_dtype_0, x = select_0)[name = tensor("cast_63")]; + tensor var_59_cast_fp16_cast_uint16_cast_uint16 = gather(axis = var_59_cast_fp16_cast_uint16_axis_0, batch_dims = var_59_batch_dims_0, indices = select_0_to_int16, validate_indices = var_59_validate_indices_0, x = model_embedding_weight_to_fp16)[name = tensor("op_59_cast_fp16_cast_uint16_cast_uint16")]; + tensor model_position_weight_to_fp16 = const()[name = tensor("model_position_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(65920)))]; + tensor src_1_cast_fp16 = add(x = var_59_cast_fp16_cast_uint16_cast_uint16, y = model_position_weight_to_fp16)[name = tensor("src_1_cast_fp16")]; + tensor var_66_promoted_to_fp16 = const()[name = tensor("op_66_promoted_to_fp16"), val = tensor(0x0p+0)]; + tensor mask_1_cast_fp16 = equal(x = safe_context_ids_cast_fp16, y = var_66_promoted_to_fp16)[name = tensor("mask_1_cast_fp16")]; + tensor var_82_to_fp16 = const()[name = tensor("op_82_to_fp16"), val = tensor(-inf)]; + tensor var_90_to_fp16 = const()[name = tensor("op_90_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(123328)))]; + tensor key_padding_mask_1_cast_fp16 = select(a = var_82_to_fp16, b = var_90_to_fp16, cond = mask_1_cast_fp16)[name = tensor("key_padding_mask_1_cast_fp16")]; + tensor query_1_axes_0 = const()[name = tensor("query_1_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_0_norm1_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_norm1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(123840)))]; + tensor model_encoder_layers_0_norm1_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_norm1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(124160)))]; + tensor var_69_to_fp16 = const()[name = tensor("op_69_to_fp16"), val = tensor(0x1.5p-17)]; + tensor query_1_cast_fp16 = layer_norm(axes = query_1_axes_0, beta = model_encoder_layers_0_norm1_bias_to_fp16, epsilon = var_69_to_fp16, gamma = model_encoder_layers_0_norm1_weight_to_fp16, x = src_1_cast_fp16)[name = tensor("query_1_cast_fp16")]; + tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; + tensor model_encoder_layers_0_self_attn_in_proj_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_self_attn_in_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(124480)))]; + tensor model_encoder_layers_0_self_attn_in_proj_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_self_attn_in_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(222848)))]; + tensor query_3_cast_fp16 = transpose(perm = query_3_perm_0, x = query_1_cast_fp16)[name = tensor("transpose_24")]; + tensor linear_0_cast_fp16 = linear(bias = model_encoder_layers_0_self_attn_in_proj_bias_to_fp16, weight = model_encoder_layers_0_self_attn_in_proj_weight_to_fp16, x = query_3_cast_fp16)[name = tensor("linear_0_cast_fp16")]; + tensor concat_0 = const()[name = tensor("concat_0"), val = tensor([224, 1, 3, 128])]; + tensor var_124_cast_fp16 = reshape(shape = concat_0, x = linear_0_cast_fp16)[name = tensor("op_124_cast_fp16")]; + tensor var_125_axes_0 = const()[name = tensor("op_125_axes_0"), val = tensor([0])]; + tensor var_125_cast_fp16 = expand_dims(axes = var_125_axes_0, x = var_124_cast_fp16)[name = tensor("op_125_cast_fp16")]; + tensor var_126_perm_0 = const()[name = tensor("op_126_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_127_axes_0 = const()[name = tensor("op_127_axes_0"), val = tensor([-2])]; + tensor var_126_cast_fp16 = transpose(perm = var_126_perm_0, x = var_125_cast_fp16)[name = tensor("transpose_23")]; + tensor var_127_cast_fp16 = squeeze(axes = var_127_axes_0, x = var_126_cast_fp16)[name = tensor("op_127_cast_fp16")]; + tensor q_1_begin_0 = const()[name = tensor("q_1_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor q_1_end_0 = const()[name = tensor("q_1_end_0"), val = tensor([1, 224, 1, 128])]; + tensor q_1_end_mask_0 = const()[name = tensor("q_1_end_mask_0"), val = tensor([false, true, true, true])]; + tensor q_1_squeeze_mask_0 = const()[name = tensor("q_1_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor q_1_cast_fp16 = slice_by_index(begin = q_1_begin_0, end = q_1_end_0, end_mask = q_1_end_mask_0, squeeze_mask = q_1_squeeze_mask_0, x = var_127_cast_fp16)[name = tensor("q_1_cast_fp16")]; + tensor k_1_begin_0 = const()[name = tensor("k_1_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor k_1_end_0 = const()[name = tensor("k_1_end_0"), val = tensor([2, 224, 1, 128])]; + tensor k_1_end_mask_0 = const()[name = tensor("k_1_end_mask_0"), val = tensor([false, true, true, true])]; + tensor k_1_squeeze_mask_0 = const()[name = tensor("k_1_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor k_1_cast_fp16 = slice_by_index(begin = k_1_begin_0, end = k_1_end_0, end_mask = k_1_end_mask_0, squeeze_mask = k_1_squeeze_mask_0, x = var_127_cast_fp16)[name = tensor("k_1_cast_fp16")]; + tensor v_1_begin_0 = const()[name = tensor("v_1_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor v_1_end_0 = const()[name = tensor("v_1_end_0"), val = tensor([3, 224, 1, 128])]; + tensor v_1_end_mask_0 = const()[name = tensor("v_1_end_mask_0"), val = tensor([false, true, true, true])]; + tensor v_1_squeeze_mask_0 = const()[name = tensor("v_1_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor v_1_cast_fp16 = slice_by_index(begin = v_1_begin_0, end = v_1_end_0, end_mask = v_1_end_mask_0, squeeze_mask = v_1_squeeze_mask_0, x = var_127_cast_fp16)[name = tensor("v_1_cast_fp16")]; + tensor var_135 = const()[name = tensor("op_135"), val = tensor([224, 4, 32])]; + tensor var_136_cast_fp16 = reshape(shape = var_135, x = q_1_cast_fp16)[name = tensor("op_136_cast_fp16")]; + tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([1, 0, 2])]; + tensor var_142 = const()[name = tensor("op_142"), val = tensor([224, 4, 32])]; + tensor var_143_cast_fp16 = reshape(shape = var_142, x = k_1_cast_fp16)[name = tensor("op_143_cast_fp16")]; + tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([1, 0, 2])]; + tensor var_149 = const()[name = tensor("op_149"), val = tensor([224, 4, 32])]; + tensor var_150_cast_fp16 = reshape(shape = var_149, x = v_1_cast_fp16)[name = tensor("op_150_cast_fp16")]; + tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([1, 0, 2])]; + tensor var_153 = const()[name = tensor("op_153"), val = tensor([1, 1, 1, 224])]; + tensor var_154_cast_fp16 = reshape(shape = var_153, x = key_padding_mask_1_cast_fp16)[name = tensor("op_154_cast_fp16")]; + tensor var_156_reps_0 = const()[name = tensor("op_156_reps_0"), val = tensor([1, 4, 1, 1])]; + tensor var_156_cast_fp16 = tile(reps = var_156_reps_0, x = var_154_cast_fp16)[name = tensor("op_156_cast_fp16")]; + tensor var_164 = const()[name = tensor("op_164"), val = tensor([1, 4, 224, 32])]; + tensor q_3_cast_fp16 = transpose(perm = q_3_perm_0, x = var_136_cast_fp16)[name = tensor("transpose_22")]; + tensor q_5_cast_fp16 = reshape(shape = var_164, x = q_3_cast_fp16)[name = tensor("q_5_cast_fp16")]; + tensor var_166 = const()[name = tensor("op_166"), val = tensor([1, 4, 224, 32])]; + tensor k_3_cast_fp16 = transpose(perm = k_3_perm_0, x = var_143_cast_fp16)[name = tensor("transpose_21")]; + tensor k_5_cast_fp16 = reshape(shape = var_166, x = k_3_cast_fp16)[name = tensor("k_5_cast_fp16")]; + tensor var_168 = const()[name = tensor("op_168"), val = tensor([1, 4, 224, 32])]; + tensor v_3_cast_fp16 = transpose(perm = v_3_perm_0, x = var_150_cast_fp16)[name = tensor("transpose_20")]; + tensor v_5_cast_fp16 = reshape(shape = var_168, x = v_3_cast_fp16)[name = tensor("v_5_cast_fp16")]; + tensor mul_1_y_0_to_fp16 = const()[name = tensor("mul_1_y_0_to_fp16"), val = tensor(0x1.6ap-3)]; + tensor mul_1_cast_fp16 = mul(x = q_5_cast_fp16, y = mul_1_y_0_to_fp16)[name = tensor("mul_1_cast_fp16")]; + tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; + tensor matmul_0_transpose_x_0 = const()[name = tensor("matmul_0_transpose_x_0"), val = tensor(false)]; + tensor matmul_0_cast_fp16 = matmul(transpose_x = matmul_0_transpose_x_0, transpose_y = matmul_0_transpose_y_0, x = mul_1_cast_fp16, y = k_5_cast_fp16)[name = tensor("matmul_0_cast_fp16")]; + tensor add_0_cast_fp16 = add(x = matmul_0_cast_fp16, y = var_156_cast_fp16)[name = tensor("add_0_cast_fp16")]; + tensor softmax_0_axis_0 = const()[name = tensor("softmax_0_axis_0"), val = tensor(-1)]; + tensor softmax_0_cast_fp16 = softmax(axis = softmax_0_axis_0, x = add_0_cast_fp16)[name = tensor("softmax_0_cast_fp16")]; + tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; + tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; + tensor attn_output_1_cast_fp16 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0_cast_fp16, y = v_5_cast_fp16)[name = tensor("attn_output_1_cast_fp16")]; + tensor var_171 = const()[name = tensor("op_171"), val = tensor([2, 0, 1, 3])]; + tensor var_176 = const()[name = tensor("op_176"), val = tensor([224, 128])]; + tensor var_172_cast_fp16 = transpose(perm = var_171, x = attn_output_1_cast_fp16)[name = tensor("transpose_19")]; + tensor attn_output_3_cast_fp16 = reshape(shape = var_176, x = var_172_cast_fp16)[name = tensor("attn_output_3_cast_fp16")]; + tensor model_encoder_layers_0_self_attn_out_proj_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_self_attn_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(223680)))]; + tensor model_encoder_layers_0_self_attn_out_proj_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_self_attn_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(256512)))]; + tensor linear_1_cast_fp16 = linear(bias = model_encoder_layers_0_self_attn_out_proj_bias_to_fp16, weight = model_encoder_layers_0_self_attn_out_proj_weight_to_fp16, x = attn_output_3_cast_fp16)[name = tensor("linear_1_cast_fp16")]; + tensor var_180 = const()[name = tensor("op_180"), val = tensor([224, 1, 128])]; + tensor attn_output_7_cast_fp16 = reshape(shape = var_180, x = linear_1_cast_fp16)[name = tensor("attn_output_7_cast_fp16")]; + tensor input_3_perm_0 = const()[name = tensor("input_3_perm_0"), val = tensor([1, 0, 2])]; + tensor input_3_cast_fp16 = transpose(perm = input_3_perm_0, x = attn_output_7_cast_fp16)[name = tensor("transpose_18")]; + tensor input_5_cast_fp16 = add(x = src_1_cast_fp16, y = input_3_cast_fp16)[name = tensor("input_5_cast_fp16")]; + tensor input_7_axes_0 = const()[name = tensor("input_7_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_0_norm2_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_norm2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(256832)))]; + tensor model_encoder_layers_0_norm2_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_norm2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(257152)))]; + tensor input_7_cast_fp16 = layer_norm(axes = input_7_axes_0, beta = model_encoder_layers_0_norm2_bias_to_fp16, epsilon = var_69_to_fp16, gamma = model_encoder_layers_0_norm2_weight_to_fp16, x = input_5_cast_fp16)[name = tensor("input_7_cast_fp16")]; + tensor model_encoder_layers_0_linear1_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_linear1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(257472)))]; + tensor model_encoder_layers_0_linear1_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_linear1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(388608)))]; + tensor linear_2_cast_fp16 = linear(bias = model_encoder_layers_0_linear1_bias_to_fp16, weight = model_encoder_layers_0_linear1_weight_to_fp16, x = input_7_cast_fp16)[name = tensor("linear_2_cast_fp16")]; + tensor input_11_cast_fp16 = relu(x = linear_2_cast_fp16)[name = tensor("input_11_cast_fp16")]; + tensor model_encoder_layers_0_linear2_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_linear2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(389696)))]; + tensor model_encoder_layers_0_linear2_bias_to_fp16 = const()[name = tensor("model_encoder_layers_0_linear2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(520832)))]; + tensor linear_3_cast_fp16 = linear(bias = model_encoder_layers_0_linear2_bias_to_fp16, weight = model_encoder_layers_0_linear2_weight_to_fp16, x = input_11_cast_fp16)[name = tensor("linear_3_cast_fp16")]; + tensor input_17_cast_fp16 = add(x = input_5_cast_fp16, y = linear_3_cast_fp16)[name = tensor("input_17_cast_fp16")]; + tensor query_5_axes_0 = const()[name = tensor("query_5_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_1_norm1_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_norm1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(521152)))]; + tensor model_encoder_layers_1_norm1_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_norm1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(521472)))]; + tensor query_5_cast_fp16 = layer_norm(axes = query_5_axes_0, beta = model_encoder_layers_1_norm1_bias_to_fp16, epsilon = var_69_to_fp16, gamma = model_encoder_layers_1_norm1_weight_to_fp16, x = input_17_cast_fp16)[name = tensor("query_5_cast_fp16")]; + tensor query_7_perm_0 = const()[name = tensor("query_7_perm_0"), val = tensor([1, 0, 2])]; + tensor model_encoder_layers_1_self_attn_in_proj_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_self_attn_in_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(521792)))]; + tensor model_encoder_layers_1_self_attn_in_proj_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_self_attn_in_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(620160)))]; + tensor query_7_cast_fp16 = transpose(perm = query_7_perm_0, x = query_5_cast_fp16)[name = tensor("transpose_17")]; + tensor linear_4_cast_fp16 = linear(bias = model_encoder_layers_1_self_attn_in_proj_bias_to_fp16, weight = model_encoder_layers_1_self_attn_in_proj_weight_to_fp16, x = query_7_cast_fp16)[name = tensor("linear_4_cast_fp16")]; + tensor concat_2 = const()[name = tensor("concat_2"), val = tensor([224, 1, 3, 128])]; + tensor var_231_cast_fp16 = reshape(shape = concat_2, x = linear_4_cast_fp16)[name = tensor("op_231_cast_fp16")]; + tensor var_232_axes_0 = const()[name = tensor("op_232_axes_0"), val = tensor([0])]; + tensor var_232_cast_fp16 = expand_dims(axes = var_232_axes_0, x = var_231_cast_fp16)[name = tensor("op_232_cast_fp16")]; + tensor var_233_perm_0 = const()[name = tensor("op_233_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_234_axes_0 = const()[name = tensor("op_234_axes_0"), val = tensor([-2])]; + tensor var_233_cast_fp16 = transpose(perm = var_233_perm_0, x = var_232_cast_fp16)[name = tensor("transpose_16")]; + tensor var_234_cast_fp16 = squeeze(axes = var_234_axes_0, x = var_233_cast_fp16)[name = tensor("op_234_cast_fp16")]; + tensor q_7_begin_0 = const()[name = tensor("q_7_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor q_7_end_0 = const()[name = tensor("q_7_end_0"), val = tensor([1, 224, 1, 128])]; + tensor q_7_end_mask_0 = const()[name = tensor("q_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor q_7_squeeze_mask_0 = const()[name = tensor("q_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor q_7_cast_fp16 = slice_by_index(begin = q_7_begin_0, end = q_7_end_0, end_mask = q_7_end_mask_0, squeeze_mask = q_7_squeeze_mask_0, x = var_234_cast_fp16)[name = tensor("q_7_cast_fp16")]; + tensor k_7_begin_0 = const()[name = tensor("k_7_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor k_7_end_0 = const()[name = tensor("k_7_end_0"), val = tensor([2, 224, 1, 128])]; + tensor k_7_end_mask_0 = const()[name = tensor("k_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor k_7_squeeze_mask_0 = const()[name = tensor("k_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor k_7_cast_fp16 = slice_by_index(begin = k_7_begin_0, end = k_7_end_0, end_mask = k_7_end_mask_0, squeeze_mask = k_7_squeeze_mask_0, x = var_234_cast_fp16)[name = tensor("k_7_cast_fp16")]; + tensor v_7_begin_0 = const()[name = tensor("v_7_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor v_7_end_0 = const()[name = tensor("v_7_end_0"), val = tensor([3, 224, 1, 128])]; + tensor v_7_end_mask_0 = const()[name = tensor("v_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor v_7_squeeze_mask_0 = const()[name = tensor("v_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor v_7_cast_fp16 = slice_by_index(begin = v_7_begin_0, end = v_7_end_0, end_mask = v_7_end_mask_0, squeeze_mask = v_7_squeeze_mask_0, x = var_234_cast_fp16)[name = tensor("v_7_cast_fp16")]; + tensor var_242 = const()[name = tensor("op_242"), val = tensor([224, 4, 32])]; + tensor var_243_cast_fp16 = reshape(shape = var_242, x = q_7_cast_fp16)[name = tensor("op_243_cast_fp16")]; + tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([1, 0, 2])]; + tensor var_249 = const()[name = tensor("op_249"), val = tensor([224, 4, 32])]; + tensor var_250_cast_fp16 = reshape(shape = var_249, x = k_7_cast_fp16)[name = tensor("op_250_cast_fp16")]; + tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([1, 0, 2])]; + tensor var_256 = const()[name = tensor("op_256"), val = tensor([224, 4, 32])]; + tensor var_257_cast_fp16 = reshape(shape = var_256, x = v_7_cast_fp16)[name = tensor("op_257_cast_fp16")]; + tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([1, 0, 2])]; + tensor var_271 = const()[name = tensor("op_271"), val = tensor([1, 4, 224, 32])]; + tensor q_9_cast_fp16 = transpose(perm = q_9_perm_0, x = var_243_cast_fp16)[name = tensor("transpose_15")]; + tensor q_11_cast_fp16 = reshape(shape = var_271, x = q_9_cast_fp16)[name = tensor("q_11_cast_fp16")]; + tensor var_273 = const()[name = tensor("op_273"), val = tensor([1, 4, 224, 32])]; + tensor k_9_cast_fp16 = transpose(perm = k_9_perm_0, x = var_250_cast_fp16)[name = tensor("transpose_14")]; + tensor k_11_cast_fp16 = reshape(shape = var_273, x = k_9_cast_fp16)[name = tensor("k_11_cast_fp16")]; + tensor var_275 = const()[name = tensor("op_275"), val = tensor([1, 4, 224, 32])]; + tensor v_9_cast_fp16 = transpose(perm = v_9_perm_0, x = var_257_cast_fp16)[name = tensor("transpose_13")]; + tensor v_11_cast_fp16 = reshape(shape = var_275, x = v_9_cast_fp16)[name = tensor("v_11_cast_fp16")]; + tensor mul_3_y_0_to_fp16 = const()[name = tensor("mul_3_y_0_to_fp16"), val = tensor(0x1.6ap-3)]; + tensor mul_3_cast_fp16 = mul(x = q_11_cast_fp16, y = mul_3_y_0_to_fp16)[name = tensor("mul_3_cast_fp16")]; + tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; + tensor matmul_1_transpose_x_0 = const()[name = tensor("matmul_1_transpose_x_0"), val = tensor(false)]; + tensor matmul_1_cast_fp16 = matmul(transpose_x = matmul_1_transpose_x_0, transpose_y = matmul_1_transpose_y_0, x = mul_3_cast_fp16, y = k_11_cast_fp16)[name = tensor("matmul_1_cast_fp16")]; + tensor add_1_cast_fp16 = add(x = matmul_1_cast_fp16, y = var_156_cast_fp16)[name = tensor("add_1_cast_fp16")]; + tensor softmax_1_axis_0 = const()[name = tensor("softmax_1_axis_0"), val = tensor(-1)]; + tensor softmax_1_cast_fp16 = softmax(axis = softmax_1_axis_0, x = add_1_cast_fp16)[name = tensor("softmax_1_cast_fp16")]; + tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; + tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; + tensor attn_output_9_cast_fp16 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1_cast_fp16, y = v_11_cast_fp16)[name = tensor("attn_output_9_cast_fp16")]; + tensor var_278 = const()[name = tensor("op_278"), val = tensor([2, 0, 1, 3])]; + tensor var_283 = const()[name = tensor("op_283"), val = tensor([224, 128])]; + tensor var_279_cast_fp16 = transpose(perm = var_278, x = attn_output_9_cast_fp16)[name = tensor("transpose_12")]; + tensor attn_output_11_cast_fp16 = reshape(shape = var_283, x = var_279_cast_fp16)[name = tensor("attn_output_11_cast_fp16")]; + tensor model_encoder_layers_1_self_attn_out_proj_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_self_attn_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(620992)))]; + tensor model_encoder_layers_1_self_attn_out_proj_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_self_attn_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(653824)))]; + tensor linear_5_cast_fp16 = linear(bias = model_encoder_layers_1_self_attn_out_proj_bias_to_fp16, weight = model_encoder_layers_1_self_attn_out_proj_weight_to_fp16, x = attn_output_11_cast_fp16)[name = tensor("linear_5_cast_fp16")]; + tensor var_287 = const()[name = tensor("op_287"), val = tensor([224, 1, 128])]; + tensor attn_output_15_cast_fp16 = reshape(shape = var_287, x = linear_5_cast_fp16)[name = tensor("attn_output_15_cast_fp16")]; + tensor input_19_perm_0 = const()[name = tensor("input_19_perm_0"), val = tensor([1, 0, 2])]; + tensor input_19_cast_fp16 = transpose(perm = input_19_perm_0, x = attn_output_15_cast_fp16)[name = tensor("transpose_11")]; + tensor input_21_cast_fp16 = add(x = input_17_cast_fp16, y = input_19_cast_fp16)[name = tensor("input_21_cast_fp16")]; + tensor input_23_axes_0 = const()[name = tensor("input_23_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_1_norm2_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_norm2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(654144)))]; + tensor model_encoder_layers_1_norm2_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_norm2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(654464)))]; + tensor input_23_cast_fp16 = layer_norm(axes = input_23_axes_0, beta = model_encoder_layers_1_norm2_bias_to_fp16, epsilon = var_69_to_fp16, gamma = model_encoder_layers_1_norm2_weight_to_fp16, x = input_21_cast_fp16)[name = tensor("input_23_cast_fp16")]; + tensor model_encoder_layers_1_linear1_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_linear1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(654784)))]; + tensor model_encoder_layers_1_linear1_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_linear1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(785920)))]; + tensor linear_6_cast_fp16 = linear(bias = model_encoder_layers_1_linear1_bias_to_fp16, weight = model_encoder_layers_1_linear1_weight_to_fp16, x = input_23_cast_fp16)[name = tensor("linear_6_cast_fp16")]; + tensor input_27_cast_fp16 = relu(x = linear_6_cast_fp16)[name = tensor("input_27_cast_fp16")]; + tensor model_encoder_layers_1_linear2_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_linear2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(787008)))]; + tensor model_encoder_layers_1_linear2_bias_to_fp16 = const()[name = tensor("model_encoder_layers_1_linear2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(918144)))]; + tensor linear_7_cast_fp16 = linear(bias = model_encoder_layers_1_linear2_bias_to_fp16, weight = model_encoder_layers_1_linear2_weight_to_fp16, x = input_27_cast_fp16)[name = tensor("linear_7_cast_fp16")]; + tensor context_cast_fp16 = add(x = input_21_cast_fp16, y = linear_7_cast_fp16)[name = tensor("context_cast_fp16")]; + tensor var_320 = const()[name = tensor("op_320"), val = tensor([32, 96])]; + tensor flat_ids = reshape(shape = var_320, x = option_ids)[name = tensor("flat_ids")]; + tensor var_322 = const()[name = tensor("op_322"), val = tensor(0)]; + tensor flat_mask = not_equal(x = flat_ids, y = var_322)[name = tensor("flat_mask")]; + tensor var_333_begin_0 = const()[name = tensor("op_333_begin_0"), val = tensor([0, 0])]; + tensor var_333_end_0 = const()[name = tensor("op_333_end_0"), val = tensor([32, 1])]; + tensor var_333_end_mask_0 = const()[name = tensor("op_333_end_mask_0"), val = tensor([true, false])]; + tensor var_333 = slice_by_index(begin = var_333_begin_0, end = var_333_end_0, end_mask = var_333_end_mask_0, x = flat_ids)[name = tensor("op_333")]; + tensor fill_like_1_value_0_to_fp16 = const()[name = tensor("fill_like_1_value_0_to_fp16"), val = tensor(0x1p+0)]; + tensor fill_like_1_cast_fp16 = fill_like(ref_tensor = var_333, value = fill_like_1_value_0_to_fp16)[name = tensor("fill_like_1_cast_fp16")]; + tensor var_349_begin_0 = const()[name = tensor("op_349_begin_0"), val = tensor([0, 1])]; + tensor var_349_end_0 = const()[name = tensor("op_349_end_0"), val = tensor([32, 96])]; + tensor var_349_end_mask_0 = const()[name = tensor("op_349_end_mask_0"), val = tensor([true, true])]; + tensor var_349 = slice_by_index(begin = var_349_begin_0, end = var_349_end_0, end_mask = var_349_end_mask_0, x = flat_ids)[name = tensor("op_349")]; + tensor var_351 = const()[name = tensor("op_351"), val = tensor(1)]; + tensor safe_ids_interleave_0 = const()[name = tensor("safe_ids_interleave_0"), val = tensor(false)]; + tensor var_349_promoted_to_fp16_dtype_0 = const()[name = tensor("op_349_promoted_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor var_349_to_fp16 = cast(dtype = var_349_promoted_to_fp16_dtype_0, x = var_349)[name = tensor("cast_62")]; + tensor safe_ids_cast_fp16 = concat(axis = var_351, interleave = safe_ids_interleave_0, values = (fill_like_1_cast_fp16, var_349_to_fp16))[name = tensor("safe_ids_cast_fp16")]; + tensor var_365_batch_dims_0 = const()[name = tensor("op_365_batch_dims_0"), val = tensor(0)]; + tensor var_365_validate_indices_0 = const()[name = tensor("op_365_validate_indices_0"), val = tensor(false)]; + tensor flat_ids_to_int16_dtype_0 = const()[name = tensor("flat_ids_to_int16_dtype_0"), val = tensor("int16")]; + tensor cast_55_dtype_0 = const()[name = tensor("cast_55_dtype_0"), val = tensor("int32")]; + tensor greater_equal_1_y_0 = const()[name = tensor("greater_equal_1_y_0"), val = tensor(0)]; + tensor flat_ids_to_int16 = cast(dtype = flat_ids_to_int16_dtype_0, x = flat_ids)[name = tensor("cast_61")]; + tensor cast_55 = cast(dtype = cast_55_dtype_0, x = flat_ids_to_int16)[name = tensor("cast_60")]; + tensor greater_equal_1 = greater_equal(x = cast_55, y = greater_equal_1_y_0)[name = tensor("greater_equal_1")]; + tensor slice_by_index_19 = const()[name = tensor("slice_by_index_19"), val = tensor(257)]; + tensor add_4 = add(x = cast_55, y = slice_by_index_19)[name = tensor("add_4")]; + tensor select_1 = select(a = cast_55, b = add_4, cond = greater_equal_1)[name = tensor("select_1")]; + tensor var_365_cast_fp16_cast_uint16_axis_0 = const()[name = tensor("op_365_cast_fp16_cast_uint16_axis_0"), val = tensor(0)]; + tensor select_1_to_int16_dtype_0 = const()[name = tensor("select_1_to_int16_dtype_0"), val = tensor("int16")]; + tensor select_1_to_int16 = cast(dtype = select_1_to_int16_dtype_0, x = select_1)[name = tensor("cast_59")]; + tensor var_365_cast_fp16_cast_uint16_cast_uint16 = gather(axis = var_365_cast_fp16_cast_uint16_axis_0, batch_dims = var_365_batch_dims_0, indices = select_1_to_int16, validate_indices = var_365_validate_indices_0, x = model_embedding_weight_to_fp16)[name = tensor("op_365_cast_fp16_cast_uint16_cast_uint16")]; + tensor var_369_to_fp16 = const()[name = tensor("op_369_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(918464)))]; + tensor src_cast_fp16 = add(x = var_365_cast_fp16_cast_uint16_cast_uint16, y = var_369_to_fp16)[name = tensor("src_cast_fp16")]; + tensor var_372_promoted_to_fp16 = const()[name = tensor("op_372_promoted_to_fp16"), val = tensor(0x0p+0)]; + tensor mask_cast_fp16 = equal(x = safe_ids_cast_fp16, y = var_372_promoted_to_fp16)[name = tensor("mask_cast_fp16")]; + tensor var_388_to_fp16 = const()[name = tensor("op_388_to_fp16"), val = tensor(-inf)]; + tensor var_394_to_fp16 = const()[name = tensor("op_394_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(943104)))]; + tensor key_padding_mask_7_cast_fp16 = select(a = var_388_to_fp16, b = var_394_to_fp16, cond = mask_cast_fp16)[name = tensor("key_padding_mask_7_cast_fp16")]; + tensor query_9_axes_0 = const()[name = tensor("query_9_axes_0"), val = tensor([-1])]; + tensor model_option_encoder_layers_0_norm1_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_norm1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(949312)))]; + tensor model_option_encoder_layers_0_norm1_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_norm1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(949632)))]; + tensor var_375_to_fp16 = const()[name = tensor("op_375_to_fp16"), val = tensor(0x1.5p-17)]; + tensor query_9_cast_fp16 = layer_norm(axes = query_9_axes_0, beta = model_option_encoder_layers_0_norm1_bias_to_fp16, epsilon = var_375_to_fp16, gamma = model_option_encoder_layers_0_norm1_weight_to_fp16, x = src_cast_fp16)[name = tensor("query_9_cast_fp16")]; + tensor query_11_perm_0 = const()[name = tensor("query_11_perm_0"), val = tensor([1, 0, 2])]; + tensor model_option_encoder_layers_0_self_attn_in_proj_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_self_attn_in_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(949952)))]; + tensor model_option_encoder_layers_0_self_attn_in_proj_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_self_attn_in_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1048320)))]; + tensor query_11_cast_fp16 = transpose(perm = query_11_perm_0, x = query_9_cast_fp16)[name = tensor("transpose_10")]; + tensor linear_8_cast_fp16 = linear(bias = model_option_encoder_layers_0_self_attn_in_proj_bias_to_fp16, weight = model_option_encoder_layers_0_self_attn_in_proj_weight_to_fp16, x = query_11_cast_fp16)[name = tensor("linear_8_cast_fp16")]; + tensor concat_4 = const()[name = tensor("concat_4"), val = tensor([96, 32, 3, 128])]; + tensor var_428_cast_fp16 = reshape(shape = concat_4, x = linear_8_cast_fp16)[name = tensor("op_428_cast_fp16")]; + tensor var_429_axes_0 = const()[name = tensor("op_429_axes_0"), val = tensor([0])]; + tensor var_429_cast_fp16 = expand_dims(axes = var_429_axes_0, x = var_428_cast_fp16)[name = tensor("op_429_cast_fp16")]; + tensor var_430_perm_0 = const()[name = tensor("op_430_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_431_axes_0 = const()[name = tensor("op_431_axes_0"), val = tensor([-2])]; + tensor var_430_cast_fp16 = transpose(perm = var_430_perm_0, x = var_429_cast_fp16)[name = tensor("transpose_9")]; + tensor var_431_cast_fp16 = squeeze(axes = var_431_axes_0, x = var_430_cast_fp16)[name = tensor("op_431_cast_fp16")]; + tensor q_13_begin_0 = const()[name = tensor("q_13_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor q_13_end_0 = const()[name = tensor("q_13_end_0"), val = tensor([1, 96, 32, 128])]; + tensor q_13_end_mask_0 = const()[name = tensor("q_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor q_13_squeeze_mask_0 = const()[name = tensor("q_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor q_13_cast_fp16 = slice_by_index(begin = q_13_begin_0, end = q_13_end_0, end_mask = q_13_end_mask_0, squeeze_mask = q_13_squeeze_mask_0, x = var_431_cast_fp16)[name = tensor("q_13_cast_fp16")]; + tensor k_13_begin_0 = const()[name = tensor("k_13_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor k_13_end_0 = const()[name = tensor("k_13_end_0"), val = tensor([2, 96, 32, 128])]; + tensor k_13_end_mask_0 = const()[name = tensor("k_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor k_13_squeeze_mask_0 = const()[name = tensor("k_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor k_13_cast_fp16 = slice_by_index(begin = k_13_begin_0, end = k_13_end_0, end_mask = k_13_end_mask_0, squeeze_mask = k_13_squeeze_mask_0, x = var_431_cast_fp16)[name = tensor("k_13_cast_fp16")]; + tensor v_13_begin_0 = const()[name = tensor("v_13_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor v_13_end_0 = const()[name = tensor("v_13_end_0"), val = tensor([3, 96, 32, 128])]; + tensor v_13_end_mask_0 = const()[name = tensor("v_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor v_13_squeeze_mask_0 = const()[name = tensor("v_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor v_13_cast_fp16 = slice_by_index(begin = v_13_begin_0, end = v_13_end_0, end_mask = v_13_end_mask_0, squeeze_mask = v_13_squeeze_mask_0, x = var_431_cast_fp16)[name = tensor("v_13_cast_fp16")]; + tensor var_439 = const()[name = tensor("op_439"), val = tensor([96, 128, 32])]; + tensor var_440_cast_fp16 = reshape(shape = var_439, x = q_13_cast_fp16)[name = tensor("op_440_cast_fp16")]; + tensor q_15_perm_0 = const()[name = tensor("q_15_perm_0"), val = tensor([1, 0, 2])]; + tensor var_446 = const()[name = tensor("op_446"), val = tensor([96, 128, 32])]; + tensor var_447_cast_fp16 = reshape(shape = var_446, x = k_13_cast_fp16)[name = tensor("op_447_cast_fp16")]; + tensor k_15_perm_0 = const()[name = tensor("k_15_perm_0"), val = tensor([1, 0, 2])]; + tensor var_453 = const()[name = tensor("op_453"), val = tensor([96, 128, 32])]; + tensor var_454_cast_fp16 = reshape(shape = var_453, x = v_13_cast_fp16)[name = tensor("op_454_cast_fp16")]; + tensor v_15_perm_0 = const()[name = tensor("v_15_perm_0"), val = tensor([1, 0, 2])]; + tensor var_457 = const()[name = tensor("op_457"), val = tensor([32, 1, 1, 96])]; + tensor var_458_cast_fp16 = reshape(shape = var_457, x = key_padding_mask_7_cast_fp16)[name = tensor("op_458_cast_fp16")]; + tensor var_460_reps_0 = const()[name = tensor("op_460_reps_0"), val = tensor([1, 4, 1, 1])]; + tensor var_460_cast_fp16 = tile(reps = var_460_reps_0, x = var_458_cast_fp16)[name = tensor("op_460_cast_fp16")]; + tensor var_468 = const()[name = tensor("op_468"), val = tensor([32, 4, 96, 32])]; + tensor q_15_cast_fp16 = transpose(perm = q_15_perm_0, x = var_440_cast_fp16)[name = tensor("transpose_8")]; + tensor q_cast_fp16 = reshape(shape = var_468, x = q_15_cast_fp16)[name = tensor("q_cast_fp16")]; + tensor var_470 = const()[name = tensor("op_470"), val = tensor([32, 4, 96, 32])]; + tensor k_15_cast_fp16 = transpose(perm = k_15_perm_0, x = var_447_cast_fp16)[name = tensor("transpose_7")]; + tensor k_cast_fp16 = reshape(shape = var_470, x = k_15_cast_fp16)[name = tensor("k_cast_fp16")]; + tensor var_472 = const()[name = tensor("op_472"), val = tensor([32, 4, 96, 32])]; + tensor v_15_cast_fp16 = transpose(perm = v_15_perm_0, x = var_454_cast_fp16)[name = tensor("transpose_6")]; + tensor v_cast_fp16 = reshape(shape = var_472, x = v_15_cast_fp16)[name = tensor("v_cast_fp16")]; + tensor mul_5_y_0_to_fp16 = const()[name = tensor("mul_5_y_0_to_fp16"), val = tensor(0x1.6ap-3)]; + tensor mul_5_cast_fp16 = mul(x = q_cast_fp16, y = mul_5_y_0_to_fp16)[name = tensor("mul_5_cast_fp16")]; + tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(true)]; + tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; + tensor matmul_2_cast_fp16 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = mul_5_cast_fp16, y = k_cast_fp16)[name = tensor("matmul_2_cast_fp16")]; + tensor add_2_cast_fp16 = add(x = matmul_2_cast_fp16, y = var_460_cast_fp16)[name = tensor("add_2_cast_fp16")]; + tensor softmax_2_axis_0 = const()[name = tensor("softmax_2_axis_0"), val = tensor(-1)]; + tensor softmax_2_cast_fp16 = softmax(axis = softmax_2_axis_0, x = add_2_cast_fp16)[name = tensor("softmax_2_cast_fp16")]; + tensor attn_output_17_transpose_x_0 = const()[name = tensor("attn_output_17_transpose_x_0"), val = tensor(false)]; + tensor attn_output_17_transpose_y_0 = const()[name = tensor("attn_output_17_transpose_y_0"), val = tensor(false)]; + tensor attn_output_17_cast_fp16 = matmul(transpose_x = attn_output_17_transpose_x_0, transpose_y = attn_output_17_transpose_y_0, x = softmax_2_cast_fp16, y = v_cast_fp16)[name = tensor("attn_output_17_cast_fp16")]; + tensor var_475 = const()[name = tensor("op_475"), val = tensor([2, 0, 1, 3])]; + tensor var_480 = const()[name = tensor("op_480"), val = tensor([3072, 128])]; + tensor var_476_cast_fp16 = transpose(perm = var_475, x = attn_output_17_cast_fp16)[name = tensor("transpose_5")]; + tensor attn_output_19_cast_fp16 = reshape(shape = var_480, x = var_476_cast_fp16)[name = tensor("attn_output_19_cast_fp16")]; + tensor model_option_encoder_layers_0_self_attn_out_proj_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_self_attn_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1049152)))]; + tensor model_option_encoder_layers_0_self_attn_out_proj_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_self_attn_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1081984)))]; + tensor linear_9_cast_fp16 = linear(bias = model_option_encoder_layers_0_self_attn_out_proj_bias_to_fp16, weight = model_option_encoder_layers_0_self_attn_out_proj_weight_to_fp16, x = attn_output_19_cast_fp16)[name = tensor("linear_9_cast_fp16")]; + tensor var_484 = const()[name = tensor("op_484"), val = tensor([96, 32, 128])]; + tensor attn_output_cast_fp16 = reshape(shape = var_484, x = linear_9_cast_fp16)[name = tensor("attn_output_cast_fp16")]; + tensor input_35_perm_0 = const()[name = tensor("input_35_perm_0"), val = tensor([1, 0, 2])]; + tensor input_35_cast_fp16 = transpose(perm = input_35_perm_0, x = attn_output_cast_fp16)[name = tensor("transpose_4")]; + tensor input_37_cast_fp16 = add(x = src_cast_fp16, y = input_35_cast_fp16)[name = tensor("input_37_cast_fp16")]; + tensor input_39_axes_0 = const()[name = tensor("input_39_axes_0"), val = tensor([-1])]; + tensor model_option_encoder_layers_0_norm2_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_norm2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1082304)))]; + tensor model_option_encoder_layers_0_norm2_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_norm2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1082624)))]; + tensor input_39_cast_fp16 = layer_norm(axes = input_39_axes_0, beta = model_option_encoder_layers_0_norm2_bias_to_fp16, epsilon = var_375_to_fp16, gamma = model_option_encoder_layers_0_norm2_weight_to_fp16, x = input_37_cast_fp16)[name = tensor("input_39_cast_fp16")]; + tensor model_option_encoder_layers_0_linear1_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_linear1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1082944)))]; + tensor model_option_encoder_layers_0_linear1_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_linear1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1214080)))]; + tensor linear_10_cast_fp16 = linear(bias = model_option_encoder_layers_0_linear1_bias_to_fp16, weight = model_option_encoder_layers_0_linear1_weight_to_fp16, x = input_39_cast_fp16)[name = tensor("linear_10_cast_fp16")]; + tensor input_43_cast_fp16 = relu(x = linear_10_cast_fp16)[name = tensor("input_43_cast_fp16")]; + tensor model_option_encoder_layers_0_linear2_weight_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_linear2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1215168)))]; + tensor model_option_encoder_layers_0_linear2_bias_to_fp16 = const()[name = tensor("model_option_encoder_layers_0_linear2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1346304)))]; + tensor linear_11_cast_fp16 = linear(bias = model_option_encoder_layers_0_linear2_bias_to_fp16, weight = model_option_encoder_layers_0_linear2_weight_to_fp16, x = input_43_cast_fp16)[name = tensor("linear_11_cast_fp16")]; + tensor hidden_cast_fp16 = add(x = input_37_cast_fp16, y = linear_11_cast_fp16)[name = tensor("hidden_cast_fp16")]; + tensor var_504_axes_0 = const()[name = tensor("op_504_axes_0"), val = tensor([-1])]; + tensor var_504 = expand_dims(axes = var_504_axes_0, x = flat_mask)[name = tensor("op_504")]; + tensor weights_to_fp16_dtype_0 = const()[name = tensor("weights_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor var_504_to_fp16 = cast(dtype = weights_to_fp16_dtype_0, x = var_504)[name = tensor("cast_58")]; + tensor var_510_cast_fp16 = mul(x = hidden_cast_fp16, y = var_504_to_fp16)[name = tensor("op_510_cast_fp16")]; + tensor var_515_axes_0 = const()[name = tensor("op_515_axes_0"), val = tensor([1])]; + tensor var_515_keep_dims_0 = const()[name = tensor("op_515_keep_dims_0"), val = tensor(false)]; + tensor var_515_cast_fp16 = reduce_sum(axes = var_515_axes_0, keep_dims = var_515_keep_dims_0, x = var_510_cast_fp16)[name = tensor("op_515_cast_fp16")]; + tensor var_520_axes_0 = const()[name = tensor("op_520_axes_0"), val = tensor([1])]; + tensor var_520_keep_dims_0 = const()[name = tensor("op_520_keep_dims_0"), val = tensor(false)]; + tensor var_520_cast_fp16 = reduce_sum(axes = var_520_axes_0, keep_dims = var_520_keep_dims_0, x = var_504_to_fp16)[name = tensor("op_520_cast_fp16")]; + tensor var_521_to_fp16 = const()[name = tensor("op_521_to_fp16"), val = tensor(0x1p+0)]; + tensor var_522_cast_fp16 = maximum(x = var_520_cast_fp16, y = var_521_to_fp16)[name = tensor("op_522_cast_fp16")]; + tensor pooled_cast_fp16 = real_div(x = var_515_cast_fp16, y = var_522_cast_fp16)[name = tensor("pooled_cast_fp16")]; + tensor var_525 = const()[name = tensor("op_525"), val = tensor([1, 32, -1])]; + tensor options_cast_fp16 = reshape(shape = var_525, x = pooled_cast_fp16)[name = tensor("options_cast_fp16")]; + tensor var_528 = const()[name = tensor("op_528"), val = tensor(-1)]; + tensor input_axes_0 = const()[name = tensor("input_axes_0"), val = tensor([-1])]; + tensor model_head_context_norm_weight_to_fp16 = const()[name = tensor("model_head_context_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1346624)))]; + tensor model_head_context_norm_bias_to_fp16 = const()[name = tensor("model_head_context_norm_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1346944)))]; + tensor var_536_to_fp16 = const()[name = tensor("op_536_to_fp16"), val = tensor(0x1.5p-17)]; + tensor input_cast_fp16 = layer_norm(axes = input_axes_0, beta = model_head_context_norm_bias_to_fp16, epsilon = var_536_to_fp16, gamma = model_head_context_norm_weight_to_fp16, x = context_cast_fp16)[name = tensor("input_cast_fp16")]; + tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; + tensor model_head_option_norm_weight_to_fp16 = const()[name = tensor("model_head_option_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1347264)))]; + tensor model_head_option_norm_bias_to_fp16 = const()[name = tensor("model_head_option_norm_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1347584)))]; + tensor input_53_cast_fp16 = layer_norm(axes = input_53_axes_0, beta = model_head_option_norm_bias_to_fp16, epsilon = var_536_to_fp16, gamma = model_head_option_norm_weight_to_fp16, x = options_cast_fp16)[name = tensor("input_53_cast_fp16")]; + tensor model_head_query_weight_to_fp16 = const()[name = tensor("model_head_query_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1347904)))]; + tensor linear_12_bias_0_to_fp16 = const()[name = tensor("linear_12_bias_0_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1380736)))]; + tensor linear_12_cast_fp16 = linear(bias = linear_12_bias_0_to_fp16, weight = model_head_query_weight_to_fp16, x = input_53_cast_fp16)[name = tensor("linear_12_cast_fp16")]; + tensor model_head_key_weight_to_fp16 = const()[name = tensor("model_head_key_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1381056)))]; + tensor linear_13_cast_fp16 = linear(bias = linear_12_bias_0_to_fp16, weight = model_head_key_weight_to_fp16, x = input_cast_fp16)[name = tensor("linear_13_cast_fp16")]; + tensor model_head_value_weight_to_fp16 = const()[name = tensor("model_head_value_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1413888)))]; + tensor linear_14_cast_fp16 = linear(bias = linear_12_bias_0_to_fp16, weight = model_head_value_weight_to_fp16, x = input_cast_fp16)[name = tensor("linear_14_cast_fp16")]; + tensor matmul_3_transpose_x_1 = const()[name = tensor("matmul_3_transpose_x_1"), val = tensor(false)]; + tensor matmul_3_transpose_y_1 = const()[name = tensor("matmul_3_transpose_y_1"), val = tensor(true)]; + tensor matmul_3_cast_fp16 = matmul(transpose_x = matmul_3_transpose_x_1, transpose_y = matmul_3_transpose_y_1, x = linear_12_cast_fp16, y = linear_13_cast_fp16)[name = tensor("matmul_3_cast_fp16")]; + tensor _inversed_scores_1_y_0_to_fp16 = const()[name = tensor("_inversed_scores_1_y_0_to_fp16"), val = tensor(0x1.6ap-4)]; + tensor _inversed_scores_1_cast_fp16 = mul(x = matmul_3_cast_fp16, y = _inversed_scores_1_y_0_to_fp16)[name = tensor("_inversed_scores_1_cast_fp16")]; + tensor var_567_axes_0 = const()[name = tensor("op_567_axes_0"), val = tensor([1])]; + tensor var_567 = expand_dims(axes = var_567_axes_0, x = context_mask)[name = tensor("op_567")]; + tensor var_569 = logical_not(x = var_567)[name = tensor("op_569")]; + tensor var_529_to_fp16 = const()[name = tensor("op_529_to_fp16"), val = tensor(-inf)]; + tensor scores_cast_fp16 = select(a = var_529_to_fp16, b = _inversed_scores_1_cast_fp16, cond = var_569)[name = tensor("scores_cast_fp16")]; + tensor var_571_cast_fp16 = softmax(axis = var_528, x = scores_cast_fp16)[name = tensor("op_571_cast_fp16")]; + tensor matmul_4_transpose_x_0 = const()[name = tensor("matmul_4_transpose_x_0"), val = tensor(false)]; + tensor matmul_4_transpose_y_0 = const()[name = tensor("matmul_4_transpose_y_0"), val = tensor(false)]; + tensor matmul_4_cast_fp16 = matmul(transpose_x = matmul_4_transpose_x_0, transpose_y = matmul_4_transpose_y_0, x = var_571_cast_fp16, y = linear_14_cast_fp16)[name = tensor("matmul_4_cast_fp16")]; + tensor var_574_cast_fp16 = mul(x = linear_12_cast_fp16, y = matmul_4_cast_fp16)[name = tensor("op_574_cast_fp16")]; + tensor var_576_axes_0 = const()[name = tensor("op_576_axes_0"), val = tensor([-1])]; + tensor var_576_keep_dims_0 = const()[name = tensor("op_576_keep_dims_0"), val = tensor(false)]; + tensor var_576_cast_fp16 = reduce_sum(axes = var_576_axes_0, keep_dims = var_576_keep_dims_0, x = var_574_cast_fp16)[name = tensor("op_576_cast_fp16")]; + tensor _inversed_logits_1_y_0_to_fp16 = const()[name = tensor("_inversed_logits_1_y_0_to_fp16"), val = tensor(0x1.6ap-4)]; + tensor _inversed_logits_1_cast_fp16 = mul(x = var_576_cast_fp16, y = _inversed_logits_1_y_0_to_fp16)[name = tensor("_inversed_logits_1_cast_fp16")]; + tensor var_579 = logical_not(x = option_mask_1)[name = tensor("op_579")]; + tensor logits_3_cast_fp16 = select(a = var_529_to_fp16, b = _inversed_logits_1_cast_fp16, cond = var_579)[name = tensor("logits_3_cast_fp16")]; + tensor var_587_value_0_to_fp16 = const()[name = tensor("op_587_value_0_to_fp16"), val = tensor(-0x1.388p+13)]; + tensor var_587_cast_fp16 = fill_like(ref_tensor = logits_3_cast_fp16, value = var_587_value_0_to_fp16)[name = tensor("op_587_cast_fp16")]; + tensor logits_cast_fp16 = select(a = logits_3_cast_fp16, b = var_587_cast_fp16, cond = option_mask_1)[name = tensor("logits_cast_fp16")]; + tensor logits_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("logits_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; + tensor var_589 = const()[name = tensor("op_589"), val = tensor(-1)]; + tensor var_591_cast_fp16 = softmax(axis = var_589, x = logits_cast_fp16)[name = tensor("op_591_cast_fp16")]; + tensor var_591_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("op_591_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; + tensor probabilities = cast(dtype = var_591_cast_fp16_to_fp32_dtype_0, x = var_591_cast_fp16)[name = tensor("cast_56")]; + tensor logits = cast(dtype = logits_cast_fp16_to_fp32_dtype_0, x = logits_cast_fp16)[name = tensor("cast_57")]; + } -> (logits, probabilities); +} \ No newline at end of file diff --git a/cua_s1_forms_fp16_options32.mlmodelc/weights/weight.bin b/cua_s1_forms_fp16_options32.mlmodelc/weights/weight.bin new file mode 100644 index 0000000000000000000000000000000000000000..bff6eee099e8b8b4e659c073c6b48112ba92dfb6 --- /dev/null +++ b/cua_s1_forms_fp16_options32.mlmodelc/weights/weight.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895 +size 1446720 diff --git a/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel b/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel new file mode 100644 index 0000000000000000000000000000000000000000..e66c556884bb15addfdd5d0a5691417d06a2c94a --- /dev/null +++ b/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0 +size 63826 diff --git a/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin b/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin new file mode 100644 index 0000000000000000000000000000000000000000..bff6eee099e8b8b4e659c073c6b48112ba92dfb6 --- /dev/null +++ b/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895 +size 1446720 diff --git a/cua_s1_forms_fp16_options32.mlpackage/Manifest.json b/cua_s1_forms_fp16_options32.mlpackage/Manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..8ea7c82615bc9a763cb50d15d043e692b356f502 --- /dev/null +++ b/cua_s1_forms_fp16_options32.mlpackage/Manifest.json @@ -0,0 +1,18 @@ +{ + "fileFormatVersion": "1.0.0", + "itemInfoEntries": { + "A4ADBA80-1EEA-488D-A01F-25F39ECC9001": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" + }, + "B61C4C42-FF5A-4E18-BC04-DD11C9CE38DE": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" + } + }, + "rootModelIdentifier": "B61C4C42-FF5A-4E18-BC04-DD11C9CE38DE" +} diff --git a/demo/browser-demo.gif b/demo/browser-demo.gif new file mode 100644 index 0000000000000000000000000000000000000000..33690527a8e50be532645dbfa486f4fb77a499af --- /dev/null +++ b/demo/browser-demo.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cf597927e7bcb5617719453eecb86c7c06ab9a91fcaf46de272b42abe12d4fc +size 1797482 diff --git a/demo/browser-demo.mp4 b/demo/browser-demo.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..947185c49aa2df9a9482b86a8166239121713705 --- /dev/null +++ b/demo/browser-demo.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fbb3925c4824d15e6c24c504db91a96d318f4e751a20a070fdedf64e587d1a0 +size 1108747 diff --git a/demo/browser-preview.png b/demo/browser-preview.png new file mode 100644 index 0000000000000000000000000000000000000000..67e39e6967aa9ab41d3a647c867d2d0b377bc806 --- /dev/null +++ b/demo/browser-preview.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e624ec2a797d0b3b050e66055d6a9fb09a1f6a9ace67b8eb071ce5f2f9e282b1 +size 1359763 diff --git a/int4-source-fp16/conversion.json b/int4-source-fp16/conversion.json new file mode 100644 index 0000000000000000000000000000000000000000..ce53928d73ba784e57304af45f62c489a91bbe4c --- /dev/null +++ b/int4-source-fp16/conversion.json @@ -0,0 +1,73 @@ +{ + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "minimum_target": "iOS18/macOS15", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "e5b1c7dc66b2014b3985c180a75548a709788c6e314d9d2ae14e4e6d441c3c68" + }, + "target_upgrade": { + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "method": "Load original MIL at specification 9; empty pass pipeline; retain decomposed attention", + "operation_counts": { + "add": 13, + "cast": 11, + "concat": 2, + "const": 239, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 2, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 7, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f" + } +} diff --git a/int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel b/int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel new file mode 100644 index 0000000000000000000000000000000000000000..1a1474818487095faf5ac73064725c557d70b2a7 --- /dev/null +++ b/int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627 +size 63743 diff --git a/int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin b/int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin new file mode 100644 index 0000000000000000000000000000000000000000..bff6eee099e8b8b4e659c073c6b48112ba92dfb6 --- /dev/null +++ b/int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895 +size 1446720 diff --git a/int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Manifest.json b/int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..46a176e6556982fe48ad48826d2ec54dd682683b --- /dev/null +++ b/int4-source-fp16/cua_s1_forms_fp16_options32.mlpackage/Manifest.json @@ -0,0 +1,18 @@ +{ + "fileFormatVersion": "1.0.0", + "itemInfoEntries": { + "15C1AFE1-37B2-4975-949D-B817E3C00088": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" + }, + "B997D799-579D-437C-9F80-DE24FF3EFB11": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" + } + }, + "rootModelIdentifier": "15C1AFE1-37B2-4975-949D-B817E3C00088" +} diff --git a/int4-weights/conversion.json b/int4-weights/conversion.json new file mode 100644 index 0000000000000000000000000000000000000000..e865eba4ec51d527efbfd54dae260141a31e3939 --- /dev/null +++ b/int4-weights/conversion.json @@ -0,0 +1,124 @@ +{ + "model": "cua_s1_forms_int4_options32.mlpackage", + "precision": "int4_weights_float16_compute", + "minimum_target": "iOS18/macOS15", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "cadad6bfa6875383a7ee3de68d0d22871148aa21cf117e7fe44c2c971287c4e5", + "Data/com.apple.CoreML/weights/weight.bin": "0f766fcb9dbd612ed4f6c7a40117a5993a4789b3ae94afba0df37193c831f63c", + "Manifest.json": "fb91c8b5ec3168111567de663bb3140d2790c6e7c48f32e0df3f5135af0673af" + }, + "target_upgrade": { + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "method": "Load original MIL at specification 9; empty pass pipeline; retain decomposed attention", + "operation_counts": { + "add": 13, + "cast": 11, + "concat": 2, + "const": 239, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 2, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 7, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f" + }, + "quantization": { + "name": "int4-weights", + "settings": { + "mode": "linear_symmetric", + "dtype": "int4", + "granularity": "per_channel", + "weight_threshold": 2048 + }, + "activations": "float16", + "calibration": "None; weight-only, data-free quantization", + "source_conversion_sha256": "01105dc8bef7a30fc3430e150caa2a143dc0ba23a295b6280de3b8ff3b40af55", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "e5b1c7dc66b2014b3985c180a75548a709788c6e314d9d2ae14e4e6d441c3c68" + }, + "source_package_bytes": 1511080, + "package_bytes": 451595, + "compression_seconds": 0.2996572910051327, + "packed_int4_tensors": 19, + "operation_counts": { + "add": 15, + "cast": 13, + "concat": 2, + "const": 226, + "constexpr_blockwise_shift_scale": 19, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 4, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 9, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "a91636e539d8934253b5e7628ca706346fbee538fa1aa9e3eaf794f440d49dcc" + } +} diff --git a/int4-weights/cua_s1_forms_int4_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel b/int4-weights/cua_s1_forms_int4_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel new file mode 100644 index 0000000000000000000000000000000000000000..b90aff190fbc397c1b85fa0ba800cd35ac7a2ef2 --- /dev/null +++ b/int4-weights/cua_s1_forms_int4_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cadad6bfa6875383a7ee3de68d0d22871148aa21cf117e7fe44c2c971287c4e5 +size 67938 diff --git a/int4-weights/cua_s1_forms_int4_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin b/int4-weights/cua_s1_forms_int4_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin new file mode 100644 index 0000000000000000000000000000000000000000..bfdc59e78027d06d8f33eea1f61db40c86874fce --- /dev/null +++ b/int4-weights/cua_s1_forms_int4_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f766fcb9dbd612ed4f6c7a40117a5993a4789b3ae94afba0df37193c831f63c +size 383040 diff --git a/int4-weights/cua_s1_forms_int4_options32.mlpackage/Manifest.json b/int4-weights/cua_s1_forms_int4_options32.mlpackage/Manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a5db2c90fb5cd29098a2e6443f085af7f6ae34df --- /dev/null +++ b/int4-weights/cua_s1_forms_int4_options32.mlpackage/Manifest.json @@ -0,0 +1,18 @@ +{ + "fileFormatVersion": "1.0.0", + "itemInfoEntries": { + "D6D01355-3106-40A1-999B-B56148C73691": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" + }, + "F0E58006-D8D0-4891-AD75-BB9E48868A99": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" + } + }, + "rootModelIdentifier": "F0E58006-D8D0-4891-AD75-BB9E48868A99" +} diff --git a/int8-weights/conversion.json b/int8-weights/conversion.json new file mode 100644 index 0000000000000000000000000000000000000000..fe9abf25c24600ddc9dc882c52c6b3e33bcfb913 --- /dev/null +++ b/int8-weights/conversion.json @@ -0,0 +1,85 @@ +{ + "model": "cua_s1_forms_int8_options32.mlpackage", + "precision": "int8_weights_float16_compute", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "a7d9747308b8b754a46f3e70c8465b821f90e12376b3a71dc909f7364ae73c3a", + "Data/com.apple.CoreML/weights/weight.bin": "f600c4434b7f7e3e789de8f1a4aa54a55f84df51fa9b788e18f5c97b68e4840f", + "Manifest.json": "2206ec01787a85c12f1376fa5a832c5ac912f06172c6ff06b26ae8415470d246" + }, + "quantization": { + "name": "int8-weights", + "settings": { + "mode": "linear_symmetric", + "dtype": "int8", + "granularity": "per_channel", + "weight_threshold": 2048 + }, + "activations": "float16", + "calibration": "None; weight-only, data-free quantization", + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "source_package_bytes": 1511163, + "package_bytes": 813422, + "compression_seconds": 0.3180606249952689, + "operation_counts": { + "add": 15, + "cast": 13, + "concat": 2, + "const": 226, + "constexpr_affine_dequantize": 19, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 4, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 9, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "22071babfcb02ef77225203a479d9bb14b9fa35462ba38bcc4bf07d29c125a97" + } +} diff --git a/int8-weights/cua_s1_forms_int8_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel b/int8-weights/cua_s1_forms_int8_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel new file mode 100644 index 0000000000000000000000000000000000000000..d7ff287335be936c5cbf05eb320032cc121c49ae --- /dev/null +++ b/int8-weights/cua_s1_forms_int8_options32.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7d9747308b8b754a46f3e70c8465b821f90e12376b3a71dc909f7364ae73c3a +size 69573 diff --git a/int8-weights/cua_s1_forms_int8_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin b/int8-weights/cua_s1_forms_int8_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin new file mode 100644 index 0000000000000000000000000000000000000000..6d885cd7f98d0a0551315f7578318feb92608af0 --- /dev/null +++ b/int8-weights/cua_s1_forms_int8_options32.mlpackage/Data/com.apple.CoreML/weights/weight.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f600c4434b7f7e3e789de8f1a4aa54a55f84df51fa9b788e18f5c97b68e4840f +size 743232 diff --git a/int8-weights/cua_s1_forms_int8_options32.mlpackage/Manifest.json b/int8-weights/cua_s1_forms_int8_options32.mlpackage/Manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3c9eb75da5fb06c53bac4daf577988e419ebd9c6 --- /dev/null +++ b/int8-weights/cua_s1_forms_int8_options32.mlpackage/Manifest.json @@ -0,0 +1,18 @@ +{ + "fileFormatVersion": "1.0.0", + "itemInfoEntries": { + "6D5496F1-54BB-4AAA-8375-1DFA1FFE6768": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" + }, + "B0C97583-069E-42B8-897B-30A05F77B14A": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" + } + }, + "rootModelIdentifier": "6D5496F1-54BB-4AAA-8375-1DFA1FFE6768" +} diff --git a/preprocessing.py b/preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..be2c62883dda893327fe75f18832770903c323ad --- /dev/null +++ b/preprocessing.py @@ -0,0 +1,46 @@ +"""Host-side UTF-8 byte encoding for the fixed-shape Core ML interface.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + + +@dataclass(frozen=True) +class InputLimits: + context_bytes: int = 224 + option_bytes: int = 96 + max_options: int = 32 + + def __post_init__(self): + if self.context_bytes < 1 or self.option_bytes < 1 or self.max_options < 2: + raise ValueError("Positive byte limits and at least two option slots are required") + + +def byte_ids(text: str, length: int) -> np.ndarray: + """Match upstream: truncate UTF-8 bytes, offset byte values by one, zero-pad.""" + data = text.encode("utf-8", errors="replace")[:length] + return np.frombuffer(data, dtype=np.uint8).astype(np.int32) + 1 + + +def prepare_inputs(context: str, options: list[str] | tuple[str, ...], limits: InputLimits) -> dict[str, np.ndarray]: + """Encode one decision; options are never silently removed to fit the model.""" + if not isinstance(context, str) or not context: + raise ValueError("context must be a nonempty string") + if not isinstance(options, (list, tuple)) or len(options) < 2: + raise ValueError("At least two options are required") + if len(options) > limits.max_options: + raise ValueError(f"{len(options)} options exceed {limits.max_options}; re-export with a larger --max-options") + if any(not isinstance(option, str) or not option for option in options): + raise ValueError("Options must be nonempty strings") + context_ids = np.zeros((1, limits.context_bytes), dtype=np.int32) + option_ids = np.zeros((1, limits.max_options, limits.option_bytes), dtype=np.int32) + option_mask = np.zeros((1, limits.max_options), dtype=np.int32) + tokens = byte_ids(context, limits.context_bytes) + context_ids[0, : len(tokens)] = tokens + for index, option in enumerate(options): + tokens = byte_ids(option, limits.option_bytes) + option_ids[0, index, : len(tokens)] = tokens + option_mask[0, : len(options)] = 1 + return {"context_ids": context_ids, "option_ids": option_ids, "option_mask": option_mask} diff --git a/reports/ane-comparison.json b/reports/ane-comparison.json new file mode 100644 index 0000000000000000000000000000000000000000..3415c2e6aa77de6da5fe7f7de77557ff923faf36 --- /dev/null +++ b/reports/ane-comparison.json @@ -0,0 +1,784 @@ +{ + "created_utc": "2026-09-19T20:07:16.396741+00:00", + "purpose": "Matched warm model-call comparison, not a power or utilization measurement", + "environment": { + "chip": "Apple M5 Pro", + "macos": "27.0", + "os_build": "26A428", + "python": "3.11.11", + "coremltools": "9.0" + }, + "protocol": { + "compute_units": "CPU_AND_NE", + "rows": [ + 0, + 68, + 130 + ], + "schedule": [ + "baseline", + "candidate", + "candidate", + "baseline" + ], + "warmup_passes_per_block": 2, + "timed_passes_per_block": 10, + "timing_scope": "Synchronous Python predict with pre-encoded inputs; validation outside timer", + "model_loading": "Both models resident in one process; loads excluded and system caches retained" + }, + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "package_files": { + "baseline": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "candidate": { + "Data/com.apple.CoreML/model.mlmodel": "de18e313c3b625e35d008ed8b6b24108edc6df7bf2fae9533af03519eca11b63", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "38f812a04eb2322080634ba788c61df362336168466ca67e5549058d346ac793" + } + }, + "summary": { + "baseline": { + "count": 60, + "median_ms": 0.9150624828180298, + "p95_ms": 0.9682062489446251, + "min_ms": 0.88170898379758, + "max_ms": 1.0281660070177168 + }, + "candidate": { + "count": 60, + "median_ms": 0.9703329997137189, + "p95_ms": 0.9877315780613571, + "min_ms": 0.9434169915039092, + "max_ms": 1.0195420181844383 + } + }, + "candidate_over_baseline_median": 1.0604008118937165, + "correct_timed_predictions": 120, + "calls": [ + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 0.919707992579788 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9307919826824218 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 0.9576249867677689 + }, + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 0.9234579920303077 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9247499983757734 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 0.9314999915659428 + }, + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 0.9133330022450536 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9190000128000975 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 0.9215839963871986 + }, + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 0.9041670127771795 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9001670114230365 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 0.9362080018036067 + }, + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 1.0092500015161932 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9958750160876662 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 1.0281660070177168 + }, + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 0.96674999804236 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9511670214124024 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 0.9152919810730964 + }, + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 0.9129999962169677 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9089170198421925 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 0.9163749928120524 + }, + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 0.9026250045280904 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9094580018427223 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 0.9331250039394945 + }, + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 0.9327089937869459 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9123749914579093 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 0.9423340088687837 + }, + { + "model": "baseline", + "block": 0, + "row": 0, + "milliseconds": 0.9080420131795108 + }, + { + "model": "baseline", + "block": 0, + "row": 68, + "milliseconds": 0.9152499842457473 + }, + { + "model": "baseline", + "block": 0, + "row": 130, + "milliseconds": 0.9090829989872873 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.9569999820087105 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.9605420054867864 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.971125002251938 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.9833329822868109 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.9731249883770943 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.9744580020196736 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.9531670075375587 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.960625009611249 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.9434169915039092 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.965375016676262 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.9999169851653278 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.9775409998837858 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.983749981969595 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.9759170061443001 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.979083008132875 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.9849590132944286 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.9607080137357116 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.9490000084042549 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.9506659989710897 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.9708750003483146 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.9733340120874345 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.9872919763438404 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.970749999396503 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.9687079873401672 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.977249990683049 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.9785419970285147 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.9744160051923245 + }, + { + "model": "candidate", + "block": 1, + "row": 0, + "milliseconds": 0.9864169987849891 + }, + { + "model": "candidate", + "block": 1, + "row": 68, + "milliseconds": 0.982333003776148 + }, + { + "model": "candidate", + "block": 1, + "row": 130, + "milliseconds": 0.9689999860711396 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.9796250087674707 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 0.9837090037763119 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9523750049993396 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.9744999988470227 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 0.9580420155543834 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9489580115769058 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.9510410018265247 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 0.9449580102227628 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9497080172877759 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.9529579838272184 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 0.9495420090388507 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9463340102229267 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.9437499975319952 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 0.9775829967111349 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9694590116851032 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.996084010694176 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 1.0195420181844383 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9871670044958591 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.9699160000309348 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 0.9592499991413206 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9731660247780383 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.9729999874252826 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 0.9556250006426126 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9530829847790301 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.9495830163359642 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 0.9697090135887265 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9829580085352063 + }, + { + "model": "candidate", + "block": 2, + "row": 0, + "milliseconds": 0.9604159859009087 + }, + { + "model": "candidate", + "block": 2, + "row": 68, + "milliseconds": 0.9743330010678619 + }, + { + "model": "candidate", + "block": 2, + "row": 130, + "milliseconds": 0.9570419788360596 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.927665998460725 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.9297500073444098 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.9148749813903123 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.8912909834180027 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.88170898379758 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.9040829900186509 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.8952920034062117 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.9061660093721002 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.9120829927269369 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.9074159897863865 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.9095839923247695 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.883542001247406 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.8904590213205665 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.8850830199662596 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.8904580026865005 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.9044160251505673 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.9244170214515179 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.9261659870389849 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.9326669969595969 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.9306249849032611 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.8886670111678541 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.8824579999782145 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.8989160123746842 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.8977499965112656 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.8965420129243284 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.9054580004885793 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.93029101844877 + }, + { + "model": "baseline", + "block": 3, + "row": 0, + "milliseconds": 0.9197920153383166 + }, + { + "model": "baseline", + "block": 3, + "row": 68, + "milliseconds": 0.9319579985458404 + }, + { + "model": "baseline", + "block": 3, + "row": 130, + "milliseconds": 0.9407080069649965 + } + ], + "script_sha256": "4633df7ad5f90d6aed37d5ef4f7a2d2231fdadd272ef1aa3057c237b2f937df8" +} diff --git a/reports/ane-fallback.json b/reports/ane-fallback.json new file mode 100644 index 0000000000000000000000000000000000000000..840caf55157f362e78c3c37b15cc9c211b982e4d --- /dev/null +++ b/reports/ane-fallback.json @@ -0,0 +1,84 @@ +{ + "hardware": { + "device": "arm64", + "chip": "Apple M5 Pro", + "ram": "24GB", + "os_version": "macOS 27.0", + "timestamp": "2026-09-19T19:41:20.560977+00:00" + }, + "models": [ + { + "model_path": "build/cua_s1_forms_fp16_options32.mlmodelc", + "model_name": "cua_s1_forms_fp16_options32", + "fallback": { + "compute_units": "cpu_and_neural_engine", + "total_ops": 173, + "ane_ops": 149, + "gpu_ops": 0, + "cpu_ops": 24, + "ane_percent": 86.1, + "reasons": [ + { + "reason": "Unsupported tensor data type: int32", + "count": 17, + "estimated_cpu_runtime_ms": 0.017, + "op_types": { + "ios17.cast": 8, + "ios17.slice_by_index": 4, + "ios17.add": 2, + "select": 2, + "ios17.reshape": 1 + }, + "ops": [ + "var_27", + "var_43", + "var_43_to_fp16", + "context_ids_to_int16", + "cast_54", + "add_3", + "select_0", + "select_0_to_int16", + "flat_ids", + "var_333", + "var_349", + "var_349_to_fp16", + "flat_ids_to_int16", + "cast_55", + "add_4", + "select_1", + "select_1_to_int16" + ] + }, + { + "reason": "Unable to resolve operation input \"y\".", + "count": 5, + "estimated_cpu_runtime_ms": 0.005, + "op_types": { + "ios17.not_equal": 3, + "ios17.greater_equal": 2 + }, + "ops": [ + "option_mask_1", + "context_mask", + "greater_equal_0", + "flat_mask", + "greater_equal_1" + ] + }, + { + "reason": "Unsupported gather index type", + "count": 2, + "estimated_cpu_runtime_ms": 0.0144, + "op_types": { + "ios17.gather": 2 + }, + "ops": [ + "var_59_cast_fp16_cast_uint16_cast_uint16", + "var_365_cast_fp16_cast_uint16_cast_uint16" + ] + } + ] + } + } + ] +} diff --git a/reports/ane-gather-fallback.json b/reports/ane-gather-fallback.json new file mode 100644 index 0000000000000000000000000000000000000000..3a4d726fdb6fa13330625f9143b48948020a9b0b --- /dev/null +++ b/reports/ane-gather-fallback.json @@ -0,0 +1,38 @@ +{ + "hardware": { + "device": "arm64", + "chip": "Apple M5 Pro", + "ram": "24GB", + "os_version": "macOS 27.0", + "timestamp": "2026-09-19T20:08:52.988799+00:00" + }, + "models": [ + { + "model_path": "build/ane-gather/cua_s1_forms_fp16_options32.mlmodelc", + "model_name": "cua_s1_forms_fp16_options32", + "fallback": { + "compute_units": "cpu_and_neural_engine", + "total_ops": 165, + "ane_ops": 162, + "gpu_ops": 0, + "cpu_ops": 3, + "ane_percent": 98.2, + "reasons": [ + { + "reason": "Unsupported tensor data type: int32", + "count": 3, + "estimated_cpu_runtime_ms": 0.003, + "op_types": { + "ios17.cast": 3 + }, + "ops": [ + "option_mask_to_fp16", + "context_ids_to_fp16", + "option_ids_to_fp16" + ] + } + ] + } + } + ] +} diff --git a/reports/ane-gather-profile.json b/reports/ane-gather-profile.json new file mode 100644 index 0000000000000000000000000000000000000000..9247dbeede7fd74b006cfbf09ba7723d91499783 --- /dev/null +++ b/reports/ane-gather-profile.json @@ -0,0 +1,3949 @@ +{ + "created_utc": "2026-09-19T20:07:40.458353+00:00", + "purpose": "Bounded real-input latency and scheduler placement; not a general accuracy benchmark", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "de18e313c3b625e35d008ed8b6b24108edc6df7bf2fae9533af03519eca11b63", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "38f812a04eb2322080634ba788c61df362336168466ca67e5549058d346ac793" + }, + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "environment": { + "chip": "Apple M5 Pro", + "memory_bytes": 25769803776, + "macos": "27.0", + "os_build": "26A428", + "python": "3.11.11", + "coremltools": "9.0" + }, + "protocol": { + "rows": [ + 0, + 68, + 130 + ], + "pages": [ + "patient-registration", + "job-application", + "auto-claim" + ], + "option_counts": [ + 27, + 21, + 19 + ], + "first_call_row": 0, + "warmup_passes": 2, + "timed_passes": 10, + "backend_order": [ + "CPU_ONLY", + "CPU_AND_GPU", + "CPU_AND_NE", + "ALL" + ], + "timing_scope": "Synchronous Python MLModel.predict; pre-encoded inputs; output checks outside timer", + "load_scope": "MLModel construction; system caches retained, not first-install cold start", + "first_call_scope": "First predict after this load; may use existing system/ANE caches", + "plan_order": "Compute plan loaded after timing each backend" + }, + "backends": { + "CPU_ONLY": { + "load_ms": 77.25858301273547, + "first_prediction_ms": 17.035917000612244, + "warm_prediction": { + "count": 30, + "median_ms": 1.6528124979231507, + "p95_ms": 1.7509769095340744, + "min_ms": 1.4510419860016555, + "max_ms": 1.7862079839687794 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 1.6987500130198896 + }, + { + "row": 68, + "milliseconds": 1.7862079839687794 + }, + { + "row": 130, + "milliseconds": 1.65049999486655 + }, + { + "row": 0, + "milliseconds": 1.716375001706183 + }, + { + "row": 68, + "milliseconds": 1.7633330135140568 + }, + { + "row": 130, + "milliseconds": 1.4510419860016555 + }, + { + "row": 0, + "milliseconds": 1.6525420069228858 + }, + { + "row": 68, + "milliseconds": 1.556083996547386 + }, + { + "row": 130, + "milliseconds": 1.6807909996714443 + }, + { + "row": 0, + "milliseconds": 1.7177920090034604 + }, + { + "row": 68, + "milliseconds": 1.7270410025957972 + }, + { + "row": 130, + "milliseconds": 1.7195830005221069 + }, + { + "row": 0, + "milliseconds": 1.6070419806055725 + }, + { + "row": 68, + "milliseconds": 1.664000010350719 + }, + { + "row": 130, + "milliseconds": 1.6511249996256083 + }, + { + "row": 0, + "milliseconds": 1.7358750046696514 + }, + { + "row": 68, + "milliseconds": 1.682458008872345 + }, + { + "row": 130, + "milliseconds": 1.6572079912293702 + }, + { + "row": 0, + "milliseconds": 1.581791992066428 + }, + { + "row": 68, + "milliseconds": 1.6358340217266232 + }, + { + "row": 130, + "milliseconds": 1.625500008231029 + }, + { + "row": 0, + "milliseconds": 1.6221669793594629 + }, + { + "row": 68, + "milliseconds": 1.6681250126566738 + }, + { + "row": 130, + "milliseconds": 1.6530829889234155 + }, + { + "row": 0, + "milliseconds": 1.535832998342812 + }, + { + "row": 68, + "milliseconds": 1.5366250008810312 + }, + { + "row": 130, + "milliseconds": 1.575667003635317 + }, + { + "row": 0, + "milliseconds": 1.6152499883901328 + }, + { + "row": 68, + "milliseconds": 1.7054580093827099 + }, + { + "row": 130, + "milliseconds": 1.6136250051204115 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 165, + "device_counts": { + "cpu": 165, + "gpu": 0, + "ane": 0, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 233 + }, + "operations": [ + { + "name": "option_mask_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "option_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "context_ids_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "context_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "var_42_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "cpu" + }, + { + "name": "var_58_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios17.concat", + "device": "cpu" + }, + { + "name": "cast_0", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "gather_0", + "type": "ios17.gather", + "device": "cpu" + }, + { + "name": "src_1_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios17.equal", + "device": "cpu" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "query_1_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "query_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "var_139_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_140_cast_fp16", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_141_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "var_142_cast_fp16", + "type": "ios17.squeeze", + "device": "cpu" + }, + { + "name": "q_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "k_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "v_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_151_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_158_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_165_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_169_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_171_cast_fp16", + "type": "tile", + "device": "cpu" + }, + { + "name": "q_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "q_5_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "k_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "k_5_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "v_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "v_5_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "add_0_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "var_187_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "input_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "input_5_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "input_7_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "input_11_cast_fp16", + "type": "ios16.relu", + "device": "cpu" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "input_17_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "query_5_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "query_7_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "var_246_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_247_cast_fp16", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_248_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "var_249_cast_fp16", + "type": "ios17.squeeze", + "device": "cpu" + }, + { + "name": "q_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "k_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "v_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_258_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_265_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_272_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "q_9_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "q_11_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "k_9_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "k_11_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "v_9_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "v_11_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "add_1_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "var_294_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "input_19_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "input_21_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "input_23_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "input_27_cast_fp16", + "type": "ios16.relu", + "device": "cpu" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "context_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "option_ids_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "mask_flat_ids_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "flat_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "var_353_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "cpu" + }, + { + "name": "var_369_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios17.concat", + "device": "cpu" + }, + { + "name": "reshape_0", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "cast_1", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "gather_1", + "type": "ios17.gather", + "device": "cpu" + }, + { + "name": "src_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "mask_cast_fp16", + "type": "ios17.equal", + "device": "cpu" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "query_9_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "query_11_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "var_448_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_449_cast_fp16", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_450_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "var_451_cast_fp16", + "type": "ios17.squeeze", + "device": "cpu" + }, + { + "name": "q_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "k_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "v_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_460_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_467_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_474_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_478_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_480_cast_fp16", + "type": "tile", + "device": "cpu" + }, + { + "name": "q_15_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "q_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "k_15_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "k_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "v_15_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "v_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "add_2_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "var_496_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "input_35_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "input_37_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "input_39_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "input_43_cast_fp16", + "type": "ios16.relu", + "device": "cpu" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "hidden_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "var_524", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_524_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "var_530_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "var_535_cast_fp16", + "type": "ios16.reduce_sum", + "device": "cpu" + }, + { + "name": "var_540_cast_fp16", + "type": "ios16.reduce_sum", + "device": "cpu" + }, + { + "name": "var_542_cast_fp16", + "type": "ios17.maximum", + "device": "cpu" + }, + { + "name": "pooled_cast_fp16", + "type": "ios17.real_div", + "device": "cpu" + }, + { + "name": "options_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "input_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "input_53_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "var_587", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_589", + "type": "ios17.logical_not", + "device": "cpu" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "var_591_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "var_594_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "var_596_cast_fp16", + "type": "ios16.reduce_sum", + "device": "cpu" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "var_599", + "type": "ios17.logical_not", + "device": "cpu" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "var_607_cast_fp16", + "type": "ios16.fill_like", + "device": "cpu" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "var_611_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "probabilities", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "logits", + "type": "ios17.cast", + "device": "cpu" + } + ] + } + }, + "CPU_AND_GPU": { + "load_ms": 83.79112498369068, + "first_prediction_ms": 442.5840419717133, + "warm_prediction": { + "count": 30, + "median_ms": 2.237292006611824, + "p95_ms": 2.9901315414463165, + "min_ms": 1.8634590087458491, + "max_ms": 3.190709016053006 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 2.2560000070370734 + }, + { + "row": 68, + "milliseconds": 2.181834017392248 + }, + { + "row": 130, + "milliseconds": 2.287249983055517 + }, + { + "row": 0, + "milliseconds": 2.241292007965967 + }, + { + "row": 68, + "milliseconds": 1.8634590087458491 + }, + { + "row": 130, + "milliseconds": 2.1740839874837548 + }, + { + "row": 0, + "milliseconds": 2.291666984092444 + }, + { + "row": 68, + "milliseconds": 2.3829590063542128 + }, + { + "row": 130, + "milliseconds": 1.964750001206994 + }, + { + "row": 0, + "milliseconds": 1.899958006106317 + }, + { + "row": 68, + "milliseconds": 2.066874993033707 + }, + { + "row": 130, + "milliseconds": 2.374832984060049 + }, + { + "row": 0, + "milliseconds": 2.396042022155598 + }, + { + "row": 68, + "milliseconds": 3.0421249975916 + }, + { + "row": 130, + "milliseconds": 2.23837501835078 + }, + { + "row": 0, + "milliseconds": 2.1047500194981694 + }, + { + "row": 68, + "milliseconds": 2.236208994872868 + }, + { + "row": 130, + "milliseconds": 2.1448749757837504 + }, + { + "row": 0, + "milliseconds": 2.16958302189596 + }, + { + "row": 68, + "milliseconds": 2.2922919888515025 + }, + { + "row": 130, + "milliseconds": 2.0957090018782765 + }, + { + "row": 0, + "milliseconds": 3.190709016053006 + }, + { + "row": 68, + "milliseconds": 2.9265839839354157 + }, + { + "row": 130, + "milliseconds": 2.6843750092666596 + }, + { + "row": 0, + "milliseconds": 2.227958000730723 + }, + { + "row": 68, + "milliseconds": 2.6705830241553485 + }, + { + "row": 130, + "milliseconds": 2.2067499812692404 + }, + { + "row": 0, + "milliseconds": 2.042457985226065 + }, + { + "row": 68, + "milliseconds": 2.5028749951161444 + }, + { + "row": 130, + "milliseconds": 2.067749999696389 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 165, + "device_counts": { + "cpu": 0, + "gpu": 165, + "ane": 0, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 233 + }, + "operations": [ + { + "name": "option_mask_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "option_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "gpu" + }, + { + "name": "context_ids_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "context_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "gpu" + }, + { + "name": "var_42_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "gpu" + }, + { + "name": "var_58_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios17.concat", + "device": "gpu" + }, + { + "name": "cast_0", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "gather_0", + "type": "ios17.gather", + "device": "gpu" + }, + { + "name": "src_1_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios17.equal", + "device": "gpu" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "query_1_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "query_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "var_139_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_140_cast_fp16", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_141_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "var_142_cast_fp16", + "type": "ios17.squeeze", + "device": "gpu" + }, + { + "name": "q_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "k_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "v_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_151_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_158_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_165_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_169_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_171_cast_fp16", + "type": "tile", + "device": "gpu" + }, + { + "name": "q_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "q_5_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "k_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "k_5_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "v_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "v_5_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "add_0_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_187_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "input_5_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "input_7_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_11_cast_fp16", + "type": "ios16.relu", + "device": "gpu" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_17_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "query_5_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "query_7_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "var_246_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_247_cast_fp16", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_248_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "var_249_cast_fp16", + "type": "ios17.squeeze", + "device": "gpu" + }, + { + "name": "q_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "k_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "v_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_258_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_265_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_272_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "q_9_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "q_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "k_9_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "k_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "v_9_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "v_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "add_1_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_294_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_19_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "input_21_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "input_23_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_27_cast_fp16", + "type": "ios16.relu", + "device": "gpu" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "context_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "option_ids_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "mask_flat_ids_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "flat_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "gpu" + }, + { + "name": "var_353_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "gpu" + }, + { + "name": "var_369_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios17.concat", + "device": "gpu" + }, + { + "name": "reshape_0", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "cast_1", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "gather_1", + "type": "ios17.gather", + "device": "gpu" + }, + { + "name": "src_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "mask_cast_fp16", + "type": "ios17.equal", + "device": "gpu" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "query_9_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "query_11_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "var_448_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_449_cast_fp16", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_450_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "var_451_cast_fp16", + "type": "ios17.squeeze", + "device": "gpu" + }, + { + "name": "q_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "k_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "v_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_460_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_467_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_474_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_478_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_480_cast_fp16", + "type": "tile", + "device": "gpu" + }, + { + "name": "q_15_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "q_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "k_15_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "k_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "v_15_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "v_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "add_2_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_496_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_35_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "input_37_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "input_39_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_43_cast_fp16", + "type": "ios16.relu", + "device": "gpu" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "hidden_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "var_524", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_524_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "var_530_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_535_cast_fp16", + "type": "ios16.reduce_sum", + "device": "gpu" + }, + { + "name": "var_540_cast_fp16", + "type": "ios16.reduce_sum", + "device": "gpu" + }, + { + "name": "var_542_cast_fp16", + "type": "ios17.maximum", + "device": "gpu" + }, + { + "name": "pooled_cast_fp16", + "type": "ios17.real_div", + "device": "gpu" + }, + { + "name": "options_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "input_53_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_587", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_589", + "type": "ios17.logical_not", + "device": "gpu" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "var_591_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_594_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_596_cast_fp16", + "type": "ios16.reduce_sum", + "device": "gpu" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_599", + "type": "ios17.logical_not", + "device": "gpu" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "var_607_cast_fp16", + "type": "ios16.fill_like", + "device": "gpu" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "var_611_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "probabilities", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "logits", + "type": "ios17.cast", + "device": "gpu" + } + ] + } + }, + "CPU_AND_NE": { + "load_ms": 651.0066249757074, + "first_prediction_ms": 1.54916598694399, + "warm_prediction": { + "count": 30, + "median_ms": 1.0025210067396984, + "p95_ms": 1.0631562676280737, + "min_ms": 0.9642079821787775, + "max_ms": 1.0732910013757646 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 1.0022920032497495 + }, + { + "row": 68, + "milliseconds": 0.995707989204675 + }, + { + "row": 130, + "milliseconds": 1.0302080190740526 + }, + { + "row": 0, + "milliseconds": 1.0732910013757646 + }, + { + "row": 68, + "milliseconds": 0.9841250139288604 + }, + { + "row": 130, + "milliseconds": 0.978750002104789 + }, + { + "row": 0, + "milliseconds": 1.0079999919980764 + }, + { + "row": 68, + "milliseconds": 0.9719169756863266 + }, + { + "row": 130, + "milliseconds": 1.0007499950006604 + }, + { + "row": 0, + "milliseconds": 1.0027500102296472 + }, + { + "row": 68, + "milliseconds": 1.0640000109560788 + }, + { + "row": 130, + "milliseconds": 1.0217910166829824 + }, + { + "row": 0, + "milliseconds": 0.9772079938556999 + }, + { + "row": 68, + "milliseconds": 0.9737089858390391 + }, + { + "row": 130, + "milliseconds": 0.9642079821787775 + }, + { + "row": 0, + "milliseconds": 0.9794160141609609 + }, + { + "row": 68, + "milliseconds": 1.0061249777209014 + }, + { + "row": 130, + "milliseconds": 1.0047079995274544 + }, + { + "row": 0, + "milliseconds": 1.038791990140453 + }, + { + "row": 68, + "milliseconds": 1.0125830012839288 + }, + { + "row": 130, + "milliseconds": 1.0621250257827342 + }, + { + "row": 0, + "milliseconds": 1.0089580027852207 + }, + { + "row": 68, + "milliseconds": 0.9832920040935278 + }, + { + "row": 130, + "milliseconds": 0.9904580074362457 + }, + { + "row": 0, + "milliseconds": 0.9932499960996211 + }, + { + "row": 68, + "milliseconds": 1.0070830176118761 + }, + { + "row": 130, + "milliseconds": 1.0052089928649366 + }, + { + "row": 0, + "milliseconds": 1.004124991595745 + }, + { + "row": 68, + "milliseconds": 0.9940839954651892 + }, + { + "row": 130, + "milliseconds": 0.9704580006655306 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 165, + "device_counts": { + "cpu": 3, + "gpu": 0, + "ane": 162, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 233 + }, + "operations": [ + { + "name": "option_mask_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "option_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "ane" + }, + { + "name": "context_ids_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "context_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "ane" + }, + { + "name": "var_42_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_58_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios17.concat", + "device": "ane" + }, + { + "name": "cast_0", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "gather_0", + "type": "ios17.gather", + "device": "ane" + }, + { + "name": "src_1_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios17.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_1_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_139_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_140_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_141_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_142_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_151_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_158_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_165_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_169_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_171_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_0_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_187_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_5_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_7_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_11_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_17_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "query_5_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_7_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_246_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_247_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_248_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_249_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_258_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_265_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_272_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "q_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_1_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_294_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_19_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_21_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_23_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_27_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "context_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "option_ids_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "mask_flat_ids_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "flat_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "ane" + }, + { + "name": "var_353_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_369_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios17.concat", + "device": "ane" + }, + { + "name": "reshape_0", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "cast_1", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "gather_1", + "type": "ios17.gather", + "device": "ane" + }, + { + "name": "src_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "mask_cast_fp16", + "type": "ios17.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_9_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_11_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_448_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_449_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_450_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_451_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_460_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_467_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_474_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_478_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_480_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_2_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_496_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_35_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_37_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_39_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_43_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "hidden_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "var_524", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_524_to_fp16", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "var_530_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_535_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_540_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_542_cast_fp16", + "type": "ios17.maximum", + "device": "ane" + }, + { + "name": "pooled_cast_fp16", + "type": "ios17.real_div", + "device": "ane" + }, + { + "name": "options_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "input_53_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_587", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_589", + "type": "ios17.logical_not", + "device": "ane" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_591_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_594_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_596_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_599", + "type": "ios17.logical_not", + "device": "ane" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_607_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_611_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "probabilities", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "logits", + "type": "ios17.cast", + "device": "ane" + } + ] + } + }, + "ALL": { + "load_ms": 616.2445000081789, + "first_prediction_ms": 1.4469160232692957, + "warm_prediction": { + "count": 30, + "median_ms": 0.9694789914647117, + "p95_ms": 0.9983017560443841, + "min_ms": 0.947958993492648, + "max_ms": 1.0204589925706387 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 0.9807920141611248 + }, + { + "row": 68, + "milliseconds": 0.9690829901956022 + }, + { + "row": 130, + "milliseconds": 0.9684590040706098 + }, + { + "row": 0, + "milliseconds": 0.9725840063765645 + }, + { + "row": 68, + "milliseconds": 0.9725420095492154 + }, + { + "row": 130, + "milliseconds": 0.9714580082800239 + }, + { + "row": 0, + "milliseconds": 0.9822079737205058 + }, + { + "row": 68, + "milliseconds": 0.9763329871930182 + }, + { + "row": 130, + "milliseconds": 0.9671660081949085 + }, + { + "row": 0, + "milliseconds": 0.9611659916117787 + }, + { + "row": 68, + "milliseconds": 0.9519580053165555 + }, + { + "row": 130, + "milliseconds": 0.9669169958215207 + }, + { + "row": 0, + "milliseconds": 0.9737499931361526 + }, + { + "row": 68, + "milliseconds": 0.9952080145012587 + }, + { + "row": 130, + "milliseconds": 1.0204589925706387 + }, + { + "row": 0, + "milliseconds": 0.9872920054476708 + }, + { + "row": 68, + "milliseconds": 0.9513749973848462 + }, + { + "row": 130, + "milliseconds": 0.9527500078547746 + }, + { + "row": 0, + "milliseconds": 1.000832999125123 + }, + { + "row": 68, + "milliseconds": 0.9625419916119426 + }, + { + "row": 130, + "milliseconds": 0.9820419945754111 + }, + { + "row": 0, + "milliseconds": 0.9630420245230198 + }, + { + "row": 68, + "milliseconds": 0.9737079963088036 + }, + { + "row": 130, + "milliseconds": 0.978750002104789 + }, + { + "row": 0, + "milliseconds": 0.9698749927338213 + }, + { + "row": 68, + "milliseconds": 0.9592919959686697 + }, + { + "row": 130, + "milliseconds": 0.947958993492648 + }, + { + "row": 0, + "milliseconds": 0.9595000010449439 + }, + { + "row": 68, + "milliseconds": 0.9607499814592302 + }, + { + "row": 130, + "milliseconds": 0.9605830127838999 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 165, + "device_counts": { + "cpu": 3, + "gpu": 0, + "ane": 162, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 233 + }, + "operations": [ + { + "name": "option_mask_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "option_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "ane" + }, + { + "name": "context_ids_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "context_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "ane" + }, + { + "name": "var_42_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_58_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios17.concat", + "device": "ane" + }, + { + "name": "cast_0", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "gather_0", + "type": "ios17.gather", + "device": "ane" + }, + { + "name": "src_1_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios17.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_1_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_139_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_140_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_141_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_142_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_151_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_158_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_165_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_169_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_171_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_0_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_187_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_5_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_7_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_11_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_17_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "query_5_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_7_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_246_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_247_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_248_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_249_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_258_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_265_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_272_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "q_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_1_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_294_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_19_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_21_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_23_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_27_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "context_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "option_ids_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "mask_flat_ids_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "flat_mask_cast_fp16", + "type": "ios17.not_equal", + "device": "ane" + }, + { + "name": "var_353_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_369_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios17.concat", + "device": "ane" + }, + { + "name": "reshape_0", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "cast_1", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "gather_1", + "type": "ios17.gather", + "device": "ane" + }, + { + "name": "src_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "mask_cast_fp16", + "type": "ios17.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_9_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_11_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_448_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_449_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_450_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_451_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_460_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_467_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_474_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_478_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_480_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_2_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_496_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_35_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_37_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_39_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_43_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "hidden_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "var_524", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_524_to_fp16", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "var_530_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_535_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_540_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_542_cast_fp16", + "type": "ios17.maximum", + "device": "ane" + }, + { + "name": "pooled_cast_fp16", + "type": "ios17.real_div", + "device": "ane" + }, + { + "name": "options_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "input_53_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_587", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_589", + "type": "ios17.logical_not", + "device": "ane" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_591_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_594_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_596_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_599", + "type": "ios17.logical_not", + "device": "ane" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_607_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_611_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "probabilities", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "logits", + "type": "ios17.cast", + "device": "ane" + } + ] + } + } + }, + "script_sha256": "d5e1386f54fc6c482647120e593e7ffbbe4bbb277c2cc6d4949154660fe239fd" +} diff --git a/reports/ane-gather-verification.json b/reports/ane-gather-verification.json new file mode 100644 index 0000000000000000000000000000000000000000..05510a20da6bc689ff2701abf44522d45d14c5fd --- /dev/null +++ b/reports/ane-gather-verification.json @@ -0,0 +1,3314 @@ +{ + "purpose": "Local conversion parity on the upstream demo; not a generalization or live GUI benchmark", + "created_utc": "2026-09-19T20:05:25.654377+00:00", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_file": "demo.jsonl", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "conversion": { + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "optimization": "ane-gather", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.6771400420111604, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "de18e313c3b625e35d008ed8b6b24108edc6df7bf2fae9533af03519eca11b63", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "38f812a04eb2322080634ba788c61df362336168466ca67e5549058d346ac793" + } + }, + "environment": { + "machine": "Apple M5 Pro", + "os": "27.0", + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "torch_cpu_threads": 2 + }, + "rows": 196, + "upstream": { + "correct": 196, + "accuracy": 1.0, + "inference_latency_ms": { + "count": 196, + "median_ms": 2.46754199906718, + "p95_ms": 3.106593489064835, + "min_ms": 1.7850839940365404, + "max_ms": 3.8927909918129444 + } + }, + "export_adapter_fp32": { + "argmax_agreement": 196, + "max_abs_probability_error": 1.1324882507324219e-06 + }, + "host_preprocessing_latency_ms": { + "count": 196, + "median_ms": 0.03879200085066259, + "p95_ms": 0.050938004278577864, + "min_ms": 0.026541994884610176, + "max_ms": 0.06945800851099193 + }, + "thresholds": { + "argmax_agreement": 1.0, + "max_abs_probability_error": 0.005, + "export_adapter_max_abs_probability_error": 0.0001, + "allow_accuracy_loss": false + }, + "backends": { + "ALL": { + "passed": true, + "correct": 196, + "accuracy": 1.0, + "argmax_agreement": 196, + "max_abs_probability_error": 0.00233614444732666, + "mean_row_max_abs_probability_error": 2.934475840582507e-05, + "max_abs_logit_error": 0.07230281829833984, + "load_seconds": 0.6175256659917068, + "warm_inference_latency_ms": { + "count": 196, + "median_ms": 0.9540419996483251, + "p95_ms": 1.0018329994636588, + "min_ms": 0.9285419946536422, + "max_ms": 1.5723749820608646 + }, + "reversed_option_order": { + "rows": 6, + "max_abs_probability_error": 3.1828880310058594e-05 + }, + "per_action": { + "fill": { + "rows": 36, + "correct": 36 + }, + "skip": { + "rows": 150, + "correct": 150 + }, + "check": { + "rows": 4, + "correct": 4 + }, + "click": { + "rows": 6, + "correct": 6 + } + }, + "decisions": [ + { + "row": 0, + "label": 22, + "upstream": 22, + "coreml": 22, + "max_abs_probability_error": 3.1828880310058594e-05, + "max_abs_logit_error": 0.03086566925048828 + }, + { + "row": 1, + "label": 23, + "upstream": 23, + "coreml": 23, + "max_abs_probability_error": 3.814697265625e-05, + "max_abs_logit_error": 0.020841598510742188 + }, + { + "row": 2, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.024377822875976562 + }, + { + "row": 3, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 3.218650817871094e-05, + "max_abs_logit_error": 0.030775785446166992 + }, + { + "row": 4, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 2.944469451904297e-05, + "max_abs_logit_error": 0.03573417663574219 + }, + { + "row": 5, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2636184692382812e-05, + "max_abs_logit_error": 0.026729822158813477 + }, + { + "row": 6, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 2.3365020751953125e-05, + "max_abs_logit_error": 0.023685455322265625 + }, + { + "row": 7, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.00014638900756835938, + "max_abs_logit_error": 0.04038810729980469 + }, + { + "row": 8, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.02460002899169922 + }, + { + "row": 9, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 2.1457672119140625e-05, + "max_abs_logit_error": 0.018064022064208984 + }, + { + "row": 10, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.047691673040390015 + }, + { + "row": 11, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 3.2186508178710938e-06, + "max_abs_logit_error": 0.03340768814086914 + }, + { + "row": 12, + "label": 15, + "upstream": 15, + "coreml": 15, + "max_abs_probability_error": 2.384185791015625e-06, + "max_abs_logit_error": 0.053191184997558594 + }, + { + "row": 13, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 0.0005360841751098633, + "max_abs_logit_error": 0.03362560272216797 + }, + { + "row": 14, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.024669170379638672 + }, + { + "row": 15, + "label": 24, + "upstream": 24, + "coreml": 24, + "max_abs_probability_error": 1.6689300537109375e-05, + "max_abs_logit_error": 0.034501075744628906 + }, + { + "row": 16, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.033290743827819824 + }, + { + "row": 17, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03323173522949219 + }, + { + "row": 18, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.022179126739501953 + }, + { + "row": 19, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.029880523681640625 + }, + { + "row": 20, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.031145095825195312 + }, + { + "row": 21, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.03861522674560547 + }, + { + "row": 22, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.027164459228515625 + }, + { + "row": 23, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.058716535568237305 + }, + { + "row": 24, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.035381317138671875 + }, + { + "row": 25, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.04862499237060547 + }, + { + "row": 26, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.029034733772277832 + }, + { + "row": 27, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.03808116912841797 + }, + { + "row": 28, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.03319191932678223 + }, + { + "row": 29, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.030045509338378906 + }, + { + "row": 30, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026231765747070312 + }, + { + "row": 31, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.033112093806266785 + }, + { + "row": 32, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.022128582000732422 + }, + { + "row": 33, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.038306236267089844 + }, + { + "row": 34, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.0004820823669433594, + "max_abs_logit_error": 0.032683372497558594 + }, + { + "row": 35, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.462750809366753e-09, + "max_abs_logit_error": 0.029730796813964844 + }, + { + "row": 36, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.7510341227277024e-10, + "max_abs_logit_error": 0.02333354949951172 + }, + { + "row": 37, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.657592190757896e-08, + "max_abs_logit_error": 0.022855758666992188 + }, + { + "row": 38, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 9.552543478452691e-11, + "max_abs_logit_error": 0.023164749145507812 + }, + { + "row": 39, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.04716482549361e-10, + "max_abs_logit_error": 0.02880859375 + }, + { + "row": 40, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.00021719932556152344, + "max_abs_logit_error": 0.04735088348388672 + }, + { + "row": 41, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.146233744175333e-09, + "max_abs_logit_error": 0.03671276569366455 + }, + { + "row": 42, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8819494851385343e-09, + "max_abs_logit_error": 0.023815155029296875 + }, + { + "row": 43, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.02039623260498047 + }, + { + "row": 44, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.780525865635596e-10, + "max_abs_logit_error": 0.022927284240722656 + }, + { + "row": 45, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.414331317674879e-11, + "max_abs_logit_error": 0.030417919158935547 + }, + { + "row": 46, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.599257724142603e-10, + "max_abs_logit_error": 0.025608062744140625 + }, + { + "row": 47, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.212314607705437e-11, + "max_abs_logit_error": 0.022733211517333984 + }, + { + "row": 48, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.024669170379638672 + }, + { + "row": 49, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.932842080831776e-10, + "max_abs_logit_error": 0.030317306518554688 + }, + { + "row": 50, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.033290743827819824 + }, + { + "row": 51, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03323173522949219 + }, + { + "row": 52, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.022179126739501953 + }, + { + "row": 53, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.029880523681640625 + }, + { + "row": 54, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.031145095825195312 + }, + { + "row": 55, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.03861522674560547 + }, + { + "row": 56, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.027164459228515625 + }, + { + "row": 57, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.058716535568237305 + }, + { + "row": 58, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.035381317138671875 + }, + { + "row": 59, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.04862499237060547 + }, + { + "row": 60, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.029034733772277832 + }, + { + "row": 61, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.03808116912841797 + }, + { + "row": 62, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.03319191932678223 + }, + { + "row": 63, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.030045509338378906 + }, + { + "row": 64, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026231765747070312 + }, + { + "row": 65, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.033112093806266785 + }, + { + "row": 66, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.022128582000732422 + }, + { + "row": 67, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.038306236267089844 + }, + { + "row": 68, + "label": 0, + "upstream": 0, + "coreml": 0, + "max_abs_probability_error": 3.147125244140625e-05, + "max_abs_logit_error": 0.03265953063964844 + }, + { + "row": 69, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 5.030632019042969e-05, + "max_abs_logit_error": 0.038794517517089844 + }, + { + "row": 70, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.040470123291015625 + }, + { + "row": 71, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 5.2928924560546875e-05, + "max_abs_logit_error": 0.040035247802734375 + }, + { + "row": 72, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 4.649162292480469e-06, + "max_abs_logit_error": 0.042186737060546875 + }, + { + "row": 73, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 2.1576881408691406e-05, + "max_abs_logit_error": 0.033161163330078125 + }, + { + "row": 74, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 4.470348358154297e-05, + "max_abs_logit_error": 0.0293731689453125 + }, + { + "row": 75, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 1.1563301086425781e-05, + "max_abs_logit_error": 0.03249359130859375 + }, + { + "row": 76, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 2.86102294921875e-06, + "max_abs_logit_error": 0.03498554229736328 + }, + { + "row": 77, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 0.0003635883331298828, + "max_abs_logit_error": 0.023473739624023438 + }, + { + "row": 78, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.034421443939208984 + }, + { + "row": 79, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 0.0003609657287597656, + "max_abs_logit_error": 0.03937721252441406 + }, + { + "row": 80, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.03752422332763672 + }, + { + "row": 81, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.025417327880859375 + }, + { + "row": 82, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.04453086853027344 + }, + { + "row": 83, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.023950576782226562 + }, + { + "row": 84, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.022706031799316406 + }, + { + "row": 85, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.035375118255615234 + }, + { + "row": 86, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.03717994689941406 + }, + { + "row": 87, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.04058122634887695 + }, + { + "row": 88, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.032048702239990234 + }, + { + "row": 89, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.030427932739257812 + }, + { + "row": 90, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.0281219482421875 + }, + { + "row": 91, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.030226707458496094 + }, + { + "row": 92, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.04007530212402344 + }, + { + "row": 93, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.02728891372680664 + }, + { + "row": 94, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.0217437744140625 + }, + { + "row": 95, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.03214454650878906 + }, + { + "row": 96, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.03669023513793945 + }, + { + "row": 97, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.019012451171875 + }, + { + "row": 98, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026651382446289062 + }, + { + "row": 99, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.1457672119140625e-06, + "max_abs_logit_error": 0.04161381721496582 + }, + { + "row": 100, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.04680633544921875 + }, + { + "row": 101, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.789079107084035e-08, + "max_abs_logit_error": 0.029542922973632812 + }, + { + "row": 102, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5862225694339713e-09, + "max_abs_logit_error": 0.023795127868652344 + }, + { + "row": 103, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 0.00024586915969848633, + "max_abs_logit_error": 0.030111312866210938 + }, + { + "row": 104, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.841255187988281e-06, + "max_abs_logit_error": 0.03126239776611328 + }, + { + "row": 105, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.928799035575082e-10, + "max_abs_logit_error": 0.026149272918701172 + }, + { + "row": 106, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.4136979082101107e-09, + "max_abs_logit_error": 0.024585723876953125 + }, + { + "row": 107, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.302839361538304e-10, + "max_abs_logit_error": 0.024765491485595703 + }, + { + "row": 108, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03492164611816406 + }, + { + "row": 109, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.034421443939208984 + }, + { + "row": 110, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.0320439338684082 + }, + { + "row": 111, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.811981201171875e-05, + "max_abs_logit_error": 0.032814979553222656 + }, + { + "row": 112, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.025417327880859375 + }, + { + "row": 113, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.04453086853027344 + }, + { + "row": 114, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.023950576782226562 + }, + { + "row": 115, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.022706031799316406 + }, + { + "row": 116, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.035375118255615234 + }, + { + "row": 117, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.03717994689941406 + }, + { + "row": 118, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.04058122634887695 + }, + { + "row": 119, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.032048702239990234 + }, + { + "row": 120, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.030427932739257812 + }, + { + "row": 121, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.0281219482421875 + }, + { + "row": 122, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.030226707458496094 + }, + { + "row": 123, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.04007530212402344 + }, + { + "row": 124, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.02728891372680664 + }, + { + "row": 125, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.0217437744140625 + }, + { + "row": 126, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.03214454650878906 + }, + { + "row": 127, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.03669023513793945 + }, + { + "row": 128, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.019012451171875 + }, + { + "row": 129, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026651382446289062 + }, + { + "row": 130, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 3.170967102050781e-05, + "max_abs_logit_error": 0.071075439453125 + }, + { + "row": 131, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.051357269287109375 + }, + { + "row": 132, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 2.7894973754882812e-05, + "max_abs_logit_error": 0.033801641315221786 + }, + { + "row": 133, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 1.0967254638671875e-05, + "max_abs_logit_error": 0.039325714111328125 + }, + { + "row": 134, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 6.258487701416016e-05, + "max_abs_logit_error": 0.033056676387786865 + }, + { + "row": 135, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2993812561035156e-05, + "max_abs_logit_error": 0.02561807632446289 + }, + { + "row": 136, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 6.198883056640625e-06, + "max_abs_logit_error": 0.03448677062988281 + }, + { + "row": 137, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.00233614444732666, + "max_abs_logit_error": 0.04230833053588867 + }, + { + "row": 138, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 0.00022917985916137695, + "max_abs_logit_error": 0.05339241027832031 + }, + { + "row": 139, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 4.851818084716797e-05, + "max_abs_logit_error": 0.032756805419921875 + }, + { + "row": 140, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 3.0994415283203125e-06, + "max_abs_logit_error": 0.029809951782226562 + }, + { + "row": 141, + "label": 12, + "upstream": 12, + "coreml": 12, + "max_abs_probability_error": 9.34600830078125e-05, + "max_abs_logit_error": 0.026587963104248047 + }, + { + "row": 142, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.03671073913574219 + }, + { + "row": 143, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.05281543731689453 + }, + { + "row": 144, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.04511451721191406 + }, + { + "row": 145, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.07230281829833984 + }, + { + "row": 146, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03114461898803711 + }, + { + "row": 147, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.020772457122802734 + }, + { + "row": 148, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.03475606441497803 + }, + { + "row": 149, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.020298004150390625 + }, + { + "row": 150, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.04283332824707031 + }, + { + "row": 151, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.028450965881347656 + }, + { + "row": 152, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.02250051498413086 + }, + { + "row": 153, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.06335592269897461 + }, + { + "row": 154, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.02735137939453125 + }, + { + "row": 155, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.015047073364257812 + }, + { + "row": 156, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.04868888854980469 + }, + { + "row": 157, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.026497364044189453 + }, + { + "row": 158, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.022236347198486328 + }, + { + "row": 159, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.020044326782226562 + }, + { + "row": 160, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.05533123016357422 + }, + { + "row": 161, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.0247042179107666 + }, + { + "row": 162, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.028873443603515625 + }, + { + "row": 163, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9131388140358752e-10, + "max_abs_logit_error": 0.015494346618652344 + }, + { + "row": 164, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0254025184508464e-08, + "max_abs_logit_error": 0.02497100830078125 + }, + { + "row": 165, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.15840287151903e-11, + "max_abs_logit_error": 0.02679443359375 + }, + { + "row": 166, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.6797479790729994e-09, + "max_abs_logit_error": 0.028203964233398438 + }, + { + "row": 167, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.54509399403247e-10, + "max_abs_logit_error": 0.03371429443359375 + }, + { + "row": 168, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.028280355036258698 + }, + { + "row": 169, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.4169340140378637e-11, + "max_abs_logit_error": 0.038245439529418945 + }, + { + "row": 170, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.0329603412093675e-11, + "max_abs_logit_error": 0.030043363571166992 + }, + { + "row": 171, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.02963542938232422 + }, + { + "row": 172, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.66337235494052e-09, + "max_abs_logit_error": 0.03510093688964844 + }, + { + "row": 173, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2885730260592254e-09, + "max_abs_logit_error": 0.04009199142456055 + }, + { + "row": 174, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1279766010119374e-09, + "max_abs_logit_error": 0.031810760498046875 + }, + { + "row": 175, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.03671073913574219 + }, + { + "row": 176, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0728836059570312e-06, + "max_abs_logit_error": 0.05123615264892578 + }, + { + "row": 177, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2799782567185503e-08, + "max_abs_logit_error": 0.03157186508178711 + }, + { + "row": 178, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.07230281829833984 + }, + { + "row": 179, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03114461898803711 + }, + { + "row": 180, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.020772457122802734 + }, + { + "row": 181, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.03475606441497803 + }, + { + "row": 182, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.020298004150390625 + }, + { + "row": 183, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.04283332824707031 + }, + { + "row": 184, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.028450965881347656 + }, + { + "row": 185, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.02250051498413086 + }, + { + "row": 186, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.06335592269897461 + }, + { + "row": 187, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.02735137939453125 + }, + { + "row": 188, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.015047073364257812 + }, + { + "row": 189, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.04868888854980469 + }, + { + "row": 190, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.026497364044189453 + }, + { + "row": 191, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.022236347198486328 + }, + { + "row": 192, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.020044326782226562 + }, + { + "row": 193, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.05533123016357422 + }, + { + "row": 194, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.0247042179107666 + }, + { + "row": 195, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.028873443603515625 + } + ] + }, + "CPU_AND_NE": { + "passed": true, + "correct": 196, + "accuracy": 1.0, + "argmax_agreement": 196, + "max_abs_probability_error": 0.00233614444732666, + "mean_row_max_abs_probability_error": 2.934475840582507e-05, + "max_abs_logit_error": 0.07230281829833984, + "load_seconds": 0.578353041986702, + "warm_inference_latency_ms": { + "count": 196, + "median_ms": 0.9623960067983717, + "p95_ms": 0.9909170039463788, + "min_ms": 0.9270830196328461, + "max_ms": 1.0197919909842312 + }, + "reversed_option_order": { + "rows": 6, + "max_abs_probability_error": 3.1828880310058594e-05 + }, + "per_action": { + "fill": { + "rows": 36, + "correct": 36 + }, + "skip": { + "rows": 150, + "correct": 150 + }, + "check": { + "rows": 4, + "correct": 4 + }, + "click": { + "rows": 6, + "correct": 6 + } + }, + "decisions": [ + { + "row": 0, + "label": 22, + "upstream": 22, + "coreml": 22, + "max_abs_probability_error": 3.1828880310058594e-05, + "max_abs_logit_error": 0.03086566925048828 + }, + { + "row": 1, + "label": 23, + "upstream": 23, + "coreml": 23, + "max_abs_probability_error": 3.814697265625e-05, + "max_abs_logit_error": 0.020841598510742188 + }, + { + "row": 2, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.024377822875976562 + }, + { + "row": 3, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 3.218650817871094e-05, + "max_abs_logit_error": 0.030775785446166992 + }, + { + "row": 4, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 2.944469451904297e-05, + "max_abs_logit_error": 0.03573417663574219 + }, + { + "row": 5, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2636184692382812e-05, + "max_abs_logit_error": 0.026729822158813477 + }, + { + "row": 6, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 2.3365020751953125e-05, + "max_abs_logit_error": 0.023685455322265625 + }, + { + "row": 7, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.00014638900756835938, + "max_abs_logit_error": 0.04038810729980469 + }, + { + "row": 8, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.02460002899169922 + }, + { + "row": 9, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 2.1457672119140625e-05, + "max_abs_logit_error": 0.018064022064208984 + }, + { + "row": 10, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.047691673040390015 + }, + { + "row": 11, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 3.2186508178710938e-06, + "max_abs_logit_error": 0.03340768814086914 + }, + { + "row": 12, + "label": 15, + "upstream": 15, + "coreml": 15, + "max_abs_probability_error": 2.384185791015625e-06, + "max_abs_logit_error": 0.053191184997558594 + }, + { + "row": 13, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 0.0005360841751098633, + "max_abs_logit_error": 0.03362560272216797 + }, + { + "row": 14, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.024669170379638672 + }, + { + "row": 15, + "label": 24, + "upstream": 24, + "coreml": 24, + "max_abs_probability_error": 1.6689300537109375e-05, + "max_abs_logit_error": 0.034501075744628906 + }, + { + "row": 16, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.033290743827819824 + }, + { + "row": 17, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03323173522949219 + }, + { + "row": 18, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.022179126739501953 + }, + { + "row": 19, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.029880523681640625 + }, + { + "row": 20, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.031145095825195312 + }, + { + "row": 21, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.03861522674560547 + }, + { + "row": 22, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.027164459228515625 + }, + { + "row": 23, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.058716535568237305 + }, + { + "row": 24, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.035381317138671875 + }, + { + "row": 25, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.04862499237060547 + }, + { + "row": 26, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.029034733772277832 + }, + { + "row": 27, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.03808116912841797 + }, + { + "row": 28, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.03319191932678223 + }, + { + "row": 29, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.030045509338378906 + }, + { + "row": 30, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026231765747070312 + }, + { + "row": 31, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.033112093806266785 + }, + { + "row": 32, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.022128582000732422 + }, + { + "row": 33, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.038306236267089844 + }, + { + "row": 34, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.0004820823669433594, + "max_abs_logit_error": 0.032683372497558594 + }, + { + "row": 35, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.462750809366753e-09, + "max_abs_logit_error": 0.029730796813964844 + }, + { + "row": 36, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.7510341227277024e-10, + "max_abs_logit_error": 0.02333354949951172 + }, + { + "row": 37, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.657592190757896e-08, + "max_abs_logit_error": 0.022855758666992188 + }, + { + "row": 38, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 9.552543478452691e-11, + "max_abs_logit_error": 0.023164749145507812 + }, + { + "row": 39, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.04716482549361e-10, + "max_abs_logit_error": 0.02880859375 + }, + { + "row": 40, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.00021719932556152344, + "max_abs_logit_error": 0.04735088348388672 + }, + { + "row": 41, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.146233744175333e-09, + "max_abs_logit_error": 0.03671276569366455 + }, + { + "row": 42, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8819494851385343e-09, + "max_abs_logit_error": 0.023815155029296875 + }, + { + "row": 43, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.02039623260498047 + }, + { + "row": 44, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.780525865635596e-10, + "max_abs_logit_error": 0.022927284240722656 + }, + { + "row": 45, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.414331317674879e-11, + "max_abs_logit_error": 0.030417919158935547 + }, + { + "row": 46, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.599257724142603e-10, + "max_abs_logit_error": 0.025608062744140625 + }, + { + "row": 47, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.212314607705437e-11, + "max_abs_logit_error": 0.022733211517333984 + }, + { + "row": 48, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.024669170379638672 + }, + { + "row": 49, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.932842080831776e-10, + "max_abs_logit_error": 0.030317306518554688 + }, + { + "row": 50, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.033290743827819824 + }, + { + "row": 51, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03323173522949219 + }, + { + "row": 52, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.022179126739501953 + }, + { + "row": 53, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.029880523681640625 + }, + { + "row": 54, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.031145095825195312 + }, + { + "row": 55, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.03861522674560547 + }, + { + "row": 56, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.027164459228515625 + }, + { + "row": 57, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.058716535568237305 + }, + { + "row": 58, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.035381317138671875 + }, + { + "row": 59, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.04862499237060547 + }, + { + "row": 60, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.029034733772277832 + }, + { + "row": 61, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.03808116912841797 + }, + { + "row": 62, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.03319191932678223 + }, + { + "row": 63, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.030045509338378906 + }, + { + "row": 64, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026231765747070312 + }, + { + "row": 65, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.033112093806266785 + }, + { + "row": 66, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.022128582000732422 + }, + { + "row": 67, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.038306236267089844 + }, + { + "row": 68, + "label": 0, + "upstream": 0, + "coreml": 0, + "max_abs_probability_error": 3.147125244140625e-05, + "max_abs_logit_error": 0.03265953063964844 + }, + { + "row": 69, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 5.030632019042969e-05, + "max_abs_logit_error": 0.038794517517089844 + }, + { + "row": 70, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.040470123291015625 + }, + { + "row": 71, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 5.2928924560546875e-05, + "max_abs_logit_error": 0.040035247802734375 + }, + { + "row": 72, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 4.649162292480469e-06, + "max_abs_logit_error": 0.042186737060546875 + }, + { + "row": 73, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 2.1576881408691406e-05, + "max_abs_logit_error": 0.033161163330078125 + }, + { + "row": 74, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 4.470348358154297e-05, + "max_abs_logit_error": 0.0293731689453125 + }, + { + "row": 75, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 1.1563301086425781e-05, + "max_abs_logit_error": 0.03249359130859375 + }, + { + "row": 76, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 2.86102294921875e-06, + "max_abs_logit_error": 0.03498554229736328 + }, + { + "row": 77, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 0.0003635883331298828, + "max_abs_logit_error": 0.023473739624023438 + }, + { + "row": 78, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.034421443939208984 + }, + { + "row": 79, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 0.0003609657287597656, + "max_abs_logit_error": 0.03937721252441406 + }, + { + "row": 80, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.03752422332763672 + }, + { + "row": 81, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.025417327880859375 + }, + { + "row": 82, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.04453086853027344 + }, + { + "row": 83, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.023950576782226562 + }, + { + "row": 84, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.022706031799316406 + }, + { + "row": 85, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.035375118255615234 + }, + { + "row": 86, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.03717994689941406 + }, + { + "row": 87, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.04058122634887695 + }, + { + "row": 88, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.032048702239990234 + }, + { + "row": 89, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.030427932739257812 + }, + { + "row": 90, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.0281219482421875 + }, + { + "row": 91, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.030226707458496094 + }, + { + "row": 92, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.04007530212402344 + }, + { + "row": 93, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.02728891372680664 + }, + { + "row": 94, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.0217437744140625 + }, + { + "row": 95, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.03214454650878906 + }, + { + "row": 96, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.03669023513793945 + }, + { + "row": 97, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.019012451171875 + }, + { + "row": 98, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026651382446289062 + }, + { + "row": 99, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.1457672119140625e-06, + "max_abs_logit_error": 0.04161381721496582 + }, + { + "row": 100, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.04680633544921875 + }, + { + "row": 101, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.789079107084035e-08, + "max_abs_logit_error": 0.029542922973632812 + }, + { + "row": 102, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5862225694339713e-09, + "max_abs_logit_error": 0.023795127868652344 + }, + { + "row": 103, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 0.00024586915969848633, + "max_abs_logit_error": 0.030111312866210938 + }, + { + "row": 104, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.841255187988281e-06, + "max_abs_logit_error": 0.03126239776611328 + }, + { + "row": 105, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.928799035575082e-10, + "max_abs_logit_error": 0.026149272918701172 + }, + { + "row": 106, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.4136979082101107e-09, + "max_abs_logit_error": 0.024585723876953125 + }, + { + "row": 107, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.302839361538304e-10, + "max_abs_logit_error": 0.024765491485595703 + }, + { + "row": 108, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03492164611816406 + }, + { + "row": 109, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.034421443939208984 + }, + { + "row": 110, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.0320439338684082 + }, + { + "row": 111, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.811981201171875e-05, + "max_abs_logit_error": 0.032814979553222656 + }, + { + "row": 112, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.025417327880859375 + }, + { + "row": 113, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.04453086853027344 + }, + { + "row": 114, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.023950576782226562 + }, + { + "row": 115, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.022706031799316406 + }, + { + "row": 116, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.035375118255615234 + }, + { + "row": 117, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.03717994689941406 + }, + { + "row": 118, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.04058122634887695 + }, + { + "row": 119, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.032048702239990234 + }, + { + "row": 120, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.030427932739257812 + }, + { + "row": 121, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.0281219482421875 + }, + { + "row": 122, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.030226707458496094 + }, + { + "row": 123, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.04007530212402344 + }, + { + "row": 124, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.02728891372680664 + }, + { + "row": 125, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.0217437744140625 + }, + { + "row": 126, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.03214454650878906 + }, + { + "row": 127, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.03669023513793945 + }, + { + "row": 128, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.019012451171875 + }, + { + "row": 129, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026651382446289062 + }, + { + "row": 130, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 3.170967102050781e-05, + "max_abs_logit_error": 0.071075439453125 + }, + { + "row": 131, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.051357269287109375 + }, + { + "row": 132, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 2.7894973754882812e-05, + "max_abs_logit_error": 0.033801641315221786 + }, + { + "row": 133, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 1.0967254638671875e-05, + "max_abs_logit_error": 0.039325714111328125 + }, + { + "row": 134, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 6.258487701416016e-05, + "max_abs_logit_error": 0.033056676387786865 + }, + { + "row": 135, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2993812561035156e-05, + "max_abs_logit_error": 0.02561807632446289 + }, + { + "row": 136, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 6.198883056640625e-06, + "max_abs_logit_error": 0.03448677062988281 + }, + { + "row": 137, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.00233614444732666, + "max_abs_logit_error": 0.04230833053588867 + }, + { + "row": 138, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 0.00022917985916137695, + "max_abs_logit_error": 0.05339241027832031 + }, + { + "row": 139, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 4.851818084716797e-05, + "max_abs_logit_error": 0.032756805419921875 + }, + { + "row": 140, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 3.0994415283203125e-06, + "max_abs_logit_error": 0.029809951782226562 + }, + { + "row": 141, + "label": 12, + "upstream": 12, + "coreml": 12, + "max_abs_probability_error": 9.34600830078125e-05, + "max_abs_logit_error": 0.026587963104248047 + }, + { + "row": 142, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.03671073913574219 + }, + { + "row": 143, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.05281543731689453 + }, + { + "row": 144, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.04511451721191406 + }, + { + "row": 145, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.07230281829833984 + }, + { + "row": 146, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03114461898803711 + }, + { + "row": 147, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.020772457122802734 + }, + { + "row": 148, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.03475606441497803 + }, + { + "row": 149, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.020298004150390625 + }, + { + "row": 150, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.04283332824707031 + }, + { + "row": 151, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.028450965881347656 + }, + { + "row": 152, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.02250051498413086 + }, + { + "row": 153, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.06335592269897461 + }, + { + "row": 154, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.02735137939453125 + }, + { + "row": 155, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.015047073364257812 + }, + { + "row": 156, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.04868888854980469 + }, + { + "row": 157, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.026497364044189453 + }, + { + "row": 158, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.022236347198486328 + }, + { + "row": 159, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.020044326782226562 + }, + { + "row": 160, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.05533123016357422 + }, + { + "row": 161, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.0247042179107666 + }, + { + "row": 162, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.028873443603515625 + }, + { + "row": 163, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9131388140358752e-10, + "max_abs_logit_error": 0.015494346618652344 + }, + { + "row": 164, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0254025184508464e-08, + "max_abs_logit_error": 0.02497100830078125 + }, + { + "row": 165, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.15840287151903e-11, + "max_abs_logit_error": 0.02679443359375 + }, + { + "row": 166, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.6797479790729994e-09, + "max_abs_logit_error": 0.028203964233398438 + }, + { + "row": 167, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.54509399403247e-10, + "max_abs_logit_error": 0.03371429443359375 + }, + { + "row": 168, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.028280355036258698 + }, + { + "row": 169, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.4169340140378637e-11, + "max_abs_logit_error": 0.038245439529418945 + }, + { + "row": 170, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.0329603412093675e-11, + "max_abs_logit_error": 0.030043363571166992 + }, + { + "row": 171, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.02963542938232422 + }, + { + "row": 172, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.66337235494052e-09, + "max_abs_logit_error": 0.03510093688964844 + }, + { + "row": 173, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2885730260592254e-09, + "max_abs_logit_error": 0.04009199142456055 + }, + { + "row": 174, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1279766010119374e-09, + "max_abs_logit_error": 0.031810760498046875 + }, + { + "row": 175, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.03671073913574219 + }, + { + "row": 176, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0728836059570312e-06, + "max_abs_logit_error": 0.05123615264892578 + }, + { + "row": 177, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2799782567185503e-08, + "max_abs_logit_error": 0.03157186508178711 + }, + { + "row": 178, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.07230281829833984 + }, + { + "row": 179, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03114461898803711 + }, + { + "row": 180, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.020772457122802734 + }, + { + "row": 181, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.03475606441497803 + }, + { + "row": 182, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.020298004150390625 + }, + { + "row": 183, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.04283332824707031 + }, + { + "row": 184, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.028450965881347656 + }, + { + "row": 185, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.02250051498413086 + }, + { + "row": 186, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.06335592269897461 + }, + { + "row": 187, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.02735137939453125 + }, + { + "row": 188, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.015047073364257812 + }, + { + "row": 189, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.04868888854980469 + }, + { + "row": 190, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.026497364044189453 + }, + { + "row": 191, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.022236347198486328 + }, + { + "row": 192, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.020044326782226562 + }, + { + "row": 193, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.05533123016357422 + }, + { + "row": 194, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.0247042179107666 + }, + { + "row": 195, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.028873443603515625 + } + ] + } + }, + "passed": true, + "code_sha256": { + "ane_gather.py": "c3379a3797041dcfed82eaaea3990771ec91f1dff7ec146b025988ad7a8c9aae", + "assets.py": "f7a5bc931075237e00c73bf21fee3575be35531f934f23b5e22627d50a79f92f", + "convert-coreml.py": "46fe0194df924033126286e601214b6789f5d667aae38c153cf4dbc1b0e629b0", + "export-reference.py": "ff9eab4d6e4e72ffaf074d82610f41fbea1e802da7c0e987ce562a89eb561b99", + "export_model.py": "c61da5d678c77030d1e9a78359626063ea31da46fb6434e49517fdeaa7800949", + "preprocessing.py": "18c74ede43f95f91cd638c7e9631923a353e3a24d3b522ddb8b18b480f9c2846", + "profile-coreml.py": "d5e1386f54fc6c482647120e593e7ffbbe4bbb277c2cc6d4949154660fe239fd", + "score-report.py": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e", + "verify.py": "bc20ec9df47708e9b48e411076abe4eab7c18313f67e11334de2d9e3ecb3efa7" + } +} diff --git a/reports/ane-profile.json b/reports/ane-profile.json new file mode 100644 index 0000000000000000000000000000000000000000..b47f76e07a2f67b23b1d918a1d8e339cafbe3849 --- /dev/null +++ b/reports/ane-profile.json @@ -0,0 +1,4109 @@ +{ + "created_utc": "2026-09-19T19:43:44.834712+00:00", + "purpose": "Bounded real-input latency and scheduler placement; not a general accuracy benchmark", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "environment": { + "chip": "Apple M5 Pro", + "memory_bytes": 25769803776, + "macos": "27.0", + "os_build": "26A428", + "python": "3.11.11", + "coremltools": "9.0" + }, + "protocol": { + "rows": [ + 0, + 68, + 130 + ], + "pages": [ + "patient-registration", + "job-application", + "auto-claim" + ], + "option_counts": [ + 27, + 21, + 19 + ], + "first_call_row": 0, + "warmup_passes": 2, + "timed_passes": 10, + "backend_order": [ + "CPU_ONLY", + "CPU_AND_GPU", + "CPU_AND_NE", + "ALL" + ], + "timing_scope": "Synchronous Python MLModel.predict; pre-encoded inputs; output checks outside timer", + "load_scope": "MLModel construction; system caches retained, not first-install cold start", + "first_call_scope": "First predict after this load; may use existing system/ANE caches", + "plan_order": "Compute plan loaded after timing each backend" + }, + "backends": { + "CPU_ONLY": { + "load_ms": 71.21945900144055, + "first_prediction_ms": 15.096832998096943, + "warm_prediction": { + "count": 30, + "median_ms": 1.5270625008270144, + "p95_ms": 1.601899994420819, + "min_ms": 1.3285410241223872, + "max_ms": 1.6374590049963444 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 1.578499999595806 + }, + { + "row": 68, + "milliseconds": 1.561041979584843 + }, + { + "row": 130, + "milliseconds": 1.5606670058332384 + }, + { + "row": 0, + "milliseconds": 1.4908749726600945 + }, + { + "row": 68, + "milliseconds": 1.5069579822011292 + }, + { + "row": 130, + "milliseconds": 1.412415993399918 + }, + { + "row": 0, + "milliseconds": 1.5242919907905161 + }, + { + "row": 68, + "milliseconds": 1.5393749927170575 + }, + { + "row": 130, + "milliseconds": 1.3988339924253523 + }, + { + "row": 0, + "milliseconds": 1.5052919916342944 + }, + { + "row": 68, + "milliseconds": 1.4890410238876939 + }, + { + "row": 130, + "milliseconds": 1.5105829806998372 + }, + { + "row": 0, + "milliseconds": 1.6374590049963444 + }, + { + "row": 68, + "milliseconds": 1.6122499946504831 + }, + { + "row": 130, + "milliseconds": 1.5023329760879278 + }, + { + "row": 0, + "milliseconds": 1.5473749954253435 + }, + { + "row": 68, + "milliseconds": 1.3900829944759607 + }, + { + "row": 130, + "milliseconds": 1.5310419839806855 + }, + { + "row": 0, + "milliseconds": 1.3285410241223872 + }, + { + "row": 68, + "milliseconds": 1.5892499941401184 + }, + { + "row": 130, + "milliseconds": 1.5656249888706952 + }, + { + "row": 0, + "milliseconds": 1.546334009617567 + }, + { + "row": 68, + "milliseconds": 1.5298330108635128 + }, + { + "row": 130, + "milliseconds": 1.543457998195663 + }, + { + "row": 0, + "milliseconds": 1.5623749932274222 + }, + { + "row": 68, + "milliseconds": 1.4965840091463178 + }, + { + "row": 130, + "milliseconds": 1.5304579865187407 + }, + { + "row": 0, + "milliseconds": 1.465249981265515 + }, + { + "row": 68, + "milliseconds": 1.4851669839117676 + }, + { + "row": 130, + "milliseconds": 1.4999579871073365 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 173, + "device_counts": { + "cpu": 173, + "gpu": 0, + "ane": 0, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 239 + }, + "operations": [ + { + "name": "option_mask_1", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "context_mask", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "var_27", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "cpu" + }, + { + "name": "var_43", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_43_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios17.concat", + "device": "cpu" + }, + { + "name": "context_ids_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "cast_54", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "greater_equal_0", + "type": "ios17.greater_equal", + "device": "cpu" + }, + { + "name": "add_3", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "select_0", + "type": "select", + "device": "cpu" + }, + { + "name": "select_0_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "var_59_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "cpu" + }, + { + "name": "src_1_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios17.equal", + "device": "cpu" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "query_1_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "query_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "var_124_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_125_cast_fp16", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_126_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "var_127_cast_fp16", + "type": "ios17.squeeze", + "device": "cpu" + }, + { + "name": "q_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "k_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "v_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_136_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_143_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_150_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_154_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_156_cast_fp16", + "type": "tile", + "device": "cpu" + }, + { + "name": "q_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "q_5_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "k_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "k_5_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "v_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "v_5_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "add_0_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "var_172_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "input_3_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "input_5_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "input_7_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "input_11_cast_fp16", + "type": "ios16.relu", + "device": "cpu" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "input_17_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "query_5_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "query_7_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "var_231_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_232_cast_fp16", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_233_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "var_234_cast_fp16", + "type": "ios17.squeeze", + "device": "cpu" + }, + { + "name": "q_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "k_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "v_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_243_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_250_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_257_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "q_9_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "q_11_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "k_9_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "k_11_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "v_9_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "v_11_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "add_1_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "var_279_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "input_19_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "input_21_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "input_23_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "input_27_cast_fp16", + "type": "ios16.relu", + "device": "cpu" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "context_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "flat_ids", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "flat_mask", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "var_333", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "cpu" + }, + { + "name": "var_349", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_349_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios17.concat", + "device": "cpu" + }, + { + "name": "flat_ids_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "cast_55", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "greater_equal_1", + "type": "ios17.greater_equal", + "device": "cpu" + }, + { + "name": "add_4", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "select_1", + "type": "select", + "device": "cpu" + }, + { + "name": "select_1_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "var_365_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "cpu" + }, + { + "name": "src_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "mask_cast_fp16", + "type": "ios17.equal", + "device": "cpu" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "query_9_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "query_11_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "var_428_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_429_cast_fp16", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_430_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "var_431_cast_fp16", + "type": "ios17.squeeze", + "device": "cpu" + }, + { + "name": "q_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "k_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "v_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_440_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_447_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_454_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_458_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "var_460_cast_fp16", + "type": "tile", + "device": "cpu" + }, + { + "name": "q_15_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "q_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "k_15_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "k_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "v_15_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "v_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "add_2_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "var_476_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "input_35_cast_fp16", + "type": "ios17.transpose", + "device": "cpu" + }, + { + "name": "input_37_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "input_39_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "input_43_cast_fp16", + "type": "ios16.relu", + "device": "cpu" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "hidden_cast_fp16", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "var_504", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_504_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "var_510_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "var_515_cast_fp16", + "type": "ios16.reduce_sum", + "device": "cpu" + }, + { + "name": "var_520_cast_fp16", + "type": "ios16.reduce_sum", + "device": "cpu" + }, + { + "name": "var_522_cast_fp16", + "type": "ios17.maximum", + "device": "cpu" + }, + { + "name": "pooled_cast_fp16", + "type": "ios17.real_div", + "device": "cpu" + }, + { + "name": "options_cast_fp16", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "input_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "input_53_cast_fp16", + "type": "ios17.layer_norm", + "device": "cpu" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios17.linear", + "device": "cpu" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "var_567", + "type": "ios17.expand_dims", + "device": "cpu" + }, + { + "name": "var_569", + "type": "ios17.logical_not", + "device": "cpu" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "var_571_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios17.matmul", + "device": "cpu" + }, + { + "name": "var_574_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "var_576_cast_fp16", + "type": "ios16.reduce_sum", + "device": "cpu" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios17.mul", + "device": "cpu" + }, + { + "name": "var_579", + "type": "ios17.logical_not", + "device": "cpu" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "var_587_cast_fp16", + "type": "ios16.fill_like", + "device": "cpu" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "cpu" + }, + { + "name": "var_591_cast_fp16", + "type": "ios16.softmax", + "device": "cpu" + }, + { + "name": "probabilities", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "logits", + "type": "ios17.cast", + "device": "cpu" + } + ] + } + }, + "CPU_AND_GPU": { + "load_ms": 70.07474999409169, + "first_prediction_ms": 66.33504197816364, + "warm_prediction": { + "count": 30, + "median_ms": 0.9288130095228553, + "p95_ms": 2.3798043403076, + "min_ms": 0.8875419734977186, + "max_ms": 2.6384999800939113 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 2.5134169845841825 + }, + { + "row": 68, + "milliseconds": 2.6384999800939113 + }, + { + "row": 130, + "milliseconds": 1.9895419827662408 + }, + { + "row": 0, + "milliseconds": 2.21649999730289 + }, + { + "row": 68, + "milliseconds": 1.2297500215936452 + }, + { + "row": 130, + "milliseconds": 1.1009590234607458 + }, + { + "row": 0, + "milliseconds": 0.9331250039394945 + }, + { + "row": 68, + "milliseconds": 0.9082920150831342 + }, + { + "row": 130, + "milliseconds": 0.9552080009598285 + }, + { + "row": 0, + "milliseconds": 1.4009579899720848 + }, + { + "row": 68, + "milliseconds": 1.207541994517669 + }, + { + "row": 130, + "milliseconds": 0.9119999886024743 + }, + { + "row": 0, + "milliseconds": 0.9255839977413416 + }, + { + "row": 68, + "milliseconds": 1.0290830105077475 + }, + { + "row": 130, + "milliseconds": 0.932042021304369 + }, + { + "row": 0, + "milliseconds": 0.9105000062845647 + }, + { + "row": 68, + "milliseconds": 0.9429579949937761 + }, + { + "row": 130, + "milliseconds": 0.9207910043187439 + }, + { + "row": 0, + "milliseconds": 0.8967499888967723 + }, + { + "row": 68, + "milliseconds": 1.2767080042976886 + }, + { + "row": 130, + "milliseconds": 0.9552080009598285 + }, + { + "row": 0, + "milliseconds": 0.9043329919222742 + }, + { + "row": 68, + "milliseconds": 0.9005409956444055 + }, + { + "row": 130, + "milliseconds": 0.8939160034060478 + }, + { + "row": 0, + "milliseconds": 0.8917910163290799 + }, + { + "row": 68, + "milliseconds": 0.8989170019049197 + }, + { + "row": 130, + "milliseconds": 0.8875419734977186 + }, + { + "row": 0, + "milliseconds": 0.88900001719594 + }, + { + "row": 68, + "milliseconds": 0.8909160096663982 + }, + { + "row": 130, + "milliseconds": 0.8875839994288981 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 173, + "device_counts": { + "cpu": 0, + "gpu": 173, + "ane": 0, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 239 + }, + "operations": [ + { + "name": "option_mask_1", + "type": "ios17.not_equal", + "device": "gpu" + }, + { + "name": "context_mask", + "type": "ios17.not_equal", + "device": "gpu" + }, + { + "name": "var_27", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "gpu" + }, + { + "name": "var_43", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_43_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios17.concat", + "device": "gpu" + }, + { + "name": "context_ids_to_int16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "cast_54", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "greater_equal_0", + "type": "ios17.greater_equal", + "device": "gpu" + }, + { + "name": "add_3", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "select_0", + "type": "select", + "device": "gpu" + }, + { + "name": "select_0_to_int16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "var_59_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "gpu" + }, + { + "name": "src_1_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios17.equal", + "device": "gpu" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "query_1_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "query_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "var_124_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_125_cast_fp16", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_126_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "var_127_cast_fp16", + "type": "ios17.squeeze", + "device": "gpu" + }, + { + "name": "q_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "k_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "v_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_136_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_143_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_150_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_154_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_156_cast_fp16", + "type": "tile", + "device": "gpu" + }, + { + "name": "q_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "q_5_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "k_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "k_5_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "v_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "v_5_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "add_0_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_172_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "input_5_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "input_7_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_11_cast_fp16", + "type": "ios16.relu", + "device": "gpu" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_17_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "query_5_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "query_7_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "var_231_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_232_cast_fp16", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_233_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "var_234_cast_fp16", + "type": "ios17.squeeze", + "device": "gpu" + }, + { + "name": "q_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "k_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "v_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_243_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_250_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_257_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "q_9_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "q_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "k_9_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "k_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "v_9_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "v_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "add_1_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_279_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_19_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "input_21_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "input_23_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_27_cast_fp16", + "type": "ios16.relu", + "device": "gpu" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "context_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "flat_ids", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "flat_mask", + "type": "ios17.not_equal", + "device": "gpu" + }, + { + "name": "var_333", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "gpu" + }, + { + "name": "var_349", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_349_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios17.concat", + "device": "gpu" + }, + { + "name": "flat_ids_to_int16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "cast_55", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "greater_equal_1", + "type": "ios17.greater_equal", + "device": "gpu" + }, + { + "name": "add_4", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "select_1", + "type": "select", + "device": "gpu" + }, + { + "name": "select_1_to_int16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "var_365_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "gpu" + }, + { + "name": "src_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "mask_cast_fp16", + "type": "ios17.equal", + "device": "gpu" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "query_9_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "query_11_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "var_428_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_429_cast_fp16", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_430_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "var_431_cast_fp16", + "type": "ios17.squeeze", + "device": "gpu" + }, + { + "name": "q_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "k_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "v_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_440_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_447_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_454_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_458_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_460_cast_fp16", + "type": "tile", + "device": "gpu" + }, + { + "name": "q_15_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "q_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "k_15_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "k_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "v_15_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "v_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "add_2_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_476_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_35_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "input_37_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "input_39_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_43_cast_fp16", + "type": "ios16.relu", + "device": "gpu" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "hidden_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "var_504", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_504_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "var_510_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_515_cast_fp16", + "type": "ios16.reduce_sum", + "device": "gpu" + }, + { + "name": "var_520_cast_fp16", + "type": "ios16.reduce_sum", + "device": "gpu" + }, + { + "name": "var_522_cast_fp16", + "type": "ios17.maximum", + "device": "gpu" + }, + { + "name": "pooled_cast_fp16", + "type": "ios17.real_div", + "device": "gpu" + }, + { + "name": "options_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "input_53_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_567", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_569", + "type": "ios17.logical_not", + "device": "gpu" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "var_571_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_574_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_576_cast_fp16", + "type": "ios16.reduce_sum", + "device": "gpu" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_579", + "type": "ios17.logical_not", + "device": "gpu" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "var_587_cast_fp16", + "type": "ios16.fill_like", + "device": "gpu" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "var_591_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "probabilities", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "logits", + "type": "ios17.cast", + "device": "gpu" + } + ] + } + }, + "CPU_AND_NE": { + "load_ms": 566.8206670088693, + "first_prediction_ms": 1.6522080113645643, + "warm_prediction": { + "count": 30, + "median_ms": 0.9285830019507557, + "p95_ms": 0.9725897951284423, + "min_ms": 0.9064170008059591, + "max_ms": 1.2301660026423633 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 0.9575410222169012 + }, + { + "row": 68, + "milliseconds": 0.9647910192143172 + }, + { + "row": 130, + "milliseconds": 0.9704580006655306 + }, + { + "row": 0, + "milliseconds": 0.9583340142853558 + }, + { + "row": 68, + "milliseconds": 0.9467090130783617 + }, + { + "row": 130, + "milliseconds": 1.2301660026423633 + }, + { + "row": 0, + "milliseconds": 0.9743339905980974 + }, + { + "row": 68, + "milliseconds": 0.9361249976791441 + }, + { + "row": 130, + "milliseconds": 0.9265419794246554 + }, + { + "row": 0, + "milliseconds": 0.9583329956512898 + }, + { + "row": 68, + "milliseconds": 0.9652920125517994 + }, + { + "row": 130, + "milliseconds": 0.9257500059902668 + }, + { + "row": 0, + "milliseconds": 0.9283330000471324 + }, + { + "row": 68, + "milliseconds": 0.9066659840755165 + }, + { + "row": 130, + "milliseconds": 0.9307079890277237 + }, + { + "row": 0, + "milliseconds": 0.9274999902117997 + }, + { + "row": 68, + "milliseconds": 0.9069579828064889 + }, + { + "row": 130, + "milliseconds": 0.9290000016335398 + }, + { + "row": 0, + "milliseconds": 0.927332992432639 + }, + { + "row": 68, + "milliseconds": 0.917499972274527 + }, + { + "row": 130, + "milliseconds": 0.9173330035991967 + }, + { + "row": 0, + "milliseconds": 0.9288330038543791 + }, + { + "row": 68, + "milliseconds": 0.9251659794244915 + }, + { + "row": 130, + "milliseconds": 0.9205000242218375 + }, + { + "row": 0, + "milliseconds": 0.9272919851355255 + }, + { + "row": 68, + "milliseconds": 0.9192500147037208 + }, + { + "row": 130, + "milliseconds": 0.933834002353251 + }, + { + "row": 0, + "milliseconds": 0.9269170113839209 + }, + { + "row": 68, + "milliseconds": 0.9406250028405339 + }, + { + "row": 130, + "milliseconds": 0.9064170008059591 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 173, + "device_counts": { + "cpu": 24, + "gpu": 0, + "ane": 149, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 239 + }, + "operations": [ + { + "name": "option_mask_1", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "context_mask", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "var_27", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_43", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_43_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios17.concat", + "device": "ane" + }, + { + "name": "context_ids_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "cast_54", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "greater_equal_0", + "type": "ios17.greater_equal", + "device": "cpu" + }, + { + "name": "add_3", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "select_0", + "type": "select", + "device": "cpu" + }, + { + "name": "select_0_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "var_59_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "cpu" + }, + { + "name": "src_1_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios17.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_1_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_124_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_125_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_126_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_127_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_136_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_143_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_150_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_154_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_156_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_0_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_172_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_5_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_7_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_11_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_17_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "query_5_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_7_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_231_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_232_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_233_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_234_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_243_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_250_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_257_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "q_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_1_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_279_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_19_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_21_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_23_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_27_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "context_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "flat_ids", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "flat_mask", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "var_333", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_349", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_349_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios17.concat", + "device": "ane" + }, + { + "name": "flat_ids_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "cast_55", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "greater_equal_1", + "type": "ios17.greater_equal", + "device": "cpu" + }, + { + "name": "add_4", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "select_1", + "type": "select", + "device": "cpu" + }, + { + "name": "select_1_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "var_365_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "cpu" + }, + { + "name": "src_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "mask_cast_fp16", + "type": "ios17.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_9_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_11_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_428_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_429_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_430_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_431_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_440_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_447_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_454_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_458_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_460_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_2_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_476_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_35_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_37_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_39_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_43_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "hidden_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "var_504", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_504_to_fp16", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "var_510_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_515_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_520_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_522_cast_fp16", + "type": "ios17.maximum", + "device": "ane" + }, + { + "name": "pooled_cast_fp16", + "type": "ios17.real_div", + "device": "ane" + }, + { + "name": "options_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "input_53_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_567", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_569", + "type": "ios17.logical_not", + "device": "ane" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_571_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_574_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_576_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_579", + "type": "ios17.logical_not", + "device": "ane" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_587_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_591_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "probabilities", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "logits", + "type": "ios17.cast", + "device": "ane" + } + ] + } + }, + "ALL": { + "load_ms": 115.82287499913946, + "first_prediction_ms": 17.531749996123835, + "warm_prediction": { + "count": 30, + "median_ms": 0.9119170135818422, + "p95_ms": 1.2285901117138565, + "min_ms": 0.8736249874345958, + "max_ms": 1.7777080065570772 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 1.7777080065570772 + }, + { + "row": 68, + "milliseconds": 1.2005419994238764 + }, + { + "row": 130, + "milliseconds": 0.9608750115148723 + }, + { + "row": 0, + "milliseconds": 1.0891249985434115 + }, + { + "row": 68, + "milliseconds": 1.2131670082453638 + }, + { + "row": 130, + "milliseconds": 1.241209014551714 + }, + { + "row": 0, + "milliseconds": 1.1116660025436431 + }, + { + "row": 68, + "milliseconds": 1.179958024295047 + }, + { + "row": 130, + "milliseconds": 0.8888749871402979 + }, + { + "row": 0, + "milliseconds": 0.8894159982446581 + }, + { + "row": 68, + "milliseconds": 0.9866250038612634 + }, + { + "row": 130, + "milliseconds": 0.9067920036613941 + }, + { + "row": 0, + "milliseconds": 0.8964589796960354 + }, + { + "row": 68, + "milliseconds": 0.901333027286455 + }, + { + "row": 130, + "milliseconds": 0.9048339852597564 + }, + { + "row": 0, + "milliseconds": 0.9170420235022902 + }, + { + "row": 68, + "milliseconds": 0.8820410002954304 + }, + { + "row": 130, + "milliseconds": 0.8736249874345958 + }, + { + "row": 0, + "milliseconds": 0.8920000109355897 + }, + { + "row": 68, + "milliseconds": 0.8972920186351985 + }, + { + "row": 130, + "milliseconds": 0.888916983967647 + }, + { + "row": 0, + "milliseconds": 0.8755000017117709 + }, + { + "row": 68, + "milliseconds": 0.9061669989023358 + }, + { + "row": 130, + "milliseconds": 1.0763329919427633 + }, + { + "row": 0, + "milliseconds": 0.9489590011071414 + }, + { + "row": 68, + "milliseconds": 0.9267920104321092 + }, + { + "row": 130, + "milliseconds": 0.9253750031348318 + }, + { + "row": 0, + "milliseconds": 0.9227920090779662 + }, + { + "row": 68, + "milliseconds": 0.8931249903980643 + }, + { + "row": 130, + "milliseconds": 0.8973750227596611 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 173, + "device_counts": { + "cpu": 0, + "gpu": 173, + "ane": 0, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 239 + }, + "operations": [ + { + "name": "option_mask_1", + "type": "ios17.not_equal", + "device": "gpu" + }, + { + "name": "context_mask", + "type": "ios17.not_equal", + "device": "gpu" + }, + { + "name": "var_27", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "gpu" + }, + { + "name": "var_43", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_43_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios17.concat", + "device": "gpu" + }, + { + "name": "context_ids_to_int16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "cast_54", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "greater_equal_0", + "type": "ios17.greater_equal", + "device": "gpu" + }, + { + "name": "add_3", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "select_0", + "type": "select", + "device": "gpu" + }, + { + "name": "select_0_to_int16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "var_59_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "gpu" + }, + { + "name": "src_1_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios17.equal", + "device": "gpu" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "query_1_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "query_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "var_124_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_125_cast_fp16", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_126_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "var_127_cast_fp16", + "type": "ios17.squeeze", + "device": "gpu" + }, + { + "name": "q_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "k_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "v_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_136_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_143_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_150_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_154_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_156_cast_fp16", + "type": "tile", + "device": "gpu" + }, + { + "name": "q_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "q_5_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "k_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "k_5_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "v_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "v_5_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "add_0_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_172_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_3_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "input_5_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "input_7_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_11_cast_fp16", + "type": "ios16.relu", + "device": "gpu" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_17_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "query_5_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "query_7_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "var_231_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_232_cast_fp16", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_233_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "var_234_cast_fp16", + "type": "ios17.squeeze", + "device": "gpu" + }, + { + "name": "q_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "k_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "v_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_243_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_250_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_257_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "q_9_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "q_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "k_9_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "k_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "v_9_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "v_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "add_1_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_279_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_19_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "input_21_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "input_23_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_27_cast_fp16", + "type": "ios16.relu", + "device": "gpu" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "context_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "flat_ids", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "flat_mask", + "type": "ios17.not_equal", + "device": "gpu" + }, + { + "name": "var_333", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "gpu" + }, + { + "name": "var_349", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_349_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios17.concat", + "device": "gpu" + }, + { + "name": "flat_ids_to_int16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "cast_55", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "greater_equal_1", + "type": "ios17.greater_equal", + "device": "gpu" + }, + { + "name": "add_4", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "select_1", + "type": "select", + "device": "gpu" + }, + { + "name": "select_1_to_int16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "var_365_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "gpu" + }, + { + "name": "src_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "mask_cast_fp16", + "type": "ios17.equal", + "device": "gpu" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "query_9_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "query_11_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "var_428_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_429_cast_fp16", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_430_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "var_431_cast_fp16", + "type": "ios17.squeeze", + "device": "gpu" + }, + { + "name": "q_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "k_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "v_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "gpu" + }, + { + "name": "var_440_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_447_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_454_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_458_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "var_460_cast_fp16", + "type": "tile", + "device": "gpu" + }, + { + "name": "q_15_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "q_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "k_15_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "k_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "v_15_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "v_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "add_2_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_476_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_35_cast_fp16", + "type": "ios17.transpose", + "device": "gpu" + }, + { + "name": "input_37_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "input_39_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "input_43_cast_fp16", + "type": "ios16.relu", + "device": "gpu" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "hidden_cast_fp16", + "type": "ios17.add", + "device": "gpu" + }, + { + "name": "var_504", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_504_to_fp16", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "var_510_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_515_cast_fp16", + "type": "ios16.reduce_sum", + "device": "gpu" + }, + { + "name": "var_520_cast_fp16", + "type": "ios16.reduce_sum", + "device": "gpu" + }, + { + "name": "var_522_cast_fp16", + "type": "ios17.maximum", + "device": "gpu" + }, + { + "name": "pooled_cast_fp16", + "type": "ios17.real_div", + "device": "gpu" + }, + { + "name": "options_cast_fp16", + "type": "ios17.reshape", + "device": "gpu" + }, + { + "name": "input_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "input_53_cast_fp16", + "type": "ios17.layer_norm", + "device": "gpu" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios17.linear", + "device": "gpu" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_567", + "type": "ios17.expand_dims", + "device": "gpu" + }, + { + "name": "var_569", + "type": "ios17.logical_not", + "device": "gpu" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "var_571_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios17.matmul", + "device": "gpu" + }, + { + "name": "var_574_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_576_cast_fp16", + "type": "ios16.reduce_sum", + "device": "gpu" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios17.mul", + "device": "gpu" + }, + { + "name": "var_579", + "type": "ios17.logical_not", + "device": "gpu" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "var_587_cast_fp16", + "type": "ios16.fill_like", + "device": "gpu" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "gpu" + }, + { + "name": "var_591_cast_fp16", + "type": "ios16.softmax", + "device": "gpu" + }, + { + "name": "probabilities", + "type": "ios17.cast", + "device": "gpu" + }, + { + "name": "logits", + "type": "ios17.cast", + "device": "gpu" + } + ] + } + } + }, + "script_sha256": "d5e1386f54fc6c482647120e593e7ffbbe4bbb277c2cc6d4949154660fe239fd" +} diff --git a/reports/browser-validation.json b/reports/browser-validation.json new file mode 100644 index 0000000000000000000000000000000000000000..3162a8b7bceb6bd2633e5be088a9867b6635c68f --- /dev/null +++ b/reports/browser-validation.json @@ -0,0 +1,3641 @@ +{ + "datasetSHA256" : "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "description" : "Actual WKWebView DOM observations, real Core ML choices, dispatched events, independent DOM readback. Public examples only.", + "integrationChecks" : [ + "100/100 original choices", + "Independent DOM readback after every action", + "Input/change events match fill/check actions", + "No model-triggered submissions", + "Stale DOM observations rejected", + "Explicit local button clicks produce receipts" + ], + "runs" : [ + { + "correct" : 18, + "decisions" : [ + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"First name\" value=\"\"", + "controlID" : "control-0", + "effect" : "fill", + "loopMilliseconds" : 10.183375, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 3.235125, + "selectedIndex" : 22, + "selectedOption" : "fill First name: Amara", + "title" : "First name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Last name\" value=\"\"", + "controlID" : "control-1", + "effect" : "fill", + "loopMilliseconds" : 6.169709, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.915459, + "selectedIndex" : 23, + "selectedOption" : "fill Last name: Okafor", + "title" : "Last name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Date of birth\" value=\"\"", + "controlID" : "control-2", + "effect" : "fill", + "loopMilliseconds" : 5.879125, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.493958, + "selectedIndex" : 2, + "selectedOption" : "fill DOB: 03/14/1987", + "title" : "Date of birth" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Email address\" value=\"\"", + "controlID" : "control-3", + "effect" : "fill", + "loopMilliseconds" : 6.116791, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.579125, + "selectedIndex" : 5, + "selectedOption" : "fill E-mail: amara.okafor@fastmail.com", + "title" : "Email address" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Phone number\" value=\"\"", + "controlID" : "control-4", + "effect" : "fill", + "loopMilliseconds" : 5.651792, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.414833, + "selectedIndex" : 4, + "selectedOption" : "fill Tel: (503) 555-0142", + "title" : "Phone number" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Street address\" value=\"\"", + "controlID" : "control-5", + "effect" : "fill", + "loopMilliseconds" : 6.211917, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.912667, + "selectedIndex" : 6, + "selectedOption" : "fill Address: 4821 Lakeview Drive", + "title" : "Street address" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"City\" value=\"\"", + "controlID" : "control-6", + "effect" : "fill", + "loopMilliseconds" : 5.497875, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.494375, + "selectedIndex" : 7, + "selectedOption" : "fill City: Portland", + "title" : "City" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"State\" value=\"\"", + "controlID" : "control-7", + "effect" : "fill", + "loopMilliseconds" : 6.191459, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.159167, + "selectedIndex" : 8, + "selectedOption" : "fill State: OR", + "title" : "State" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"ZIP code\" value=\"\"", + "controlID" : "control-8", + "effect" : "fill", + "loopMilliseconds" : 6.161958, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.719875, + "selectedIndex" : 9, + "selectedOption" : "fill ZIP: 97205", + "title" : "ZIP code" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Insurance provider\" value=\"\"", + "controlID" : "control-9", + "effect" : "fill", + "loopMilliseconds" : 6.071208, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.387458, + "selectedIndex" : 10, + "selectedOption" : "fill Insurer: Blue Cross", + "title" : "Insurance provider" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Policy number\" value=\"\"", + "controlID" : "control-10", + "effect" : "fill", + "loopMilliseconds" : 6.052333, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.621791, + "selectedIndex" : 11, + "selectedOption" : "fill Policy #: POL-55018827", + "title" : "Policy number" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Emergency contact name\" value=\"\"", + "controlID" : "control-11", + "effect" : "fill", + "loopMilliseconds" : 6.140875, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.816458, + "selectedIndex" : 13, + "selectedOption" : "fill Emergency contact: Chidi Okafor", + "title" : "Emergency contact name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Emergency contact phone\" value=\"\"", + "controlID" : "control-12", + "effect" : "fill", + "loopMilliseconds" : 5.870875, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.703875, + "selectedIndex" : 15, + "selectedOption" : "fill ICE phone: (503) 555-0199", + "title" : "Emergency contact phone" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Reason for visit\" value=\"\"", + "controlID" : "control-13", + "effect" : "fill", + "loopMilliseconds" : 5.660875, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 0.99902344, + "scoreMilliseconds" : 1.387, + "selectedIndex" : 16, + "selectedOption" : "fill Reason for visit: Persistent cough for two weeks", + "title" : "Reason for visit" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Promo code (optional)\" value=\"\"", + "controlID" : "control-14", + "effect" : "skip", + "loopMilliseconds" : 5.711625, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.434083, + "selectedIndex" : 26, + "selectedOption" : "skip", + "title" : "Promo code (optional)" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT CheckBox \"I consent to treatment and agree to the privacy policy\" unchecked", + "controlID" : "control-15", + "effect" : "check", + "loopMilliseconds" : 5.672333, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.448958, + "selectedIndex" : 24, + "selectedOption" : "check", + "title" : "I consent to treatment and agree to the privacy policy" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Button \"Clear form\" value=\"\"", + "controlID" : "control-16", + "effect" : "skip", + "loopMilliseconds" : 5.482583, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.759083, + "selectedIndex" : 26, + "selectedOption" : "skip", + "title" : "Clear form" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Button \"Submit registration\" value=\"\"", + "controlID" : "control-17", + "effect" : "review", + "loopMilliseconds" : 6.024375, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.840916, + "selectedIndex" : 25, + "selectedOption" : "click", + "title" : "Submit registration" + } + ], + "events" : { + "change" : 15, + "input" : 15, + "submit" : 0 + }, + "filledFields" : 14, + "form" : "patient-registration", + "variant" : "baseline" + }, + { + "correct" : 18, + "decisions" : [ + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"First name\" value=\"\"", + "controlID" : "control-0", + "effect" : "fill", + "loopMilliseconds" : 8.078875, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.602042, + "selectedIndex" : 22, + "selectedOption" : "fill First name: Amara", + "title" : "First name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Last name\" value=\"\"", + "controlID" : "control-1", + "effect" : "fill", + "loopMilliseconds" : 6.896833, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.691792, + "selectedIndex" : 23, + "selectedOption" : "fill Last name: Okafor", + "title" : "Last name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Date of birth\" value=\"\"", + "controlID" : "control-2", + "effect" : "fill", + "loopMilliseconds" : 6.925166, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.655292, + "selectedIndex" : 2, + "selectedOption" : "fill DOB: 03/14/1987", + "title" : "Date of birth" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Email address\" value=\"\"", + "controlID" : "control-3", + "effect" : "fill", + "loopMilliseconds" : 5.820166, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.825958, + "selectedIndex" : 5, + "selectedOption" : "fill E-mail: amara.okafor@fastmail.com", + "title" : "Email address" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Phone number\" value=\"\"", + "controlID" : "control-4", + "effect" : "fill", + "loopMilliseconds" : 5.496542, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.49975, + "selectedIndex" : 4, + "selectedOption" : "fill Tel: (503) 555-0142", + "title" : "Phone number" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Street address\" value=\"\"", + "controlID" : "control-5", + "effect" : "fill", + "loopMilliseconds" : 7.937416, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.715542, + "selectedIndex" : 6, + "selectedOption" : "fill Address: 4821 Lakeview Drive", + "title" : "Street address" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"City\" value=\"\"", + "controlID" : "control-6", + "effect" : "fill", + "loopMilliseconds" : 6.481875, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.626417, + "selectedIndex" : 7, + "selectedOption" : "fill City: Portland", + "title" : "City" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"State\" value=\"\"", + "controlID" : "control-7", + "effect" : "fill", + "loopMilliseconds" : 5.75225, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.873334, + "selectedIndex" : 8, + "selectedOption" : "fill State: OR", + "title" : "State" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"ZIP code\" value=\"\"", + "controlID" : "control-8", + "effect" : "fill", + "loopMilliseconds" : 8.218333, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.964792, + "selectedIndex" : 9, + "selectedOption" : "fill ZIP: 97205", + "title" : "ZIP code" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Insurance provider\" value=\"\"", + "controlID" : "control-9", + "effect" : "fill", + "loopMilliseconds" : 6.425791, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.866792, + "selectedIndex" : 10, + "selectedOption" : "fill Insurer: Blue Cross", + "title" : "Insurance provider" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Policy number\" value=\"\"", + "controlID" : "control-10", + "effect" : "fill", + "loopMilliseconds" : 6.026916, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.236667, + "selectedIndex" : 11, + "selectedOption" : "fill Policy #: POL-55018827", + "title" : "Policy number" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Emergency contact name\" value=\"\"", + "controlID" : "control-11", + "effect" : "fill", + "loopMilliseconds" : 8.008208, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.506792, + "selectedIndex" : 13, + "selectedOption" : "fill Emergency contact: Chidi Okafor", + "title" : "Emergency contact name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Emergency contact phone\" value=\"\"", + "controlID" : "control-12", + "effect" : "fill", + "loopMilliseconds" : 6.63675, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.645042, + "selectedIndex" : 15, + "selectedOption" : "fill ICE phone: (503) 555-0199", + "title" : "Emergency contact phone" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Reason for visit\" value=\"\"", + "controlID" : "control-13", + "effect" : "fill", + "loopMilliseconds" : 7.698625, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 0.99902344, + "scoreMilliseconds" : 1.266458, + "selectedIndex" : 16, + "selectedOption" : "fill Reason for visit: Persistent cough for two weeks", + "title" : "Reason for visit" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Promo code (optional)\" value=\"\"", + "controlID" : "control-14", + "effect" : "skip", + "loopMilliseconds" : 6.407667, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.33375, + "selectedIndex" : 26, + "selectedOption" : "skip", + "title" : "Promo code (optional)" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT CheckBox \"I consent to treatment and agree to the privacy policy\" unchecked", + "controlID" : "control-15", + "effect" : "check", + "loopMilliseconds" : 7.717416, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.495084, + "selectedIndex" : 24, + "selectedOption" : "check", + "title" : "I consent to treatment and agree to the privacy policy" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Button \"Clear form\" value=\"\"", + "controlID" : "control-16", + "effect" : "skip", + "loopMilliseconds" : 4.246209, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.464583, + "selectedIndex" : 26, + "selectedOption" : "skip", + "title" : "Clear form" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Button \"Submit registration\" value=\"\"", + "controlID" : "control-17", + "effect" : "review", + "loopMilliseconds" : 5.310375, + "options" : [ + "fill Referral Letter: Riverside Family Practice", + "fill Patient: Amara Okafor", + "fill DOB: 03/14/1987", + "fill Sex: Female", + "fill Tel: (503) 555-0142", + "fill E-mail: amara.okafor@fastmail.com", + "fill Address: 4821 Lakeview Drive", + "fill City: Portland", + "fill State: OR", + "fill ZIP: 97205", + "fill Insurer: Blue Cross", + "fill Policy #: POL-55018827", + "fill Group: 004412", + "fill Emergency contact: Chidi Okafor", + "fill Relationship: Spouse", + "fill ICE phone: (503) 555-0199", + "fill Reason for visit: Persistent cough for two weeks", + "fill Allergies: Penicillin", + "fill Blood type: O+", + "fill Referred by: Dr. Lindqvist", + "fill Printed: 09/17/2026", + "fill Document ID: DOC-118204", + "fill First name: Amara", + "fill Last name: Okafor", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.5045, + "selectedIndex" : 25, + "selectedOption" : "click", + "title" : "Submit registration" + } + ], + "events" : { + "change" : 15, + "input" : 15, + "submit" : 0 + }, + "filledFields" : 14, + "form" : "patient-registration", + "variant" : "ane-gather" + }, + { + "correct" : 15, + "decisions" : [ + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Full name\" value=\"\"", + "controlID" : "control-68", + "effect" : "fill", + "loopMilliseconds" : 8.486084, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.43625, + "selectedIndex" : 0, + "selectedOption" : "fill Name: Mateo Delgado", + "title" : "Full name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Email\" value=\"\"", + "controlID" : "control-69", + "effect" : "fill", + "loopMilliseconds" : 5.892417, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.901792, + "selectedIndex" : 1, + "selectedOption" : "fill Email: mateo.delgado@proton.me", + "title" : "Email" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Mobile phone\" value=\"\"", + "controlID" : "control-70", + "effect" : "fill", + "loopMilliseconds" : 5.433167, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.393041, + "selectedIndex" : 2, + "selectedOption" : "fill Mobile: 512-555-0177", + "title" : "Mobile phone" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"LinkedIn profile URL\" value=\"\"", + "controlID" : "control-71", + "effect" : "fill", + "loopMilliseconds" : 5.500917, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.4015, + "selectedIndex" : 3, + "selectedOption" : "fill LinkedIn: linkedin.com/in/mateo-delgado", + "title" : "LinkedIn profile URL" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Current employer\" value=\"\"", + "controlID" : "control-72", + "effect" : "fill", + "loopMilliseconds" : 5.516458, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.504292, + "selectedIndex" : 5, + "selectedOption" : "fill Current employer: Initech", + "title" : "Current employer" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Current job title\" value=\"\"", + "controlID" : "control-73", + "effect" : "fill", + "loopMilliseconds" : 5.526083, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.405125, + "selectedIndex" : 6, + "selectedOption" : "fill Title: Robotics Technician", + "title" : "Current job title" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Years of experience\" value=\"\"", + "controlID" : "control-74", + "effect" : "fill", + "loopMilliseconds" : 5.729708, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.630708, + "selectedIndex" : 7, + "selectedOption" : "fill Years of experience: 7", + "title" : "Years of experience" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Desired salary\" value=\"\"", + "controlID" : "control-75", + "effect" : "fill", + "loopMilliseconds" : 5.508709, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.445834, + "selectedIndex" : 8, + "selectedOption" : "fill Expected salary: $92,000", + "title" : "Desired salary" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Earliest start date\" value=\"\"", + "controlID" : "control-76", + "effect" : "fill", + "loopMilliseconds" : 5.614541, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.436833, + "selectedIndex" : 9, + "selectedOption" : "fill Available: 2026-11-02", + "title" : "Earliest start date" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Cover letter\" value=\"\"", + "controlID" : "control-77", + "effect" : "fill", + "loopMilliseconds" : 5.681291, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.449417, + "selectedIndex" : 13, + "selectedOption" : "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "title" : "Cover letter" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Referral code (optional)\" value=\"\"", + "controlID" : "control-78", + "effect" : "skip", + "loopMilliseconds" : 5.941208, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.713208, + "selectedIndex" : 20, + "selectedOption" : "skip", + "title" : "Referral code (optional)" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", + "controlID" : "control-79", + "effect" : "check", + "loopMilliseconds" : 6.24725, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 0.99902344, + "scoreMilliseconds" : 2.055166, + "selectedIndex" : 18, + "selectedOption" : "check", + "title" : "I am legally authorized to work in this country" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT CheckBox \"I certify the information above is accurate\" unchecked", + "controlID" : "control-80", + "effect" : "check", + "loopMilliseconds" : 5.326166, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.440916, + "selectedIndex" : 18, + "selectedOption" : "check", + "title" : "I certify the information above is accurate" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Button \"Cancel\" value=\"\"", + "controlID" : "control-81", + "effect" : "skip", + "loopMilliseconds" : 5.507792, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.481125, + "selectedIndex" : 20, + "selectedOption" : "skip", + "title" : "Cancel" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Button \"Submit application\" value=\"\"", + "controlID" : "control-82", + "effect" : "review", + "loopMilliseconds" : 5.818625, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.872208, + "selectedIndex" : 19, + "selectedOption" : "click", + "title" : "Submit application" + } + ], + "events" : { + "change" : 12, + "input" : 12, + "submit" : 0 + }, + "filledFields" : 10, + "form" : "job-application", + "variant" : "baseline" + }, + { + "correct" : 15, + "decisions" : [ + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Full name\" value=\"\"", + "controlID" : "control-68", + "effect" : "fill", + "loopMilliseconds" : 8.233375, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.241375, + "selectedIndex" : 0, + "selectedOption" : "fill Name: Mateo Delgado", + "title" : "Full name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Email\" value=\"\"", + "controlID" : "control-69", + "effect" : "fill", + "loopMilliseconds" : 5.800833, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.805375, + "selectedIndex" : 1, + "selectedOption" : "fill Email: mateo.delgado@proton.me", + "title" : "Email" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Mobile phone\" value=\"\"", + "controlID" : "control-70", + "effect" : "fill", + "loopMilliseconds" : 5.839667, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.187708, + "selectedIndex" : 2, + "selectedOption" : "fill Mobile: 512-555-0177", + "title" : "Mobile phone" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"LinkedIn profile URL\" value=\"\"", + "controlID" : "control-71", + "effect" : "fill", + "loopMilliseconds" : 4.211541, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.283458, + "selectedIndex" : 3, + "selectedOption" : "fill LinkedIn: linkedin.com/in/mateo-delgado", + "title" : "LinkedIn profile URL" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Current employer\" value=\"\"", + "controlID" : "control-72", + "effect" : "fill", + "loopMilliseconds" : 6.520292, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.729416, + "selectedIndex" : 5, + "selectedOption" : "fill Current employer: Initech", + "title" : "Current employer" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Current job title\" value=\"\"", + "controlID" : "control-73", + "effect" : "fill", + "loopMilliseconds" : 6.080833, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.641375, + "selectedIndex" : 6, + "selectedOption" : "fill Title: Robotics Technician", + "title" : "Current job title" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Years of experience\" value=\"\"", + "controlID" : "control-74", + "effect" : "fill", + "loopMilliseconds" : 6.047791, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.65175, + "selectedIndex" : 7, + "selectedOption" : "fill Years of experience: 7", + "title" : "Years of experience" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Desired salary\" value=\"\"", + "controlID" : "control-75", + "effect" : "fill", + "loopMilliseconds" : 7.888875, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.783709, + "selectedIndex" : 8, + "selectedOption" : "fill Expected salary: $92,000", + "title" : "Desired salary" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Earliest start date\" value=\"\"", + "controlID" : "control-76", + "effect" : "fill", + "loopMilliseconds" : 5.914333, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.209875, + "selectedIndex" : 9, + "selectedOption" : "fill Available: 2026-11-02", + "title" : "Earliest start date" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Cover letter\" value=\"\"", + "controlID" : "control-77", + "effect" : "fill", + "loopMilliseconds" : 6.635375, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.755, + "selectedIndex" : 13, + "selectedOption" : "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "title" : "Cover letter" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Referral code (optional)\" value=\"\"", + "controlID" : "control-78", + "effect" : "skip", + "loopMilliseconds" : 5.980958, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.178416, + "selectedIndex" : 20, + "selectedOption" : "skip", + "title" : "Referral code (optional)" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", + "controlID" : "control-79", + "effect" : "check", + "loopMilliseconds" : 6.144917, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 0.99902344, + "scoreMilliseconds" : 1.353916, + "selectedIndex" : 18, + "selectedOption" : "check", + "title" : "I am legally authorized to work in this country" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT CheckBox \"I certify the information above is accurate\" unchecked", + "controlID" : "control-80", + "effect" : "check", + "loopMilliseconds" : 5.907459, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.449417, + "selectedIndex" : 18, + "selectedOption" : "check", + "title" : "I certify the information above is accurate" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Button \"Cancel\" value=\"\"", + "controlID" : "control-81", + "effect" : "skip", + "loopMilliseconds" : 5.812125, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.267459, + "selectedIndex" : 20, + "selectedOption" : "skip", + "title" : "Cancel" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Button \"Submit application\" value=\"\"", + "controlID" : "control-82", + "effect" : "review", + "loopMilliseconds" : 7.78025, + "options" : [ + "fill Name: Mateo Delgado", + "fill Email: mateo.delgado@proton.me", + "fill Mobile: 512-555-0177", + "fill LinkedIn: linkedin.com/in/mateo-delgado", + "fill Website: https://delgado.dev", + "fill Current employer: Initech", + "fill Title: Robotics Technician", + "fill Years of experience: 7", + "fill Expected salary: $92,000", + "fill Available: 2026-11-02", + "fill Education: State University", + "fill Degree: BEng Mechanical", + "fill Graduated: 2019", + "fill Statement: I am excited to apply for this role and bring my experience to your team.", + "fill Reference: Priya Patel, Engineering Manager", + "fill Language: English", + "fill First name: Mateo", + "fill Last name: Delgado", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.673625, + "selectedIndex" : 19, + "selectedOption" : "click", + "title" : "Submit application" + } + ], + "events" : { + "change" : 12, + "input" : 12, + "submit" : 0 + }, + "filledFields" : 10, + "form" : "job-application", + "variant" : "ane-gather" + }, + { + "correct" : 17, + "decisions" : [ + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Full name\" value=\"\"", + "controlID" : "control-130", + "effect" : "fill", + "loopMilliseconds" : 8.6865, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.414042, + "selectedIndex" : 1, + "selectedOption" : "fill Insured: Kenji Tanaka", + "title" : "Full name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Policy number\" value=\"\"", + "controlID" : "control-131", + "effect" : "fill", + "loopMilliseconds" : 5.618083, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.543417, + "selectedIndex" : 2, + "selectedOption" : "fill Policy: POL-77120944", + "title" : "Policy number" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Email\" value=\"\"", + "controlID" : "control-132", + "effect" : "fill", + "loopMilliseconds" : 5.575959, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.463167, + "selectedIndex" : 3, + "selectedOption" : "fill E-mail: ktanaka42@outlook.com", + "title" : "Email" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Daytime phone\" value=\"\"", + "controlID" : "control-133", + "effect" : "fill", + "loopMilliseconds" : 5.640875, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.583542, + "selectedIndex" : 4, + "selectedOption" : "fill Phone: (720) 555-0186", + "title" : "Daytime phone" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Vehicle make\" value=\"\"", + "controlID" : "control-134", + "effect" : "fill", + "loopMilliseconds" : 5.965625, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.96425, + "selectedIndex" : 5, + "selectedOption" : "fill Make: Subaru", + "title" : "Vehicle make" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Vehicle model\" value=\"\"", + "controlID" : "control-135", + "effect" : "fill", + "loopMilliseconds" : 5.659833, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.488916, + "selectedIndex" : 6, + "selectedOption" : "fill Model: Outback", + "title" : "Vehicle model" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Model year\" value=\"\"", + "controlID" : "control-136", + "effect" : "fill", + "loopMilliseconds" : 5.295916, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.362375, + "selectedIndex" : 7, + "selectedOption" : "fill Year: 2021", + "title" : "Model year" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"License plate\" value=\"\"", + "controlID" : "control-137", + "effect" : "fill", + "loopMilliseconds" : 6.967209, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 0.75927734, + "scoreMilliseconds" : 2.041708, + "selectedIndex" : 8, + "selectedOption" : "fill Plate: KLM-4471", + "title" : "License plate" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"VIN\" value=\"\"", + "controlID" : "control-138", + "effect" : "fill", + "loopMilliseconds" : 5.984333, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.913084, + "selectedIndex" : 9, + "selectedOption" : "fill VIN: 4S4BTANC2M3187721", + "title" : "VIN" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Date of incident\" value=\"\"", + "controlID" : "control-139", + "effect" : "fill", + "loopMilliseconds" : 5.831792, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.672958, + "selectedIndex" : 10, + "selectedOption" : "fill Date of loss: 2026-08-29", + "title" : "Date of incident" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Claim number\" value=\"\"", + "controlID" : "control-140", + "effect" : "fill", + "loopMilliseconds" : 5.989167, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.499542, + "selectedIndex" : 11, + "selectedOption" : "fill Claim: CLM-402917", + "title" : "Claim number" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Description of incident\" value=\"\"", + "controlID" : "control-141", + "effect" : "fill", + "loopMilliseconds" : 6.22725, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.980791, + "selectedIndex" : 12, + "selectedOption" : "fill Summary: Rear-ended at a stop light", + "title" : "Description of incident" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Agent code\" value=\"\"", + "controlID" : "control-142", + "effect" : "skip", + "loopMilliseconds" : 5.470208, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.479709, + "selectedIndex" : 18, + "selectedOption" : "skip", + "title" : "Agent code" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT CheckBox \"I confirm the details are correct\" unchecked", + "controlID" : "control-143", + "effect" : "check", + "loopMilliseconds" : 6.18375, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.976333, + "selectedIndex" : 16, + "selectedOption" : "check", + "title" : "I confirm the details are correct" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT CheckBox \"Text me claim status updates\" unchecked", + "controlID" : "control-144", + "effect" : "skip", + "loopMilliseconds" : 5.920167, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.948209, + "selectedIndex" : 18, + "selectedOption" : "skip", + "title" : "Text me claim status updates" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Button \"Discard\" value=\"\"", + "controlID" : "control-145", + "effect" : "skip", + "loopMilliseconds" : 6.310834, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.15975, + "selectedIndex" : 18, + "selectedOption" : "skip", + "title" : "Discard" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Button \"Submit claim\" value=\"\"", + "controlID" : "control-146", + "effect" : "review", + "loopMilliseconds" : 5.565916, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.579417, + "selectedIndex" : 17, + "selectedOption" : "click", + "title" : "Submit claim" + } + ], + "events" : { + "change" : 13, + "input" : 13, + "submit" : 0 + }, + "filledFields" : 12, + "form" : "auto-claim", + "variant" : "baseline" + }, + { + "correct" : 17, + "decisions" : [ + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Full name\" value=\"\"", + "controlID" : "control-130", + "effect" : "fill", + "loopMilliseconds" : 8.486792, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.203875, + "selectedIndex" : 1, + "selectedOption" : "fill Insured: Kenji Tanaka", + "title" : "Full name" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Policy number\" value=\"\"", + "controlID" : "control-131", + "effect" : "fill", + "loopMilliseconds" : 7.892, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.797, + "selectedIndex" : 2, + "selectedOption" : "fill Policy: POL-77120944", + "title" : "Policy number" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Email\" value=\"\"", + "controlID" : "control-132", + "effect" : "fill", + "loopMilliseconds" : 7.798584, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.809417, + "selectedIndex" : 3, + "selectedOption" : "fill E-mail: ktanaka42@outlook.com", + "title" : "Email" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Daytime phone\" value=\"\"", + "controlID" : "control-133", + "effect" : "fill", + "loopMilliseconds" : 7.63125, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.784833, + "selectedIndex" : 4, + "selectedOption" : "fill Phone: (720) 555-0186", + "title" : "Daytime phone" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Vehicle make\" value=\"\"", + "controlID" : "control-134", + "effect" : "fill", + "loopMilliseconds" : 7.0895, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.276791, + "selectedIndex" : 5, + "selectedOption" : "fill Make: Subaru", + "title" : "Vehicle make" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Vehicle model\" value=\"\"", + "controlID" : "control-135", + "effect" : "fill", + "loopMilliseconds" : 6.453125, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.761292, + "selectedIndex" : 6, + "selectedOption" : "fill Model: Outback", + "title" : "Vehicle model" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Model year\" value=\"\"", + "controlID" : "control-136", + "effect" : "fill", + "loopMilliseconds" : 6.379333, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.762083, + "selectedIndex" : 7, + "selectedOption" : "fill Year: 2021", + "title" : "Model year" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"License plate\" value=\"\"", + "controlID" : "control-137", + "effect" : "fill", + "loopMilliseconds" : 8.22125, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 0.75927734, + "scoreMilliseconds" : 1.879542, + "selectedIndex" : 8, + "selectedOption" : "fill Plate: KLM-4471", + "title" : "License plate" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"VIN\" value=\"\"", + "controlID" : "control-138", + "effect" : "fill", + "loopMilliseconds" : 6.757625, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.752583, + "selectedIndex" : 9, + "selectedOption" : "fill VIN: 4S4BTANC2M3187721", + "title" : "VIN" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Date of incident\" value=\"\"", + "controlID" : "control-139", + "effect" : "fill", + "loopMilliseconds" : 4.160083, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.276083, + "selectedIndex" : 10, + "selectedOption" : "fill Date of loss: 2026-08-29", + "title" : "Date of incident" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Claim number\" value=\"\"", + "controlID" : "control-140", + "effect" : "fill", + "loopMilliseconds" : 6.037917, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.210334, + "selectedIndex" : 11, + "selectedOption" : "fill Claim: CLM-402917", + "title" : "Claim number" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Description of incident\" value=\"\"", + "controlID" : "control-141", + "effect" : "fill", + "loopMilliseconds" : 6.763083, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.920458, + "selectedIndex" : 12, + "selectedOption" : "fill Summary: Rear-ended at a stop light", + "title" : "Description of incident" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Agent code\" value=\"\"", + "controlID" : "control-142", + "effect" : "skip", + "loopMilliseconds" : 5.811333, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.489875, + "selectedIndex" : 18, + "selectedOption" : "skip", + "title" : "Agent code" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT CheckBox \"I confirm the details are correct\" unchecked", + "controlID" : "control-143", + "effect" : "check", + "loopMilliseconds" : 4.982916, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.275875, + "selectedIndex" : 16, + "selectedOption" : "check", + "title" : "I confirm the details are correct" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT CheckBox \"Text me claim status updates\" unchecked", + "controlID" : "control-144", + "effect" : "skip", + "loopMilliseconds" : 5.898417, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.340208, + "selectedIndex" : 18, + "selectedOption" : "skip", + "title" : "Text me claim status updates" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Button \"Discard\" value=\"\"", + "controlID" : "control-145", + "effect" : "skip", + "loopMilliseconds" : 6.430375, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 1.781208, + "selectedIndex" : 18, + "selectedOption" : "skip", + "title" : "Discard" + }, + { + "context" : "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Button \"Submit claim\" value=\"\"", + "controlID" : "control-146", + "effect" : "review", + "loopMilliseconds" : 5.631208, + "options" : [ + "fill Police Incident Report: Copy for Insurer", + "fill Insured: Kenji Tanaka", + "fill Policy: POL-77120944", + "fill E-mail: ktanaka42@outlook.com", + "fill Phone: (720) 555-0186", + "fill Make: Subaru", + "fill Model: Outback", + "fill Year: 2021", + "fill Plate: KLM-4471", + "fill VIN: 4S4BTANC2M3187721", + "fill Date of loss: 2026-08-29", + "fill Claim: CLM-402917", + "fill Summary: Rear-ended at a stop light", + "fill Officer: Sgt. Moreau", + "fill Report number: RPT-5581", + "fill Weather: Clear", + "check", + "click", + "skip" + ], + "probability" : 1, + "scoreMilliseconds" : 2.8355, + "selectedIndex" : 17, + "selectedOption" : "click", + "title" : "Submit claim" + } + ], + "events" : { + "change" : 13, + "input" : 13, + "submit" : 0 + }, + "filledFields" : 12, + "form" : "auto-claim", + "variant" : "ane-gather" + } + ], + "timingScope" : "Trace timings include rendering activity and are not the isolated variant benchmark. Presentation paused 250 ms per control pair." +} \ No newline at end of file diff --git a/reports/int4-demo-verification.json b/reports/int4-demo-verification.json new file mode 100644 index 0000000000000000000000000000000000000000..82a3bd5cb36481d284f1967492241dd05b1182c3 --- /dev/null +++ b/reports/int4-demo-verification.json @@ -0,0 +1,1799 @@ +{ + "purpose": "Local conversion parity on the upstream demo; not a generalization or live GUI benchmark", + "created_utc": "2026-09-19T22:25:44.244422+00:00", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_file": "demo.jsonl", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "conversion": { + "model": "cua_s1_forms_int4_options32.mlpackage", + "precision": "int4_weights_float16_compute", + "minimum_target": "iOS18/macOS15", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "cadad6bfa6875383a7ee3de68d0d22871148aa21cf117e7fe44c2c971287c4e5", + "Data/com.apple.CoreML/weights/weight.bin": "0f766fcb9dbd612ed4f6c7a40117a5993a4789b3ae94afba0df37193c831f63c", + "Manifest.json": "fb91c8b5ec3168111567de663bb3140d2790c6e7c48f32e0df3f5135af0673af" + }, + "target_upgrade": { + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "method": "Load original MIL at specification 9; empty pass pipeline; retain decomposed attention", + "operation_counts": { + "add": 13, + "cast": 11, + "concat": 2, + "const": 239, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 2, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 7, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f" + }, + "quantization": { + "name": "int4-weights", + "settings": { + "mode": "linear_symmetric", + "dtype": "int4", + "granularity": "per_channel", + "weight_threshold": 2048 + }, + "activations": "float16", + "calibration": "None; weight-only, data-free quantization", + "source_conversion_sha256": "01105dc8bef7a30fc3430e150caa2a143dc0ba23a295b6280de3b8ff3b40af55", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "e5b1c7dc66b2014b3985c180a75548a709788c6e314d9d2ae14e4e6d441c3c68" + }, + "source_package_bytes": 1511080, + "package_bytes": 451595, + "compression_seconds": 0.2996572910051327, + "packed_int4_tensors": 19, + "operation_counts": { + "add": 15, + "cast": 13, + "concat": 2, + "const": 226, + "constexpr_blockwise_shift_scale": 19, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 4, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 9, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "a91636e539d8934253b5e7628ca706346fbee538fa1aa9e3eaf794f440d49dcc" + } + }, + "environment": { + "machine": "Apple M5 Pro", + "os": "27.0", + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "torch_cpu_threads": 2 + }, + "rows": 196, + "upstream": { + "correct": 196, + "accuracy": 1.0, + "inference_latency_ms": { + "count": 196, + "median_ms": 2.3931670002639294, + "p95_ms": 3.033687506103888, + "min_ms": 1.7705829814076424, + "max_ms": 3.819000005023554 + } + }, + "export_adapter_fp32": { + "argmax_agreement": 196, + "max_abs_probability_error": 1.1324882507324219e-06 + }, + "host_preprocessing_latency_ms": { + "count": 196, + "median_ms": 0.02962502185255289, + "p95_ms": 0.038823003706056625, + "min_ms": 0.024125009076669812, + "max_ms": 0.06787499296478927 + }, + "thresholds": { + "argmax_agreement": 1.0, + "max_abs_probability_error": 0.005, + "export_adapter_max_abs_probability_error": 0.0001, + "allow_accuracy_loss": false + }, + "backends": { + "CPU_AND_NE": { + "passed": false, + "correct": 196, + "accuracy": 1.0, + "argmax_agreement": 196, + "max_abs_probability_error": 0.20076976716518402, + "mean_row_max_abs_probability_error": 0.0011279110022456198, + "max_abs_logit_error": 6.7208967208862305, + "load_seconds": 0.6122147499991115, + "warm_inference_latency_ms": { + "count": 196, + "median_ms": 0.9312290058005601, + "p95_ms": 1.0691460047382861, + "min_ms": 0.8852920145727694, + "max_ms": 1.2105409987270832 + }, + "reversed_option_order": { + "rows": 6, + "max_abs_probability_error": 6.0313243011478335e-05 + }, + "per_action": { + "fill": { + "rows": 36, + "correct": 36 + }, + "skip": { + "rows": 150, + "correct": 150 + }, + "check": { + "rows": 4, + "correct": 4 + }, + "click": { + "rows": 6, + "correct": 6 + } + }, + "decisions": [ + { + "row": 0, + "label": 22, + "upstream": 22, + "coreml": 22, + "max_abs_probability_error": 6.0313243011478335e-05, + "max_abs_logit_error": 2.864961624145508 + }, + { + "row": 1, + "label": 23, + "upstream": 23, + "coreml": 23, + "max_abs_probability_error": 3.814697265625e-05, + "max_abs_logit_error": 3.1371307373046875 + }, + { + "row": 2, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 2.247527599334717 + }, + { + "row": 3, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 0.00022571229783352464, + "max_abs_logit_error": 4.824214935302734 + }, + { + "row": 4, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 3.3578715374460444e-05, + "max_abs_logit_error": 3.618511438369751 + }, + { + "row": 5, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 6.828465848229825e-05, + "max_abs_logit_error": 3.7253708839416504 + }, + { + "row": 6, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 2.3365020751953125e-05, + "max_abs_logit_error": 2.4195241928100586 + }, + { + "row": 7, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.004736423492431641, + "max_abs_logit_error": 3.612703323364258 + }, + { + "row": 8, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 3.931535482406616 + }, + { + "row": 9, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 2.1457672119140625e-05, + "max_abs_logit_error": 3.5902247428894043 + }, + { + "row": 10, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 9.810033816393116e-07, + "max_abs_logit_error": 4.762610912322998 + }, + { + "row": 11, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 4.136032657697797e-06, + "max_abs_logit_error": 3.376967430114746 + }, + { + "row": 12, + "label": 15, + "upstream": 15, + "coreml": 15, + "max_abs_probability_error": 3.272867388659506e-06, + "max_abs_logit_error": 2.41459059715271 + }, + { + "row": 13, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 0.006299853324890137, + "max_abs_logit_error": 2.80385684967041 + }, + { + "row": 14, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.480397471226752e-05, + "max_abs_logit_error": 2.1806278228759766 + }, + { + "row": 15, + "label": 24, + "upstream": 24, + "coreml": 24, + "max_abs_probability_error": 1.6689300537109375e-05, + "max_abs_logit_error": 1.947547435760498 + }, + { + "row": 16, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.5203719878372794e-08, + "max_abs_logit_error": 3.743783950805664 + }, + { + "row": 17, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 5.519996193470433e-06, + "max_abs_logit_error": 3.9063854217529297 + }, + { + "row": 18, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 1.3515470027923584 + }, + { + "row": 19, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 3.501469850540161 + }, + { + "row": 20, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 3.2016849517822266 + }, + { + "row": 21, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 2.692106246948242 + }, + { + "row": 22, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 2.046893835067749 + }, + { + "row": 23, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 4.133575439453125 + }, + { + "row": 24, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 2.4618735313415527 + }, + { + "row": 25, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 3.3376502990722656 + }, + { + "row": 26, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 2.2433438301086426 + }, + { + "row": 27, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 2.810356616973877 + }, + { + "row": 28, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 3.7122511863708496 + }, + { + "row": 29, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 2.806028366088867 + }, + { + "row": 30, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.498983349141781e-07, + "max_abs_logit_error": 2.376333236694336 + }, + { + "row": 31, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 2.9533097743988037 + }, + { + "row": 32, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 2.1958632469177246 + }, + { + "row": 33, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 3.4985671043395996 + }, + { + "row": 34, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.0004820823669433594, + "max_abs_logit_error": 3.9588379859924316 + }, + { + "row": 35, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.462750809366753e-09, + "max_abs_logit_error": 2.490422248840332 + }, + { + "row": 36, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.7510341227277024e-10, + "max_abs_logit_error": 1.6093449592590332 + }, + { + "row": 37, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.74394488744656e-08, + "max_abs_logit_error": 1.8185267448425293 + }, + { + "row": 38, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 9.552543478452691e-11, + "max_abs_logit_error": 2.2058591842651367 + }, + { + "row": 39, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.04716482549361e-10, + "max_abs_logit_error": 2.799337863922119 + }, + { + "row": 40, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.0019283411093056202, + "max_abs_logit_error": 3.8344945907592773 + }, + { + "row": 41, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.345841103121529e-08, + "max_abs_logit_error": 2.8801822662353516 + }, + { + "row": 42, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8819494851385343e-09, + "max_abs_logit_error": 1.621840476989746 + }, + { + "row": 43, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 2.8215017318725586 + }, + { + "row": 44, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.780525865635596e-10, + "max_abs_logit_error": 1.614362359046936 + }, + { + "row": 45, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.414331317674879e-11, + "max_abs_logit_error": 1.6064977645874023 + }, + { + "row": 46, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.599257724142603e-10, + "max_abs_logit_error": 1.4032363891601562 + }, + { + "row": 47, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.212314607705437e-11, + "max_abs_logit_error": 1.6732616424560547 + }, + { + "row": 48, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.480397471226752e-05, + "max_abs_logit_error": 2.1806278228759766 + }, + { + "row": 49, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.932842080831776e-10, + "max_abs_logit_error": 1.8353347778320312 + }, + { + "row": 50, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.5203719878372794e-08, + "max_abs_logit_error": 3.743783950805664 + }, + { + "row": 51, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 5.519996193470433e-06, + "max_abs_logit_error": 3.9063854217529297 + }, + { + "row": 52, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 1.3515470027923584 + }, + { + "row": 53, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 3.501469850540161 + }, + { + "row": 54, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 3.2016849517822266 + }, + { + "row": 55, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 2.692106246948242 + }, + { + "row": 56, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 2.046893835067749 + }, + { + "row": 57, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 4.133575439453125 + }, + { + "row": 58, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 2.4618735313415527 + }, + { + "row": 59, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 3.3376502990722656 + }, + { + "row": 60, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 2.2433438301086426 + }, + { + "row": 61, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 2.810356616973877 + }, + { + "row": 62, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 3.7122511863708496 + }, + { + "row": 63, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 2.806028366088867 + }, + { + "row": 64, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.498983349141781e-07, + "max_abs_logit_error": 2.376333236694336 + }, + { + "row": 65, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 2.9533097743988037 + }, + { + "row": 66, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 2.1958632469177246 + }, + { + "row": 67, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 3.4985671043395996 + }, + { + "row": 68, + "label": 0, + "upstream": 0, + "coreml": 0, + "max_abs_probability_error": 0.00011065257422160357, + "max_abs_logit_error": 2.1242733001708984 + }, + { + "row": 69, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 5.5948574299691245e-05, + "max_abs_logit_error": 4.156556606292725 + }, + { + "row": 70, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 3.2760276553744916e-06, + "max_abs_logit_error": 4.688884735107422 + }, + { + "row": 71, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 5.781622166978195e-05, + "max_abs_logit_error": 2.7651166915893555 + }, + { + "row": 72, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 1.0177071089856327e-05, + "max_abs_logit_error": 2.8724193572998047 + }, + { + "row": 73, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 2.4447179384878837e-05, + "max_abs_logit_error": 2.4996681213378906 + }, + { + "row": 74, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 4.470348358154297e-05, + "max_abs_logit_error": 6.7208967208862305 + }, + { + "row": 75, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 3.379677946213633e-05, + "max_abs_logit_error": 5.287045478820801 + }, + { + "row": 76, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 4.347872618382098e-06, + "max_abs_logit_error": 3.2474308013916016 + }, + { + "row": 77, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 0.0006129741668701172, + "max_abs_logit_error": 4.197287559509277 + }, + { + "row": 78, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.0681543446698925e-06, + "max_abs_logit_error": 2.6077210903167725 + }, + { + "row": 79, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 0.0035958904772996902, + "max_abs_logit_error": 1.8085737228393555 + }, + { + "row": 80, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3343336604521028e-06, + "max_abs_logit_error": 1.546478271484375 + }, + { + "row": 81, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 1.7384634017944336 + }, + { + "row": 82, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 6.477808369709237e-07, + "max_abs_logit_error": 2.3306055068969727 + }, + { + "row": 83, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 1.5485305786132812 + }, + { + "row": 84, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 1.5241689682006836 + }, + { + "row": 85, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 2.1916251182556152 + }, + { + "row": 86, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.236582296674896e-07, + "max_abs_logit_error": 3.8295583724975586 + }, + { + "row": 87, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 3.709770679473877 + }, + { + "row": 88, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 3.0971665382385254 + }, + { + "row": 89, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 4.972865104675293 + }, + { + "row": 90, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 1.8527463674545288 + }, + { + "row": 91, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 2.7531187534332275 + }, + { + "row": 92, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 3.0166378021240234 + }, + { + "row": 93, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 3.1289987564086914 + }, + { + "row": 94, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 1.6400773525238037 + }, + { + "row": 95, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 1.7170515060424805 + }, + { + "row": 96, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 2.2562785148620605 + }, + { + "row": 97, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 1.4498934745788574 + }, + { + "row": 98, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 2.764932632446289 + }, + { + "row": 99, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.352701585550676e-06, + "max_abs_logit_error": 3.0918569564819336 + }, + { + "row": 100, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 2.7188186645507812 + }, + { + "row": 101, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.1713853704550274e-08, + "max_abs_logit_error": 1.6199064254760742 + }, + { + "row": 102, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5862225694339713e-09, + "max_abs_logit_error": 4.226329803466797 + }, + { + "row": 103, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 0.00024586915969848633, + "max_abs_logit_error": 2.840273380279541 + }, + { + "row": 104, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.841255187988281e-06, + "max_abs_logit_error": 3.6580047607421875 + }, + { + "row": 105, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.3792570402747515e-07, + "max_abs_logit_error": 6.307399272918701 + }, + { + "row": 106, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.4136979082101107e-09, + "max_abs_logit_error": 1.3745036125183105 + }, + { + "row": 107, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.302839361538304e-10, + "max_abs_logit_error": 1.8023838996887207 + }, + { + "row": 108, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 3.241445541381836 + }, + { + "row": 109, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.0681543446698925e-06, + "max_abs_logit_error": 2.6077210903167725 + }, + { + "row": 110, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 2.4543886184692383 + }, + { + "row": 111, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.811981201171875e-05, + "max_abs_logit_error": 1.956678867340088 + }, + { + "row": 112, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 1.7384634017944336 + }, + { + "row": 113, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 6.477808369709237e-07, + "max_abs_logit_error": 2.3306055068969727 + }, + { + "row": 114, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 1.5485305786132812 + }, + { + "row": 115, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 1.5241689682006836 + }, + { + "row": 116, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 2.1916251182556152 + }, + { + "row": 117, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.236582296674896e-07, + "max_abs_logit_error": 3.8295583724975586 + }, + { + "row": 118, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 3.709770679473877 + }, + { + "row": 119, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 3.0971665382385254 + }, + { + "row": 120, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 4.972865104675293 + }, + { + "row": 121, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 1.8527463674545288 + }, + { + "row": 122, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 2.7531187534332275 + }, + { + "row": 123, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 3.0166378021240234 + }, + { + "row": 124, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 3.1289987564086914 + }, + { + "row": 125, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 1.6400773525238037 + }, + { + "row": 126, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 1.7170515060424805 + }, + { + "row": 127, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 2.2562785148620605 + }, + { + "row": 128, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 1.4498934745788574 + }, + { + "row": 129, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 2.764932632446289 + }, + { + "row": 130, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 4.3291383917676285e-05, + "max_abs_logit_error": 3.559276580810547 + }, + { + "row": 131, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 3.2550487518310547 + }, + { + "row": 132, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 2.7894973754882812e-05, + "max_abs_logit_error": 3.068411111831665 + }, + { + "row": 133, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 0.0002166023332392797, + "max_abs_logit_error": 3.101825714111328 + }, + { + "row": 134, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 6.258487701416016e-05, + "max_abs_logit_error": 2.6334457397460938 + }, + { + "row": 135, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2993812561035156e-05, + "max_abs_logit_error": 3.064577102661133 + }, + { + "row": 136, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 3.9360515074804425e-05, + "max_abs_logit_error": 3.498318672180176 + }, + { + "row": 137, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.20076976716518402, + "max_abs_logit_error": 4.9444427490234375 + }, + { + "row": 138, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 0.000747382640838623, + "max_abs_logit_error": 2.80314302444458 + }, + { + "row": 139, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 5.860753663000651e-05, + "max_abs_logit_error": 4.217677116394043 + }, + { + "row": 140, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 1.1063334568461869e-05, + "max_abs_logit_error": 3.9767203330993652 + }, + { + "row": 141, + "label": 12, + "upstream": 12, + "coreml": 12, + "max_abs_probability_error": 9.34600830078125e-05, + "max_abs_logit_error": 2.8596105575561523 + }, + { + "row": 142, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 3.8807570934295654 + }, + { + "row": 143, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 1.6191354461625451e-06, + "max_abs_logit_error": 2.416942596435547 + }, + { + "row": 144, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.6463640097063035e-06, + "max_abs_logit_error": 2.728714942932129 + }, + { + "row": 145, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 3.8880889415740967 + }, + { + "row": 146, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 5.8316572904004715e-06, + "max_abs_logit_error": 4.496463298797607 + }, + { + "row": 147, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 2.1004066467285156 + }, + { + "row": 148, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 1.8194046020507812 + }, + { + "row": 149, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 4.05654764175415 + }, + { + "row": 150, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 2.169252395629883 + }, + { + "row": 151, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 2.1067094802856445 + }, + { + "row": 152, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 3.3871490955352783 + }, + { + "row": 153, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0083449808462319e-07, + "max_abs_logit_error": 4.667112827301025 + }, + { + "row": 154, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 2.944230079650879 + }, + { + "row": 155, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 2.179288864135742 + }, + { + "row": 156, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 2.217021942138672 + }, + { + "row": 157, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 2.0631823539733887 + }, + { + "row": 158, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 3.4993996620178223 + }, + { + "row": 159, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 2.16013240814209 + }, + { + "row": 160, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.588627788848498e-08, + "max_abs_logit_error": 3.608731269836426 + }, + { + "row": 161, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 3.2034151554107666 + }, + { + "row": 162, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 3.6694984436035156 + }, + { + "row": 163, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9131388140358752e-10, + "max_abs_logit_error": 2.9220056533813477 + }, + { + "row": 164, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.935061781452532e-08, + "max_abs_logit_error": 1.4873247146606445 + }, + { + "row": 165, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.15840287151903e-11, + "max_abs_logit_error": 1.6720411777496338 + }, + { + "row": 166, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.6797479790729994e-09, + "max_abs_logit_error": 1.2103171348571777 + }, + { + "row": 167, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.54509399403247e-10, + "max_abs_logit_error": 2.835115432739258 + }, + { + "row": 168, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 3.596881866455078 + }, + { + "row": 169, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.4169340140378637e-11, + "max_abs_logit_error": 1.5558462142944336 + }, + { + "row": 170, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.0329603412093675e-11, + "max_abs_logit_error": 2.1443252563476562 + }, + { + "row": 171, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 1.609299659729004 + }, + { + "row": 172, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.66337235494052e-09, + "max_abs_logit_error": 2.1106224060058594 + }, + { + "row": 173, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2885730260592254e-09, + "max_abs_logit_error": 1.5084967613220215 + }, + { + "row": 174, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1279766010119374e-09, + "max_abs_logit_error": 2.5634703636169434 + }, + { + "row": 175, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 3.8807570934295654 + }, + { + "row": 176, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0728836059570312e-06, + "max_abs_logit_error": 2.2860074043273926 + }, + { + "row": 177, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.665992202670168e-08, + "max_abs_logit_error": 2.732346534729004 + }, + { + "row": 178, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 3.8880889415740967 + }, + { + "row": 179, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 5.8316572904004715e-06, + "max_abs_logit_error": 4.496463298797607 + }, + { + "row": 180, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 2.1004066467285156 + }, + { + "row": 181, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 1.8194046020507812 + }, + { + "row": 182, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 4.05654764175415 + }, + { + "row": 183, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 2.169252395629883 + }, + { + "row": 184, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 2.1067094802856445 + }, + { + "row": 185, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 3.3871490955352783 + }, + { + "row": 186, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0083449808462319e-07, + "max_abs_logit_error": 4.667112827301025 + }, + { + "row": 187, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 2.944230079650879 + }, + { + "row": 188, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 2.179288864135742 + }, + { + "row": 189, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 2.217021942138672 + }, + { + "row": 190, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 2.0631823539733887 + }, + { + "row": 191, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 3.4993996620178223 + }, + { + "row": 192, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 2.16013240814209 + }, + { + "row": 193, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.588627788848498e-08, + "max_abs_logit_error": 3.608731269836426 + }, + { + "row": 194, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 3.2034151554107666 + }, + { + "row": 195, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 3.6694984436035156 + } + ] + } + }, + "passed": false, + "code_sha256": { + "ane_gather.py": "c3379a3797041dcfed82eaaea3990771ec91f1dff7ec146b025988ad7a8c9aae", + "assets.py": "f7a5bc931075237e00c73bf21fee3575be35531f934f23b5e22627d50a79f92f", + "benchmark-synthetic.py": "ba8255fd3f63d87ef340c26c4c251800f2e20878d80858ad1e6557cf59f249a2", + "compare-ane.py": "4633df7ad5f90d6aed37d5ef4f7a2d2231fdadd272ef1aa3057c237b2f937df8", + "convert-coreml.py": "46fe0194df924033126286e601214b6789f5d667aae38c153cf4dbc1b0e629b0", + "export-reference.py": "ff9eab4d6e4e72ffaf074d82610f41fbea1e802da7c0e987ce562a89eb561b99", + "export_model.py": "c61da5d678c77030d1e9a78359626063ea31da46fb6434e49517fdeaa7800949", + "prepare-int4-source.py": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f", + "preprocessing.py": "18c74ede43f95f91cd638c7e9631923a353e3a24d3b522ddb8b18b480f9c2846", + "profile-coreml.py": "d5e1386f54fc6c482647120e593e7ffbbe4bbb277c2cc6d4949154660fe239fd", + "quantize-int4.py": "a91636e539d8934253b5e7628ca706346fbee538fa1aa9e3eaf794f440d49dcc", + "quantize-int8.py": "22071babfcb02ef77225203a479d9bb14b9fa35462ba38bcc4bf07d29c125a97", + "score-report.py": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e", + "synthetic_test.py": "e2b68725236e6827eb422480c8cc8aa189b2490c1594650d8f2928771d1bc626", + "verify.py": "bc20ec9df47708e9b48e411076abe4eab7c18313f67e11334de2d9e3ecb3efa7" + } +} diff --git a/reports/int4-fallback.json b/reports/int4-fallback.json new file mode 100644 index 0000000000000000000000000000000000000000..60a6e8b33d37da4f8d2f3983ca87c5e0b45f7f4a --- /dev/null +++ b/reports/int4-fallback.json @@ -0,0 +1,112 @@ +{ + "hardware": { + "device": "arm64", + "chip": "Apple M5 Pro", + "ram": "24GB", + "os_version": "macOS 27.0", + "timestamp": "2026-09-19T22:28:18.483796+00:00" + }, + "models": [ + { + "model_path": "/Users/hanweng/Documents/FluidAudio/.mobius/cua-s1-prs/mobius/models/computer-use/cua-s1-forms/coreml/build/int4-weights/cua_s1_forms_int4_options32.mlmodelc", + "model_name": "cua_s1_forms_int4_options32", + "fallback": { + "compute_units": "cpu_and_neural_engine", + "total_ops": 200, + "ane_ops": 149, + "gpu_ops": 0, + "cpu_ops": 51, + "ane_percent": 74.5, + "reasons": [ + { + "reason": "Unsupported tensor data type: int32", + "count": 25, + "estimated_cpu_runtime_ms": 0.0375, + "op_types": { + "ios18.cast": 10, + "ios18.slice_by_index": 4, + "ios18.add": 4, + "select": 4, + "ios18.gather": 2, + "ios18.reshape": 1 + }, + "ops": [ + "var_27", + "var_43", + "var_43_to_fp16", + "context_ids_to_int16", + "cast_54", + "add_3", + "select_0", + "select_0_to_int16", + "cast_0", + "add_0", + "select_0_1", + "op_59_cast_fp16_cast_uint16_cast_uint16", + "flat_ids", + "var_333", + "var_349", + "var_349_to_fp16", + "flat_ids_to_int16", + "cast_55", + "add_4", + "select_1", + "select_1_to_int16", + "cast_1", + "add_1", + "select_1_1", + "op_365_cast_fp16_cast_uint16_cast_uint16" + ] + }, + { + "reason": "ANE not available for this op", + "count": 19, + "estimated_cpu_runtime_ms": 0, + "op_types": { + "ios18.constexpr_blockwise_shift_scale": 19 + }, + "ops": [ + "model_embedding_weight_to_fp16_quantized", + "model_position_weight_to_fp16_quantized", + "model_encoder_layers_0_self_attn_in_proj_weight_to_fp16_quantized", + "model_encoder_layers_0_self_attn_out_proj_weight_to_fp16_quantized", + "model_encoder_layers_0_linear1_weight_to_fp16_quantized", + "model_encoder_layers_0_linear2_weight_to_fp16_quantized", + "model_encoder_layers_1_self_attn_in_proj_weight_to_fp16_quantized", + "model_encoder_layers_1_self_attn_out_proj_weight_to_fp16_quantized", + "model_encoder_layers_1_linear1_weight_to_fp16_quantized", + "model_encoder_layers_1_linear2_weight_to_fp16_quantized", + "op_369_to_fp16_quantized", + "op_394_to_fp16_quantized", + "model_option_encoder_layers_0_self_attn_in_proj_weight_to_fp16_quantized", + "model_option_encoder_layers_0_self_attn_out_proj_weight_to_fp16_quantized", + "model_option_encoder_layers_0_linear1_weight_to_fp16_quantized", + "model_option_encoder_layers_0_linear2_weight_to_fp16_quantized", + "model_head_query_weight_to_fp16_quantized", + "model_head_key_weight_to_fp16_quantized", + "model_head_value_weight_to_fp16_quantized" + ] + }, + { + "reason": "Unable to resolve operation input \"y\".", + "count": 7, + "estimated_cpu_runtime_ms": 0.007, + "op_types": { + "ios18.greater_equal": 4, + "ios18.not_equal": 3 + }, + "ops": [ + "option_mask_1", + "context_mask", + "greater_equal_0", + "greater_equal_0_1", + "flat_mask", + "greater_equal_1", + "greater_equal_1_1" + ] + } + ] + } + } + ] +} diff --git a/reports/int4-profile.json b/reports/int4-profile.json new file mode 100644 index 0000000000000000000000000000000000000000..76284076def407a8a4d39d0f6e760b8caa6bfecc --- /dev/null +++ b/reports/int4-profile.json @@ -0,0 +1,1105 @@ +{ + "created_utc": "2026-09-19T22:27:49.746625+00:00", + "purpose": "Bounded real-input latency and scheduler placement; not a general accuracy benchmark", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "cadad6bfa6875383a7ee3de68d0d22871148aa21cf117e7fe44c2c971287c4e5", + "Data/com.apple.CoreML/weights/weight.bin": "0f766fcb9dbd612ed4f6c7a40117a5993a4789b3ae94afba0df37193c831f63c", + "Manifest.json": "fb91c8b5ec3168111567de663bb3140d2790c6e7c48f32e0df3f5135af0673af" + }, + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "environment": { + "chip": "Apple M5 Pro", + "memory_bytes": 25769803776, + "macos": "27.0", + "os_build": "26A428", + "python": "3.11.11", + "coremltools": "9.0" + }, + "protocol": { + "rows": [ + 0, + 68, + 130 + ], + "pages": [ + "patient-registration", + "job-application", + "auto-claim" + ], + "option_counts": [ + 27, + 21, + 19 + ], + "first_call_row": 0, + "warmup_passes": 2, + "timed_passes": 10, + "backend_order": [ + "CPU_AND_NE" + ], + "timing_scope": "Synchronous Python MLModel.predict; pre-encoded inputs; output checks outside timer", + "load_scope": "MLModel construction; system caches retained, not first-install cold start", + "first_call_scope": "First predict after this load; may use existing system/ANE caches", + "plan_order": "Compute plan loaded after timing each backend" + }, + "backends": { + "CPU_AND_NE": { + "load_ms": 628.8241670117714, + "first_prediction_ms": 2.4197499733418226, + "warm_prediction": { + "count": 30, + "median_ms": 0.9370829939143732, + "p95_ms": 0.969368452206254, + "min_ms": 0.8955409866757691, + "max_ms": 0.985999999102205 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 0.9536660218145698 + }, + { + "row": 68, + "milliseconds": 0.9417080145794898 + }, + { + "row": 130, + "milliseconds": 0.9293750044889748 + }, + { + "row": 0, + "milliseconds": 0.9304159902967513 + }, + { + "row": 68, + "milliseconds": 0.9457910200580955 + }, + { + "row": 130, + "milliseconds": 0.9614159935154021 + }, + { + "row": 0, + "milliseconds": 0.9506249916739762 + }, + { + "row": 68, + "milliseconds": 0.9356669906992465 + }, + { + "row": 130, + "milliseconds": 0.9454160172026604 + }, + { + "row": 0, + "milliseconds": 0.9370410116389394 + }, + { + "row": 68, + "milliseconds": 0.9322499972768128 + }, + { + "row": 130, + "milliseconds": 0.975875009316951 + }, + { + "row": 0, + "milliseconds": 0.985999999102205 + }, + { + "row": 68, + "milliseconds": 0.960250006755814 + }, + { + "row": 130, + "milliseconds": 0.9522080072201788 + }, + { + "row": 0, + "milliseconds": 0.9541249892208725 + }, + { + "row": 68, + "milliseconds": 0.9371249761898071 + }, + { + "row": 130, + "milliseconds": 0.9184999798890203 + }, + { + "row": 0, + "milliseconds": 0.9398749971296638 + }, + { + "row": 68, + "milliseconds": 0.9219589992426336 + }, + { + "row": 130, + "milliseconds": 0.9396669920533895 + }, + { + "row": 0, + "milliseconds": 0.9407500037923455 + }, + { + "row": 68, + "milliseconds": 0.9344589780084789 + }, + { + "row": 130, + "milliseconds": 0.931665999814868 + }, + { + "row": 0, + "milliseconds": 0.9232910233549774 + }, + { + "row": 68, + "milliseconds": 0.9075840061996132 + }, + { + "row": 130, + "milliseconds": 0.8955409866757691 + }, + { + "row": 0, + "milliseconds": 0.9247090201824903 + }, + { + "row": 68, + "milliseconds": 0.9097080037463456 + }, + { + "row": 130, + "milliseconds": 0.9349999891128391 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 181, + "device_counts": { + "cpu": 32, + "gpu": 0, + "ane": 149, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 226, + "ios18.constexpr_blockwise_shift_scale": 19 + }, + "operations": [ + { + "name": "option_mask_1", + "type": "ios18.not_equal", + "device": "cpu" + }, + { + "name": "context_mask", + "type": "ios18.not_equal", + "device": "cpu" + }, + { + "name": "var_27", + "type": "ios18.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_43", + "type": "ios18.slice_by_index", + "device": "cpu" + }, + { + "name": "var_43_to_fp16", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios18.concat", + "device": "ane" + }, + { + "name": "context_ids_to_int16", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "cast_54", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "greater_equal_0", + "type": "ios18.greater_equal", + "device": "cpu" + }, + { + "name": "add_3", + "type": "ios18.add", + "device": "cpu" + }, + { + "name": "select_0", + "type": "select", + "device": "cpu" + }, + { + "name": "select_0_to_int16", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "cast_0", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "greater_equal_0_1", + "type": "ios18.greater_equal", + "device": "cpu" + }, + { + "name": "add_0", + "type": "ios18.add", + "device": "cpu" + }, + { + "name": "select_0_1", + "type": "select", + "device": "cpu" + }, + { + "name": "op_59_cast_fp16_cast_uint16_cast_uint16", + "type": "ios18.gather", + "device": "cpu" + }, + { + "name": "src_1_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios18.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_1_cast_fp16", + "type": "ios18.layer_norm", + "device": "ane" + }, + { + "name": "query_3_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "var_124_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_125_cast_fp16", + "type": "ios18.expand_dims", + "device": "ane" + }, + { + "name": "var_126_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "var_127_cast_fp16", + "type": "ios18.squeeze", + "device": "ane" + }, + { + "name": "q_1_cast_fp16", + "type": "ios18.slice_by_index", + "device": "ane" + }, + { + "name": "k_1_cast_fp16", + "type": "ios18.slice_by_index", + "device": "ane" + }, + { + "name": "v_1_cast_fp16", + "type": "ios18.slice_by_index", + "device": "ane" + }, + { + "name": "var_136_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_143_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_150_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_154_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_156_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_3_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "q_5_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "k_3_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "k_5_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "v_3_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "v_5_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios18.mul", + "device": "ane" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios18.matmul", + "device": "ane" + }, + { + "name": "add_0_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios18.softmax", + "device": "ane" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios18.matmul", + "device": "ane" + }, + { + "name": "var_172_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "input_3_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "input_5_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "input_7_cast_fp16", + "type": "ios18.layer_norm", + "device": "ane" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "input_11_cast_fp16", + "type": "ios18.relu", + "device": "ane" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "input_17_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "query_5_cast_fp16", + "type": "ios18.layer_norm", + "device": "ane" + }, + { + "name": "query_7_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "var_231_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_232_cast_fp16", + "type": "ios18.expand_dims", + "device": "ane" + }, + { + "name": "var_233_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "var_234_cast_fp16", + "type": "ios18.squeeze", + "device": "ane" + }, + { + "name": "q_7_cast_fp16", + "type": "ios18.slice_by_index", + "device": "ane" + }, + { + "name": "k_7_cast_fp16", + "type": "ios18.slice_by_index", + "device": "ane" + }, + { + "name": "v_7_cast_fp16", + "type": "ios18.slice_by_index", + "device": "ane" + }, + { + "name": "var_243_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_250_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_257_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "q_9_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "q_11_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "k_9_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "k_11_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "v_9_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "v_11_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios18.mul", + "device": "ane" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios18.matmul", + "device": "ane" + }, + { + "name": "add_1_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios18.softmax", + "device": "ane" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios18.matmul", + "device": "ane" + }, + { + "name": "var_279_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "input_19_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "input_21_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "input_23_cast_fp16", + "type": "ios18.layer_norm", + "device": "ane" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "input_27_cast_fp16", + "type": "ios18.relu", + "device": "ane" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "context_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "flat_ids", + "type": "ios18.reshape", + "device": "cpu" + }, + { + "name": "flat_mask", + "type": "ios18.not_equal", + "device": "cpu" + }, + { + "name": "var_333", + "type": "ios18.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_349", + "type": "ios18.slice_by_index", + "device": "cpu" + }, + { + "name": "var_349_to_fp16", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios18.concat", + "device": "ane" + }, + { + "name": "flat_ids_to_int16", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "cast_55", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "greater_equal_1", + "type": "ios18.greater_equal", + "device": "cpu" + }, + { + "name": "add_4", + "type": "ios18.add", + "device": "cpu" + }, + { + "name": "select_1", + "type": "select", + "device": "cpu" + }, + { + "name": "select_1_to_int16", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "cast_1", + "type": "ios18.cast", + "device": "cpu" + }, + { + "name": "greater_equal_1_1", + "type": "ios18.greater_equal", + "device": "cpu" + }, + { + "name": "add_1", + "type": "ios18.add", + "device": "cpu" + }, + { + "name": "select_1_1", + "type": "select", + "device": "cpu" + }, + { + "name": "op_365_cast_fp16_cast_uint16_cast_uint16", + "type": "ios18.gather", + "device": "cpu" + }, + { + "name": "src_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "mask_cast_fp16", + "type": "ios18.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_9_cast_fp16", + "type": "ios18.layer_norm", + "device": "ane" + }, + { + "name": "query_11_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "var_428_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_429_cast_fp16", + "type": "ios18.expand_dims", + "device": "ane" + }, + { + "name": "var_430_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "var_431_cast_fp16", + "type": "ios18.squeeze", + "device": "ane" + }, + { + "name": "q_13_cast_fp16", + "type": "ios18.slice_by_index", + "device": "ane" + }, + { + "name": "k_13_cast_fp16", + "type": "ios18.slice_by_index", + "device": "ane" + }, + { + "name": "v_13_cast_fp16", + "type": "ios18.slice_by_index", + "device": "ane" + }, + { + "name": "var_440_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_447_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_454_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_458_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "var_460_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_15_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "q_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "k_15_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "k_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "v_15_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "v_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios18.mul", + "device": "ane" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios18.matmul", + "device": "ane" + }, + { + "name": "add_2_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios18.softmax", + "device": "ane" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios18.matmul", + "device": "ane" + }, + { + "name": "var_476_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "input_35_cast_fp16", + "type": "ios18.transpose", + "device": "ane" + }, + { + "name": "input_37_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "input_39_cast_fp16", + "type": "ios18.layer_norm", + "device": "ane" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "input_43_cast_fp16", + "type": "ios18.relu", + "device": "ane" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "hidden_cast_fp16", + "type": "ios18.add", + "device": "ane" + }, + { + "name": "var_504", + "type": "ios18.expand_dims", + "device": "ane" + }, + { + "name": "var_504_to_fp16", + "type": "ios18.cast", + "device": "ane" + }, + { + "name": "var_510_cast_fp16", + "type": "ios18.mul", + "device": "ane" + }, + { + "name": "var_515_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_520_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_522_cast_fp16", + "type": "ios18.maximum", + "device": "ane" + }, + { + "name": "pooled_cast_fp16", + "type": "ios18.real_div", + "device": "ane" + }, + { + "name": "options_cast_fp16", + "type": "ios18.reshape", + "device": "ane" + }, + { + "name": "input_cast_fp16", + "type": "ios18.layer_norm", + "device": "ane" + }, + { + "name": "input_53_cast_fp16", + "type": "ios18.layer_norm", + "device": "ane" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios18.linear", + "device": "ane" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios18.matmul", + "device": "ane" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios18.mul", + "device": "ane" + }, + { + "name": "var_567", + "type": "ios18.expand_dims", + "device": "ane" + }, + { + "name": "var_569", + "type": "ios18.logical_not", + "device": "ane" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_571_cast_fp16", + "type": "ios18.softmax", + "device": "ane" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios18.matmul", + "device": "ane" + }, + { + "name": "var_574_cast_fp16", + "type": "ios18.mul", + "device": "ane" + }, + { + "name": "var_576_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios18.mul", + "device": "ane" + }, + { + "name": "var_579", + "type": "ios18.logical_not", + "device": "ane" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_587_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_591_cast_fp16", + "type": "ios18.softmax", + "device": "ane" + }, + { + "name": "logits", + "type": "ios18.cast", + "device": "ane" + }, + { + "name": "probabilities", + "type": "ios18.cast", + "device": "ane" + } + ] + } + } + }, + "script_sha256": "d5e1386f54fc6c482647120e593e7ffbbe4bbb277c2cc6d4949154660fe239fd" +} diff --git a/reports/int4-source-demo-verification.json b/reports/int4-source-demo-verification.json new file mode 100644 index 0000000000000000000000000000000000000000..e0d27b91ef5b65fbd3485887ad8fd07cd257c516 --- /dev/null +++ b/reports/int4-source-demo-verification.json @@ -0,0 +1,1748 @@ +{ + "purpose": "Local conversion parity on the upstream demo; not a generalization or live GUI benchmark", + "created_utc": "2026-09-19T22:24:56.262849+00:00", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_file": "demo.jsonl", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "conversion": { + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "minimum_target": "iOS18/macOS15", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "e5b1c7dc66b2014b3985c180a75548a709788c6e314d9d2ae14e4e6d441c3c68" + }, + "target_upgrade": { + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "method": "Load original MIL at specification 9; empty pass pipeline; retain decomposed attention", + "operation_counts": { + "add": 13, + "cast": 11, + "concat": 2, + "const": 239, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 2, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 7, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f" + } + }, + "environment": { + "machine": "Apple M5 Pro", + "os": "27.0", + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "torch_cpu_threads": 2 + }, + "rows": 196, + "upstream": { + "correct": 196, + "accuracy": 1.0, + "inference_latency_ms": { + "count": 196, + "median_ms": 2.572771001723595, + "p95_ms": 3.2190517595154233, + "min_ms": 1.7389580025337636, + "max_ms": 4.086540982825682 + } + }, + "export_adapter_fp32": { + "argmax_agreement": 196, + "max_abs_probability_error": 1.1324882507324219e-06 + }, + "host_preprocessing_latency_ms": { + "count": 196, + "median_ms": 0.041624996811151505, + "p95_ms": 0.052635499741882086, + "min_ms": 0.029125018045306206, + "max_ms": 0.08079098188318312 + }, + "thresholds": { + "argmax_agreement": 1.0, + "max_abs_probability_error": 0.005, + "export_adapter_max_abs_probability_error": 0.0001, + "allow_accuracy_loss": false + }, + "backends": { + "CPU_AND_NE": { + "passed": true, + "correct": 196, + "accuracy": 1.0, + "argmax_agreement": 196, + "max_abs_probability_error": 0.00233614444732666, + "mean_row_max_abs_probability_error": 2.934475840582507e-05, + "max_abs_logit_error": 0.07230281829833984, + "load_seconds": 0.6436757920018863, + "warm_inference_latency_ms": { + "count": 196, + "median_ms": 0.9285830019507557, + "p95_ms": 0.9653437373344786, + "min_ms": 0.8930829935707152, + "max_ms": 1.0020830086432397 + }, + "reversed_option_order": { + "rows": 6, + "max_abs_probability_error": 3.1828880310058594e-05 + }, + "per_action": { + "fill": { + "rows": 36, + "correct": 36 + }, + "skip": { + "rows": 150, + "correct": 150 + }, + "check": { + "rows": 4, + "correct": 4 + }, + "click": { + "rows": 6, + "correct": 6 + } + }, + "decisions": [ + { + "row": 0, + "label": 22, + "upstream": 22, + "coreml": 22, + "max_abs_probability_error": 3.1828880310058594e-05, + "max_abs_logit_error": 0.03086566925048828 + }, + { + "row": 1, + "label": 23, + "upstream": 23, + "coreml": 23, + "max_abs_probability_error": 3.814697265625e-05, + "max_abs_logit_error": 0.020841598510742188 + }, + { + "row": 2, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.024377822875976562 + }, + { + "row": 3, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 3.218650817871094e-05, + "max_abs_logit_error": 0.030775785446166992 + }, + { + "row": 4, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 2.944469451904297e-05, + "max_abs_logit_error": 0.03573417663574219 + }, + { + "row": 5, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2636184692382812e-05, + "max_abs_logit_error": 0.026729822158813477 + }, + { + "row": 6, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 2.3365020751953125e-05, + "max_abs_logit_error": 0.023685455322265625 + }, + { + "row": 7, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.00014638900756835938, + "max_abs_logit_error": 0.04038810729980469 + }, + { + "row": 8, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.02460002899169922 + }, + { + "row": 9, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 2.1457672119140625e-05, + "max_abs_logit_error": 0.018064022064208984 + }, + { + "row": 10, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.047691673040390015 + }, + { + "row": 11, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 3.2186508178710938e-06, + "max_abs_logit_error": 0.03340768814086914 + }, + { + "row": 12, + "label": 15, + "upstream": 15, + "coreml": 15, + "max_abs_probability_error": 2.384185791015625e-06, + "max_abs_logit_error": 0.053191184997558594 + }, + { + "row": 13, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 0.0005360841751098633, + "max_abs_logit_error": 0.03362560272216797 + }, + { + "row": 14, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.024669170379638672 + }, + { + "row": 15, + "label": 24, + "upstream": 24, + "coreml": 24, + "max_abs_probability_error": 1.6689300537109375e-05, + "max_abs_logit_error": 0.034501075744628906 + }, + { + "row": 16, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.033290743827819824 + }, + { + "row": 17, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03323173522949219 + }, + { + "row": 18, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.022179126739501953 + }, + { + "row": 19, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.029880523681640625 + }, + { + "row": 20, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.031145095825195312 + }, + { + "row": 21, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.03861522674560547 + }, + { + "row": 22, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.027164459228515625 + }, + { + "row": 23, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.058716535568237305 + }, + { + "row": 24, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.035381317138671875 + }, + { + "row": 25, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.04862499237060547 + }, + { + "row": 26, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.029034733772277832 + }, + { + "row": 27, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.03808116912841797 + }, + { + "row": 28, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.03319191932678223 + }, + { + "row": 29, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.030045509338378906 + }, + { + "row": 30, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026231765747070312 + }, + { + "row": 31, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.033112093806266785 + }, + { + "row": 32, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.022128582000732422 + }, + { + "row": 33, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.038306236267089844 + }, + { + "row": 34, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.0004820823669433594, + "max_abs_logit_error": 0.032683372497558594 + }, + { + "row": 35, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.462750809366753e-09, + "max_abs_logit_error": 0.029730796813964844 + }, + { + "row": 36, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.7510341227277024e-10, + "max_abs_logit_error": 0.02333354949951172 + }, + { + "row": 37, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.657592190757896e-08, + "max_abs_logit_error": 0.022855758666992188 + }, + { + "row": 38, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 9.552543478452691e-11, + "max_abs_logit_error": 0.023164749145507812 + }, + { + "row": 39, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.04716482549361e-10, + "max_abs_logit_error": 0.02880859375 + }, + { + "row": 40, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.00021719932556152344, + "max_abs_logit_error": 0.04735088348388672 + }, + { + "row": 41, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.146233744175333e-09, + "max_abs_logit_error": 0.03671276569366455 + }, + { + "row": 42, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8819494851385343e-09, + "max_abs_logit_error": 0.023815155029296875 + }, + { + "row": 43, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.02039623260498047 + }, + { + "row": 44, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.780525865635596e-10, + "max_abs_logit_error": 0.022927284240722656 + }, + { + "row": 45, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.414331317674879e-11, + "max_abs_logit_error": 0.030417919158935547 + }, + { + "row": 46, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.599257724142603e-10, + "max_abs_logit_error": 0.025608062744140625 + }, + { + "row": 47, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.212314607705437e-11, + "max_abs_logit_error": 0.022733211517333984 + }, + { + "row": 48, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.024669170379638672 + }, + { + "row": 49, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.932842080831776e-10, + "max_abs_logit_error": 0.030317306518554688 + }, + { + "row": 50, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.033290743827819824 + }, + { + "row": 51, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03323173522949219 + }, + { + "row": 52, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.022179126739501953 + }, + { + "row": 53, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.029880523681640625 + }, + { + "row": 54, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.031145095825195312 + }, + { + "row": 55, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.03861522674560547 + }, + { + "row": 56, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.027164459228515625 + }, + { + "row": 57, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.058716535568237305 + }, + { + "row": 58, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.035381317138671875 + }, + { + "row": 59, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.04862499237060547 + }, + { + "row": 60, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.029034733772277832 + }, + { + "row": 61, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.03808116912841797 + }, + { + "row": 62, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.03319191932678223 + }, + { + "row": 63, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.030045509338378906 + }, + { + "row": 64, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026231765747070312 + }, + { + "row": 65, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.033112093806266785 + }, + { + "row": 66, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.022128582000732422 + }, + { + "row": 67, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.038306236267089844 + }, + { + "row": 68, + "label": 0, + "upstream": 0, + "coreml": 0, + "max_abs_probability_error": 3.147125244140625e-05, + "max_abs_logit_error": 0.03265953063964844 + }, + { + "row": 69, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 5.030632019042969e-05, + "max_abs_logit_error": 0.038794517517089844 + }, + { + "row": 70, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.040470123291015625 + }, + { + "row": 71, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 5.2928924560546875e-05, + "max_abs_logit_error": 0.040035247802734375 + }, + { + "row": 72, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 4.649162292480469e-06, + "max_abs_logit_error": 0.042186737060546875 + }, + { + "row": 73, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 2.1576881408691406e-05, + "max_abs_logit_error": 0.033161163330078125 + }, + { + "row": 74, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 4.470348358154297e-05, + "max_abs_logit_error": 0.0293731689453125 + }, + { + "row": 75, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 1.1563301086425781e-05, + "max_abs_logit_error": 0.03249359130859375 + }, + { + "row": 76, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 2.86102294921875e-06, + "max_abs_logit_error": 0.03498554229736328 + }, + { + "row": 77, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 0.0003635883331298828, + "max_abs_logit_error": 0.023473739624023438 + }, + { + "row": 78, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.034421443939208984 + }, + { + "row": 79, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 0.0003609657287597656, + "max_abs_logit_error": 0.03937721252441406 + }, + { + "row": 80, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.03752422332763672 + }, + { + "row": 81, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.025417327880859375 + }, + { + "row": 82, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.04453086853027344 + }, + { + "row": 83, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.023950576782226562 + }, + { + "row": 84, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.022706031799316406 + }, + { + "row": 85, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.035375118255615234 + }, + { + "row": 86, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.03717994689941406 + }, + { + "row": 87, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.04058122634887695 + }, + { + "row": 88, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.032048702239990234 + }, + { + "row": 89, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.030427932739257812 + }, + { + "row": 90, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.0281219482421875 + }, + { + "row": 91, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.030226707458496094 + }, + { + "row": 92, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.04007530212402344 + }, + { + "row": 93, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.02728891372680664 + }, + { + "row": 94, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.0217437744140625 + }, + { + "row": 95, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.03214454650878906 + }, + { + "row": 96, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.03669023513793945 + }, + { + "row": 97, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.019012451171875 + }, + { + "row": 98, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026651382446289062 + }, + { + "row": 99, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.1457672119140625e-06, + "max_abs_logit_error": 0.04161381721496582 + }, + { + "row": 100, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.04680633544921875 + }, + { + "row": 101, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.789079107084035e-08, + "max_abs_logit_error": 0.029542922973632812 + }, + { + "row": 102, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5862225694339713e-09, + "max_abs_logit_error": 0.023795127868652344 + }, + { + "row": 103, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 0.00024586915969848633, + "max_abs_logit_error": 0.030111312866210938 + }, + { + "row": 104, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.841255187988281e-06, + "max_abs_logit_error": 0.03126239776611328 + }, + { + "row": 105, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.928799035575082e-10, + "max_abs_logit_error": 0.026149272918701172 + }, + { + "row": 106, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.4136979082101107e-09, + "max_abs_logit_error": 0.024585723876953125 + }, + { + "row": 107, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.302839361538304e-10, + "max_abs_logit_error": 0.024765491485595703 + }, + { + "row": 108, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03492164611816406 + }, + { + "row": 109, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.034421443939208984 + }, + { + "row": 110, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.0320439338684082 + }, + { + "row": 111, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.811981201171875e-05, + "max_abs_logit_error": 0.032814979553222656 + }, + { + "row": 112, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.025417327880859375 + }, + { + "row": 113, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.04453086853027344 + }, + { + "row": 114, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.023950576782226562 + }, + { + "row": 115, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.022706031799316406 + }, + { + "row": 116, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.035375118255615234 + }, + { + "row": 117, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.03717994689941406 + }, + { + "row": 118, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.04058122634887695 + }, + { + "row": 119, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.032048702239990234 + }, + { + "row": 120, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.030427932739257812 + }, + { + "row": 121, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.0281219482421875 + }, + { + "row": 122, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.030226707458496094 + }, + { + "row": 123, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.04007530212402344 + }, + { + "row": 124, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.02728891372680664 + }, + { + "row": 125, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.0217437744140625 + }, + { + "row": 126, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.03214454650878906 + }, + { + "row": 127, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.03669023513793945 + }, + { + "row": 128, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.019012451171875 + }, + { + "row": 129, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026651382446289062 + }, + { + "row": 130, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 3.170967102050781e-05, + "max_abs_logit_error": 0.071075439453125 + }, + { + "row": 131, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.051357269287109375 + }, + { + "row": 132, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 2.7894973754882812e-05, + "max_abs_logit_error": 0.033801641315221786 + }, + { + "row": 133, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 1.0967254638671875e-05, + "max_abs_logit_error": 0.039325714111328125 + }, + { + "row": 134, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 6.258487701416016e-05, + "max_abs_logit_error": 0.033056676387786865 + }, + { + "row": 135, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2993812561035156e-05, + "max_abs_logit_error": 0.02561807632446289 + }, + { + "row": 136, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 6.198883056640625e-06, + "max_abs_logit_error": 0.03448677062988281 + }, + { + "row": 137, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.00233614444732666, + "max_abs_logit_error": 0.04230833053588867 + }, + { + "row": 138, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 0.00022917985916137695, + "max_abs_logit_error": 0.05339241027832031 + }, + { + "row": 139, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 4.851818084716797e-05, + "max_abs_logit_error": 0.032756805419921875 + }, + { + "row": 140, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 3.0994415283203125e-06, + "max_abs_logit_error": 0.029809951782226562 + }, + { + "row": 141, + "label": 12, + "upstream": 12, + "coreml": 12, + "max_abs_probability_error": 9.34600830078125e-05, + "max_abs_logit_error": 0.026587963104248047 + }, + { + "row": 142, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.03671073913574219 + }, + { + "row": 143, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.05281543731689453 + }, + { + "row": 144, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.04511451721191406 + }, + { + "row": 145, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.07230281829833984 + }, + { + "row": 146, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03114461898803711 + }, + { + "row": 147, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.020772457122802734 + }, + { + "row": 148, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.03475606441497803 + }, + { + "row": 149, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.020298004150390625 + }, + { + "row": 150, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.04283332824707031 + }, + { + "row": 151, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.028450965881347656 + }, + { + "row": 152, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.02250051498413086 + }, + { + "row": 153, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.06335592269897461 + }, + { + "row": 154, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.02735137939453125 + }, + { + "row": 155, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.015047073364257812 + }, + { + "row": 156, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.04868888854980469 + }, + { + "row": 157, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.026497364044189453 + }, + { + "row": 158, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.022236347198486328 + }, + { + "row": 159, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.020044326782226562 + }, + { + "row": 160, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.05533123016357422 + }, + { + "row": 161, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.0247042179107666 + }, + { + "row": 162, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.028873443603515625 + }, + { + "row": 163, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9131388140358752e-10, + "max_abs_logit_error": 0.015494346618652344 + }, + { + "row": 164, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0254025184508464e-08, + "max_abs_logit_error": 0.02497100830078125 + }, + { + "row": 165, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.15840287151903e-11, + "max_abs_logit_error": 0.02679443359375 + }, + { + "row": 166, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.6797479790729994e-09, + "max_abs_logit_error": 0.028203964233398438 + }, + { + "row": 167, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.54509399403247e-10, + "max_abs_logit_error": 0.03371429443359375 + }, + { + "row": 168, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.028280355036258698 + }, + { + "row": 169, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.4169340140378637e-11, + "max_abs_logit_error": 0.038245439529418945 + }, + { + "row": 170, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.0329603412093675e-11, + "max_abs_logit_error": 0.030043363571166992 + }, + { + "row": 171, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.02963542938232422 + }, + { + "row": 172, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.66337235494052e-09, + "max_abs_logit_error": 0.03510093688964844 + }, + { + "row": 173, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2885730260592254e-09, + "max_abs_logit_error": 0.04009199142456055 + }, + { + "row": 174, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1279766010119374e-09, + "max_abs_logit_error": 0.031810760498046875 + }, + { + "row": 175, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.03671073913574219 + }, + { + "row": 176, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0728836059570312e-06, + "max_abs_logit_error": 0.05123615264892578 + }, + { + "row": 177, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2799782567185503e-08, + "max_abs_logit_error": 0.03157186508178711 + }, + { + "row": 178, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.07230281829833984 + }, + { + "row": 179, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03114461898803711 + }, + { + "row": 180, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.020772457122802734 + }, + { + "row": 181, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.03475606441497803 + }, + { + "row": 182, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.020298004150390625 + }, + { + "row": 183, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.04283332824707031 + }, + { + "row": 184, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.028450965881347656 + }, + { + "row": 185, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.02250051498413086 + }, + { + "row": 186, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.06335592269897461 + }, + { + "row": 187, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.02735137939453125 + }, + { + "row": 188, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.015047073364257812 + }, + { + "row": 189, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.04868888854980469 + }, + { + "row": 190, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.026497364044189453 + }, + { + "row": 191, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.022236347198486328 + }, + { + "row": 192, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.020044326782226562 + }, + { + "row": 193, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.05533123016357422 + }, + { + "row": 194, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.0247042179107666 + }, + { + "row": 195, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.028873443603515625 + } + ] + } + }, + "passed": true, + "code_sha256": { + "ane_gather.py": "c3379a3797041dcfed82eaaea3990771ec91f1dff7ec146b025988ad7a8c9aae", + "assets.py": "f7a5bc931075237e00c73bf21fee3575be35531f934f23b5e22627d50a79f92f", + "benchmark-synthetic.py": "ba8255fd3f63d87ef340c26c4c251800f2e20878d80858ad1e6557cf59f249a2", + "compare-ane.py": "4633df7ad5f90d6aed37d5ef4f7a2d2231fdadd272ef1aa3057c237b2f937df8", + "convert-coreml.py": "46fe0194df924033126286e601214b6789f5d667aae38c153cf4dbc1b0e629b0", + "export-reference.py": "ff9eab4d6e4e72ffaf074d82610f41fbea1e802da7c0e987ce562a89eb561b99", + "export_model.py": "c61da5d678c77030d1e9a78359626063ea31da46fb6434e49517fdeaa7800949", + "prepare-int4-source.py": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f", + "preprocessing.py": "18c74ede43f95f91cd638c7e9631923a353e3a24d3b522ddb8b18b480f9c2846", + "profile-coreml.py": "d5e1386f54fc6c482647120e593e7ffbbe4bbb277c2cc6d4949154660fe239fd", + "quantize-int4.py": "a91636e539d8934253b5e7628ca706346fbee538fa1aa9e3eaf794f440d49dcc", + "quantize-int8.py": "22071babfcb02ef77225203a479d9bb14b9fa35462ba38bcc4bf07d29c125a97", + "score-report.py": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e", + "synthetic_test.py": "e2b68725236e6827eb422480c8cc8aa189b2490c1594650d8f2928771d1bc626", + "verify.py": "bc20ec9df47708e9b48e411076abe4eab7c18313f67e11334de2d9e3ecb3efa7" + } +} diff --git a/reports/int4-synthetic-test-decisions.jsonl.gz b/reports/int4-synthetic-test-decisions.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..bb55b89121fc0b65a243e6a323d109404be979e1 --- /dev/null +++ b/reports/int4-synthetic-test-decisions.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6922913041f95ac8872958b92312cb5b869b9ce88116c4684b36ea7753284674 +size 2410463 diff --git a/reports/int4-synthetic-test.json b/reports/int4-synthetic-test.json new file mode 100644 index 0000000000000000000000000000000000000000..c3ad8ac78c3628eedee2275618d77589ce8517bd --- /dev/null +++ b/reports/int4-synthetic-test.json @@ -0,0 +1,4733 @@ +{ + "purpose": "Full published synthetic test; no fitting, filtering, resampling or test-based tuning", + "created_utc": "2026-09-19T22:27:41.071082+00:00", + "dataset": { + "dataset_repository": "cua-ai/cua-s1-forms", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "path": "artifacts/test.jsonl", + "url": "https://huggingface.co/datasets/cua-ai/cua-s1-forms/resolve/8273f34778b99ac2e12d9f6e7d57dad99ae20845/test.jsonl", + "sha256": "d63a7e0db195d4d20154a40b2f8dd09ce3bb65487a158c638da5c609d4475e7c", + "rows": 24370, + "bytes": 18710924, + "upstream_split_claim": "Synthetic test, form-signature-disjoint from train/validation. Train and validation signatures have not been independently re-audited here.", + "model_card_claim": { + "synthetic_test_top1": 0.9995, + "approximate_decisions": 15000, + "note": "The released file has 24370 rows. No matching full result ledger or exact 15000-row manifest is supplied by the linked model card." + } + }, + "manifest_sha256": "e582b951ecc07809a0b06a2bc0790827b90a86ce57c70d5b6177ef04f0b4f1ca", + "input_census": { + "rows": 24370, + "unique_episode_seeds": 1040, + "maxima": { + "context_bytes": 217, + "option_bytes": 92, + "max_options": 32 + }, + "per_action": { + "check": 816, + "click": 1040, + "fill": 9802, + "skip": 12712 + }, + "option_count_histogram": { + "7": 17, + "8": 46, + "9": 80, + "10": 227, + "11": 361, + "12": 589, + "13": 888, + "14": 1254, + "15": 1513, + "16": 1761, + "17": 2373, + "18": 2014, + "19": 2277, + "20": 2506, + "21": 2396, + "22": 1818, + "23": 1422, + "24": 907, + "25": 688, + "26": 524, + "27": 483, + "28": 89, + "29": 33, + "30": 29, + "31": 35, + "32": 40 + }, + "excluded_rows": 0, + "truncated_contexts": 0, + "truncated_options": 0 + }, + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "conversions": { + "baseline": { + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "minimum_target": "iOS18/macOS15", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "e5b1c7dc66b2014b3985c180a75548a709788c6e314d9d2ae14e4e6d441c3c68" + }, + "target_upgrade": { + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "method": "Load original MIL at specification 9; empty pass pipeline; retain decomposed attention", + "operation_counts": { + "add": 13, + "cast": 11, + "concat": 2, + "const": 239, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 2, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 7, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f" + } + }, + "int4-weights": { + "model": "cua_s1_forms_int4_options32.mlpackage", + "precision": "int4_weights_float16_compute", + "minimum_target": "iOS18/macOS15", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "cadad6bfa6875383a7ee3de68d0d22871148aa21cf117e7fe44c2c971287c4e5", + "Data/com.apple.CoreML/weights/weight.bin": "0f766fcb9dbd612ed4f6c7a40117a5993a4789b3ae94afba0df37193c831f63c", + "Manifest.json": "fb91c8b5ec3168111567de663bb3140d2790c6e7c48f32e0df3f5135af0673af" + }, + "target_upgrade": { + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "method": "Load original MIL at specification 9; empty pass pipeline; retain decomposed attention", + "operation_counts": { + "add": 13, + "cast": 11, + "concat": 2, + "const": 239, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 2, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 7, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f" + }, + "quantization": { + "name": "int4-weights", + "settings": { + "mode": "linear_symmetric", + "dtype": "int4", + "granularity": "per_channel", + "weight_threshold": 2048 + }, + "activations": "float16", + "calibration": "None; weight-only, data-free quantization", + "source_conversion_sha256": "01105dc8bef7a30fc3430e150caa2a143dc0ba23a295b6280de3b8ff3b40af55", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "e5b1c7dc66b2014b3985c180a75548a709788c6e314d9d2ae14e4e6d441c3c68" + }, + "source_package_bytes": 1511080, + "package_bytes": 451595, + "compression_seconds": 0.2996572910051327, + "packed_int4_tensors": 19, + "operation_counts": { + "add": 15, + "cast": 13, + "concat": 2, + "const": 226, + "constexpr_blockwise_shift_scale": 19, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 4, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 9, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "a91636e539d8934253b5e7628ca706346fbee538fa1aa9e3eaf794f440d49dcc" + } + } + }, + "environment": { + "machine": "Apple M5 Pro", + "memory_bytes": 25769803776, + "os": "27.0", + "os_build": "26A428", + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "torch_cpu_threads": 2, + "torch_interop_threads": 1, + "torch_mha_fastpath": false + }, + "protocol": { + "batch_size": 1, + "warmups_per_model": 3, + "timed_passes": 1, + "coreml_compute_units": "CPU_AND_NE", + "coreml_order": "AB on even rows, BA on odd rows", + "timing": "PyTorch CPU forward+softmax / synchronous Core ML predict; excludes encoding, validation, GUI and network", + "preprocessing": "Upstream variable-length collator; independent fixed-shape Core ML encoder shared by both exports", + "load_note": "Process/model creation can use existing system caches; not a cold-start experiment", + "output_audit": "Imperfect finite probability sums are retained unchanged and fail the original normalization gate", + "calibration_note": "NLL/ECE use raw emitted scores; no renormalization or calibration was applied" + }, + "thresholds": { + "argmax_agreement": 1.0, + "max_abs_probability_error": 0.005, + "allow_accuracy_loss": false, + "probability_sum_atol": 0.001, + "probability_sum_rtol": 1e-05 + }, + "host_preprocessing_ms": { + "upstream_collator_ms": { + "count": 24370, + "median_ms": 0.09124999633058906, + "p95_ms": 0.12510610686149445, + "min_ms": 0.04041698412038386, + "max_ms": 0.5723749927710742 + }, + "shared_coreml_encoding_ms": { + "count": 24370, + "median_ms": 0.026124995201826096, + "p95_ms": 0.036083016311749816, + "min_ms": 0.011541007552295923, + "max_ms": 0.15366601292043924 + } + }, + "results": { + "upstream_pytorch": { + "correct": 24359, + "rows": 24370, + "accuracy": 0.999548625359048, + "model_call_latency_ms": { + "count": 24370, + "median_ms": 1.722437998978421, + "p95_ms": 2.9158063945942554, + "min_ms": 1.0111250157933682, + "max_ms": 7.711042009759694 + }, + "load_ms": 11.409584025386721, + "first_call_ms": 7.722499984083697, + "calibration": { + "nll": 0.0021302032038990836, + "ece": 0.00035450826133425167, + "equal_width_bins": 15, + "bins": [ + { + "bin": 7, + "count": 2, + "accuracy": 0.5, + "mean_confidence": 0.5216051042079926 + }, + { + "bin": 8, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.575071656703949 + }, + { + "bin": 9, + "count": 7, + "accuracy": 0.7142857142857143, + "mean_confidence": 0.6404258779117039 + }, + { + "bin": 10, + "count": 4, + "accuracy": 1.0, + "mean_confidence": 0.6963506639003754 + }, + { + "bin": 11, + "count": 11, + "accuracy": 0.9090909090909091, + "mean_confidence": 0.7664791941642761 + }, + { + "bin": 12, + "count": 21, + "accuracy": 0.9523809523809523, + "mean_confidence": 0.8310647464933849 + }, + { + "bin": 13, + "count": 22, + "accuracy": 0.9545454545454546, + "mean_confidence": 0.9173552746122534 + }, + { + "bin": 14, + "count": 24298, + "accuracy": 0.9998353773973166, + "mean_confidence": 0.999802232897422 + } + ], + "nll_probability_floor": 1.1754943508222875e-38 + }, + "upstream_action_metrics": { + "examples": 24370, + "accuracy": 0.999548625359048, + "coverage": 0.4788264259335248, + "abstention_rate": 0.5211735740664751, + "selective_accuracy": 0.9990573313908647, + "wrong_action_rate": 0.00045137464095199015, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.00045137464095199015, + "counts": { + "total": 24370, + "correct": 24359, + "abstained": 12701, + "acted": 11669, + "acted_correct": 11658, + "wrong_action": 11, + "unsafe_action": 11 + }, + "per_action": { + "abstain": { + "accuracy": 0.9991346758967904, + "abstention_rate": 0.9991346758967904, + "examples": 12712 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 816 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 1040 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 9802 + } + } + }, + "macro_action_accuracy": 0.9997836689741976, + "wrong_rows": [ + 137, + 3216, + 6631, + 6748, + 7280, + 10703, + 12255, + 13480, + 15214, + 21022, + 23671 + ] + }, + "baseline": { + "correct": 24359, + "rows": 24370, + "accuracy": 0.999548625359048, + "model_call_latency_ms": { + "count": 24370, + "median_ms": 0.9818960097618401, + "p95_ms": 1.0811670086695813, + "min_ms": 0.8811249863356352, + "max_ms": 1.690416014753282 + }, + "load_ms": 611.4340419881046, + "first_call_ms": 2.451416978146881, + "calibration": { + "nll": 0.002108235388127147, + "ece": 0.000331678613561794, + "equal_width_bins": 15, + "bins": [ + { + "bin": 7, + "count": 2, + "accuracy": 0.5, + "mean_confidence": 0.5224609375 + }, + { + "bin": 8, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.57607421875 + }, + { + "bin": 9, + "count": 6, + "accuracy": 0.8333333333333334, + "mean_confidence": 0.637451171875 + }, + { + "bin": 10, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.69248046875 + }, + { + "bin": 11, + "count": 12, + "accuracy": 0.9166666666666666, + "mean_confidence": 0.7687581380208334 + }, + { + "bin": 12, + "count": 20, + "accuracy": 0.95, + "mean_confidence": 0.8322265625 + }, + { + "bin": 13, + "count": 22, + "accuracy": 0.9545454545454546, + "mean_confidence": 0.9176136363636364 + }, + { + "bin": 14, + "count": 24298, + "accuracy": 0.9998353773973166, + "mean_confidence": 0.9998245660008025 + } + ], + "nll_probability_floor": 1.1754943508222875e-38 + }, + "upstream_action_metrics": { + "examples": 24370, + "accuracy": 0.999548625359048, + "coverage": 0.4788264259335248, + "abstention_rate": 0.5211735740664751, + "selective_accuracy": 0.9990573313908647, + "wrong_action_rate": 0.00045137464095199015, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.00045137464095199015, + "counts": { + "total": 24370, + "correct": 24359, + "abstained": 12701, + "acted": 11669, + "acted_correct": 11658, + "wrong_action": 11, + "unsafe_action": 11 + }, + "per_action": { + "abstain": { + "accuracy": 0.9991346758967904, + "abstention_rate": 0.9991346758967904, + "examples": 12712 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 816 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 1040 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 9802 + } + } + }, + "macro_action_accuracy": 0.9997836689741976, + "wrong_rows": [ + 137, + 3216, + 6631, + 6748, + 7280, + 10703, + 12255, + 13480, + 15214, + 21022, + 23671 + ], + "paired_with_upstream": { + "rows": 24370, + "argmax_agreement": 24370, + "agreement_rate": 1.0, + "reference_correct_candidate_wrong": 0, + "reference_wrong_candidate_correct": 0, + "both_wrong": 11, + "disagreement_rows": [] + }, + "max_abs_probability_error": 0.020487427711486816, + "mean_row_max_abs_probability_error": 4.673297645567538e-05, + "probability_tolerance_violation_rows": [ + 137, + 884, + 2945, + 3216, + 8706, + 9193, + 12725, + 13480, + 13768, + 15698, + 23992 + ], + "conversion_parity_passed": false, + "output_validation_failures": { + "19270": [ + "Live option probabilities do not sum to one" + ] + } + }, + "int4-weights": { + "correct": 24353, + "rows": 24370, + "accuracy": 0.9993024210094378, + "model_call_latency_ms": { + "count": 24370, + "median_ms": 0.9815625235205516, + "p95_ms": 1.08248109172564, + "min_ms": 0.8774579910095781, + "max_ms": 2.3330000112764537 + }, + "load_ms": 601.2531660089735, + "first_call_ms": 1.561125012813136, + "calibration": { + "nll": 0.002501404265778786, + "ece": 0.00041435783301703634, + "equal_width_bins": 15, + "bins": [ + { + "bin": 7, + "count": 3, + "accuracy": 0.6666666666666666, + "mean_confidence": 0.5087076822916666 + }, + { + "bin": 8, + "count": 4, + "accuracy": 0.75, + "mean_confidence": 0.58056640625 + }, + { + "bin": 9, + "count": 8, + "accuracy": 0.5, + "mean_confidence": 0.6361083984375 + }, + { + "bin": 10, + "count": 6, + "accuracy": 1.0, + "mean_confidence": 0.7128092447916666 + }, + { + "bin": 11, + "count": 11, + "accuracy": 0.7272727272727273, + "mean_confidence": 0.7681995738636364 + }, + { + "bin": 12, + "count": 11, + "accuracy": 0.8181818181818182, + "mean_confidence": 0.8423739346590909 + }, + { + "bin": 13, + "count": 34, + "accuracy": 1.0, + "mean_confidence": 0.9043255974264706 + }, + { + "bin": 14, + "count": 24293, + "accuracy": 0.999753015271889, + "mean_confidence": 0.9996638933411888 + } + ], + "nll_probability_floor": 1.1754943508222875e-38 + }, + "upstream_action_metrics": { + "examples": 24370, + "accuracy": 0.9993024210094378, + "coverage": 0.4789905621665983, + "abstention_rate": 0.5210094378334017, + "selective_accuracy": 0.998629315514435, + "wrong_action_rate": 0.0006565449322938039, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.0006565449322938039, + "counts": { + "total": 24370, + "correct": 24353, + "abstained": 12697, + "acted": 11673, + "acted_correct": 11657, + "wrong_action": 16, + "unsafe_action": 16 + }, + "per_action": { + "abstain": { + "accuracy": 0.9987413467589679, + "abstention_rate": 0.9987413467589679, + "examples": 12712 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 816 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 1040 + }, + "fill": { + "accuracy": 0.9998979800040808, + "abstention_rate": 0.00010201999591920016, + "examples": 9802 + } + } + }, + "macro_action_accuracy": 0.9996598316907622, + "wrong_rows": [ + 137, + 410, + 882, + 2093, + 3151, + 6631, + 6748, + 7280, + 9253, + 12255, + 13370, + 16920, + 21022, + 21344, + 22791, + 23671, + 23992 + ], + "paired_with_upstream": { + "rows": 24370, + "argmax_agreement": 24356, + "agreement_rate": 0.9994255231842429, + "reference_correct_candidate_wrong": 10, + "reference_wrong_candidate_correct": 4, + "both_wrong": 7, + "disagreement_rows": [ + 410, + 882, + 2093, + 3151, + 3216, + 9253, + 10703, + 13370, + 13480, + 15214, + 16920, + 21344, + 22791, + 23992 + ] + }, + "max_abs_probability_error": 0.7543594241142273, + "mean_row_max_abs_probability_error": 0.0010651588038265774, + "probability_tolerance_violation_rows": [ + 84, + 137, + 309, + 410, + 514, + 570, + 612, + 685, + 811, + 882, + 884, + 992, + 1033, + 1109, + 1195, + 1225, + 1266, + 1369, + 1398, + 1438, + 1475, + 1547, + 1656, + 1898, + 1946, + 1963, + 2076, + 2093, + 2095, + 2155, + 2187, + 2272, + 2308, + 2420, + 2584, + 2611, + 2661, + 2798, + 2818, + 2860, + 2928, + 2945, + 2976, + 3064, + 3090, + 3151, + 3209, + 3216, + 3282, + 3298, + 3313, + 3479, + 3483, + 3538, + 3733, + 3736, + 3756, + 3806, + 3878, + 3978, + 4097, + 4106, + 4123, + 4139, + 4233, + 4278, + 4469, + 4605, + 4630, + 4682, + 4745, + 4908, + 4936, + 5015, + 5243, + 5294, + 5330, + 5525, + 5527, + 5585, + 5591, + 5613, + 5687, + 5759, + 5838, + 5946, + 6113, + 6123, + 6196, + 6205, + 6299, + 6349, + 6426, + 6466, + 6587, + 6631, + 6928, + 7088, + 7177, + 7306, + 7411, + 7515, + 7641, + 7650, + 7672, + 7815, + 8017, + 8034, + 8235, + 8243, + 8385, + 8497, + 8514, + 8543, + 8664, + 8668, + 8706, + 8714, + 8891, + 8892, + 9037, + 9051, + 9193, + 9253, + 9291, + 9345, + 9445, + 9533, + 9570, + 9577, + 9581, + 9621, + 9624, + 9642, + 9730, + 9855, + 9945, + 9977, + 9981, + 10165, + 10191, + 10216, + 10345, + 10364, + 10398, + 10436, + 10458, + 10486, + 10488, + 10503, + 10620, + 10703, + 10740, + 10962, + 10993, + 11046, + 11090, + 11122, + 11191, + 11214, + 11257, + 11300, + 11337, + 11390, + 11553, + 11630, + 11813, + 11880, + 11959, + 12043, + 12217, + 12219, + 12255, + 12411, + 12516, + 12546, + 12605, + 12606, + 12699, + 12725, + 12754, + 12792, + 12936, + 12949, + 12983, + 13027, + 13031, + 13053, + 13115, + 13154, + 13204, + 13323, + 13370, + 13392, + 13423, + 13450, + 13480, + 13483, + 13533, + 13545, + 13609, + 13693, + 13766, + 13768, + 13795, + 13969, + 14043, + 14145, + 14220, + 14568, + 14608, + 14656, + 14718, + 14722, + 14868, + 14899, + 14934, + 15065, + 15132, + 15214, + 15219, + 15252, + 15259, + 15698, + 15764, + 15819, + 16041, + 16106, + 16182, + 16200, + 16205, + 16207, + 16253, + 16256, + 16392, + 16517, + 16597, + 16608, + 16678, + 16920, + 16951, + 17100, + 17122, + 17253, + 17304, + 17318, + 17351, + 17372, + 17454, + 17632, + 17665, + 17854, + 17858, + 17865, + 17873, + 17918, + 18099, + 18119, + 18154, + 18168, + 18237, + 18252, + 18290, + 18339, + 18432, + 18541, + 18549, + 18551, + 18559, + 18579, + 18726, + 18731, + 18753, + 18775, + 18802, + 18945, + 19021, + 19205, + 19276, + 19308, + 19361, + 19438, + 19439, + 19511, + 19634, + 19669, + 19704, + 19809, + 19834, + 19940, + 19957, + 20008, + 20097, + 20214, + 20326, + 20370, + 20457, + 20462, + 20476, + 20486, + 20492, + 20625, + 20776, + 20928, + 20956, + 21028, + 21029, + 21049, + 21163, + 21344, + 21352, + 21464, + 21519, + 21566, + 21631, + 21813, + 21844, + 21866, + 21871, + 21886, + 21890, + 21903, + 22046, + 22148, + 22223, + 22297, + 22336, + 22442, + 22571, + 22714, + 22785, + 22791, + 22831, + 22855, + 23012, + 23051, + 23074, + 23097, + 23276, + 23322, + 23350, + 23398, + 23670, + 23671, + 23742, + 23749, + 23794, + 23858, + 23863, + 23888, + 23992, + 24016, + 24018, + 24027, + 24142, + 24163 + ], + "conversion_parity_passed": false, + "output_validation_failures": {} + } + }, + "failures": [ + { + "row": 84, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"Contact email\" value=\"\" hint=\"name@domain.com\"", + "gold_option": "fill Email address: cmbeki19@fastmail.com", + "selected_options": { + "upstream_pytorch": "fill Email address: cmbeki19@fastmail.com", + "baseline": "fill Email address: cmbeki19@fastmail.com", + "int4-weights": "fill Email address: cmbeki19@fastmail.com" + } + }, + { + "row": 137, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"Full name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Last: Novak", + "baseline": "fill Last: Novak", + "int4-weights": "fill Last: Novak" + } + }, + { + "row": 309, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - JevBrowser\nELEMENT Edit \"Last name\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Surname: Ivanova", + "selected_options": { + "upstream_pytorch": "fill Surname: Ivanova", + "baseline": "fill Surname: Ivanova", + "int4-weights": "fill Surname: Ivanova" + } + }, + { + "row": 410, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Date of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "fill Incident: Water damage from burst pipe" + } + }, + { + "row": 514, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Car make\" value=\"\"", + "gold_option": "fill Vehicle make: Toyota", + "selected_options": { + "upstream_pytorch": "fill Vehicle make: Toyota", + "baseline": "fill Vehicle make: Toyota", + "int4-weights": "fill Vehicle make: Toyota" + } + }, + { + "row": 570, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 612, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Financial institution\" value=\"\"", + "gold_option": "fill Bank name: Capital One", + "selected_options": { + "upstream_pytorch": "fill Bank name: Capital One", + "baseline": "fill Bank name: Capital One", + "int4-weights": "fill Bank name: Capital One" + } + }, + { + "row": 685, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Claim number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 811, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Button \"Create account\" value=\"\"", + "gold_option": "click", + "selected_options": { + "upstream_pytorch": "click", + "baseline": "click", + "int4-weights": "click" + } + }, + { + "row": 882, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Surname\" value=\"old\" hint=\"Family name\"", + "gold_option": "fill Last name: Fischer", + "selected_options": { + "upstream_pytorch": "fill Last name: Fischer", + "baseline": "fill Last name: Fischer", + "int4-weights": "skip" + } + }, + { + "row": 884, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Health plan\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 992, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form - Microsoft Edge\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 1033, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer\nELEMENT Edit \"Passport no.\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1109, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Plate number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1195, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"Year\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1225, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Vehicle identification number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1266, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - Microsoft Edge\nELEMENT Edit \"Marital status\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1369, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Manufacturer\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1398, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Social security no.\" value=\"\"", + "gold_option": "fill Social Security: 271-37-2373", + "selected_options": { + "upstream_pytorch": "fill Social Security: 271-37-2373", + "baseline": "fill Social Security: 271-37-2373", + "int4-weights": "fill Social Security: 271-37-2373" + } + }, + { + "row": 1438, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Mozilla Firefox\nELEMENT Edit \"Discount code\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1475, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake - Mozilla Firefox\nELEMENT Edit \"Car model\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1547, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1656, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Relation to applicant\" value=\"\"", + "gold_option": "fill Relationship: Father", + "selected_options": { + "upstream_pytorch": "fill Relationship: Father", + "baseline": "fill Relationship: Father", + "int4-weights": "fill Relationship: Father" + } + }, + { + "row": 1898, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Reason for visit\" value=\"\"", + "gold_option": "fill Reason: Allergy testing", + "selected_options": { + "upstream_pytorch": "fill Reason: Allergy testing", + "baseline": "fill Reason: Allergy testing", + "int4-weights": "fill Reason: Allergy testing" + } + }, + { + "row": 1946, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Member ID\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 1963, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Manufacturer\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 2076, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Graduation year\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 2093, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Title\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "fill Reason: Flu vaccination" + } + }, + { + "row": 2095, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Years in role\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 2155, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "fill Summary: Slipped on icy pavement", + "selected_options": { + "upstream_pytorch": "fill Summary: Slipped on icy pavement", + "baseline": "fill Summary: Slipped on icy pavement", + "int4-weights": "fill Summary: Slipped on icy pavement" + } + }, + { + "row": 2187, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Microsoft Edge\nELEMENT Edit \"Last Name\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Family name: Okafor", + "selected_options": { + "upstream_pytorch": "fill Family name: Okafor", + "baseline": "fill Family name: Okafor", + "int4-weights": "fill Family name: Okafor" + } + }, + { + "row": 2272, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Total years experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 2308, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration\nELEMENT Edit \"Purpose of visit\" value=\"\"", + "gold_option": "fill Complaint: Knee pain when climbing stairs", + "selected_options": { + "upstream_pytorch": "fill Complaint: Knee pain when climbing stairs", + "baseline": "fill Complaint: Knee pain when climbing stairs", + "int4-weights": "fill Complaint: Knee pain when climbing stairs" + } + }, + { + "row": 2420, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Town\" value=\"\"", + "gold_option": "fill Town: Tucson", + "selected_options": { + "upstream_pytorch": "fill Town: Tucson", + "baseline": "fill Town: Tucson", + "int4-weights": "fill Town: Tucson" + } + }, + { + "row": 2584, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Salary expectation\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 2611, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - Microsoft Edge\nELEMENT Edit \"Claim #\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 2661, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Years in role\" value=\"11\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 2798, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "fill Complaint: Allergy testing", + "selected_options": { + "upstream_pytorch": "fill Complaint: Allergy testing", + "baseline": "fill Complaint: Allergy testing", + "int4-weights": "fill Complaint: Allergy testing" + } + }, + { + "row": 2818, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Your email\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 2860, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Surname\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Last: Mbeki", + "selected_options": { + "upstream_pytorch": "fill Last: Mbeki", + "baseline": "fill Last: Mbeki", + "int4-weights": "fill Last: Mbeki" + } + }, + { + "row": 2928, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"SSN\" value=\"\"", + "gold_option": "fill Social security number: 431-74-1573", + "selected_options": { + "upstream_pytorch": "fill Social security number: 431-74-1573", + "baseline": "fill Social security number: 431-74-1573", + "int4-weights": "fill Social security number: 431-74-1573" + } + }, + { + "row": 2945, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Organization\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 2976, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake - Microsoft Edge\nELEMENT Edit \"Claim reference\" value=\"\"", + "gold_option": "fill Reference: CLM-317554", + "selected_options": { + "upstream_pytorch": "fill Reference: CLM-317554", + "baseline": "fill Reference: CLM-317554", + "int4-weights": "fill Reference: CLM-317554" + } + }, + { + "row": 3064, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Make\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3090, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Discount code\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3151, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Google Chrome\nELEMENT Edit \"Date of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "fill Description: Water damage from burst pipe" + } + }, + { + "row": 3209, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - Mozilla Firefox\nELEMENT Button \"Create account\" value=\"\"", + "gold_option": "click", + "selected_options": { + "upstream_pytorch": "click", + "baseline": "click", + "int4-weights": "click" + } + }, + { + "row": 3216, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Google Chrome\nELEMENT Edit \"Email\" value=\"oivanova66@yahoo.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Email address: oivanova66@yahoo.com", + "baseline": "fill Email address: oivanova66@yahoo.com", + "int4-weights": "skip" + } + }, + { + "row": 3282, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Mozilla Firefox\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3298, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Mozilla Firefox\nELEMENT Edit \"DL number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3313, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Google Chrome\nELEMENT Edit \"Office phone\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3479, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Insurance company\" value=\"Centene\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3483, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"E-mail\" value=\"ethan.patel@yahoo.com\" hint=\"name@domain.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3538, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form - Mozilla Firefox\nELEMENT Edit \"State\" value=\"\"", + "gold_option": "fill Region: WA", + "selected_options": { + "upstream_pytorch": "fill Region: WA", + "baseline": "fill Region: WA", + "int4-weights": "fill Region: WA" + } + }, + { + "row": 3733, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"What happened?\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3736, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Degree earned\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3756, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Claim #\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3806, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - JevBrowser\nELEMENT Edit \"Next of kin\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 3878, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Surname\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Last name: Larsson", + "selected_options": { + "upstream_pytorch": "fill Last name: Larsson", + "baseline": "fill Last name: Larsson", + "int4-weights": "fill Last name: Larsson" + } + }, + { + "row": 3978, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - Microsoft Edge\nELEMENT Edit \"Family doctor\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 4097, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Vehicle model\" value=\"Civic\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 4106, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - JevBrowser\nELEMENT Edit \"Date of loss\" value=\"\"", + "gold_option": "fill Date of incident: 2026-05-02", + "selected_options": { + "upstream_pytorch": "fill Date of incident: 2026-05-02", + "baseline": "fill Date of incident: 2026-05-02", + "int4-weights": "fill Date of incident: 2026-05-02" + } + }, + { + "row": 4123, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - JevBrowser\nELEMENT Edit \"GP name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 4139, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Google Chrome\nELEMENT Edit \"Years of experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 4233, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Mozilla Firefox\nELEMENT Edit \"Nationality\" value=\"old\"", + "gold_option": "fill Citizen of: Canadian", + "selected_options": { + "upstream_pytorch": "fill Citizen of: Canadian", + "baseline": "fill Citizen of: Canadian", + "int4-weights": "fill Citizen of: Canadian" + } + }, + { + "row": 4278, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Nationality\" value=\"old\"", + "gold_option": "fill Citizenship: Japanese", + "selected_options": { + "upstream_pytorch": "fill Citizenship: Japanese", + "baseline": "fill Citizenship: Japanese", + "int4-weights": "fill Citizenship: Japanese" + } + }, + { + "row": 4469, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"City\" value=\"Portland\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 4605, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake - Mozilla Firefox\nELEMENT Edit \"VIN number\" value=\"\"", + "gold_option": "fill VIN: GLZRYQTQRWYSTHUZ2", + "selected_options": { + "upstream_pytorch": "fill VIN: GLZRYQTQRWYSTHUZ2", + "baseline": "fill VIN: GLZRYQTQRWYSTHUZ2", + "int4-weights": "fill VIN: GLZRYQTQRWYSTHUZ2" + } + }, + { + "row": 4630, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Mozilla Firefox\nELEMENT Edit \"Current salary\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 4682, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"Surname\" value=\"\"", + "gold_option": "fill Last: Schmidt", + "selected_options": { + "upstream_pytorch": "fill Last: Schmidt", + "baseline": "fill Last: Schmidt", + "int4-weights": "fill Last: Schmidt" + } + }, + { + "row": 4745, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - JevBrowser\nELEMENT Edit \"Given name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 4908, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - JevBrowser\nELEMENT Edit \"Make\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 4936, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Mozilla Firefox\nELEMENT Edit \"Years of experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 5015, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Google Chrome\nELEMENT CheckBox \"Consent to electronic communication (required)\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 5243, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Contact email\" value=\"zara.schmidt@yahoo.com\" hint=\"name@domain.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 5294, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Email\" value=\"nadia.fischer@fastmail.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 5330, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 5525, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Occupation\" value=\"\"", + "gold_option": "fill Position: Dental Hygienist", + "selected_options": { + "upstream_pytorch": "fill Position: Dental Hygienist", + "baseline": "fill Position: Dental Hygienist", + "int4-weights": "fill Position: Dental Hygienist" + } + }, + { + "row": 5527, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Telephone\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 5585, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "fill Presenting problem: Skin rash on left arm", + "selected_options": { + "upstream_pytorch": "fill Presenting problem: Skin rash on left arm", + "baseline": "fill Presenting problem: Skin rash on left arm", + "int4-weights": "fill Presenting problem: Skin rash on left arm" + } + }, + { + "row": 5591, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Last\" value=\"\"", + "gold_option": "fill Family name: Abernathy", + "selected_options": { + "upstream_pytorch": "fill Family name: Abernathy", + "baseline": "fill Family name: Abernathy", + "int4-weights": "fill Family name: Abernathy" + } + }, + { + "row": 5613, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Current medications\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 5687, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Mobile number\" value=\"\"", + "gold_option": "fill Contact number: 792.396.2253", + "selected_options": { + "upstream_pytorch": "fill Contact number: 792.396.2253", + "baseline": "fill Contact number: 792.396.2253", + "int4-weights": "fill Contact number: 792.396.2253" + } + }, + { + "row": 5759, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form\nELEMENT Edit \"Relationship to patient\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 5838, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Region\" value=\"AZ\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 5946, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Occupation\" value=\"\"", + "gold_option": "fill Position: Civil Engineer", + "selected_options": { + "upstream_pytorch": "fill Position: Civil Engineer", + "baseline": "fill Position: Civil Engineer", + "int4-weights": "fill Position: Civil Engineer" + } + }, + { + "row": 6113, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Purpose of visit\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 6123, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Primary care physician\" value=\"\"", + "gold_option": "fill GP: Dr. Becker", + "selected_options": { + "upstream_pytorch": "fill GP: Dr. Becker", + "baseline": "fill GP: Dr. Becker", + "int4-weights": "fill GP: Dr. Becker" + } + }, + { + "row": 6196, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - JevBrowser\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 6205, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - JevBrowser\nELEMENT Edit \"Last name\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Family name: Silva", + "selected_options": { + "upstream_pytorch": "fill Family name: Silva", + "baseline": "fill Family name: Silva", + "int4-weights": "fill Family name: Silva" + } + }, + { + "row": 6299, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Mozilla Firefox\nELEMENT Edit \"Birth date\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 6349, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Microsoft Edge\nELEMENT Edit \"Relationship status\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 6426, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Microsoft Edge\nELEMENT CheckBox \"Consent to electronic communication (required)\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 6466, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - JevBrowser\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 6587, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Microsoft Edge\nELEMENT Edit \"Current employer\" value=\"\"", + "gold_option": "fill Employer: Umbrella Health", + "selected_options": { + "upstream_pytorch": "fill Employer: Umbrella Health", + "baseline": "fill Employer: Umbrella Health", + "int4-weights": "fill Employer: Umbrella Health" + } + }, + { + "row": 6631, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Google Chrome\nELEMENT Edit \"Given name\" value=\"Priya\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Physician: Dr. Fischer", + "baseline": "fill Physician: Dr. Fischer", + "int4-weights": "fill Physician: Dr. Fischer" + } + }, + { + "row": 6748, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"State / Province\" value=\"NE\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill ST: NE", + "baseline": "fill ST: NE", + "int4-weights": "fill ST: NE" + } + }, + { + "row": 6928, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Google Chrome\nELEMENT Edit \"Citizenship\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 7088, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"Current salary\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 7177, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Claim #\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 7280, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Given name\" value=\"Tomas\" hint=\"First\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill First name: Tomas", + "baseline": "fill First name: Tomas", + "int4-weights": "fill First name: Tomas" + } + }, + { + "row": 7306, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Given name\" value=\"\" hint=\"Given name\"", + "gold_option": "fill First: Mateo", + "selected_options": { + "upstream_pytorch": "fill First: Mateo", + "baseline": "fill First: Mateo", + "int4-weights": "fill First: Mateo" + } + }, + { + "row": 7411, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"National ID number\" value=\"\"", + "gold_option": "fill Social Security: 638-17-6557", + "selected_options": { + "upstream_pytorch": "fill Social Security: 638-17-6557", + "baseline": "fill Social Security: 638-17-6557", + "int4-weights": "fill Social Security: 638-17-6557" + } + }, + { + "row": 7515, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Province\" value=\"GA\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 7641, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form - Mozilla Firefox\nELEMENT Edit \"Manufacturer\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 7650, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form - Mozilla Firefox\nELEMENT Edit \"VIN number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 7672, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Forename\" value=\"Kenji\" hint=\"Given name\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 7815, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"Vehicle year\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8017, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Policy number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8034, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Qualification\" value=\"\"", + "gold_option": "fill Degree: Associate of Arts", + "selected_options": { + "upstream_pytorch": "fill Degree: Associate of Arts", + "baseline": "fill Degree: Associate of Arts", + "int4-weights": "fill Degree: Associate of Arts" + } + }, + { + "row": 8235, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Reason for visit\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8243, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Passport number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8385, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration\nELEMENT Edit \"City\" value=\"Denver\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8497, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer - JevBrowser\nELEMENT Edit \"VIN number\" value=\"\"", + "gold_option": "fill Chassis number: 6X03E5GUHTB3AMH57", + "selected_options": { + "upstream_pytorch": "fill Chassis number: 6X03E5GUHTB3AMH57", + "baseline": "fill Chassis number: 6X03E5GUHTB3AMH57", + "int4-weights": "fill Chassis number: 6X03E5GUHTB3AMH57" + } + }, + { + "row": 8514, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Microsoft Edge\nELEMENT Edit \"Reason for visit\" value=\"\"", + "gold_option": "fill Reason: Skin rash on left arm", + "selected_options": { + "upstream_pytorch": "fill Reason: Skin rash on left arm", + "baseline": "fill Reason: Skin rash on left arm", + "int4-weights": "fill Reason: Skin rash on left arm" + } + }, + { + "row": 8543, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Address line 1\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8664, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - JevBrowser\nELEMENT Edit \"Experience (years)\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8668, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - JevBrowser\nELEMENT Edit \"Message to hiring manager\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8706, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Mozilla Firefox\nELEMENT CheckBox \"I authorize verification of the information provided\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 8714, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Registration number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8891, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Google Chrome\nELEMENT Edit \"Last\" value=\"Ivanova\" hint=\"Family name\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 8892, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Google Chrome\nELEMENT Edit \"Reason for visit\" value=\"\"", + "gold_option": "fill Presenting problem: Knee pain when climbing stairs", + "selected_options": { + "upstream_pytorch": "fill Presenting problem: Knee pain when climbing stairs", + "baseline": "fill Presenting problem: Knee pain when climbing stairs", + "int4-weights": "fill Presenting problem: Knee pain when climbing stairs" + } + }, + { + "row": 9037, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer\nELEMENT Edit \"Last Name\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Last: Lindqvist", + "selected_options": { + "upstream_pytorch": "fill Last: Lindqvist", + "baseline": "fill Last: Lindqvist", + "int4-weights": "fill Last: Lindqvist" + } + }, + { + "row": 9051, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding\nELEMENT Edit \"Bank account number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9193, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding\nELEMENT Edit \"DL number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9253, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Homepage\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "fill Member #: 75R6HHUVJ" + } + }, + { + "row": 9291, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form - JevBrowser\nELEMENT Edit \"Relationship status\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9345, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Registration plate\" value=\"\"", + "gold_option": "fill Reg plate: BED-9597", + "selected_options": { + "upstream_pytorch": "fill Reg plate: BED-9597", + "baseline": "fill Reg plate: BED-9597", + "int4-weights": "fill Reg plate: BED-9597" + } + }, + { + "row": 9445, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - JevBrowser\nELEMENT Edit \"Full name\" value=\"\"", + "gold_option": "fill Applicant: Lucas Becker", + "selected_options": { + "upstream_pytorch": "fill Applicant: Lucas Becker", + "baseline": "fill Applicant: Lucas Becker", + "int4-weights": "fill Applicant: Lucas Becker" + } + }, + { + "row": 9533, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Insurance ID\" value=\"\"", + "gold_option": "fill Member #: V291KY57V", + "selected_options": { + "upstream_pytorch": "fill Member #: V291KY57V", + "baseline": "fill Member #: V291KY57V", + "int4-weights": "fill Member #: V291KY57V" + } + }, + { + "row": 9570, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - JevBrowser\nELEMENT Edit \"First\" value=\"Isla\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9577, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - JevBrowser\nELEMENT Edit \"Registration plate\" value=\"\"", + "gold_option": "fill Reg plate: YV7-9984", + "selected_options": { + "upstream_pytorch": "fill Reg plate: YV7-9984", + "baseline": "fill Reg plate: YV7-9984", + "int4-weights": "fill Reg plate: YV7-9984" + } + }, + { + "row": 9581, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - JevBrowser\nELEMENT Edit \"Manufacturer\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9621, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Mozilla Firefox\nELEMENT Edit \"Mobile phone\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9624, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Mozilla Firefox\nELEMENT CheckBox \"I consent to treatment and agree to the privacy policy\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9642, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer\nELEMENT Edit \"Drug allergies\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9730, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Registration plate\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9855, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "fill Complaint: Persistent cough for two weeks", + "selected_options": { + "upstream_pytorch": "fill Complaint: Persistent cough for two weeks", + "baseline": "fill Complaint: Persistent cough for two weeks", + "int4-weights": "fill Complaint: Persistent cough for two weeks" + } + }, + { + "row": 9945, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration\nELEMENT Edit \"Financial institution\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9977, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Country\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 9981, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"City\" value=\"Burlington\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10165, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Google Chrome\nELEMENT Edit \"Insurance company\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10191, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Highest degree\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10216, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - JevBrowser\nELEMENT Edit \"Forename\" value=\"\" hint=\"First\"", + "gold_option": "fill First name: Jonas", + "selected_options": { + "upstream_pytorch": "fill First name: Jonas", + "baseline": "fill First name: Jonas", + "int4-weights": "fill First name: Jonas" + } + }, + { + "row": 10345, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Button \"Create account\" value=\"\"", + "gold_option": "click", + "selected_options": { + "upstream_pytorch": "click", + "baseline": "click", + "int4-weights": "click" + } + }, + { + "row": 10364, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "fill Complaint: Migraine headaches", + "selected_options": { + "upstream_pytorch": "fill Complaint: Migraine headaches", + "baseline": "fill Complaint: Migraine headaches", + "int4-weights": "fill Complaint: Migraine headaches" + } + }, + { + "row": 10398, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10436, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Microsoft Edge\nELEMENT Edit \"Last Name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10458, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Current salary\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10486, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "fill Presenting problem: Allergy testing", + "selected_options": { + "upstream_pytorch": "fill Presenting problem: Allergy testing", + "baseline": "fill Presenting problem: Allergy testing", + "int4-weights": "fill Presenting problem: Allergy testing" + } + }, + { + "row": 10488, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Highest degree\" value=\"\"", + "gold_option": "fill Studied: BSc Computer Science", + "selected_options": { + "upstream_pytorch": "fill Studied: BSc Computer Science", + "baseline": "fill Studied: BSc Computer Science", + "int4-weights": "fill Studied: BSc Computer Science" + } + }, + { + "row": 10503, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Discount code\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10620, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Car model\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10703, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - JevBrowser\nELEMENT Edit \"Last\" value=\"Silva\" hint=\"Family name\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Family name: Silva", + "baseline": "fill Family name: Silva", + "int4-weights": "skip" + } + }, + { + "row": 10740, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Mozilla Firefox\nELEMENT Edit \"Passport no.\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10962, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Office phone\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 10993, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Mozilla Firefox\nELEMENT Edit \"Family name\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Last name: Yamamoto", + "selected_options": { + "upstream_pytorch": "fill Last name: Yamamoto", + "baseline": "fill Last name: Yamamoto", + "int4-weights": "fill Last name: Yamamoto" + } + }, + { + "row": 11046, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake - Google Chrome\nELEMENT Edit \"Date of loss\" value=\"\"", + "gold_option": "fill Occurred: 2026-12-23", + "selected_options": { + "upstream_pytorch": "fill Occurred: 2026-12-23", + "baseline": "fill Occurred: 2026-12-23", + "int4-weights": "fill Occurred: 2026-12-23" + } + }, + { + "row": 11090, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"VIN\" value=\"\"", + "gold_option": "fill Vehicle ID: W2VC74RR3YU3WLQ3V", + "selected_options": { + "upstream_pytorch": "fill Vehicle ID: W2VC74RR3YU3WLQ3V", + "baseline": "fill Vehicle ID: W2VC74RR3YU3WLQ3V", + "int4-weights": "fill Vehicle ID: W2VC74RR3YU3WLQ3V" + } + }, + { + "row": 11122, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"EIN\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 11191, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"First\" value=\"Jonas\" hint=\"First\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 11214, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Mozilla Firefox\nELEMENT Edit \"VIN number\" value=\"\"", + "gold_option": "fill Vehicle ID: VU7X3AG831XFKJXPC", + "selected_options": { + "upstream_pytorch": "fill Vehicle ID: VU7X3AG831XFKJXPC", + "baseline": "fill Vehicle ID: VU7X3AG831XFKJXPC", + "int4-weights": "fill Vehicle ID: VU7X3AG831XFKJXPC" + } + }, + { + "row": 11257, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Microsoft Edge\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 11300, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Email Address\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 11337, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding\nELEMENT Edit \"Describe the incident\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 11390, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Your name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 11553, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"DL number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 11630, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Purpose of visit\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 11813, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - JevBrowser\nELEMENT Edit \"Surname\" value=\"\"", + "gold_option": "fill Last: Yamamoto", + "selected_options": { + "upstream_pytorch": "fill Last: Yamamoto", + "baseline": "fill Last: Yamamoto", + "int4-weights": "fill Last: Yamamoto" + } + }, + { + "row": 11880, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Google Chrome\nELEMENT Edit \"Date of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 11959, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer - Mozilla Firefox\nELEMENT Edit \"Reason for visit\" value=\"\"", + "gold_option": "fill Reason: Annual check-up", + "selected_options": { + "upstream_pytorch": "fill Reason: Annual check-up", + "baseline": "fill Reason: Annual check-up", + "int4-weights": "fill Reason: Annual check-up" + } + }, + { + "row": 12043, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Street\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 12217, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Google Chrome\nELEMENT Edit \"Forename\" value=\"\" hint=\"First\"", + "gold_option": "fill Forename: Maya", + "selected_options": { + "upstream_pytorch": "fill Forename: Maya", + "baseline": "fill Forename: Maya", + "int4-weights": "fill Forename: Maya" + } + }, + { + "row": 12219, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Google Chrome\nELEMENT Edit \"Reason for visit\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 12255, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form - JevBrowser\nELEMENT Edit \"Taxpayer identification number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill National ID: 813-20-3361", + "baseline": "fill National ID: 813-20-3361", + "int4-weights": "fill National ID: 813-20-3361" + } + }, + { + "row": 12411, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Purpose of visit\" value=\"\"", + "gold_option": "fill Presenting problem: Persistent cough for two weeks", + "selected_options": { + "upstream_pytorch": "fill Presenting problem: Persistent cough for two weeks", + "baseline": "fill Presenting problem: Persistent cough for two weeks", + "int4-weights": "fill Presenting problem: Persistent cough for two weeks" + } + }, + { + "row": 12516, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Insurance ID\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 12546, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Vehicle make\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 12605, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Forename\" value=\"old\"", + "gold_option": "fill Forename: Ravi", + "selected_options": { + "upstream_pytorch": "fill Forename: Ravi", + "baseline": "fill Forename: Ravi", + "int4-weights": "fill Forename: Ravi" + } + }, + { + "row": 12606, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"E-mail\" value=\"\" hint=\"you@example.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 12699, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Button \"Create account\" value=\"\"", + "gold_option": "click", + "selected_options": { + "upstream_pytorch": "click", + "baseline": "click", + "int4-weights": "click" + } + }, + { + "row": 12725, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Company\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 12754, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT CheckBox \"I consent to treatment and agree to the privacy policy\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 12792, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Microsoft Edge\nELEMENT Edit \"Insurance carrier\" value=\"\"", + "gold_option": "fill Health plan: Aetna", + "selected_options": { + "upstream_pytorch": "fill Health plan: Aetna", + "baseline": "fill Health plan: Aetna", + "int4-weights": "fill Health plan: Aetna" + } + }, + { + "row": 12936, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Vehicle identification number\" value=\"old\"", + "gold_option": "fill VIN: L17B14Q75FUYGNC6S", + "selected_options": { + "upstream_pytorch": "fill VIN: L17B14Q75FUYGNC6S", + "baseline": "fill VIN: L17B14Q75FUYGNC6S", + "int4-weights": "fill VIN: L17B14Q75FUYGNC6S" + } + }, + { + "row": 12949, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Insurance policy number\" value=\"\"", + "gold_option": "fill Policy #: POL-08547702", + "selected_options": { + "upstream_pytorch": "fill Policy #: POL-08547702", + "baseline": "fill Policy #: POL-08547702", + "int4-weights": "fill Policy #: POL-08547702" + } + }, + { + "row": 12983, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13027, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Town\" value=\"\"", + "gold_option": "fill City: Omaha", + "selected_options": { + "upstream_pytorch": "fill City: Omaha", + "baseline": "fill City: Omaha", + "int4-weights": "fill City: Omaha" + } + }, + { + "row": 13031, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Region\" value=\"NE\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13053, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Symptoms\" value=\"\"", + "gold_option": "fill Reason: Skin rash on left arm", + "selected_options": { + "upstream_pytorch": "fill Reason: Skin rash on left arm", + "baseline": "fill Reason: Skin rash on left arm", + "int4-weights": "fill Reason: Skin rash on left arm" + } + }, + { + "row": 13115, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"SSN\" value=\"\"", + "gold_option": "fill National ID: 663-60-6630", + "selected_options": { + "upstream_pytorch": "fill National ID: 663-60-6630", + "baseline": "fill National ID: 663-60-6630", + "int4-weights": "fill National ID: 663-60-6630" + } + }, + { + "row": 13154, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Last\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Family name: Delgado", + "selected_options": { + "upstream_pytorch": "fill Family name: Delgado", + "baseline": "fill Family name: Delgado", + "int4-weights": "fill Family name: Delgado" + } + }, + { + "row": 13204, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"Institution attended\" value=\"Tech Institute\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13323, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Postal code\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13370, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Microsoft Edge\nELEMENT Edit \"Mailing address\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "fill Status: Divorced" + } + }, + { + "row": 13392, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer\nELEMENT Edit \"Member ID\" value=\"TYB2Y2JDT\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13423, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Current job title\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13450, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Last Name\" value=\"\"", + "gold_option": "fill Last: Andersen", + "selected_options": { + "upstream_pytorch": "fill Last: Andersen", + "baseline": "fill Last: Andersen", + "int4-weights": "fill Last: Andersen" + } + }, + { + "row": 13480, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Current employer\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Company reg: REG3878384", + "baseline": "fill Company reg: REG3878384", + "int4-weights": "skip" + } + }, + { + "row": 13483, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Medicines taken\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13533, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"First\" value=\"Nadia\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13545, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Google Chrome\nELEMENT Edit \"Last Name\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Last: Schmidt", + "selected_options": { + "upstream_pytorch": "fill Last: Schmidt", + "baseline": "fill Last: Schmidt", + "int4-weights": "fill Last: Schmidt" + } + }, + { + "row": 13609, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Applicant name\" value=\"Isla Quinn\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13693, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Mozilla Firefox\nELEMENT Edit \"Organization\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13766, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Last\" value=\"\"", + "gold_option": "fill Surname: Andersen", + "selected_options": { + "upstream_pytorch": "fill Surname: Andersen", + "baseline": "fill Surname: Andersen", + "int4-weights": "fill Surname: Andersen" + } + }, + { + "row": 13768, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Legal name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13795, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Microsoft Edge\nELEMENT Edit \"Forename\" value=\"Leila\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 13969, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form - Mozilla Firefox\nELEMENT Edit \"Email address\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 14043, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form - JevBrowser\nELEMENT Edit \"Health plan\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 14145, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - JevBrowser\nELEMENT Edit \"Sex\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 14220, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"First name\" value=\"Leila\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 14568, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT Button \"Create account\" value=\"\"", + "gold_option": "click", + "selected_options": { + "upstream_pytorch": "click", + "baseline": "click", + "int4-weights": "click" + } + }, + { + "row": 14608, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"SSN\" value=\"\"", + "gold_option": "fill Social security number: 003-11-8534", + "selected_options": { + "upstream_pytorch": "fill Social security number: 003-11-8534", + "baseline": "fill Social security number: 003-11-8534", + "int4-weights": "fill Social security number: 003-11-8534" + } + }, + { + "row": 14656, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Google Chrome\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 14718, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form - Microsoft Edge\nELEMENT Edit \"Case number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 14722, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form - Microsoft Edge\nELEMENT Edit \"Nationality\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 14868, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Microsoft Edge\nELEMENT Edit \"Medicines taken\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 14899, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Insurance carrier\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 14934, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Marital status\" value=\"Divorced\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 15065, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - JevBrowser\nELEMENT Edit \"Title\" value=\"Nurse Practitioner\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 15132, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Mozilla Firefox\nELEMENT Edit \"Town\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 15214, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Relation to applicant\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Reg plate: WN9-8147", + "baseline": "fill Reg plate: WN9-8147", + "int4-weights": "skip" + } + }, + { + "row": 15219, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Model\" value=\"Model 3\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 15252, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"VIN number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 15259, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Mozilla Firefox\nELEMENT Edit \"VIN number\" value=\"\"", + "gold_option": "fill Vehicle ID: UF6M521K3G9J023CS", + "selected_options": { + "upstream_pytorch": "fill Vehicle ID: UF6M521K3G9J023CS", + "baseline": "fill Vehicle ID: UF6M521K3G9J023CS", + "int4-weights": "fill Vehicle ID: UF6M521K3G9J023CS" + } + }, + { + "row": 15698, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer - Mozilla Firefox\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 15764, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer\nELEMENT Edit \"First Name\" value=\"\" hint=\"Given name\"", + "gold_option": "fill Forename: Kenji", + "selected_options": { + "upstream_pytorch": "fill Forename: Kenji", + "baseline": "fill Forename: Kenji", + "int4-weights": "fill Forename: Kenji" + } + }, + { + "row": 15819, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"Address line 1\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 16041, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Last\" value=\"\"", + "gold_option": "fill Surname: Lindqvist", + "selected_options": { + "upstream_pytorch": "fill Surname: Lindqvist", + "baseline": "fill Surname: Lindqvist", + "int4-weights": "fill Surname: Lindqvist" + } + }, + { + "row": 16106, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Microsoft Edge\nELEMENT Edit \"Home address\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 16182, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form - Microsoft Edge\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 16200, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - JevBrowser\nELEMENT Edit \"Start date\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 16205, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - JevBrowser\nELEMENT Edit \"State\" value=\"\"", + "gold_option": "fill ST: OR", + "selected_options": { + "upstream_pytorch": "fill ST: OR", + "baseline": "fill ST: OR", + "int4-weights": "fill ST: OR" + } + }, + { + "row": 16207, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - JevBrowser\nELEMENT Edit \"Surname\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Surname: Moreau", + "selected_options": { + "upstream_pytorch": "fill Surname: Moreau", + "baseline": "fill Surname: Moreau", + "int4-weights": "fill Surname: Moreau" + } + }, + { + "row": 16253, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Relation to applicant\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 16256, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Role\" value=\"\"", + "gold_option": "fill Position: Project Manager", + "selected_options": { + "upstream_pytorch": "fill Position: Project Manager", + "baseline": "fill Position: Project Manager", + "int4-weights": "fill Position: Project Manager" + } + }, + { + "row": 16392, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer - Google Chrome\nELEMENT Edit \"Your email\" value=\"\" hint=\"name@domain.com\"", + "gold_option": "fill Email: nandersen99@fastmail.com", + "selected_options": { + "upstream_pytorch": "fill Email: nandersen99@fastmail.com", + "baseline": "fill Email: nandersen99@fastmail.com", + "int4-weights": "fill Email: nandersen99@fastmail.com" + } + }, + { + "row": 16517, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"First Name\" value=\"old\" hint=\"First\"", + "gold_option": "fill First name: Hannah", + "selected_options": { + "upstream_pytorch": "fill First name: Hannah", + "baseline": "fill First name: Hannah", + "int4-weights": "fill First name: Hannah" + } + }, + { + "row": 16597, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Google Chrome\nELEMENT Edit \"Telephone\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 16608, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Google Chrome\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 16678, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake - Mozilla Firefox\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "fill Complaint: Skin rash on left arm", + "selected_options": { + "upstream_pytorch": "fill Complaint: Skin rash on left arm", + "baseline": "fill Complaint: Skin rash on left arm", + "int4-weights": "fill Complaint: Skin rash on left arm" + } + }, + { + "row": 16920, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application - Google Chrome\nELEMENT Edit \"Total years experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "fill Expected salary: $130k" + } + }, + { + "row": 16951, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"State\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 17100, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"National ID number\" value=\"\"", + "gold_option": "fill Social Security: 513-12-8044", + "selected_options": { + "upstream_pytorch": "fill Social Security: 513-12-8044", + "baseline": "fill Social Security: 513-12-8044", + "int4-weights": "fill Social Security: 513-12-8044" + } + }, + { + "row": 17122, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form - Mozilla Firefox\nELEMENT Edit \"Plate #\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 17253, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Primary phone\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 17304, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - Google Chrome\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 17318, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Google Chrome\nELEMENT Edit \"Last name\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Last: Abernathy", + "selected_options": { + "upstream_pytorch": "fill Last: Abernathy", + "baseline": "fill Last: Abernathy", + "int4-weights": "fill Last: Abernathy" + } + }, + { + "row": 17351, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Marital status\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 17372, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Last name\" value=\"old\"", + "gold_option": "fill Last: Delgado", + "selected_options": { + "upstream_pytorch": "fill Last: Delgado", + "baseline": "fill Last: Delgado", + "int4-weights": "fill Last: Delgado" + } + }, + { + "row": 17454, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Forename\" value=\"\" hint=\"First\"", + "gold_option": "fill First: Jonas", + "selected_options": { + "upstream_pytorch": "fill First: Jonas", + "baseline": "fill First: Jonas", + "int4-weights": "fill First: Jonas" + } + }, + { + "row": 17632, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Mobile number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 17665, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form - JevBrowser\nELEMENT Edit \"Contact email\" value=\"\"", + "gold_option": "fill Contact: olivia.garcia@gmail.com", + "selected_options": { + "upstream_pytorch": "fill Contact: olivia.garcia@gmail.com", + "baseline": "fill Contact: olivia.garcia@gmail.com", + "int4-weights": "fill Contact: olivia.garcia@gmail.com" + } + }, + { + "row": 17854, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT Edit \"Purpose of visit\" value=\"\"", + "gold_option": "fill Complaint: Annual check-up", + "selected_options": { + "upstream_pytorch": "fill Complaint: Annual check-up", + "baseline": "fill Complaint: Annual check-up", + "int4-weights": "fill Complaint: Annual check-up" + } + }, + { + "row": 17858, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT Edit \"Contact phone\" value=\"\"", + "gold_option": "fill Phone: +1 779 550 3974", + "selected_options": { + "upstream_pytorch": "fill Phone: +1 779 550 3974", + "baseline": "fill Phone: +1 779 550 3974", + "int4-weights": "fill Phone: +1 779 550 3974" + } + }, + { + "row": 17865, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT Edit \"Apartment or unit\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 17873, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT CheckBox \"I acknowledge the cancellation policy\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 17918, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Mozilla Firefox\nELEMENT Edit \"Gender\" value=\"Female\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18099, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "fill Presenting problem: Knee pain when climbing stairs", + "selected_options": { + "upstream_pytorch": "fill Presenting problem: Knee pain when climbing stairs", + "baseline": "fill Presenting problem: Knee pain when climbing stairs", + "int4-weights": "fill Presenting problem: Knee pain when climbing stairs" + } + }, + { + "row": 18119, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application - Google Chrome\nELEMENT Edit \"Surname\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Family name: Abernathy", + "selected_options": { + "upstream_pytorch": "fill Family name: Abernathy", + "baseline": "fill Family name: Abernathy", + "int4-weights": "fill Family name: Abernathy" + } + }, + { + "row": 18154, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form - JevBrowser\nELEMENT Edit \"Last\" value=\"\"", + "gold_option": "fill Surname: Garcia", + "selected_options": { + "upstream_pytorch": "fill Surname: Garcia", + "baseline": "fill Surname: Garcia", + "int4-weights": "fill Surname: Garcia" + } + }, + { + "row": 18168, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form - JevBrowser\nELEMENT Edit \"Model\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18237, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Microsoft Edge\nELEMENT Edit \"Primary care physician\" value=\"Dr. Chen\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18252, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Marital status\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18290, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding\nELEMENT Edit \"Purpose of visit\" value=\"\"", + "gold_option": "fill Presenting problem: Knee pain when climbing stairs", + "selected_options": { + "upstream_pytorch": "fill Presenting problem: Knee pain when climbing stairs", + "baseline": "fill Presenting problem: Knee pain when climbing stairs", + "int4-weights": "fill Presenting problem: Knee pain when climbing stairs" + } + }, + { + "row": 18339, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Mozilla Firefox\nELEMENT Edit \"Last name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18432, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"Search\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18541, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"GP name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18549, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"ABA routing number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18551, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18559, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"Search\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18579, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration\nELEMENT Edit \"Member number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18726, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Last Name\" value=\"old\" hint=\"Family name\"", + "gold_option": "fill Last: Tanaka", + "selected_options": { + "upstream_pytorch": "fill Last: Tanaka", + "baseline": "fill Last: Tanaka", + "int4-weights": "fill Last: Tanaka" + } + }, + { + "row": 18731, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Button \"Create account\" value=\"\"", + "gold_option": "click", + "selected_options": { + "upstream_pytorch": "click", + "baseline": "click", + "int4-weights": "click" + } + }, + { + "row": 18753, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Microsoft Edge\nELEMENT Edit \"Job title\" value=\"\"", + "gold_option": "fill Occupation: Registered Nurse", + "selected_options": { + "upstream_pytorch": "fill Occupation: Registered Nurse", + "baseline": "fill Occupation: Registered Nurse", + "int4-weights": "fill Occupation: Registered Nurse" + } + }, + { + "row": 18775, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form\nELEMENT Edit \"Name (as on ID)\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18802, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Unit number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 18945, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT Button \"Create account\" value=\"\"", + "gold_option": "click", + "selected_options": { + "upstream_pytorch": "click", + "baseline": "click", + "int4-weights": "click" + } + }, + { + "row": 19021, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form\nELEMENT Edit \"Surname\" value=\"\"", + "gold_option": "fill Surname: Novak", + "selected_options": { + "upstream_pytorch": "fill Surname: Novak", + "baseline": "fill Surname: Novak", + "int4-weights": "fill Surname: Novak" + } + }, + { + "row": 19205, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form - Microsoft Edge\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int4-weights": "check" + } + }, + { + "row": 19270, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Plate number\" value=\"old\"", + "gold_option": "fill License plate: SJ3-1115", + "selected_options": { + "upstream_pytorch": "fill License plate: SJ3-1115", + "baseline": "fill License plate: SJ3-1115", + "int4-weights": "fill License plate: SJ3-1115" + } + }, + { + "row": 19276, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Routing number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 19308, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Google Chrome\nELEMENT Edit \"Last\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 19361, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Mozilla Firefox\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 19438, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Last\" value=\"Rossi\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 19439, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Describe the incident\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 19511, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Last Name\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Last name: Silva", + "selected_options": { + "upstream_pytorch": "fill Last name: Silva", + "baseline": "fill Last name: Silva", + "int4-weights": "fill Last name: Silva" + } + }, + { + "row": 19634, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "fill Summary: Slipped on icy pavement", + "selected_options": { + "upstream_pytorch": "fill Summary: Slipped on icy pavement", + "baseline": "fill Summary: Slipped on icy pavement", + "int4-weights": "fill Summary: Slipped on icy pavement" + } + }, + { + "row": 19669, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"School\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 19704, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Occupation\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 19809, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - JevBrowser\nELEMENT Edit \"Total years experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 19834, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - JevBrowser\nELEMENT Edit \"Tax number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 19940, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Surname\" value=\"\"", + "gold_option": "fill Last: Kowalski", + "selected_options": { + "upstream_pytorch": "fill Last: Kowalski", + "baseline": "fill Last: Kowalski", + "int4-weights": "fill Last: Kowalski" + } + }, + { + "row": 19957, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Plate number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20008, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form - Mozilla Firefox\nELEMENT Edit \"First name\" value=\"\" hint=\"First\"", + "gold_option": "fill Forename: Ibrahim", + "selected_options": { + "upstream_pytorch": "fill Forename: Ibrahim", + "baseline": "fill Forename: Ibrahim", + "int4-weights": "fill Forename: Ibrahim" + } + }, + { + "row": 20097, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20214, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer\nELEMENT Edit \"State\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20326, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration\nELEMENT Edit \"Current medications\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20370, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - Microsoft Edge\nELEMENT Edit \"Name (as on ID)\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20457, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Mozilla Firefox\nELEMENT Edit \"Town\" value=\"Seattle\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20462, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Mozilla Firefox\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20476, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Sort code\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20486, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Social Security Number\" value=\"\"", + "gold_option": "fill National ID: 858-41-8534", + "selected_options": { + "upstream_pytorch": "fill National ID: 858-41-8534", + "baseline": "fill National ID: 858-41-8534", + "int4-weights": "fill National ID: 858-41-8534" + } + }, + { + "row": 20492, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20625, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Street\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20776, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Given name\" value=\"\"", + "gold_option": "fill Forename: Ethan", + "selected_options": { + "upstream_pytorch": "fill Forename: Ethan", + "baseline": "fill Forename: Ethan", + "int4-weights": "fill Forename: Ethan" + } + }, + { + "row": 20928, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Next of kin\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 20956, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21022, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - JevBrowser\nELEMENT Edit \"State\" value=\"NE\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill National ID: 794-03-1212", + "baseline": "fill National ID: 794-03-1212", + "int4-weights": "fill National ID: 794-03-1212" + } + }, + { + "row": 21028, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - JevBrowser\nELEMENT Edit \"DL number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21029, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - JevBrowser\nELEMENT Edit \"Taxpayer identification number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21049, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Nation\" value=\"\"", + "gold_option": "fill Country: Japan", + "selected_options": { + "upstream_pytorch": "fill Country: Japan", + "baseline": "fill Country: Japan", + "int4-weights": "fill Country: Japan" + } + }, + { + "row": 21163, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Google Chrome\nELEMENT Edit \"School\" value=\"Tech Institute\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21344, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Job title\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "fill Phone: 403.964.9030" + } + }, + { + "row": 21352, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT Edit \"Last Name\" value=\"\" hint=\"Family name\"", + "gold_option": "fill Family name: Abernathy", + "selected_options": { + "upstream_pytorch": "fill Family name: Abernathy", + "baseline": "fill Family name: Abernathy", + "int4-weights": "fill Family name: Abernathy" + } + }, + { + "row": 21464, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"National ID number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21519, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - JevBrowser\nELEMENT Edit \"Region\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21566, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Relationship\" value=\"\"", + "gold_option": "fill Relationship: Daughter", + "selected_options": { + "upstream_pytorch": "fill Relationship: Daughter", + "baseline": "fill Relationship: Daughter", + "int4-weights": "fill Relationship: Daughter" + } + }, + { + "row": 21631, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Mozilla Firefox\nELEMENT Edit \"Medicines taken\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21813, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form - Microsoft Edge\nELEMENT Edit \"Insurer\" value=\"\"", + "gold_option": "fill Insurance company: Centene", + "selected_options": { + "upstream_pytorch": "fill Insurance company: Centene", + "baseline": "fill Insurance company: Centene", + "int4-weights": "fill Insurance company: Centene" + } + }, + { + "row": 21844, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - Mozilla Firefox\nELEMENT Edit \"Gender\" value=\"Female\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21866, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"DOB\" value=\"\" hint=\"YYYY-MM-DD\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21871, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Sort code\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21886, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Sort code\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 21890, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Province\" value=\"\"", + "gold_option": "fill Province: GA", + "selected_options": { + "upstream_pytorch": "fill Province: GA", + "baseline": "fill Province: GA", + "int4-weights": "fill Province: GA" + } + }, + { + "row": 21903, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding\nELEMENT Edit \"Bank account number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22046, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Nationality\" value=\"Indian\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22148, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Google Chrome\nELEMENT Edit \"Purpose of visit\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22223, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - Microsoft Edge\nELEMENT Edit \"Marital status\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22297, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Microsoft Edge\nELEMENT Edit \"Last\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22336, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Car model\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22442, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "fill Complaint: Knee pain when climbing stairs", + "selected_options": { + "upstream_pytorch": "fill Complaint: Knee pain when climbing stairs", + "baseline": "fill Complaint: Knee pain when climbing stairs", + "int4-weights": "fill Complaint: Knee pain when climbing stairs" + } + }, + { + "row": 22571, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Phone\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22714, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Describe your concern\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22785, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake - Mozilla Firefox\nELEMENT Edit \"Gender\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22791, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake - Mozilla Firefox\nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "check" + } + }, + { + "row": 22831, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 22855, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Reason for visit\" value=\"\"", + "gold_option": "fill Reason: Back pain", + "selected_options": { + "upstream_pytorch": "fill Reason: Back pain", + "baseline": "fill Reason: Back pain", + "int4-weights": "fill Reason: Back pain" + } + }, + { + "row": 23012, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Chief complaint\" value=\"\"", + "gold_option": "fill Complaint: Migraine headaches", + "selected_options": { + "upstream_pytorch": "fill Complaint: Migraine headaches", + "baseline": "fill Complaint: Migraine headaches", + "int4-weights": "fill Complaint: Migraine headaches" + } + }, + { + "row": 23051, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT Edit \"Total years experience\" value=\"\"", + "gold_option": "fill Years: 6", + "selected_options": { + "upstream_pytorch": "fill Years: 6", + "baseline": "fill Years: 6", + "int4-weights": "fill Years: 6" + } + }, + { + "row": 23074, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer\nELEMENT Edit \"Degree\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23097, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form - Microsoft Edge\nELEMENT Edit \"Surname\" value=\"\"", + "gold_option": "fill Last name: Haddad", + "selected_options": { + "upstream_pytorch": "fill Last name: Haddad", + "baseline": "fill Last name: Haddad", + "int4-weights": "fill Last name: Haddad" + } + }, + { + "row": 23276, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form - JevBrowser\nELEMENT Edit \"Relationship status\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23322, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Passport number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23350, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"E-mail\" value=\"\" hint=\"you@example.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23398, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Button \"Create account\" value=\"\"", + "gold_option": "click", + "selected_options": { + "upstream_pytorch": "click", + "baseline": "click", + "int4-weights": "click" + } + }, + { + "row": 23670, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Member number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23671, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Email\" value=\"lchen6@yahoo.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Email: lchen6@yahoo.com", + "baseline": "fill Email: lchen6@yahoo.com", + "int4-weights": "fill Email: lchen6@yahoo.com" + } + }, + { + "row": 23742, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Email\" value=\"old\"", + "gold_option": "fill Email: anovak17@gmail.com", + "selected_options": { + "upstream_pytorch": "fill Email: anovak17@gmail.com", + "baseline": "fill Email: anovak17@gmail.com", + "int4-weights": "fill Email: anovak17@gmail.com" + } + }, + { + "row": 23749, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Personal website\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23794, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Member number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23858, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Mozilla Firefox\nELEMENT Edit \"Job title\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23863, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Mozilla Firefox\nELEMENT Edit \"Years of experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23888, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form\nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 23992, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Microsoft Edge\nELEMENT Edit \"Total years experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "fill Desired compensation: $181,000" + } + }, + { + "row": 24016, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Description of incident\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 24018, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Current job title\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 24027, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Current salary\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 24142, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Next of kin phone\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int4-weights": "skip" + } + }, + { + "row": 24163, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment\nELEMENT Edit \"SSN\" value=\"\"", + "gold_option": "fill National ID: 185-82-5310", + "selected_options": { + "upstream_pytorch": "fill National ID: 185-82-5310", + "baseline": "fill National ID: 185-82-5310", + "int4-weights": "fill National ID: 185-82-5310" + } + } + ], + "trace": { + "file": "int4-synthetic-test-decisions.jsonl.gz", + "sha256": "6922913041f95ac8872958b92312cb5b869b9ce88116c4684b36ea7753284674", + "rows": 24370 + }, + "upstream_evaluator_sha256": "49dccdeb4e2b456187cdc4491a6d78fabac6a45384272f8e3ec802a58872aa6e", + "harness_sha256": { + "benchmark-synthetic.py": "ba8255fd3f63d87ef340c26c4c251800f2e20878d80858ad1e6557cf59f249a2", + "synthetic_test.py": "e2b68725236e6827eb422480c8cc8aa189b2490c1594650d8f2928771d1bc626", + "score-report.py": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e" + }, + "conversion_parity_passed": false, + "limitations": [ + "The release file has 24370 rows; the model card claims approximately 15000. This is not a reconstruction of that unspecified manifest.", + "Upstream describes form-disjoint synthetic data; this is not an unseen real-world GUI or task-completion benchmark.", + "No fresh hosted Jev calls are included. Published hosted-Jev percentages are not a matched baseline for this run.", + "Local timings use CPU for PyTorch and CPU+ANE for Core ML; they are backend/deployment measurements, not equal-hardware algorithm comparisons." + ] +} diff --git a/reports/int8-demo-verification.json b/reports/int8-demo-verification.json new file mode 100644 index 0000000000000000000000000000000000000000..38e62c4810dca652cbbf0deff60ed18d82604f68 --- /dev/null +++ b/reports/int8-demo-verification.json @@ -0,0 +1,1758 @@ +{ + "purpose": "Local conversion parity on the upstream demo; not a generalization or live GUI benchmark", + "created_utc": "2026-09-19T22:03:05.949263+00:00", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_file": "demo.jsonl", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "conversion": { + "model": "cua_s1_forms_int8_options32.mlpackage", + "precision": "int8_weights_float16_compute", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "a7d9747308b8b754a46f3e70c8465b821f90e12376b3a71dc909f7364ae73c3a", + "Data/com.apple.CoreML/weights/weight.bin": "f600c4434b7f7e3e789de8f1a4aa54a55f84df51fa9b788e18f5c97b68e4840f", + "Manifest.json": "2206ec01787a85c12f1376fa5a832c5ac912f06172c6ff06b26ae8415470d246" + }, + "quantization": { + "name": "int8-weights", + "settings": { + "mode": "linear_symmetric", + "dtype": "int8", + "granularity": "per_channel", + "weight_threshold": 2048 + }, + "activations": "float16", + "calibration": "None; weight-only, data-free quantization", + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "source_package_bytes": 1511163, + "package_bytes": 813422, + "compression_seconds": 0.3180606249952689, + "operation_counts": { + "add": 15, + "cast": 13, + "concat": 2, + "const": 226, + "constexpr_affine_dequantize": 19, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 4, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 9, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "22071babfcb02ef77225203a479d9bb14b9fa35462ba38bcc4bf07d29c125a97" + } + }, + "environment": { + "machine": "Apple M5 Pro", + "os": "27.0", + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "torch_cpu_threads": 2 + }, + "rows": 196, + "upstream": { + "correct": 196, + "accuracy": 1.0, + "inference_latency_ms": { + "count": 196, + "median_ms": 2.3908544972073287, + "p95_ms": 3.0149687590892427, + "min_ms": 1.7770830017980188, + "max_ms": 9.844208980211988 + } + }, + "export_adapter_fp32": { + "argmax_agreement": 196, + "max_abs_probability_error": 1.1324882507324219e-06 + }, + "host_preprocessing_latency_ms": { + "count": 196, + "median_ms": 0.03708299482241273, + "p95_ms": 0.04522973904386163, + "min_ms": 0.024834007490426302, + "max_ms": 0.08750002598389983 + }, + "thresholds": { + "argmax_agreement": 1.0, + "max_abs_probability_error": 0.005, + "export_adapter_max_abs_probability_error": 0.0001, + "allow_accuracy_loss": false + }, + "backends": { + "CPU_AND_NE": { + "passed": false, + "correct": 196, + "accuracy": 1.0, + "argmax_agreement": 196, + "max_abs_probability_error": 0.0053362250328063965, + "mean_row_max_abs_probability_error": 4.465129200521148e-05, + "max_abs_logit_error": 0.41709184646606445, + "load_seconds": 0.6060812089999672, + "warm_inference_latency_ms": { + "count": 196, + "median_ms": 0.9286874992540106, + "p95_ms": 1.1076562368543819, + "min_ms": 0.8750410052016377, + "max_ms": 1.3015410222578794 + }, + "reversed_option_order": { + "rows": 6, + "max_abs_probability_error": 3.1828880310058594e-05 + }, + "per_action": { + "fill": { + "rows": 36, + "correct": 36 + }, + "skip": { + "rows": 150, + "correct": 150 + }, + "check": { + "rows": 4, + "correct": 4 + }, + "click": { + "rows": 6, + "correct": 6 + } + }, + "decisions": [ + { + "row": 0, + "label": 22, + "upstream": 22, + "coreml": 22, + "max_abs_probability_error": 3.1828880310058594e-05, + "max_abs_logit_error": 0.17777347564697266 + }, + { + "row": 1, + "label": 23, + "upstream": 23, + "coreml": 23, + "max_abs_probability_error": 3.814697265625e-05, + "max_abs_logit_error": 0.13521480560302734 + }, + { + "row": 2, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.21187782287597656 + }, + { + "row": 3, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 3.218650817871094e-05, + "max_abs_logit_error": 0.11447498202323914 + }, + { + "row": 4, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 2.944469451904297e-05, + "max_abs_logit_error": 0.20640206336975098 + }, + { + "row": 5, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2636184692382812e-05, + "max_abs_logit_error": 0.26122236251831055 + }, + { + "row": 6, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 2.3365020751953125e-05, + "max_abs_logit_error": 0.15306663513183594 + }, + { + "row": 7, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.00014638900756835938, + "max_abs_logit_error": 0.14249515533447266 + }, + { + "row": 8, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.15889453887939453 + }, + { + "row": 9, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 2.1457672119140625e-05, + "max_abs_logit_error": 0.1523761749267578 + }, + { + "row": 10, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.14359092712402344 + }, + { + "row": 11, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 3.2186508178710938e-06, + "max_abs_logit_error": 0.3163447380065918 + }, + { + "row": 12, + "label": 15, + "upstream": 15, + "coreml": 15, + "max_abs_probability_error": 2.384185791015625e-06, + "max_abs_logit_error": 0.19838738441467285 + }, + { + "row": 13, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 0.0005360841751098633, + "max_abs_logit_error": 0.15249061584472656 + }, + { + "row": 14, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.08178329467773438 + }, + { + "row": 15, + "label": 24, + "upstream": 24, + "coreml": 24, + "max_abs_probability_error": 1.6689300537109375e-05, + "max_abs_logit_error": 0.19499272108078003 + }, + { + "row": 16, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.10968685150146484 + }, + { + "row": 17, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.22722625732421875 + }, + { + "row": 18, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.16490459442138672 + }, + { + "row": 19, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.09899425506591797 + }, + { + "row": 20, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.16230183839797974 + }, + { + "row": 21, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.11515188217163086 + }, + { + "row": 22, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.14793038368225098 + }, + { + "row": 23, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.23419427871704102 + }, + { + "row": 24, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.16013479232788086 + }, + { + "row": 25, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.2223043441772461 + }, + { + "row": 26, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.15115368366241455 + }, + { + "row": 27, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.1729874610900879 + }, + { + "row": 28, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.24140071868896484 + }, + { + "row": 29, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.09259410947561264 + }, + { + "row": 30, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.1546640396118164 + }, + { + "row": 31, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.11323285102844238 + }, + { + "row": 32, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.17779135704040527 + }, + { + "row": 33, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.19167208671569824 + }, + { + "row": 34, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.0004820823669433594, + "max_abs_logit_error": 0.18633151054382324 + }, + { + "row": 35, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.462750809366753e-09, + "max_abs_logit_error": 0.1198587417602539 + }, + { + "row": 36, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.7510341227277024e-10, + "max_abs_logit_error": 0.04924964904785156 + }, + { + "row": 37, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.657592190757896e-08, + "max_abs_logit_error": 0.0999135971069336 + }, + { + "row": 38, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 9.552543478452691e-11, + "max_abs_logit_error": 0.07785224914550781 + }, + { + "row": 39, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.04716482549361e-10, + "max_abs_logit_error": 0.11640000343322754 + }, + { + "row": 40, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.00021719932556152344, + "max_abs_logit_error": 0.18055367469787598 + }, + { + "row": 41, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.146233744175333e-09, + "max_abs_logit_error": 0.2431175708770752 + }, + { + "row": 42, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8819494851385343e-09, + "max_abs_logit_error": 0.14107990264892578 + }, + { + "row": 43, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.15815162658691406 + }, + { + "row": 44, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.780525865635596e-10, + "max_abs_logit_error": 0.0665130615234375 + }, + { + "row": 45, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.414331317674879e-11, + "max_abs_logit_error": 0.11009097099304199 + }, + { + "row": 46, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.599257724142603e-10, + "max_abs_logit_error": 0.10155582427978516 + }, + { + "row": 47, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.212314607705437e-11, + "max_abs_logit_error": 0.09491920471191406 + }, + { + "row": 48, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.08178329467773438 + }, + { + "row": 49, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.932842080831776e-10, + "max_abs_logit_error": 0.1341402530670166 + }, + { + "row": 50, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.10968685150146484 + }, + { + "row": 51, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.22722625732421875 + }, + { + "row": 52, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.16490459442138672 + }, + { + "row": 53, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.09899425506591797 + }, + { + "row": 54, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.16230183839797974 + }, + { + "row": 55, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.11515188217163086 + }, + { + "row": 56, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.14793038368225098 + }, + { + "row": 57, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.23419427871704102 + }, + { + "row": 58, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.16013479232788086 + }, + { + "row": 59, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.2223043441772461 + }, + { + "row": 60, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.15115368366241455 + }, + { + "row": 61, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.1729874610900879 + }, + { + "row": 62, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.24140071868896484 + }, + { + "row": 63, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.09259410947561264 + }, + { + "row": 64, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.1546640396118164 + }, + { + "row": 65, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.11323285102844238 + }, + { + "row": 66, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.17779135704040527 + }, + { + "row": 67, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.19167208671569824 + }, + { + "row": 68, + "label": 0, + "upstream": 0, + "coreml": 0, + "max_abs_probability_error": 3.147125244140625e-05, + "max_abs_logit_error": 0.16318607330322266 + }, + { + "row": 69, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 5.030632019042969e-05, + "max_abs_logit_error": 0.16801834106445312 + }, + { + "row": 70, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.12366819381713867 + }, + { + "row": 71, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 5.2928924560546875e-05, + "max_abs_logit_error": 0.23097705841064453 + }, + { + "row": 72, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 4.649162292480469e-06, + "max_abs_logit_error": 0.28166770935058594 + }, + { + "row": 73, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 2.1576881408691406e-05, + "max_abs_logit_error": 0.14033889770507812 + }, + { + "row": 74, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 4.470348358154297e-05, + "max_abs_logit_error": 0.1834554672241211 + }, + { + "row": 75, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 1.1563301086425781e-05, + "max_abs_logit_error": 0.16341686248779297 + }, + { + "row": 76, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 2.86102294921875e-06, + "max_abs_logit_error": 0.16710281372070312 + }, + { + "row": 77, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 0.0003635883331298828, + "max_abs_logit_error": 0.22506141662597656 + }, + { + "row": 78, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.10105061531066895 + }, + { + "row": 79, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 0.0003609657287597656, + "max_abs_logit_error": 0.19726498425006866 + }, + { + "row": 80, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.225250244140625 + }, + { + "row": 81, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.2853279113769531 + }, + { + "row": 82, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.2046489715576172 + }, + { + "row": 83, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.17853069305419922 + }, + { + "row": 84, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.1555185317993164 + }, + { + "row": 85, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.24918603897094727 + }, + { + "row": 86, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.1990499496459961 + }, + { + "row": 87, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.2444934844970703 + }, + { + "row": 88, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.23791265487670898 + }, + { + "row": 89, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.13373339176177979 + }, + { + "row": 90, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.22733306884765625 + }, + { + "row": 91, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.1239767074584961 + }, + { + "row": 92, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.19014644622802734 + }, + { + "row": 93, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.2004070281982422 + }, + { + "row": 94, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.13628053665161133 + }, + { + "row": 95, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.27038145065307617 + }, + { + "row": 96, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.16453242301940918 + }, + { + "row": 97, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.1947021484375 + }, + { + "row": 98, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.41709184646606445 + }, + { + "row": 99, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.1457672119140625e-06, + "max_abs_logit_error": 0.16466069221496582 + }, + { + "row": 100, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.16838455200195312 + }, + { + "row": 101, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.789079107084035e-08, + "max_abs_logit_error": 0.12038791179656982 + }, + { + "row": 102, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5862225694339713e-09, + "max_abs_logit_error": 0.16624808311462402 + }, + { + "row": 103, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 0.00024586915969848633, + "max_abs_logit_error": 0.19487690925598145 + }, + { + "row": 104, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.841255187988281e-06, + "max_abs_logit_error": 0.13808655738830566 + }, + { + "row": 105, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.928799035575082e-10, + "max_abs_logit_error": 0.13288813829421997 + }, + { + "row": 106, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.4136979082101107e-09, + "max_abs_logit_error": 0.0866403579711914 + }, + { + "row": 107, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.302839361538304e-10, + "max_abs_logit_error": 0.11413812637329102 + }, + { + "row": 108, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.18335914611816406 + }, + { + "row": 109, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.10105061531066895 + }, + { + "row": 110, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.1961498260498047 + }, + { + "row": 111, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.811981201171875e-05, + "max_abs_logit_error": 0.30625247955322266 + }, + { + "row": 112, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.2853279113769531 + }, + { + "row": 113, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.2046489715576172 + }, + { + "row": 114, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.17853069305419922 + }, + { + "row": 115, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.1555185317993164 + }, + { + "row": 116, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.24918603897094727 + }, + { + "row": 117, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.1990499496459961 + }, + { + "row": 118, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.2444934844970703 + }, + { + "row": 119, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.23791265487670898 + }, + { + "row": 120, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.13373339176177979 + }, + { + "row": 121, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.22733306884765625 + }, + { + "row": 122, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.1239767074584961 + }, + { + "row": 123, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.19014644622802734 + }, + { + "row": 124, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.2004070281982422 + }, + { + "row": 125, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.13628053665161133 + }, + { + "row": 126, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.27038145065307617 + }, + { + "row": 127, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.16453242301940918 + }, + { + "row": 128, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.1947021484375 + }, + { + "row": 129, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.41709184646606445 + }, + { + "row": 130, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 3.170967102050781e-05, + "max_abs_logit_error": 0.12571239471435547 + }, + { + "row": 131, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.0871438980102539 + }, + { + "row": 132, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 2.7894973754882812e-05, + "max_abs_logit_error": 0.16924285888671875 + }, + { + "row": 133, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 1.0967254638671875e-05, + "max_abs_logit_error": 0.21336686611175537 + }, + { + "row": 134, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 6.258487701416016e-05, + "max_abs_logit_error": 0.16839587688446045 + }, + { + "row": 135, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2993812561035156e-05, + "max_abs_logit_error": 0.12725162506103516 + }, + { + "row": 136, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 6.198883056640625e-06, + "max_abs_logit_error": 0.1457538604736328 + }, + { + "row": 137, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.0053362250328063965, + "max_abs_logit_error": 0.15258073806762695 + }, + { + "row": 138, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 0.00022917985916137695, + "max_abs_logit_error": 0.12661266326904297 + }, + { + "row": 139, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 4.851818084716797e-05, + "max_abs_logit_error": 0.2998933792114258 + }, + { + "row": 140, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 3.0994415283203125e-06, + "max_abs_logit_error": 0.19615697860717773 + }, + { + "row": 141, + "label": 12, + "upstream": 12, + "coreml": 12, + "max_abs_probability_error": 9.34600830078125e-05, + "max_abs_logit_error": 0.11831855773925781 + }, + { + "row": 142, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.14543867111206055 + }, + { + "row": 143, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.13959884643554688 + }, + { + "row": 144, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.20041000843048096 + }, + { + "row": 145, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.14644718170166016 + }, + { + "row": 146, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.3961820602416992 + }, + { + "row": 147, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.22239947319030762 + }, + { + "row": 148, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.13992595672607422 + }, + { + "row": 149, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.16092300415039062 + }, + { + "row": 150, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.15441608428955078 + }, + { + "row": 151, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.15936005115509033 + }, + { + "row": 152, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.173073410987854 + }, + { + "row": 153, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.11217308044433594 + }, + { + "row": 154, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.14353275299072266 + }, + { + "row": 155, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.1971302032470703 + }, + { + "row": 156, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.14509010314941406 + }, + { + "row": 157, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.2037668228149414 + }, + { + "row": 158, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.12031292915344238 + }, + { + "row": 159, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.18417954444885254 + }, + { + "row": 160, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.17492496967315674 + }, + { + "row": 161, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.09895849227905273 + }, + { + "row": 162, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.22567105293273926 + }, + { + "row": 163, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9131388140358752e-10, + "max_abs_logit_error": 0.08278036117553711 + }, + { + "row": 164, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0254025184508464e-08, + "max_abs_logit_error": 0.0756688117980957 + }, + { + "row": 165, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.15840287151903e-11, + "max_abs_logit_error": 0.18943119049072266 + }, + { + "row": 166, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.6797479790729994e-09, + "max_abs_logit_error": 0.08643817901611328 + }, + { + "row": 167, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.54509399403247e-10, + "max_abs_logit_error": 0.2648029327392578 + }, + { + "row": 168, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.2841672897338867 + }, + { + "row": 169, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.4169340140378637e-11, + "max_abs_logit_error": 0.10731315612792969 + }, + { + "row": 170, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.0329603412093675e-11, + "max_abs_logit_error": 0.14229869842529297 + }, + { + "row": 171, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.10416412353515625 + }, + { + "row": 172, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.66337235494052e-09, + "max_abs_logit_error": 0.08718490600585938 + }, + { + "row": 173, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2885730260592254e-09, + "max_abs_logit_error": 0.07872724533081055 + }, + { + "row": 174, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1279766010119374e-09, + "max_abs_logit_error": 0.17765092849731445 + }, + { + "row": 175, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.14543867111206055 + }, + { + "row": 176, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0728836059570312e-06, + "max_abs_logit_error": 0.14330339431762695 + }, + { + "row": 177, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2799782567185503e-08, + "max_abs_logit_error": 0.151580810546875 + }, + { + "row": 178, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.14644718170166016 + }, + { + "row": 179, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.3961820602416992 + }, + { + "row": 180, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.22239947319030762 + }, + { + "row": 181, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.13992595672607422 + }, + { + "row": 182, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.16092300415039062 + }, + { + "row": 183, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.15441608428955078 + }, + { + "row": 184, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.15936005115509033 + }, + { + "row": 185, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.173073410987854 + }, + { + "row": 186, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.11217308044433594 + }, + { + "row": 187, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.14353275299072266 + }, + { + "row": 188, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.1971302032470703 + }, + { + "row": 189, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.14509010314941406 + }, + { + "row": 190, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.2037668228149414 + }, + { + "row": 191, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.12031292915344238 + }, + { + "row": 192, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.18417954444885254 + }, + { + "row": 193, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.17492496967315674 + }, + { + "row": 194, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.09895849227905273 + }, + { + "row": 195, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.22567105293273926 + } + ] + } + }, + "passed": false, + "code_sha256": { + "ane_gather.py": "c3379a3797041dcfed82eaaea3990771ec91f1dff7ec146b025988ad7a8c9aae", + "assets.py": "f7a5bc931075237e00c73bf21fee3575be35531f934f23b5e22627d50a79f92f", + "benchmark-synthetic.py": "891da8408156e2b80849b4d799c77c9a31c091dfd999fcd044639deefac3e888", + "compare-ane.py": "4633df7ad5f90d6aed37d5ef4f7a2d2231fdadd272ef1aa3057c237b2f937df8", + "convert-coreml.py": "46fe0194df924033126286e601214b6789f5d667aae38c153cf4dbc1b0e629b0", + "export-reference.py": "ff9eab4d6e4e72ffaf074d82610f41fbea1e802da7c0e987ce562a89eb561b99", + "export_model.py": "c61da5d678c77030d1e9a78359626063ea31da46fb6434e49517fdeaa7800949", + "preprocessing.py": "18c74ede43f95f91cd638c7e9631923a353e3a24d3b522ddb8b18b480f9c2846", + "profile-coreml.py": "d5e1386f54fc6c482647120e593e7ffbbe4bbb277c2cc6d4949154660fe239fd", + "quantize-int8.py": "22071babfcb02ef77225203a479d9bb14b9fa35462ba38bcc4bf07d29c125a97", + "score-report.py": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e", + "synthetic_test.py": "e2b68725236e6827eb422480c8cc8aa189b2490c1594650d8f2928771d1bc626", + "verify.py": "bc20ec9df47708e9b48e411076abe4eab7c18313f67e11334de2d9e3ecb3efa7" + } +} diff --git a/reports/int8-fallback.json b/reports/int8-fallback.json new file mode 100644 index 0000000000000000000000000000000000000000..3703430924e94a729fc885fb53fb47fedc20bca5 --- /dev/null +++ b/reports/int8-fallback.json @@ -0,0 +1,112 @@ +{ + "hardware": { + "device": "arm64", + "chip": "Apple M5 Pro", + "ram": "24GB", + "os_version": "macOS 27.0", + "timestamp": "2026-09-19T22:05:35.064926+00:00" + }, + "models": [ + { + "model_path": "/Users/hanweng/Documents/FluidAudio/.mobius/cua-s1-prs/mobius/models/computer-use/cua-s1-forms/coreml/build/int8-weights/cua_s1_forms_int8_options32.mlmodelc", + "model_name": "cua_s1_forms_int8_options32", + "fallback": { + "compute_units": "cpu_and_neural_engine", + "total_ops": 200, + "ane_ops": 149, + "gpu_ops": 0, + "cpu_ops": 51, + "ane_percent": 74.5, + "reasons": [ + { + "reason": "Unsupported tensor data type: int32", + "count": 25, + "estimated_cpu_runtime_ms": 0.0375, + "op_types": { + "ios17.cast": 10, + "ios17.slice_by_index": 4, + "ios17.add": 4, + "select": 4, + "ios17.gather": 2, + "ios17.reshape": 1 + }, + "ops": [ + "var_27", + "var_43", + "var_43_to_fp16", + "context_ids_to_int16", + "cast_54", + "add_3", + "select_0", + "select_0_to_int16", + "cast_0", + "add_0", + "select_0_1", + "op_59_cast_fp16_cast_uint16_cast_uint16", + "flat_ids", + "var_333", + "var_349", + "var_349_to_fp16", + "flat_ids_to_int16", + "cast_55", + "add_4", + "select_1", + "select_1_to_int16", + "cast_1", + "add_1", + "select_1_1", + "op_365_cast_fp16_cast_uint16_cast_uint16" + ] + }, + { + "reason": "ANE not available for this op", + "count": 19, + "estimated_cpu_runtime_ms": 0, + "op_types": { + "ios16.constexpr_affine_dequantize": 19 + }, + "ops": [ + "model_embedding_weight_to_fp16_quantized", + "model_position_weight_to_fp16_quantized", + "model_encoder_layers_0_self_attn_in_proj_weight_to_fp16_quantized", + "model_encoder_layers_0_self_attn_out_proj_weight_to_fp16_quantized", + "model_encoder_layers_0_linear1_weight_to_fp16_quantized", + "model_encoder_layers_0_linear2_weight_to_fp16_quantized", + "model_encoder_layers_1_self_attn_in_proj_weight_to_fp16_quantized", + "model_encoder_layers_1_self_attn_out_proj_weight_to_fp16_quantized", + "model_encoder_layers_1_linear1_weight_to_fp16_quantized", + "model_encoder_layers_1_linear2_weight_to_fp16_quantized", + "op_369_to_fp16_quantized", + "op_394_to_fp16_quantized", + "model_option_encoder_layers_0_self_attn_in_proj_weight_to_fp16_quantized", + "model_option_encoder_layers_0_self_attn_out_proj_weight_to_fp16_quantized", + "model_option_encoder_layers_0_linear1_weight_to_fp16_quantized", + "model_option_encoder_layers_0_linear2_weight_to_fp16_quantized", + "model_head_query_weight_to_fp16_quantized", + "model_head_key_weight_to_fp16_quantized", + "model_head_value_weight_to_fp16_quantized" + ] + }, + { + "reason": "Unable to resolve operation input \"y\".", + "count": 7, + "estimated_cpu_runtime_ms": 0.007, + "op_types": { + "ios17.greater_equal": 4, + "ios17.not_equal": 3 + }, + "ops": [ + "option_mask_1", + "context_mask", + "greater_equal_0", + "greater_equal_0_1", + "flat_mask", + "greater_equal_1", + "greater_equal_1_1" + ] + } + ] + } + } + ] +} diff --git a/reports/int8-profile.json b/reports/int8-profile.json new file mode 100644 index 0000000000000000000000000000000000000000..169e49135a7c4653c40f8ef006ecb843d7463e8f --- /dev/null +++ b/reports/int8-profile.json @@ -0,0 +1,1105 @@ +{ + "created_utc": "2026-09-19T22:05:06.621713+00:00", + "purpose": "Bounded real-input latency and scheduler placement; not a general accuracy benchmark", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "a7d9747308b8b754a46f3e70c8465b821f90e12376b3a71dc909f7364ae73c3a", + "Data/com.apple.CoreML/weights/weight.bin": "f600c4434b7f7e3e789de8f1a4aa54a55f84df51fa9b788e18f5c97b68e4840f", + "Manifest.json": "2206ec01787a85c12f1376fa5a832c5ac912f06172c6ff06b26ae8415470d246" + }, + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "environment": { + "chip": "Apple M5 Pro", + "memory_bytes": 25769803776, + "macos": "27.0", + "os_build": "26A428", + "python": "3.11.11", + "coremltools": "9.0" + }, + "protocol": { + "rows": [ + 0, + 68, + 130 + ], + "pages": [ + "patient-registration", + "job-application", + "auto-claim" + ], + "option_counts": [ + 27, + 21, + 19 + ], + "first_call_row": 0, + "warmup_passes": 2, + "timed_passes": 10, + "backend_order": [ + "CPU_AND_NE" + ], + "timing_scope": "Synchronous Python MLModel.predict; pre-encoded inputs; output checks outside timer", + "load_scope": "MLModel construction; system caches retained, not first-install cold start", + "first_call_scope": "First predict after this load; may use existing system/ANE caches", + "plan_order": "Compute plan loaded after timing each backend" + }, + "backends": { + "CPU_AND_NE": { + "load_ms": 634.8958330054302, + "first_prediction_ms": 3.1584999815095216, + "warm_prediction": { + "count": 30, + "median_ms": 1.0110830189660192, + "p95_ms": 1.124097959836945, + "min_ms": 0.9326659783255309, + "max_ms": 1.1762090143747628 + }, + "timed_predictions_correct": 30, + "calls": [ + { + "row": 0, + "milliseconds": 1.089458994101733 + }, + { + "row": 68, + "milliseconds": 1.0625830036588013 + }, + { + "row": 130, + "milliseconds": 1.1127080069854856 + }, + { + "row": 0, + "milliseconds": 1.0287080076523125 + }, + { + "row": 68, + "milliseconds": 0.9326659783255309 + }, + { + "row": 130, + "milliseconds": 0.9901669982355088 + }, + { + "row": 0, + "milliseconds": 1.0140000085812062 + }, + { + "row": 68, + "milliseconds": 1.0877909953705966 + }, + { + "row": 130, + "milliseconds": 1.0328329808544368 + }, + { + "row": 0, + "milliseconds": 1.047459023538977 + }, + { + "row": 68, + "milliseconds": 1.0382909968029708 + }, + { + "row": 130, + "milliseconds": 1.1334170121699572 + }, + { + "row": 0, + "milliseconds": 1.1762090143747628 + }, + { + "row": 68, + "milliseconds": 1.0909580159932375 + }, + { + "row": 130, + "milliseconds": 0.9953750122804195 + }, + { + "row": 0, + "milliseconds": 1.0375829879194498 + }, + { + "row": 68, + "milliseconds": 1.0102080123033375 + }, + { + "row": 130, + "milliseconds": 0.9982500050682575 + }, + { + "row": 0, + "milliseconds": 1.0159579978790134 + }, + { + "row": 68, + "milliseconds": 0.9699999936856329 + }, + { + "row": 130, + "milliseconds": 1.0119580256287009 + }, + { + "row": 0, + "milliseconds": 0.9712500032037497 + }, + { + "row": 68, + "milliseconds": 0.980541022727266 + }, + { + "row": 130, + "milliseconds": 0.9917500137817115 + }, + { + "row": 0, + "milliseconds": 0.9814579971134663 + }, + { + "row": 68, + "milliseconds": 0.942542013945058 + }, + { + "row": 130, + "milliseconds": 0.93891701544635 + }, + { + "row": 0, + "milliseconds": 0.9628330008126795 + }, + { + "row": 68, + "milliseconds": 0.9442089940421283 + }, + { + "row": 130, + "milliseconds": 0.9703329997137189 + } + ], + "compute_plan": { + "kind": "MLComputePlan preferred device; not a runtime utilization or power trace", + "assigned_operation_count": 181, + "device_counts": { + "cpu": 32, + "gpu": 0, + "ane": 149, + "unknown": 0 + }, + "unassigned_operation_types": { + "const": 226, + "ios16.constexpr_affine_dequantize": 19 + }, + "operations": [ + { + "name": "option_mask_1", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "context_mask", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "var_27", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_0_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_43", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_43_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "safe_context_ids_cast_fp16", + "type": "ios17.concat", + "device": "ane" + }, + { + "name": "context_ids_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "cast_54", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "greater_equal_0", + "type": "ios17.greater_equal", + "device": "cpu" + }, + { + "name": "add_3", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "select_0", + "type": "select", + "device": "cpu" + }, + { + "name": "select_0_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "cast_0", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "greater_equal_0_1", + "type": "ios17.greater_equal", + "device": "cpu" + }, + { + "name": "add_0", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "select_0_1", + "type": "select", + "device": "cpu" + }, + { + "name": "op_59_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "cpu" + }, + { + "name": "src_1_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "mask_1_cast_fp16", + "type": "ios17.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_1_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_1_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_0_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_124_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_125_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_126_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_127_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_1_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_136_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_143_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_150_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_154_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_156_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_5_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_0_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_0_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_0_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_1_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_172_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_3_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_1_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_7_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_3_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_5_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_7_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_2_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_11_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_3_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_17_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "query_5_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_7_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_4_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_231_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_232_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_233_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_234_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_7_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_243_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_250_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_257_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "q_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_9_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_3_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_1_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_1_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_1_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_9_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_279_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_11_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_5_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_15_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_19_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_21_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_23_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_6_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_27_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_7_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "context_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "flat_ids", + "type": "ios17.reshape", + "device": "cpu" + }, + { + "name": "flat_mask", + "type": "ios17.not_equal", + "device": "cpu" + }, + { + "name": "var_333", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "fill_like_1_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "var_349", + "type": "ios17.slice_by_index", + "device": "cpu" + }, + { + "name": "var_349_to_fp16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "safe_ids_cast_fp16", + "type": "ios17.concat", + "device": "ane" + }, + { + "name": "flat_ids_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "cast_55", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "greater_equal_1", + "type": "ios17.greater_equal", + "device": "cpu" + }, + { + "name": "add_4", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "select_1", + "type": "select", + "device": "cpu" + }, + { + "name": "select_1_to_int16", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "cast_1", + "type": "ios17.cast", + "device": "cpu" + }, + { + "name": "greater_equal_1_1", + "type": "ios17.greater_equal", + "device": "cpu" + }, + { + "name": "add_1", + "type": "ios17.add", + "device": "cpu" + }, + { + "name": "select_1_1", + "type": "select", + "device": "cpu" + }, + { + "name": "op_365_cast_fp16_cast_uint16_cast_uint16", + "type": "ios17.gather", + "device": "cpu" + }, + { + "name": "src_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "mask_cast_fp16", + "type": "ios17.equal", + "device": "ane" + }, + { + "name": "key_padding_mask_7_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "query_9_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "query_11_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "linear_8_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "var_428_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_429_cast_fp16", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_430_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "var_431_cast_fp16", + "type": "ios17.squeeze", + "device": "ane" + }, + { + "name": "q_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "k_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "v_13_cast_fp16", + "type": "ios17.slice_by_index", + "device": "ane" + }, + { + "name": "var_440_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_447_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_454_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_458_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "var_460_cast_fp16", + "type": "tile", + "device": "ane" + }, + { + "name": "q_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "q_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "k_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "k_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "v_15_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "v_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "mul_5_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "matmul_2_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "add_2_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "softmax_2_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "attn_output_17_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_476_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "attn_output_19_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "linear_9_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "attn_output_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_35_cast_fp16", + "type": "ios17.transpose", + "device": "ane" + }, + { + "name": "input_37_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "input_39_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_10_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "input_43_cast_fp16", + "type": "ios16.relu", + "device": "ane" + }, + { + "name": "linear_11_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "hidden_cast_fp16", + "type": "ios17.add", + "device": "ane" + }, + { + "name": "var_504", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_504_to_fp16", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "var_510_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_515_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_520_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "var_522_cast_fp16", + "type": "ios17.maximum", + "device": "ane" + }, + { + "name": "pooled_cast_fp16", + "type": "ios17.real_div", + "device": "ane" + }, + { + "name": "options_cast_fp16", + "type": "ios17.reshape", + "device": "ane" + }, + { + "name": "input_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "input_53_cast_fp16", + "type": "ios17.layer_norm", + "device": "ane" + }, + { + "name": "linear_12_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "linear_13_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "linear_14_cast_fp16", + "type": "ios17.linear", + "device": "ane" + }, + { + "name": "matmul_3_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "_inversed_scores_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_567", + "type": "ios17.expand_dims", + "device": "ane" + }, + { + "name": "var_569", + "type": "ios17.logical_not", + "device": "ane" + }, + { + "name": "scores_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_571_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "matmul_4_cast_fp16", + "type": "ios17.matmul", + "device": "ane" + }, + { + "name": "var_574_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_576_cast_fp16", + "type": "ios16.reduce_sum", + "device": "ane" + }, + { + "name": "_inversed_logits_1_cast_fp16", + "type": "ios17.mul", + "device": "ane" + }, + { + "name": "var_579", + "type": "ios17.logical_not", + "device": "ane" + }, + { + "name": "logits_3_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_587_cast_fp16", + "type": "ios16.fill_like", + "device": "ane" + }, + { + "name": "logits_cast_fp16", + "type": "select", + "device": "ane" + }, + { + "name": "var_591_cast_fp16", + "type": "ios16.softmax", + "device": "ane" + }, + { + "name": "logits", + "type": "ios17.cast", + "device": "ane" + }, + { + "name": "probabilities", + "type": "ios17.cast", + "device": "ane" + } + ] + } + } + }, + "script_sha256": "d5e1386f54fc6c482647120e593e7ffbbe4bbb277c2cc6d4949154660fe239fd" +} diff --git a/reports/int8-synthetic-test-decisions.jsonl.gz b/reports/int8-synthetic-test-decisions.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..6d901128bebf0bed58e54bd70d5c09a9380211e8 --- /dev/null +++ b/reports/int8-synthetic-test-decisions.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69720b9e11430ecff7ea0718f14bf16792de6f39f114faba250cfe855e4e4156 +size 2307507 diff --git a/reports/int8-synthetic-test.json b/reports/int8-synthetic-test.json new file mode 100644 index 0000000000000000000000000000000000000000..a4a6b36df25367af292e54063e74db10b19e5337 --- /dev/null +++ b/reports/int8-synthetic-test.json @@ -0,0 +1,1473 @@ +{ + "purpose": "Full published synthetic test; no fitting, filtering, resampling or test-based tuning", + "created_utc": "2026-09-19T22:07:40.987091+00:00", + "dataset": { + "dataset_repository": "cua-ai/cua-s1-forms", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "path": "artifacts/test.jsonl", + "url": "https://huggingface.co/datasets/cua-ai/cua-s1-forms/resolve/8273f34778b99ac2e12d9f6e7d57dad99ae20845/test.jsonl", + "sha256": "d63a7e0db195d4d20154a40b2f8dd09ce3bb65487a158c638da5c609d4475e7c", + "rows": 24370, + "bytes": 18710924, + "upstream_split_claim": "Synthetic test, form-signature-disjoint from train/validation. Train and validation signatures have not been independently re-audited here.", + "model_card_claim": { + "synthetic_test_top1": 0.9995, + "approximate_decisions": 15000, + "note": "The released file has 24370 rows. No matching full result ledger or exact 15000-row manifest is supplied by the linked model card." + } + }, + "manifest_sha256": "e582b951ecc07809a0b06a2bc0790827b90a86ce57c70d5b6177ef04f0b4f1ca", + "input_census": { + "rows": 24370, + "unique_episode_seeds": 1040, + "maxima": { + "context_bytes": 217, + "option_bytes": 92, + "max_options": 32 + }, + "per_action": { + "check": 816, + "click": 1040, + "fill": 9802, + "skip": 12712 + }, + "option_count_histogram": { + "7": 17, + "8": 46, + "9": 80, + "10": 227, + "11": 361, + "12": 589, + "13": 888, + "14": 1254, + "15": 1513, + "16": 1761, + "17": 2373, + "18": 2014, + "19": 2277, + "20": 2506, + "21": 2396, + "22": 1818, + "23": 1422, + "24": 907, + "25": 688, + "26": 524, + "27": 483, + "28": 89, + "29": 33, + "30": 29, + "31": 35, + "32": 40 + }, + "excluded_rows": 0, + "truncated_contexts": 0, + "truncated_options": 0 + }, + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "conversions": { + "baseline": { + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + } + }, + "int8-weights": { + "model": "cua_s1_forms_int8_options32.mlpackage", + "precision": "int8_weights_float16_compute", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "a7d9747308b8b754a46f3e70c8465b821f90e12376b3a71dc909f7364ae73c3a", + "Data/com.apple.CoreML/weights/weight.bin": "f600c4434b7f7e3e789de8f1a4aa54a55f84df51fa9b788e18f5c97b68e4840f", + "Manifest.json": "2206ec01787a85c12f1376fa5a832c5ac912f06172c6ff06b26ae8415470d246" + }, + "quantization": { + "name": "int8-weights", + "settings": { + "mode": "linear_symmetric", + "dtype": "int8", + "granularity": "per_channel", + "weight_threshold": 2048 + }, + "activations": "float16", + "calibration": "None; weight-only, data-free quantization", + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "source_package_bytes": 1511163, + "package_bytes": 813422, + "compression_seconds": 0.3180606249952689, + "operation_counts": { + "add": 15, + "cast": 13, + "concat": 2, + "const": 226, + "constexpr_affine_dequantize": 19, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 4, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 9, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "22071babfcb02ef77225203a479d9bb14b9fa35462ba38bcc4bf07d29c125a97" + } + } + }, + "environment": { + "machine": "Apple M5 Pro", + "memory_bytes": 25769803776, + "os": "27.0", + "os_build": "26A428", + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "torch_cpu_threads": 2, + "torch_interop_threads": 1, + "torch_mha_fastpath": false + }, + "protocol": { + "batch_size": 1, + "warmups_per_model": 3, + "timed_passes": 1, + "coreml_compute_units": "CPU_AND_NE", + "coreml_order": "AB on even rows, BA on odd rows", + "timing": "PyTorch CPU forward+softmax / synchronous Core ML predict; excludes encoding, validation, GUI and network", + "preprocessing": "Upstream variable-length collator; independent fixed-shape Core ML encoder shared by both exports", + "load_note": "Process/model creation can use existing system caches; not a cold-start experiment", + "output_audit": "Imperfect finite probability sums are retained unchanged and fail the original normalization gate", + "calibration_note": "NLL/ECE use raw emitted scores; no renormalization or calibration was applied" + }, + "thresholds": { + "argmax_agreement": 1.0, + "max_abs_probability_error": 0.005, + "allow_accuracy_loss": false, + "probability_sum_atol": 0.001, + "probability_sum_rtol": 1e-05 + }, + "host_preprocessing_ms": { + "upstream_collator_ms": { + "count": 24370, + "median_ms": 0.09233300806954503, + "p95_ms": 0.12889755889773366, + "min_ms": 0.04445898230187595, + "max_ms": 1.0104170069098473 + }, + "shared_coreml_encoding_ms": { + "count": 24370, + "median_ms": 0.027417001547291875, + "p95_ms": 0.03995798761025071, + "min_ms": 0.012666976545006037, + "max_ms": 0.49662499804981053 + } + }, + "results": { + "upstream_pytorch": { + "correct": 24359, + "rows": 24370, + "accuracy": 0.999548625359048, + "model_call_latency_ms": { + "count": 24370, + "median_ms": 1.7357920005451888, + "p95_ms": 2.984285591810476, + "min_ms": 1.0271249921061099, + "max_ms": 30.681957985507324 + }, + "load_ms": 10.63050000811927, + "first_call_ms": 5.652791995089501, + "calibration": { + "nll": 0.0021302032038990836, + "ece": 0.00035450826133425167, + "equal_width_bins": 15, + "bins": [ + { + "bin": 7, + "count": 2, + "accuracy": 0.5, + "mean_confidence": 0.5216051042079926 + }, + { + "bin": 8, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.575071656703949 + }, + { + "bin": 9, + "count": 7, + "accuracy": 0.7142857142857143, + "mean_confidence": 0.6404258779117039 + }, + { + "bin": 10, + "count": 4, + "accuracy": 1.0, + "mean_confidence": 0.6963506639003754 + }, + { + "bin": 11, + "count": 11, + "accuracy": 0.9090909090909091, + "mean_confidence": 0.7664791941642761 + }, + { + "bin": 12, + "count": 21, + "accuracy": 0.9523809523809523, + "mean_confidence": 0.8310647464933849 + }, + { + "bin": 13, + "count": 22, + "accuracy": 0.9545454545454546, + "mean_confidence": 0.9173552746122534 + }, + { + "bin": 14, + "count": 24298, + "accuracy": 0.9998353773973166, + "mean_confidence": 0.999802232897422 + } + ], + "nll_probability_floor": 1.1754943508222875e-38 + }, + "upstream_action_metrics": { + "examples": 24370, + "accuracy": 0.999548625359048, + "coverage": 0.4788264259335248, + "abstention_rate": 0.5211735740664751, + "selective_accuracy": 0.9990573313908647, + "wrong_action_rate": 0.00045137464095199015, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.00045137464095199015, + "counts": { + "total": 24370, + "correct": 24359, + "abstained": 12701, + "acted": 11669, + "acted_correct": 11658, + "wrong_action": 11, + "unsafe_action": 11 + }, + "per_action": { + "abstain": { + "accuracy": 0.9991346758967904, + "abstention_rate": 0.9991346758967904, + "examples": 12712 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 816 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 1040 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 9802 + } + } + }, + "macro_action_accuracy": 0.9997836689741976, + "wrong_rows": [ + 137, + 3216, + 6631, + 6748, + 7280, + 10703, + 12255, + 13480, + 15214, + 21022, + 23671 + ] + }, + "baseline": { + "correct": 24359, + "rows": 24370, + "accuracy": 0.999548625359048, + "model_call_latency_ms": { + "count": 24370, + "median_ms": 0.9897080017253757, + "p95_ms": 1.1015420022886246, + "min_ms": 0.884292006958276, + "max_ms": 3.03837499814108 + }, + "load_ms": 640.2017919754144, + "first_call_ms": 2.522958006011322, + "calibration": { + "nll": 0.002108235388127147, + "ece": 0.000331678613561794, + "equal_width_bins": 15, + "bins": [ + { + "bin": 7, + "count": 2, + "accuracy": 0.5, + "mean_confidence": 0.5224609375 + }, + { + "bin": 8, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.57607421875 + }, + { + "bin": 9, + "count": 6, + "accuracy": 0.8333333333333334, + "mean_confidence": 0.637451171875 + }, + { + "bin": 10, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.69248046875 + }, + { + "bin": 11, + "count": 12, + "accuracy": 0.9166666666666666, + "mean_confidence": 0.7687581380208334 + }, + { + "bin": 12, + "count": 20, + "accuracy": 0.95, + "mean_confidence": 0.8322265625 + }, + { + "bin": 13, + "count": 22, + "accuracy": 0.9545454545454546, + "mean_confidence": 0.9176136363636364 + }, + { + "bin": 14, + "count": 24298, + "accuracy": 0.9998353773973166, + "mean_confidence": 0.9998245660008025 + } + ], + "nll_probability_floor": 1.1754943508222875e-38 + }, + "upstream_action_metrics": { + "examples": 24370, + "accuracy": 0.999548625359048, + "coverage": 0.4788264259335248, + "abstention_rate": 0.5211735740664751, + "selective_accuracy": 0.9990573313908647, + "wrong_action_rate": 0.00045137464095199015, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.00045137464095199015, + "counts": { + "total": 24370, + "correct": 24359, + "abstained": 12701, + "acted": 11669, + "acted_correct": 11658, + "wrong_action": 11, + "unsafe_action": 11 + }, + "per_action": { + "abstain": { + "accuracy": 0.9991346758967904, + "abstention_rate": 0.9991346758967904, + "examples": 12712 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 816 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 1040 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 9802 + } + } + }, + "macro_action_accuracy": 0.9997836689741976, + "wrong_rows": [ + 137, + 3216, + 6631, + 6748, + 7280, + 10703, + 12255, + 13480, + 15214, + 21022, + 23671 + ], + "paired_with_upstream": { + "rows": 24370, + "argmax_agreement": 24370, + "agreement_rate": 1.0, + "reference_correct_candidate_wrong": 0, + "reference_wrong_candidate_correct": 0, + "both_wrong": 11, + "disagreement_rows": [] + }, + "max_abs_probability_error": 0.020487427711486816, + "mean_row_max_abs_probability_error": 4.673297645567538e-05, + "probability_tolerance_violation_rows": [ + 137, + 884, + 2945, + 3216, + 8706, + 9193, + 12725, + 13480, + 13768, + 15698, + 23992 + ], + "conversion_parity_passed": false, + "output_validation_failures": { + "19270": [ + "Live option probabilities do not sum to one" + ] + } + }, + "int8-weights": { + "correct": 24359, + "rows": 24370, + "accuracy": 0.999548625359048, + "model_call_latency_ms": { + "count": 24370, + "median_ms": 0.9902919991873205, + "p95_ms": 1.103792006324511, + "min_ms": 0.8826660050544888, + "max_ms": 49.30691700428724 + }, + "load_ms": 615.115666994825, + "first_call_ms": 1.328666985500604, + "calibration": { + "nll": 0.002145651629876076, + "ece": 0.00048729947809817673, + "equal_width_bins": 15, + "bins": [ + { + "bin": 7, + "count": 4, + "accuracy": 0.75, + "mean_confidence": 0.520751953125 + }, + { + "bin": 8, + "count": 4, + "accuracy": 1.0, + "mean_confidence": 0.576904296875 + }, + { + "bin": 9, + "count": 6, + "accuracy": 0.8333333333333334, + "mean_confidence": 0.6383463541666666 + }, + { + "bin": 10, + "count": 3, + "accuracy": 0.3333333333333333, + "mean_confidence": 0.7058919270833334 + }, + { + "bin": 11, + "count": 15, + "accuracy": 0.9333333333333333, + "mean_confidence": 0.7711263020833333 + }, + { + "bin": 12, + "count": 15, + "accuracy": 0.9333333333333333, + "mean_confidence": 0.8310546875 + }, + { + "bin": 13, + "count": 26, + "accuracy": 1.0, + "mean_confidence": 0.9116023137019231 + }, + { + "bin": 14, + "count": 24297, + "accuracy": 0.9997942132773593, + "mean_confidence": 0.9998235539624234 + } + ], + "nll_probability_floor": 1.1754943508222875e-38 + }, + "upstream_action_metrics": { + "examples": 24370, + "accuracy": 0.999548625359048, + "coverage": 0.4788264259335248, + "abstention_rate": 0.5211735740664751, + "selective_accuracy": 0.9990573313908647, + "wrong_action_rate": 0.00045137464095199015, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.00045137464095199015, + "counts": { + "total": 24370, + "correct": 24359, + "abstained": 12701, + "acted": 11669, + "acted_correct": 11658, + "wrong_action": 11, + "unsafe_action": 11 + }, + "per_action": { + "abstain": { + "accuracy": 0.9991346758967904, + "abstention_rate": 0.9991346758967904, + "examples": 12712 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 816 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 1040 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 9802 + } + } + }, + "macro_action_accuracy": 0.9997836689741976, + "wrong_rows": [ + 137, + 3216, + 6631, + 6748, + 7280, + 10703, + 12255, + 13480, + 15214, + 21022, + 23671 + ], + "paired_with_upstream": { + "rows": 24370, + "argmax_agreement": 24370, + "agreement_rate": 1.0, + "reference_correct_candidate_wrong": 0, + "reference_wrong_candidate_correct": 0, + "both_wrong": 11, + "disagreement_rows": [] + }, + "max_abs_probability_error": 0.0677376389503479, + "mean_row_max_abs_probability_error": 8.935863356371478e-05, + "probability_tolerance_violation_rows": [ + 137, + 410, + 1225, + 2076, + 2093, + 2818, + 2945, + 3064, + 3151, + 3216, + 3298, + 3479, + 3756, + 3978, + 4469, + 5294, + 5759, + 6426, + 6466, + 7177, + 7515, + 8243, + 8706, + 8714, + 8891, + 9193, + 9253, + 9977, + 9981, + 10703, + 11122, + 11257, + 11880, + 12725, + 12754, + 13370, + 13423, + 13693, + 13768, + 14656, + 15132, + 15214, + 15252, + 15698, + 16253, + 16920, + 16951, + 17304, + 18339, + 18549, + 18802, + 19308, + 19669, + 19957, + 20462, + 20476, + 21844, + 21871, + 22336, + 22714, + 22791, + 23276, + 23670, + 23992 + ], + "conversion_parity_passed": false, + "output_validation_failures": {} + } + }, + "failures": [ + { + "row": 137, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"Full name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Last: Novak", + "baseline": "fill Last: Novak", + "int8-weights": "fill Last: Novak" + } + }, + { + "row": 410, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Date of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 884, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Health plan\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 1225, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Vehicle identification number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 2076, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Graduation year\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 2093, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Title\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 2818, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Your email\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 2945, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Organization\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 3064, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Make\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 3151, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Google Chrome\nELEMENT Edit \"Date of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 3216, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Google Chrome\nELEMENT Edit \"Email\" value=\"oivanova66@yahoo.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Email address: oivanova66@yahoo.com", + "baseline": "fill Email address: oivanova66@yahoo.com", + "int8-weights": "fill Email address: oivanova66@yahoo.com" + } + }, + { + "row": 3298, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Mozilla Firefox\nELEMENT Edit \"DL number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 3479, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Insurance company\" value=\"Centene\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 3756, + "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Claim #\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 3978, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - Microsoft Edge\nELEMENT Edit \"Family doctor\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 4469, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"City\" value=\"Portland\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 5294, + "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Email\" value=\"nadia.fischer@fastmail.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 5759, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form\nELEMENT Edit \"Relationship to patient\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 6426, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Microsoft Edge\nELEMENT CheckBox \"Consent to electronic communication (required)\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int8-weights": "check" + } + }, + { + "row": 6466, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - JevBrowser\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 6631, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Google Chrome\nELEMENT Edit \"Given name\" value=\"Priya\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Physician: Dr. Fischer", + "baseline": "fill Physician: Dr. Fischer", + "int8-weights": "fill Physician: Dr. Fischer" + } + }, + { + "row": 6748, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"State / Province\" value=\"NE\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill ST: NE", + "baseline": "fill ST: NE", + "int8-weights": "fill ST: NE" + } + }, + { + "row": 7177, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Claim #\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 7280, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Given name\" value=\"Tomas\" hint=\"First\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill First name: Tomas", + "baseline": "fill First name: Tomas", + "int8-weights": "fill First name: Tomas" + } + }, + { + "row": 7515, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Province\" value=\"GA\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 8243, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Passport number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 8706, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Mozilla Firefox\nELEMENT CheckBox \"I authorize verification of the information provided\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int8-weights": "check" + } + }, + { + "row": 8714, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Registration number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 8891, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Google Chrome\nELEMENT Edit \"Last\" value=\"Ivanova\" hint=\"Family name\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 9193, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding\nELEMENT Edit \"DL number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 9253, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Homepage\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 9977, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Country\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 9981, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"City\" value=\"Burlington\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 10703, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - JevBrowser\nELEMENT Edit \"Last\" value=\"Silva\" hint=\"Family name\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Family name: Silva", + "baseline": "fill Family name: Silva", + "int8-weights": "fill Family name: Silva" + } + }, + { + "row": 11122, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"EIN\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 11257, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Microsoft Edge\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "int8-weights": "check" + } + }, + { + "row": 11880, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Google Chrome\nELEMENT Edit \"Date of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 12255, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form - JevBrowser\nELEMENT Edit \"Taxpayer identification number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill National ID: 813-20-3361", + "baseline": "fill National ID: 813-20-3361", + "int8-weights": "fill National ID: 813-20-3361" + } + }, + { + "row": 12725, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Company\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 12754, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT CheckBox \"I consent to treatment and agree to the privacy policy\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 13370, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Microsoft Edge\nELEMENT Edit \"Mailing address\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 13423, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Current job title\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 13480, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Current employer\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Company reg: REG3878384", + "baseline": "fill Company reg: REG3878384", + "int8-weights": "fill Company reg: REG3878384" + } + }, + { + "row": 13693, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Mozilla Firefox\nELEMENT Edit \"Organization\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 13768, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Legal name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 14656, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Google Chrome\nELEMENT Edit \"Details of loss\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 15132, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Mozilla Firefox\nELEMENT Edit \"Town\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 15214, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Relation to applicant\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Reg plate: WN9-8147", + "baseline": "fill Reg plate: WN9-8147", + "int8-weights": "fill Reg plate: WN9-8147" + } + }, + { + "row": 15252, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"VIN number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 15698, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer - Mozilla Firefox\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 16253, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Relation to applicant\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 16920, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application - Google Chrome\nELEMENT Edit \"Total years experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 16951, + "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"State\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 17304, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - Google Chrome\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 18339, + "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Mozilla Firefox\nELEMENT Edit \"Last name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 18549, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"ABA routing number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 18802, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Unit number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 19270, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Plate number\" value=\"old\"", + "gold_option": "fill License plate: SJ3-1115", + "selected_options": { + "upstream_pytorch": "fill License plate: SJ3-1115", + "baseline": "fill License plate: SJ3-1115", + "int8-weights": "fill License plate: SJ3-1115" + } + }, + { + "row": 19308, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Google Chrome\nELEMENT Edit \"Last\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 19669, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"School\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 19957, + "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Plate number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 20462, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Mozilla Firefox\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 20476, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Sort code\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 21022, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - JevBrowser\nELEMENT Edit \"State\" value=\"NE\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill National ID: 794-03-1212", + "baseline": "fill National ID: 794-03-1212", + "int8-weights": "fill National ID: 794-03-1212" + } + }, + { + "row": 21844, + "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - Mozilla Firefox\nELEMENT Edit \"Gender\" value=\"Female\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 21871, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Sort code\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 22336, + "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Car model\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 22714, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Describe your concern\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 22791, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake - Mozilla Firefox\nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 23276, + "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form - JevBrowser\nELEMENT Edit \"Relationship status\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 23670, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Member number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + }, + { + "row": 23671, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Email\" value=\"lchen6@yahoo.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Email: lchen6@yahoo.com", + "baseline": "fill Email: lchen6@yahoo.com", + "int8-weights": "fill Email: lchen6@yahoo.com" + } + }, + { + "row": 23992, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Microsoft Edge\nELEMENT Edit \"Total years experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "int8-weights": "skip" + } + } + ], + "trace": { + "file": "int8-synthetic-test-decisions.jsonl.gz", + "sha256": "69720b9e11430ecff7ea0718f14bf16792de6f39f114faba250cfe855e4e4156", + "rows": 24370 + }, + "upstream_evaluator_sha256": "49dccdeb4e2b456187cdc4491a6d78fabac6a45384272f8e3ec802a58872aa6e", + "harness_sha256": { + "benchmark-synthetic.py": "891da8408156e2b80849b4d799c77c9a31c091dfd999fcd044639deefac3e888", + "synthetic_test.py": "e2b68725236e6827eb422480c8cc8aa189b2490c1594650d8f2928771d1bc626", + "score-report.py": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e" + }, + "conversion_parity_passed": false, + "limitations": [ + "The release file has 24370 rows; the model card claims approximately 15000. This is not a reconstruction of that unspecified manifest.", + "Upstream describes form-disjoint synthetic data; this is not an unseen real-world GUI or task-completion benchmark.", + "No fresh hosted Jev calls are included. Published hosted-Jev percentages are not a matched baseline for this run.", + "Local timings use CPU for PyTorch and CPU+ANE for Core ML; they are backend/deployment measurements, not equal-hardware algorithm comparisons." + ] +} diff --git a/reports/swift-ane-validation.json b/reports/swift-ane-validation.json new file mode 100644 index 0000000000000000000000000000000000000000..fa524ccd57fc21b3350ec0b9683e7d14e7f9b93d --- /dev/null +++ b/reports/swift-ane-validation.json @@ -0,0 +1,12 @@ +{ + "byte_encoding_limits_and_rejection": "passed", + "concurrent_reversed_rows": 6, + "correct": 196, + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "maximum_probability_error": 0.00233614444732666, + "model_path": "ane-gather/cua_s1_forms_fp16_options32.mlpackage", + "os": "Version 27.0 (Build 26A428)", + "purpose": "Standalone Swift validation against the real upstream demo; XCTest unavailable locally", + "rows": 196, + "shared_compiled_cache": "passed" +} diff --git a/reports/swift-runtime/decisions.jsonl.gz b/reports/swift-runtime/decisions.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..5b6c0b1c4e6aa0c78fcc409defee88e8339733f7 --- /dev/null +++ b/reports/swift-runtime/decisions.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d7ae5df86489b38e8f97167ab58c1d6bf5fb74c171a57b00d818b05d6c772b5 +size 13479628 diff --git a/reports/swift-runtime/report.json b/reports/swift-runtime/report.json new file mode 100644 index 0000000000000000000000000000000000000000..71d958770597b0c33ce48a2a07ac907ea6bbe52e --- /dev/null +++ b/reports/swift-runtime/report.json @@ -0,0 +1,340 @@ +{ + "purpose": "Stable Swift softmax validation; no model changes, training or new timing claims", + "dataset": { + "dataset_repository": "cua-ai/cua-s1-forms", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "path": "artifacts/test.jsonl", + "url": "https://huggingface.co/datasets/cua-ai/cua-s1-forms/resolve/8273f34778b99ac2e12d9f6e7d57dad99ae20845/test.jsonl", + "sha256": "d63a7e0db195d4d20154a40b2f8dd09ce3bb65487a158c638da5c609d4475e7c", + "rows": 24370, + "bytes": 18710924, + "upstream_split_claim": "Synthetic test, form-signature-disjoint from train/validation. Train and validation signatures have not been independently re-audited here.", + "model_card_claim": { + "synthetic_test_top1": 0.9995, + "approximate_decisions": 15000, + "note": "The released file has 24370 rows. No matching full result ledger or exact 15000-row manifest is supplied by the linked model card." + } + }, + "conversions": { + "fp16": { + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + } + }, + "int8": { + "model": "cua_s1_forms_int8_options32.mlpackage", + "precision": "int8_weights_float16_compute", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "a7d9747308b8b754a46f3e70c8465b821f90e12376b3a71dc909f7364ae73c3a", + "Data/com.apple.CoreML/weights/weight.bin": "f600c4434b7f7e3e789de8f1a4aa54a55f84df51fa9b788e18f5c97b68e4840f", + "Manifest.json": "2206ec01787a85c12f1376fa5a832c5ac912f06172c6ff06b26ae8415470d246" + }, + "quantization": { + "name": "int8-weights", + "settings": { + "mode": "linear_symmetric", + "dtype": "int8", + "granularity": "per_channel", + "weight_threshold": 2048 + }, + "activations": "float16", + "calibration": "None; weight-only, data-free quantization", + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "source_package_bytes": 1511163, + "package_bytes": 813422, + "compression_seconds": 0.3180606249952689, + "operation_counts": { + "add": 15, + "cast": 13, + "concat": 2, + "const": 226, + "constexpr_affine_dequantize": 19, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 4, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 9, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "22071babfcb02ef77225203a479d9bb14b9fa35462ba38bcc4bf07d29c125a97" + } + }, + "int4": { + "model": "cua_s1_forms_int4_options32.mlpackage", + "precision": "int4_weights_float16_compute", + "minimum_target": "iOS18/macOS15", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "cadad6bfa6875383a7ee3de68d0d22871148aa21cf117e7fe44c2c971287c4e5", + "Data/com.apple.CoreML/weights/weight.bin": "0f766fcb9dbd612ed4f6c7a40117a5993a4789b3ae94afba0df37193c831f63c", + "Manifest.json": "fb91c8b5ec3168111567de663bb3140d2790c6e7c48f32e0df3f5135af0673af" + }, + "target_upgrade": { + "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "method": "Load original MIL at specification 9; empty pass pipeline; retain decomposed attention", + "operation_counts": { + "add": 13, + "cast": 11, + "concat": 2, + "const": 239, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 2, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 7, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f" + }, + "quantization": { + "name": "int4-weights", + "settings": { + "mode": "linear_symmetric", + "dtype": "int4", + "granularity": "per_channel", + "weight_threshold": 2048 + }, + "activations": "float16", + "calibration": "None; weight-only, data-free quantization", + "source_conversion_sha256": "01105dc8bef7a30fc3430e150caa2a143dc0ba23a295b6280de3b8ff3b40af55", + "source_package_files": { + "Data/com.apple.CoreML/model.mlmodel": "6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "e5b1c7dc66b2014b3985c180a75548a709788c6e314d9d2ae14e4e6d441c3c68" + }, + "source_package_bytes": 1511080, + "package_bytes": 451595, + "compression_seconds": 0.2996572910051327, + "packed_int4_tensors": 19, + "operation_counts": { + "add": 15, + "cast": 13, + "concat": 2, + "const": 226, + "constexpr_blockwise_shift_scale": 19, + "equal": 2, + "expand_dims": 5, + "fill_like": 3, + "gather": 2, + "greater_equal": 4, + "layer_norm": 8, + "linear": 15, + "logical_not": 2, + "matmul": 8, + "maximum": 1, + "mul": 7, + "not_equal": 3, + "real_div": 1, + "reduce_sum": 3, + "relu": 3, + "reshape": 31, + "select": 9, + "slice_by_index": 13, + "softmax": 5, + "squeeze": 3, + "tile": 2, + "transpose": 21 + }, + "script_sha256": "a91636e539d8934253b5e7628ca706346fbee538fa1aa9e3eaf794f440d49dcc" + } + } + }, + "references": { + "fp16": { + "report_sha256": "2ae85cfeb270b4f31be4f1d9082f4c73b3b9b084d6f1ddb661712a87d46b0f59", + "trace_sha256": "b6f6edc9de34fb37b10847a1f9568b530ce2b8e1bd241f51bf2e637f489d6554" + }, + "int8": { + "report_sha256": "952ebe281d0aef0c05667d3c0dc5f9f175be709639a542028940059393078b5d", + "trace_sha256": "69720b9e11430ecff7ea0718f14bf16792de6f39f114faba250cfe855e4e4156" + }, + "int4": { + "report_sha256": "9d6a691aeb2ddc352ab7c12327472c92814991c201b67c7da8db39a76c972b32", + "trace_sha256": "6922913041f95ac8872958b92312cb5b869b9ce88116c4684b36ea7753284674" + } + }, + "results": { + "fp16": { + "rows": 24370, + "correct": 24359, + "changed_choices": [], + "raw_choice_changes": [], + "raw_sum_failures": [ + 19270 + ], + "runtime_sum_failures": [], + "max_probability_change": 0.0010759234428405762, + "max_softmax_reference_error": 2.9801512702043453e-08, + "accuracy": 0.999548625359048, + "passed": true + }, + "int8": { + "rows": 24370, + "correct": 24359, + "changed_choices": [], + "raw_choice_changes": [], + "raw_sum_failures": [], + "runtime_sum_failures": [], + "max_probability_change": 0.0008352398872375488, + "max_softmax_reference_error": 2.980182234324502e-08, + "accuracy": 0.999548625359048, + "passed": true + }, + "int4": { + "rows": 24370, + "correct": 24353, + "changed_choices": [], + "raw_choice_changes": [], + "raw_sum_failures": [], + "runtime_sum_failures": [], + "max_probability_change": 0.0008838176727294922, + "max_softmax_reference_error": 2.9797999623326632e-08, + "accuracy": 0.9993024210094378, + "passed": true + } + }, + "passed": true, + "swift_source_sha256": { + "CuaS1FormsInput.swift": "b9ad5d74bac35f24155e3250bd08b18b812a308417f5a356bbb77bd8d78cb93c", + "CuaS1FormsManager.swift": "bb852f1f8f4d95eafbe04319e95a9be225052a02898db49913d8e1dd83ad036e", + "CuaS1FormsOutput.swift": "7f6c3225321a8eedb70af2cb8620e1e84b53e46b6af5f3908498e65ff50f8bd8", + "CuaS1FormsPackage.swift": "2a61b156523a42293cc1b077fd18dd064bdbd470b9f0404a093b138127b8276a", + "CuaS1FormsTypes.swift": "3aad906f508a699fb11684aba8a1f40325af33b3846bcd81b5ec32d16d0e1aa6" + }, + "harness_sha256": { + "validate-swift-runtime.py": "c06ca16c8b381a5bb08c9404cf6b6b500cc6102d43470beaa594a2542c9c87ce", + "swift-runtime-check.swift": "0ade2d3450e79adeb3380aad00ea854fd845b69f822aab3a7de16e580542ff59" + }, + "trace": { + "file": "decisions.jsonl.gz", + "sha256": "5d7ae5df86489b38e8f97167ab58c1d6bf5fb74c171a57b00d818b05d6c772b5", + "records": 73110 + }, + "limitations": [ + "Original raw conversion-parity failures remain in the unchanged benchmarks.", + "INT4 still has its previously measured accuracy loss.", + "CPU+ANE on the local Mac; no iPhone or older-OS validation." + ] +} diff --git a/reports/swift-variant-comparison.json b/reports/swift-variant-comparison.json new file mode 100644 index 0000000000000000000000000000000000000000..a527cb19f9d21728f523785f8e1f3ce99c82e6a7 --- /dev/null +++ b/reports/swift-variant-comparison.json @@ -0,0 +1,4143 @@ +{ + "artifacts" : { + "ane-gather" : { + "filesSHA256" : { + "Data/com.apple.CoreML/model.mlmodel" : "de18e313c3b625e35d008ed8b6b24108edc6df7bf2fae9533af03519eca11b63", + "Data/com.apple.CoreML/weights/weight.bin" : "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json" : "38f812a04eb2322080634ba788c61df362336168466ca67e5549058d346ac793" + }, + "firstCallMilliseconds" : 1.552084, + "loadMilliseconds" : 583.114125 + }, + "baseline" : { + "filesSHA256" : { + "Data/com.apple.CoreML/model.mlmodel" : "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin" : "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json" : "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + }, + "firstCallMilliseconds" : 2.291, + "loadMilliseconds" : 600.17475 + } + }, + "build" : "release", + "computeUnits" : "cpuAndNeuralEngine", + "datasetSHA256" : "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "forms" : { + "auto-claim" : { + "ane-gather" : { + "calls" : 68, + "correct" : 68, + "medianMilliseconds" : 0.9610415, + "p95Milliseconds" : 0.9831149 + }, + "baseline" : { + "calls" : 68, + "correct" : 68, + "medianMilliseconds" : 0.9028955, + "p95Milliseconds" : 0.9259375 + } + }, + "job-application" : { + "ane-gather" : { + "calls" : 60, + "correct" : 60, + "medianMilliseconds" : 0.956958, + "p95Milliseconds" : 0.98495355 + }, + "baseline" : { + "calls" : 60, + "correct" : 60, + "medianMilliseconds" : 0.9150005000000001, + "p95Milliseconds" : 0.9271681 + } + }, + "patient-registration" : { + "ane-gather" : { + "calls" : 72, + "correct" : 72, + "medianMilliseconds" : 0.962709, + "p95Milliseconds" : 0.98381075 + }, + "baseline" : { + "calls" : 72, + "correct" : 72, + "medianMilliseconds" : 0.916542, + "p95Milliseconds" : 0.9402545 + } + } + }, + "hardware" : "Apple M5 Pro, 24 GB memory", + "limitations" : "Exploratory local timing on 3 supplied forms; not a held-out generalization benchmark. Load may use system caches.", + "operatingSystem" : "Version 27.0 (Build 26A428)", + "protocolDescription" : "Both resident; one 50-row warmup each; ABBA; two 50-row passes per block", + "rows" : [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146 + ], + "samples" : [ + { + "block" : 0, + "expectedIndex" : 22, + "form" : "patient-registration", + "milliseconds" : 0.968625, + "pass" : 0, + "row" : 0, + "selectedIndex" : 22, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 23, + "form" : "patient-registration", + "milliseconds" : 0.943208, + "pass" : 0, + "row" : 1, + "selectedIndex" : 23, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 2, + "form" : "patient-registration", + "milliseconds" : 0.914625, + "pass" : 0, + "row" : 2, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 5, + "form" : "patient-registration", + "milliseconds" : 0.915875, + "pass" : 0, + "row" : 3, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 4, + "form" : "patient-registration", + "milliseconds" : 0.908208, + "pass" : 0, + "row" : 4, + "selectedIndex" : 4, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 6, + "form" : "patient-registration", + "milliseconds" : 0.914125, + "pass" : 0, + "row" : 5, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 7, + "form" : "patient-registration", + "milliseconds" : 0.901125, + "pass" : 0, + "row" : 6, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 8, + "form" : "patient-registration", + "milliseconds" : 0.916875, + "pass" : 0, + "row" : 7, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 9, + "form" : "patient-registration", + "milliseconds" : 0.930459, + "pass" : 0, + "row" : 8, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 10, + "form" : "patient-registration", + "milliseconds" : 0.929291, + "pass" : 0, + "row" : 9, + "selectedIndex" : 10, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 11, + "form" : "patient-registration", + "milliseconds" : 0.933167, + "pass" : 0, + "row" : 10, + "selectedIndex" : 11, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 13, + "form" : "patient-registration", + "milliseconds" : 0.911083, + "pass" : 0, + "row" : 11, + "selectedIndex" : 13, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 15, + "form" : "patient-registration", + "milliseconds" : 0.916667, + "pass" : 0, + "row" : 12, + "selectedIndex" : 15, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 16, + "form" : "patient-registration", + "milliseconds" : 0.914792, + "pass" : 0, + "row" : 13, + "selectedIndex" : 16, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.915709, + "pass" : 0, + "row" : 14, + "selectedIndex" : 26, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 24, + "form" : "patient-registration", + "milliseconds" : 0.93525, + "pass" : 0, + "row" : 15, + "selectedIndex" : 24, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.919791, + "pass" : 0, + "row" : 16, + "selectedIndex" : 26, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 25, + "form" : "patient-registration", + "milliseconds" : 0.931334, + "pass" : 0, + "row" : 17, + "selectedIndex" : 25, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 0, + "form" : "job-application", + "milliseconds" : 0.912583, + "pass" : 0, + "row" : 68, + "selectedIndex" : 0, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 1, + "form" : "job-application", + "milliseconds" : 0.915209, + "pass" : 0, + "row" : 69, + "selectedIndex" : 1, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 2, + "form" : "job-application", + "milliseconds" : 0.910292, + "pass" : 0, + "row" : 70, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 3, + "form" : "job-application", + "milliseconds" : 0.91775, + "pass" : 0, + "row" : 71, + "selectedIndex" : 3, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 5, + "form" : "job-application", + "milliseconds" : 0.917083, + "pass" : 0, + "row" : 72, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 6, + "form" : "job-application", + "milliseconds" : 0.922833, + "pass" : 0, + "row" : 73, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 7, + "form" : "job-application", + "milliseconds" : 0.938958, + "pass" : 0, + "row" : 74, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 8, + "form" : "job-application", + "milliseconds" : 0.9265, + "pass" : 0, + "row" : 75, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 9, + "form" : "job-application", + "milliseconds" : 0.92825, + "pass" : 0, + "row" : 76, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 13, + "form" : "job-application", + "milliseconds" : 0.92425, + "pass" : 0, + "row" : 77, + "selectedIndex" : 13, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.914417, + "pass" : 0, + "row" : 78, + "selectedIndex" : 20, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.906667, + "pass" : 0, + "row" : 79, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.916375, + "pass" : 0, + "row" : 80, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.914625, + "pass" : 0, + "row" : 81, + "selectedIndex" : 20, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 19, + "form" : "job-application", + "milliseconds" : 0.911167, + "pass" : 0, + "row" : 82, + "selectedIndex" : 19, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 1, + "form" : "auto-claim", + "milliseconds" : 0.920167, + "pass" : 0, + "row" : 130, + "selectedIndex" : 1, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 2, + "form" : "auto-claim", + "milliseconds" : 0.903375, + "pass" : 0, + "row" : 131, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 3, + "form" : "auto-claim", + "milliseconds" : 0.913958, + "pass" : 0, + "row" : 132, + "selectedIndex" : 3, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 4, + "form" : "auto-claim", + "milliseconds" : 0.907667, + "pass" : 0, + "row" : 133, + "selectedIndex" : 4, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 5, + "form" : "auto-claim", + "milliseconds" : 0.909625, + "pass" : 0, + "row" : 134, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 6, + "form" : "auto-claim", + "milliseconds" : 0.912833, + "pass" : 0, + "row" : 135, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 7, + "form" : "auto-claim", + "milliseconds" : 0.905542, + "pass" : 0, + "row" : 136, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 8, + "form" : "auto-claim", + "milliseconds" : 0.901292, + "pass" : 0, + "row" : 137, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 9, + "form" : "auto-claim", + "milliseconds" : 0.898209, + "pass" : 0, + "row" : 138, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 10, + "form" : "auto-claim", + "milliseconds" : 0.901292, + "pass" : 0, + "row" : 139, + "selectedIndex" : 10, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 11, + "form" : "auto-claim", + "milliseconds" : 0.899, + "pass" : 0, + "row" : 140, + "selectedIndex" : 11, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 12, + "form" : "auto-claim", + "milliseconds" : 0.901708, + "pass" : 0, + "row" : 141, + "selectedIndex" : 12, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.898125, + "pass" : 0, + "row" : 142, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 16, + "form" : "auto-claim", + "milliseconds" : 0.903291, + "pass" : 0, + "row" : 143, + "selectedIndex" : 16, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.905625, + "pass" : 0, + "row" : 144, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.905, + "pass" : 0, + "row" : 145, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 17, + "form" : "auto-claim", + "milliseconds" : 0.905125, + "pass" : 0, + "row" : 146, + "selectedIndex" : 17, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 22, + "form" : "patient-registration", + "milliseconds" : 0.905291, + "pass" : 1, + "row" : 0, + "selectedIndex" : 22, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 23, + "form" : "patient-registration", + "milliseconds" : 0.923791, + "pass" : 1, + "row" : 1, + "selectedIndex" : 23, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 2, + "form" : "patient-registration", + "milliseconds" : 0.906583, + "pass" : 1, + "row" : 2, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 5, + "form" : "patient-registration", + "milliseconds" : 0.9105, + "pass" : 1, + "row" : 3, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 4, + "form" : "patient-registration", + "milliseconds" : 0.913083, + "pass" : 1, + "row" : 4, + "selectedIndex" : 4, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 6, + "form" : "patient-registration", + "milliseconds" : 0.926416, + "pass" : 1, + "row" : 5, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 7, + "form" : "patient-registration", + "milliseconds" : 0.928333, + "pass" : 1, + "row" : 6, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 8, + "form" : "patient-registration", + "milliseconds" : 0.913417, + "pass" : 1, + "row" : 7, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 9, + "form" : "patient-registration", + "milliseconds" : 0.908167, + "pass" : 1, + "row" : 8, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 10, + "form" : "patient-registration", + "milliseconds" : 0.916417, + "pass" : 1, + "row" : 9, + "selectedIndex" : 10, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 11, + "form" : "patient-registration", + "milliseconds" : 0.91875, + "pass" : 1, + "row" : 10, + "selectedIndex" : 11, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 13, + "form" : "patient-registration", + "milliseconds" : 0.92675, + "pass" : 1, + "row" : 11, + "selectedIndex" : 13, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 15, + "form" : "patient-registration", + "milliseconds" : 0.941833, + "pass" : 1, + "row" : 12, + "selectedIndex" : 15, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 16, + "form" : "patient-registration", + "milliseconds" : 0.913875, + "pass" : 1, + "row" : 13, + "selectedIndex" : 16, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.910833, + "pass" : 1, + "row" : 14, + "selectedIndex" : 26, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 24, + "form" : "patient-registration", + "milliseconds" : 0.916875, + "pass" : 1, + "row" : 15, + "selectedIndex" : 24, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.919875, + "pass" : 1, + "row" : 16, + "selectedIndex" : 26, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 25, + "form" : "patient-registration", + "milliseconds" : 0.93275, + "pass" : 1, + "row" : 17, + "selectedIndex" : 25, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 0, + "form" : "job-application", + "milliseconds" : 0.918625, + "pass" : 1, + "row" : 68, + "selectedIndex" : 0, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 1, + "form" : "job-application", + "milliseconds" : 0.897208, + "pass" : 1, + "row" : 69, + "selectedIndex" : 1, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 2, + "form" : "job-application", + "milliseconds" : 0.914792, + "pass" : 1, + "row" : 70, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 3, + "form" : "job-application", + "milliseconds" : 0.919, + "pass" : 1, + "row" : 71, + "selectedIndex" : 3, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 5, + "form" : "job-application", + "milliseconds" : 0.920541, + "pass" : 1, + "row" : 72, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 6, + "form" : "job-application", + "milliseconds" : 0.927208, + "pass" : 1, + "row" : 73, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 7, + "form" : "job-application", + "milliseconds" : 0.901625, + "pass" : 1, + "row" : 74, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 8, + "form" : "job-application", + "milliseconds" : 0.924541, + "pass" : 1, + "row" : 75, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 9, + "form" : "job-application", + "milliseconds" : 0.917458, + "pass" : 1, + "row" : 76, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 13, + "form" : "job-application", + "milliseconds" : 0.922292, + "pass" : 1, + "row" : 77, + "selectedIndex" : 13, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.896084, + "pass" : 1, + "row" : 78, + "selectedIndex" : 20, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.917583, + "pass" : 1, + "row" : 79, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.904875, + "pass" : 1, + "row" : 80, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.900625, + "pass" : 1, + "row" : 81, + "selectedIndex" : 20, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 19, + "form" : "job-application", + "milliseconds" : 0.923083, + "pass" : 1, + "row" : 82, + "selectedIndex" : 19, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 1, + "form" : "auto-claim", + "milliseconds" : 0.894541, + "pass" : 1, + "row" : 130, + "selectedIndex" : 1, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 2, + "form" : "auto-claim", + "milliseconds" : 0.9015, + "pass" : 1, + "row" : 131, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 3, + "form" : "auto-claim", + "milliseconds" : 0.918167, + "pass" : 1, + "row" : 132, + "selectedIndex" : 3, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 4, + "form" : "auto-claim", + "milliseconds" : 0.921167, + "pass" : 1, + "row" : 133, + "selectedIndex" : 4, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 5, + "form" : "auto-claim", + "milliseconds" : 0.916834, + "pass" : 1, + "row" : 134, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 6, + "form" : "auto-claim", + "milliseconds" : 0.874709, + "pass" : 1, + "row" : 135, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 7, + "form" : "auto-claim", + "milliseconds" : 0.873125, + "pass" : 1, + "row" : 136, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 8, + "form" : "auto-claim", + "milliseconds" : 0.896792, + "pass" : 1, + "row" : 137, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 9, + "form" : "auto-claim", + "milliseconds" : 0.886292, + "pass" : 1, + "row" : 138, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 10, + "form" : "auto-claim", + "milliseconds" : 0.882125, + "pass" : 1, + "row" : 139, + "selectedIndex" : 10, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 11, + "form" : "auto-claim", + "milliseconds" : 0.911125, + "pass" : 1, + "row" : 140, + "selectedIndex" : 11, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 12, + "form" : "auto-claim", + "milliseconds" : 0.914792, + "pass" : 1, + "row" : 141, + "selectedIndex" : 12, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.920541, + "pass" : 1, + "row" : 142, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 16, + "form" : "auto-claim", + "milliseconds" : 0.911875, + "pass" : 1, + "row" : 143, + "selectedIndex" : 16, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.901666, + "pass" : 1, + "row" : 144, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.919542, + "pass" : 1, + "row" : 145, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 0, + "expectedIndex" : 17, + "form" : "auto-claim", + "milliseconds" : 0.947458, + "pass" : 1, + "row" : 146, + "selectedIndex" : 17, + "variant" : "baseline" + }, + { + "block" : 1, + "expectedIndex" : 22, + "form" : "patient-registration", + "milliseconds" : 0.999, + "pass" : 0, + "row" : 0, + "selectedIndex" : 22, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 23, + "form" : "patient-registration", + "milliseconds" : 0.976875, + "pass" : 0, + "row" : 1, + "selectedIndex" : 23, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 2, + "form" : "patient-registration", + "milliseconds" : 0.968209, + "pass" : 0, + "row" : 2, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 5, + "form" : "patient-registration", + "milliseconds" : 0.975375, + "pass" : 0, + "row" : 3, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 4, + "form" : "patient-registration", + "milliseconds" : 0.9775, + "pass" : 0, + "row" : 4, + "selectedIndex" : 4, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 6, + "form" : "patient-registration", + "milliseconds" : 0.972, + "pass" : 0, + "row" : 5, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 7, + "form" : "patient-registration", + "milliseconds" : 0.929416, + "pass" : 0, + "row" : 6, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 8, + "form" : "patient-registration", + "milliseconds" : 0.962459, + "pass" : 0, + "row" : 7, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 9, + "form" : "patient-registration", + "milliseconds" : 0.973959, + "pass" : 0, + "row" : 8, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 10, + "form" : "patient-registration", + "milliseconds" : 0.9725, + "pass" : 0, + "row" : 9, + "selectedIndex" : 10, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 11, + "form" : "patient-registration", + "milliseconds" : 0.941292, + "pass" : 0, + "row" : 10, + "selectedIndex" : 11, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 13, + "form" : "patient-registration", + "milliseconds" : 0.942667, + "pass" : 0, + "row" : 11, + "selectedIndex" : 13, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 15, + "form" : "patient-registration", + "milliseconds" : 0.950375, + "pass" : 0, + "row" : 12, + "selectedIndex" : 15, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 16, + "form" : "patient-registration", + "milliseconds" : 0.95175, + "pass" : 0, + "row" : 13, + "selectedIndex" : 16, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.933459, + "pass" : 0, + "row" : 14, + "selectedIndex" : 26, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 24, + "form" : "patient-registration", + "milliseconds" : 0.9685, + "pass" : 0, + "row" : 15, + "selectedIndex" : 24, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.975291, + "pass" : 0, + "row" : 16, + "selectedIndex" : 26, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 25, + "form" : "patient-registration", + "milliseconds" : 0.956917, + "pass" : 0, + "row" : 17, + "selectedIndex" : 25, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 0, + "form" : "job-application", + "milliseconds" : 0.956375, + "pass" : 0, + "row" : 68, + "selectedIndex" : 0, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 1, + "form" : "job-application", + "milliseconds" : 0.957541, + "pass" : 0, + "row" : 69, + "selectedIndex" : 1, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 2, + "form" : "job-application", + "milliseconds" : 0.949584, + "pass" : 0, + "row" : 70, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 3, + "form" : "job-application", + "milliseconds" : 0.947334, + "pass" : 0, + "row" : 71, + "selectedIndex" : 3, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 5, + "form" : "job-application", + "milliseconds" : 0.983583, + "pass" : 0, + "row" : 72, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 6, + "form" : "job-application", + "milliseconds" : 0.943333, + "pass" : 0, + "row" : 73, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 7, + "form" : "job-application", + "milliseconds" : 0.941292, + "pass" : 0, + "row" : 74, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 8, + "form" : "job-application", + "milliseconds" : 0.93875, + "pass" : 0, + "row" : 75, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 9, + "form" : "job-application", + "milliseconds" : 0.9435, + "pass" : 0, + "row" : 76, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 13, + "form" : "job-application", + "milliseconds" : 0.948917, + "pass" : 0, + "row" : 77, + "selectedIndex" : 13, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.955, + "pass" : 0, + "row" : 78, + "selectedIndex" : 20, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.973917, + "pass" : 0, + "row" : 79, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.962834, + "pass" : 0, + "row" : 80, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.965792, + "pass" : 0, + "row" : 81, + "selectedIndex" : 20, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 19, + "form" : "job-application", + "milliseconds" : 0.96525, + "pass" : 0, + "row" : 82, + "selectedIndex" : 19, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 1, + "form" : "auto-claim", + "milliseconds" : 0.983334, + "pass" : 0, + "row" : 130, + "selectedIndex" : 1, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 2, + "form" : "auto-claim", + "milliseconds" : 0.970917, + "pass" : 0, + "row" : 131, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 3, + "form" : "auto-claim", + "milliseconds" : 0.972458, + "pass" : 0, + "row" : 132, + "selectedIndex" : 3, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 4, + "form" : "auto-claim", + "milliseconds" : 0.970625, + "pass" : 0, + "row" : 133, + "selectedIndex" : 4, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 5, + "form" : "auto-claim", + "milliseconds" : 0.974958, + "pass" : 0, + "row" : 134, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 6, + "form" : "auto-claim", + "milliseconds" : 0.965416, + "pass" : 0, + "row" : 135, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 7, + "form" : "auto-claim", + "milliseconds" : 0.984917, + "pass" : 0, + "row" : 136, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 8, + "form" : "auto-claim", + "milliseconds" : 0.947875, + "pass" : 0, + "row" : 137, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 9, + "form" : "auto-claim", + "milliseconds" : 0.958459, + "pass" : 0, + "row" : 138, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 10, + "form" : "auto-claim", + "milliseconds" : 0.966625, + "pass" : 0, + "row" : 139, + "selectedIndex" : 10, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 11, + "form" : "auto-claim", + "milliseconds" : 0.975541, + "pass" : 0, + "row" : 140, + "selectedIndex" : 11, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 12, + "form" : "auto-claim", + "milliseconds" : 0.960708, + "pass" : 0, + "row" : 141, + "selectedIndex" : 12, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.955125, + "pass" : 0, + "row" : 142, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 16, + "form" : "auto-claim", + "milliseconds" : 0.97375, + "pass" : 0, + "row" : 143, + "selectedIndex" : 16, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.970875, + "pass" : 0, + "row" : 144, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.965958, + "pass" : 0, + "row" : 145, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 17, + "form" : "auto-claim", + "milliseconds" : 0.946083, + "pass" : 0, + "row" : 146, + "selectedIndex" : 17, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 22, + "form" : "patient-registration", + "milliseconds" : 0.966709, + "pass" : 1, + "row" : 0, + "selectedIndex" : 22, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 23, + "form" : "patient-registration", + "milliseconds" : 0.971709, + "pass" : 1, + "row" : 1, + "selectedIndex" : 23, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 2, + "form" : "patient-registration", + "milliseconds" : 0.961916, + "pass" : 1, + "row" : 2, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 5, + "form" : "patient-registration", + "milliseconds" : 0.960667, + "pass" : 1, + "row" : 3, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 4, + "form" : "patient-registration", + "milliseconds" : 0.99375, + "pass" : 1, + "row" : 4, + "selectedIndex" : 4, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 6, + "form" : "patient-registration", + "milliseconds" : 0.961333, + "pass" : 1, + "row" : 5, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 7, + "form" : "patient-registration", + "milliseconds" : 0.983417, + "pass" : 1, + "row" : 6, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 8, + "form" : "patient-registration", + "milliseconds" : 0.950625, + "pass" : 1, + "row" : 7, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 9, + "form" : "patient-registration", + "milliseconds" : 0.949375, + "pass" : 1, + "row" : 8, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 10, + "form" : "patient-registration", + "milliseconds" : 0.932083, + "pass" : 1, + "row" : 9, + "selectedIndex" : 10, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 11, + "form" : "patient-registration", + "milliseconds" : 0.935416, + "pass" : 1, + "row" : 10, + "selectedIndex" : 11, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 13, + "form" : "patient-registration", + "milliseconds" : 0.94425, + "pass" : 1, + "row" : 11, + "selectedIndex" : 13, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 15, + "form" : "patient-registration", + "milliseconds" : 0.961917, + "pass" : 1, + "row" : 12, + "selectedIndex" : 15, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 16, + "form" : "patient-registration", + "milliseconds" : 0.967125, + "pass" : 1, + "row" : 13, + "selectedIndex" : 16, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.962292, + "pass" : 1, + "row" : 14, + "selectedIndex" : 26, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 24, + "form" : "patient-registration", + "milliseconds" : 0.976792, + "pass" : 1, + "row" : 15, + "selectedIndex" : 24, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.970041, + "pass" : 1, + "row" : 16, + "selectedIndex" : 26, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 25, + "form" : "patient-registration", + "milliseconds" : 0.972458, + "pass" : 1, + "row" : 17, + "selectedIndex" : 25, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 0, + "form" : "job-application", + "milliseconds" : 0.992625, + "pass" : 1, + "row" : 68, + "selectedIndex" : 0, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 1, + "form" : "job-application", + "milliseconds" : 0.984084, + "pass" : 1, + "row" : 69, + "selectedIndex" : 1, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 2, + "form" : "job-application", + "milliseconds" : 0.954375, + "pass" : 1, + "row" : 70, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 3, + "form" : "job-application", + "milliseconds" : 0.962583, + "pass" : 1, + "row" : 71, + "selectedIndex" : 3, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 5, + "form" : "job-application", + "milliseconds" : 0.985667, + "pass" : 1, + "row" : 72, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 6, + "form" : "job-application", + "milliseconds" : 0.989542, + "pass" : 1, + "row" : 73, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 7, + "form" : "job-application", + "milliseconds" : 0.968541, + "pass" : 1, + "row" : 74, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 8, + "form" : "job-application", + "milliseconds" : 0.948375, + "pass" : 1, + "row" : 75, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 9, + "form" : "job-application", + "milliseconds" : 0.975125, + "pass" : 1, + "row" : 76, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 13, + "form" : "job-application", + "milliseconds" : 0.971666, + "pass" : 1, + "row" : 77, + "selectedIndex" : 13, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.984, + "pass" : 1, + "row" : 78, + "selectedIndex" : 20, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.967083, + "pass" : 1, + "row" : 79, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.982334, + "pass" : 1, + "row" : 80, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.97675, + "pass" : 1, + "row" : 81, + "selectedIndex" : 20, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 19, + "form" : "job-application", + "milliseconds" : 0.960084, + "pass" : 1, + "row" : 82, + "selectedIndex" : 19, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 1, + "form" : "auto-claim", + "milliseconds" : 0.960292, + "pass" : 1, + "row" : 130, + "selectedIndex" : 1, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 2, + "form" : "auto-claim", + "milliseconds" : 0.961959, + "pass" : 1, + "row" : 131, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 3, + "form" : "auto-claim", + "milliseconds" : 0.9625, + "pass" : 1, + "row" : 132, + "selectedIndex" : 3, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 4, + "form" : "auto-claim", + "milliseconds" : 0.961709, + "pass" : 1, + "row" : 133, + "selectedIndex" : 4, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 5, + "form" : "auto-claim", + "milliseconds" : 0.968792, + "pass" : 1, + "row" : 134, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 6, + "form" : "auto-claim", + "milliseconds" : 0.982708, + "pass" : 1, + "row" : 135, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 7, + "form" : "auto-claim", + "milliseconds" : 0.981791, + "pass" : 1, + "row" : 136, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 8, + "form" : "auto-claim", + "milliseconds" : 0.966917, + "pass" : 1, + "row" : 137, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 9, + "form" : "auto-claim", + "milliseconds" : 0.950667, + "pass" : 1, + "row" : 138, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 10, + "form" : "auto-claim", + "milliseconds" : 0.942041, + "pass" : 1, + "row" : 139, + "selectedIndex" : 10, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 11, + "form" : "auto-claim", + "milliseconds" : 0.935709, + "pass" : 1, + "row" : 140, + "selectedIndex" : 11, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 12, + "form" : "auto-claim", + "milliseconds" : 0.9825, + "pass" : 1, + "row" : 141, + "selectedIndex" : 12, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.983708, + "pass" : 1, + "row" : 142, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 16, + "form" : "auto-claim", + "milliseconds" : 1.0375, + "pass" : 1, + "row" : 143, + "selectedIndex" : 16, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.954625, + "pass" : 1, + "row" : 144, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.9545, + "pass" : 1, + "row" : 145, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 1, + "expectedIndex" : 17, + "form" : "auto-claim", + "milliseconds" : 0.976208, + "pass" : 1, + "row" : 146, + "selectedIndex" : 17, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 22, + "form" : "patient-registration", + "milliseconds" : 0.960375, + "pass" : 0, + "row" : 0, + "selectedIndex" : 22, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 23, + "form" : "patient-registration", + "milliseconds" : 0.954375, + "pass" : 0, + "row" : 1, + "selectedIndex" : 23, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 2, + "form" : "patient-registration", + "milliseconds" : 0.978375, + "pass" : 0, + "row" : 2, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 5, + "form" : "patient-registration", + "milliseconds" : 0.962292, + "pass" : 0, + "row" : 3, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 4, + "form" : "patient-registration", + "milliseconds" : 0.9495, + "pass" : 0, + "row" : 4, + "selectedIndex" : 4, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 6, + "form" : "patient-registration", + "milliseconds" : 0.940625, + "pass" : 0, + "row" : 5, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 7, + "form" : "patient-registration", + "milliseconds" : 0.950166, + "pass" : 0, + "row" : 6, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 8, + "form" : "patient-registration", + "milliseconds" : 0.95125, + "pass" : 0, + "row" : 7, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 9, + "form" : "patient-registration", + "milliseconds" : 0.977875, + "pass" : 0, + "row" : 8, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 10, + "form" : "patient-registration", + "milliseconds" : 0.943959, + "pass" : 0, + "row" : 9, + "selectedIndex" : 10, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 11, + "form" : "patient-registration", + "milliseconds" : 0.938542, + "pass" : 0, + "row" : 10, + "selectedIndex" : 11, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 13, + "form" : "patient-registration", + "milliseconds" : 0.949917, + "pass" : 0, + "row" : 11, + "selectedIndex" : 13, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 15, + "form" : "patient-registration", + "milliseconds" : 0.959666, + "pass" : 0, + "row" : 12, + "selectedIndex" : 15, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 16, + "form" : "patient-registration", + "milliseconds" : 0.981167, + "pass" : 0, + "row" : 13, + "selectedIndex" : 16, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.965917, + "pass" : 0, + "row" : 14, + "selectedIndex" : 26, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 24, + "form" : "patient-registration", + "milliseconds" : 0.990125, + "pass" : 0, + "row" : 15, + "selectedIndex" : 24, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.954333, + "pass" : 0, + "row" : 16, + "selectedIndex" : 26, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 25, + "form" : "patient-registration", + "milliseconds" : 0.974792, + "pass" : 0, + "row" : 17, + "selectedIndex" : 25, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 0, + "form" : "job-application", + "milliseconds" : 0.959708, + "pass" : 0, + "row" : 68, + "selectedIndex" : 0, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 1, + "form" : "job-application", + "milliseconds" : 0.960083, + "pass" : 0, + "row" : 69, + "selectedIndex" : 1, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 2, + "form" : "job-application", + "milliseconds" : 0.962458, + "pass" : 0, + "row" : 70, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 3, + "form" : "job-application", + "milliseconds" : 0.969291, + "pass" : 0, + "row" : 71, + "selectedIndex" : 3, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 5, + "form" : "job-application", + "milliseconds" : 0.965916, + "pass" : 0, + "row" : 72, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 6, + "form" : "job-application", + "milliseconds" : 0.968125, + "pass" : 0, + "row" : 73, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 7, + "form" : "job-application", + "milliseconds" : 0.96375, + "pass" : 0, + "row" : 74, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 8, + "form" : "job-application", + "milliseconds" : 0.984916, + "pass" : 0, + "row" : 75, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 9, + "form" : "job-application", + "milliseconds" : 0.978583, + "pass" : 0, + "row" : 76, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 13, + "form" : "job-application", + "milliseconds" : 0.945958, + "pass" : 0, + "row" : 77, + "selectedIndex" : 13, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.938958, + "pass" : 0, + "row" : 78, + "selectedIndex" : 20, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.93975, + "pass" : 0, + "row" : 79, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.939125, + "pass" : 0, + "row" : 80, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.942375, + "pass" : 0, + "row" : 81, + "selectedIndex" : 20, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 19, + "form" : "job-application", + "milliseconds" : 0.946209, + "pass" : 0, + "row" : 82, + "selectedIndex" : 19, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 1, + "form" : "auto-claim", + "milliseconds" : 0.943167, + "pass" : 0, + "row" : 130, + "selectedIndex" : 1, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 2, + "form" : "auto-claim", + "milliseconds" : 0.956416, + "pass" : 0, + "row" : 131, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 3, + "form" : "auto-claim", + "milliseconds" : 0.937958, + "pass" : 0, + "row" : 132, + "selectedIndex" : 3, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 4, + "form" : "auto-claim", + "milliseconds" : 0.93875, + "pass" : 0, + "row" : 133, + "selectedIndex" : 4, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 5, + "form" : "auto-claim", + "milliseconds" : 0.937291, + "pass" : 0, + "row" : 134, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 6, + "form" : "auto-claim", + "milliseconds" : 0.94325, + "pass" : 0, + "row" : 135, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 7, + "form" : "auto-claim", + "milliseconds" : 0.947583, + "pass" : 0, + "row" : 136, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 8, + "form" : "auto-claim", + "milliseconds" : 0.975625, + "pass" : 0, + "row" : 137, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 9, + "form" : "auto-claim", + "milliseconds" : 0.957834, + "pass" : 0, + "row" : 138, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 10, + "form" : "auto-claim", + "milliseconds" : 0.954, + "pass" : 0, + "row" : 139, + "selectedIndex" : 10, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 11, + "form" : "auto-claim", + "milliseconds" : 0.963125, + "pass" : 0, + "row" : 140, + "selectedIndex" : 11, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 12, + "form" : "auto-claim", + "milliseconds" : 0.96025, + "pass" : 0, + "row" : 141, + "selectedIndex" : 12, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.939625, + "pass" : 0, + "row" : 142, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 16, + "form" : "auto-claim", + "milliseconds" : 0.95225, + "pass" : 0, + "row" : 143, + "selectedIndex" : 16, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.97625, + "pass" : 0, + "row" : 144, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.952875, + "pass" : 0, + "row" : 145, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 17, + "form" : "auto-claim", + "milliseconds" : 0.941917, + "pass" : 0, + "row" : 146, + "selectedIndex" : 17, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 22, + "form" : "patient-registration", + "milliseconds" : 0.973583, + "pass" : 1, + "row" : 0, + "selectedIndex" : 22, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 23, + "form" : "patient-registration", + "milliseconds" : 0.961291, + "pass" : 1, + "row" : 1, + "selectedIndex" : 23, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 2, + "form" : "patient-registration", + "milliseconds" : 0.95625, + "pass" : 1, + "row" : 2, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 5, + "form" : "patient-registration", + "milliseconds" : 0.968125, + "pass" : 1, + "row" : 3, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 4, + "form" : "patient-registration", + "milliseconds" : 0.969541, + "pass" : 1, + "row" : 4, + "selectedIndex" : 4, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 6, + "form" : "patient-registration", + "milliseconds" : 0.969417, + "pass" : 1, + "row" : 5, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 7, + "form" : "patient-registration", + "milliseconds" : 0.962959, + "pass" : 1, + "row" : 6, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 8, + "form" : "patient-registration", + "milliseconds" : 0.9775, + "pass" : 1, + "row" : 7, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 9, + "form" : "patient-registration", + "milliseconds" : 0.969208, + "pass" : 1, + "row" : 8, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 10, + "form" : "patient-registration", + "milliseconds" : 0.969709, + "pass" : 1, + "row" : 9, + "selectedIndex" : 10, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 11, + "form" : "patient-registration", + "milliseconds" : 0.984292, + "pass" : 1, + "row" : 10, + "selectedIndex" : 11, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 13, + "form" : "patient-registration", + "milliseconds" : 0.961291, + "pass" : 1, + "row" : 11, + "selectedIndex" : 13, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 15, + "form" : "patient-registration", + "milliseconds" : 0.969458, + "pass" : 1, + "row" : 12, + "selectedIndex" : 15, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 16, + "form" : "patient-registration", + "milliseconds" : 0.953875, + "pass" : 1, + "row" : 13, + "selectedIndex" : 16, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.963666, + "pass" : 1, + "row" : 14, + "selectedIndex" : 26, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 24, + "form" : "patient-registration", + "milliseconds" : 0.953125, + "pass" : 1, + "row" : 15, + "selectedIndex" : 24, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.967833, + "pass" : 1, + "row" : 16, + "selectedIndex" : 26, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 25, + "form" : "patient-registration", + "milliseconds" : 0.953583, + "pass" : 1, + "row" : 17, + "selectedIndex" : 25, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 0, + "form" : "job-application", + "milliseconds" : 0.95275, + "pass" : 1, + "row" : 68, + "selectedIndex" : 0, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 1, + "form" : "job-application", + "milliseconds" : 0.95525, + "pass" : 1, + "row" : 69, + "selectedIndex" : 1, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 2, + "form" : "job-application", + "milliseconds" : 0.927208, + "pass" : 1, + "row" : 70, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 3, + "form" : "job-application", + "milliseconds" : 0.943666, + "pass" : 1, + "row" : 71, + "selectedIndex" : 3, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 5, + "form" : "job-application", + "milliseconds" : 0.932833, + "pass" : 1, + "row" : 72, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 6, + "form" : "job-application", + "milliseconds" : 0.930083, + "pass" : 1, + "row" : 73, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 7, + "form" : "job-application", + "milliseconds" : 0.931708, + "pass" : 1, + "row" : 74, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 8, + "form" : "job-application", + "milliseconds" : 0.958333, + "pass" : 1, + "row" : 75, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 9, + "form" : "job-application", + "milliseconds" : 0.964708, + "pass" : 1, + "row" : 76, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 13, + "form" : "job-application", + "milliseconds" : 0.937167, + "pass" : 1, + "row" : 77, + "selectedIndex" : 13, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.943333, + "pass" : 1, + "row" : 78, + "selectedIndex" : 20, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.949375, + "pass" : 1, + "row" : 79, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.938292, + "pass" : 1, + "row" : 80, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.93825, + "pass" : 1, + "row" : 81, + "selectedIndex" : 20, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 19, + "form" : "job-application", + "milliseconds" : 0.943959, + "pass" : 1, + "row" : 82, + "selectedIndex" : 19, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 1, + "form" : "auto-claim", + "milliseconds" : 0.953208, + "pass" : 1, + "row" : 130, + "selectedIndex" : 1, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 2, + "form" : "auto-claim", + "milliseconds" : 0.982291, + "pass" : 1, + "row" : 131, + "selectedIndex" : 2, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 3, + "form" : "auto-claim", + "milliseconds" : 0.976167, + "pass" : 1, + "row" : 132, + "selectedIndex" : 3, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 4, + "form" : "auto-claim", + "milliseconds" : 0.97325, + "pass" : 1, + "row" : 133, + "selectedIndex" : 4, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 5, + "form" : "auto-claim", + "milliseconds" : 0.973542, + "pass" : 1, + "row" : 134, + "selectedIndex" : 5, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 6, + "form" : "auto-claim", + "milliseconds" : 0.961375, + "pass" : 1, + "row" : 135, + "selectedIndex" : 6, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 7, + "form" : "auto-claim", + "milliseconds" : 0.968083, + "pass" : 1, + "row" : 136, + "selectedIndex" : 7, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 8, + "form" : "auto-claim", + "milliseconds" : 0.96575, + "pass" : 1, + "row" : 137, + "selectedIndex" : 8, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 9, + "form" : "auto-claim", + "milliseconds" : 0.973333, + "pass" : 1, + "row" : 138, + "selectedIndex" : 9, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 10, + "form" : "auto-claim", + "milliseconds" : 0.959167, + "pass" : 1, + "row" : 139, + "selectedIndex" : 10, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 11, + "form" : "auto-claim", + "milliseconds" : 0.943375, + "pass" : 1, + "row" : 140, + "selectedIndex" : 11, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 12, + "form" : "auto-claim", + "milliseconds" : 0.946834, + "pass" : 1, + "row" : 141, + "selectedIndex" : 12, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.959125, + "pass" : 1, + "row" : 142, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 16, + "form" : "auto-claim", + "milliseconds" : 0.948875, + "pass" : 1, + "row" : 143, + "selectedIndex" : 16, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.950875, + "pass" : 1, + "row" : 144, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.952125, + "pass" : 1, + "row" : 145, + "selectedIndex" : 18, + "variant" : "ane-gather" + }, + { + "block" : 2, + "expectedIndex" : 17, + "form" : "auto-claim", + "milliseconds" : 0.952459, + "pass" : 1, + "row" : 146, + "selectedIndex" : 17, + "variant" : "ane-gather" + }, + { + "block" : 3, + "expectedIndex" : 22, + "form" : "patient-registration", + "milliseconds" : 0.940667, + "pass" : 0, + "row" : 0, + "selectedIndex" : 22, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 23, + "form" : "patient-registration", + "milliseconds" : 0.905042, + "pass" : 0, + "row" : 1, + "selectedIndex" : 23, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 2, + "form" : "patient-registration", + "milliseconds" : 0.891791, + "pass" : 0, + "row" : 2, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 5, + "form" : "patient-registration", + "milliseconds" : 0.916417, + "pass" : 0, + "row" : 3, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 4, + "form" : "patient-registration", + "milliseconds" : 0.918667, + "pass" : 0, + "row" : 4, + "selectedIndex" : 4, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 6, + "form" : "patient-registration", + "milliseconds" : 0.900166, + "pass" : 0, + "row" : 5, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 7, + "form" : "patient-registration", + "milliseconds" : 0.905875, + "pass" : 0, + "row" : 6, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 8, + "form" : "patient-registration", + "milliseconds" : 0.887417, + "pass" : 0, + "row" : 7, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 9, + "form" : "patient-registration", + "milliseconds" : 0.882583, + "pass" : 0, + "row" : 8, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 10, + "form" : "patient-registration", + "milliseconds" : 0.882667, + "pass" : 0, + "row" : 9, + "selectedIndex" : 10, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 11, + "form" : "patient-registration", + "milliseconds" : 0.901, + "pass" : 0, + "row" : 10, + "selectedIndex" : 11, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 13, + "form" : "patient-registration", + "milliseconds" : 0.931041, + "pass" : 0, + "row" : 11, + "selectedIndex" : 13, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 15, + "form" : "patient-registration", + "milliseconds" : 0.881625, + "pass" : 0, + "row" : 12, + "selectedIndex" : 15, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 16, + "form" : "patient-registration", + "milliseconds" : 0.909667, + "pass" : 0, + "row" : 13, + "selectedIndex" : 16, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.92225, + "pass" : 0, + "row" : 14, + "selectedIndex" : 26, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 24, + "form" : "patient-registration", + "milliseconds" : 0.877833, + "pass" : 0, + "row" : 15, + "selectedIndex" : 24, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.894459, + "pass" : 0, + "row" : 16, + "selectedIndex" : 26, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 25, + "form" : "patient-registration", + "milliseconds" : 0.898583, + "pass" : 0, + "row" : 17, + "selectedIndex" : 25, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 0, + "form" : "job-application", + "milliseconds" : 0.925625, + "pass" : 0, + "row" : 68, + "selectedIndex" : 0, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 1, + "form" : "job-application", + "milliseconds" : 0.909958, + "pass" : 0, + "row" : 69, + "selectedIndex" : 1, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 2, + "form" : "job-application", + "milliseconds" : 0.927166, + "pass" : 0, + "row" : 70, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 3, + "form" : "job-application", + "milliseconds" : 0.904958, + "pass" : 0, + "row" : 71, + "selectedIndex" : 3, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 5, + "form" : "job-application", + "milliseconds" : 0.908375, + "pass" : 0, + "row" : 72, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 6, + "form" : "job-application", + "milliseconds" : 0.919375, + "pass" : 0, + "row" : 73, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 7, + "form" : "job-application", + "milliseconds" : 0.904791, + "pass" : 0, + "row" : 74, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 8, + "form" : "job-application", + "milliseconds" : 0.889125, + "pass" : 0, + "row" : 75, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 9, + "form" : "job-application", + "milliseconds" : 0.918542, + "pass" : 0, + "row" : 76, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 13, + "form" : "job-application", + "milliseconds" : 0.924833, + "pass" : 0, + "row" : 77, + "selectedIndex" : 13, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.919083, + "pass" : 0, + "row" : 78, + "selectedIndex" : 20, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.926583, + "pass" : 0, + "row" : 79, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.926333, + "pass" : 0, + "row" : 80, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.919708, + "pass" : 0, + "row" : 81, + "selectedIndex" : 20, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 19, + "form" : "job-application", + "milliseconds" : 0.92525, + "pass" : 0, + "row" : 82, + "selectedIndex" : 19, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 1, + "form" : "auto-claim", + "milliseconds" : 0.912208, + "pass" : 0, + "row" : 130, + "selectedIndex" : 1, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 2, + "form" : "auto-claim", + "milliseconds" : 0.895083, + "pass" : 0, + "row" : 131, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 3, + "form" : "auto-claim", + "milliseconds" : 0.892791, + "pass" : 0, + "row" : 132, + "selectedIndex" : 3, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 4, + "form" : "auto-claim", + "milliseconds" : 0.890042, + "pass" : 0, + "row" : 133, + "selectedIndex" : 4, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 5, + "form" : "auto-claim", + "milliseconds" : 0.886334, + "pass" : 0, + "row" : 134, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 6, + "form" : "auto-claim", + "milliseconds" : 0.8925, + "pass" : 0, + "row" : 135, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 7, + "form" : "auto-claim", + "milliseconds" : 0.879583, + "pass" : 0, + "row" : 136, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 8, + "form" : "auto-claim", + "milliseconds" : 0.930292, + "pass" : 0, + "row" : 137, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 9, + "form" : "auto-claim", + "milliseconds" : 0.923584, + "pass" : 0, + "row" : 138, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 10, + "form" : "auto-claim", + "milliseconds" : 0.891334, + "pass" : 0, + "row" : 139, + "selectedIndex" : 10, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 11, + "form" : "auto-claim", + "milliseconds" : 0.899834, + "pass" : 0, + "row" : 140, + "selectedIndex" : 11, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 12, + "form" : "auto-claim", + "milliseconds" : 0.885292, + "pass" : 0, + "row" : 141, + "selectedIndex" : 12, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.892083, + "pass" : 0, + "row" : 142, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 16, + "form" : "auto-claim", + "milliseconds" : 0.888834, + "pass" : 0, + "row" : 143, + "selectedIndex" : 16, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.917959, + "pass" : 0, + "row" : 144, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.897667, + "pass" : 0, + "row" : 145, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 17, + "form" : "auto-claim", + "milliseconds" : 0.920625, + "pass" : 0, + "row" : 146, + "selectedIndex" : 17, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 22, + "form" : "patient-registration", + "milliseconds" : 0.926209, + "pass" : 1, + "row" : 0, + "selectedIndex" : 22, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 23, + "form" : "patient-registration", + "milliseconds" : 0.919167, + "pass" : 1, + "row" : 1, + "selectedIndex" : 23, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 2, + "form" : "patient-registration", + "milliseconds" : 0.902083, + "pass" : 1, + "row" : 2, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 5, + "form" : "patient-registration", + "milliseconds" : 0.903459, + "pass" : 1, + "row" : 3, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 4, + "form" : "patient-registration", + "milliseconds" : 0.9305, + "pass" : 1, + "row" : 4, + "selectedIndex" : 4, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 6, + "form" : "patient-registration", + "milliseconds" : 0.929, + "pass" : 1, + "row" : 5, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 7, + "form" : "patient-registration", + "milliseconds" : 0.891166, + "pass" : 1, + "row" : 6, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 8, + "form" : "patient-registration", + "milliseconds" : 0.917417, + "pass" : 1, + "row" : 7, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 9, + "form" : "patient-registration", + "milliseconds" : 0.939917, + "pass" : 1, + "row" : 8, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 10, + "form" : "patient-registration", + "milliseconds" : 0.922708, + "pass" : 1, + "row" : 9, + "selectedIndex" : 10, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 11, + "form" : "patient-registration", + "milliseconds" : 0.904541, + "pass" : 1, + "row" : 10, + "selectedIndex" : 11, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 13, + "form" : "patient-registration", + "milliseconds" : 0.91675, + "pass" : 1, + "row" : 11, + "selectedIndex" : 13, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 15, + "form" : "patient-registration", + "milliseconds" : 0.922, + "pass" : 1, + "row" : 12, + "selectedIndex" : 15, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 16, + "form" : "patient-registration", + "milliseconds" : 0.935916, + "pass" : 1, + "row" : 13, + "selectedIndex" : 16, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.931833, + "pass" : 1, + "row" : 14, + "selectedIndex" : 26, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 24, + "form" : "patient-registration", + "milliseconds" : 0.903833, + "pass" : 1, + "row" : 15, + "selectedIndex" : 24, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 26, + "form" : "patient-registration", + "milliseconds" : 0.924583, + "pass" : 1, + "row" : 16, + "selectedIndex" : 26, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 25, + "form" : "patient-registration", + "milliseconds" : 0.931542, + "pass" : 1, + "row" : 17, + "selectedIndex" : 25, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 0, + "form" : "job-application", + "milliseconds" : 0.924833, + "pass" : 1, + "row" : 68, + "selectedIndex" : 0, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 1, + "form" : "job-application", + "milliseconds" : 0.9055, + "pass" : 1, + "row" : 69, + "selectedIndex" : 1, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 2, + "form" : "job-application", + "milliseconds" : 0.91075, + "pass" : 1, + "row" : 70, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 3, + "form" : "job-application", + "milliseconds" : 0.908333, + "pass" : 1, + "row" : 71, + "selectedIndex" : 3, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 5, + "form" : "job-application", + "milliseconds" : 0.894375, + "pass" : 1, + "row" : 72, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 6, + "form" : "job-application", + "milliseconds" : 0.886333, + "pass" : 1, + "row" : 73, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 7, + "form" : "job-application", + "milliseconds" : 0.909375, + "pass" : 1, + "row" : 74, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 8, + "form" : "job-application", + "milliseconds" : 0.9085, + "pass" : 1, + "row" : 75, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 9, + "form" : "job-application", + "milliseconds" : 0.898625, + "pass" : 1, + "row" : 76, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 13, + "form" : "job-application", + "milliseconds" : 0.923709, + "pass" : 1, + "row" : 77, + "selectedIndex" : 13, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.897625, + "pass" : 1, + "row" : 78, + "selectedIndex" : 20, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.890667, + "pass" : 1, + "row" : 79, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "job-application", + "milliseconds" : 0.885042, + "pass" : 1, + "row" : 80, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 20, + "form" : "job-application", + "milliseconds" : 0.888, + "pass" : 1, + "row" : 81, + "selectedIndex" : 20, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 19, + "form" : "job-application", + "milliseconds" : 0.914083, + "pass" : 1, + "row" : 82, + "selectedIndex" : 19, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 1, + "form" : "auto-claim", + "milliseconds" : 0.878375, + "pass" : 1, + "row" : 130, + "selectedIndex" : 1, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 2, + "form" : "auto-claim", + "milliseconds" : 0.897875, + "pass" : 1, + "row" : 131, + "selectedIndex" : 2, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 3, + "form" : "auto-claim", + "milliseconds" : 0.905417, + "pass" : 1, + "row" : 132, + "selectedIndex" : 3, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 4, + "form" : "auto-claim", + "milliseconds" : 0.925125, + "pass" : 1, + "row" : 133, + "selectedIndex" : 4, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 5, + "form" : "auto-claim", + "milliseconds" : 0.926375, + "pass" : 1, + "row" : 134, + "selectedIndex" : 5, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 6, + "form" : "auto-claim", + "milliseconds" : 0.92475, + "pass" : 1, + "row" : 135, + "selectedIndex" : 6, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 7, + "form" : "auto-claim", + "milliseconds" : 0.891084, + "pass" : 1, + "row" : 136, + "selectedIndex" : 7, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 8, + "form" : "auto-claim", + "milliseconds" : 0.890417, + "pass" : 1, + "row" : 137, + "selectedIndex" : 8, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 9, + "form" : "auto-claim", + "milliseconds" : 0.892792, + "pass" : 1, + "row" : 138, + "selectedIndex" : 9, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 10, + "form" : "auto-claim", + "milliseconds" : 0.894334, + "pass" : 1, + "row" : 139, + "selectedIndex" : 10, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 11, + "form" : "auto-claim", + "milliseconds" : 0.886084, + "pass" : 1, + "row" : 140, + "selectedIndex" : 11, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 12, + "form" : "auto-claim", + "milliseconds" : 0.937167, + "pass" : 1, + "row" : 141, + "selectedIndex" : 12, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.906916, + "pass" : 1, + "row" : 142, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 16, + "form" : "auto-claim", + "milliseconds" : 0.9025, + "pass" : 1, + "row" : 143, + "selectedIndex" : 16, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.907625, + "pass" : 1, + "row" : 144, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 18, + "form" : "auto-claim", + "milliseconds" : 0.916167, + "pass" : 1, + "row" : 145, + "selectedIndex" : 18, + "variant" : "baseline" + }, + { + "block" : 3, + "expectedIndex" : 17, + "form" : "auto-claim", + "milliseconds" : 0.906625, + "pass" : 1, + "row" : 146, + "selectedIndex" : 17, + "variant" : "baseline" + } + ], + "summaries" : { + "ane-gather" : { + "calls" : 200, + "correct" : 200, + "medianMilliseconds" : 0.961354, + "p95Milliseconds" : 0.9840943999999999 + }, + "baseline" : { + "calls" : 200, + "correct" : 200, + "medianMilliseconds" : 0.9123955, + "p95Milliseconds" : 0.9332711499999999 + } + }, + "timestamp" : "2026-09-19T20:36:23Z", + "timingScope" : "Swift manager.score: encoding + actor call + Core ML + output decoding; no UI or animation" +} \ No newline at end of file diff --git a/reports/synthetic-test-decisions.jsonl.gz b/reports/synthetic-test-decisions.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..2fdb8273c14dc721fe1cf6363fd8494dafc0026a --- /dev/null +++ b/reports/synthetic-test-decisions.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6f6edc9de34fb37b10847a1f9568b530ce2b8e1bd241f51bf2e637f489d6554 +size 2231147 diff --git a/reports/synthetic-test.json b/reports/synthetic-test.json new file mode 100644 index 0000000000000000000000000000000000000000..f4b0bb2d45ed535544b64a90df789f3930c21419 --- /dev/null +++ b/reports/synthetic-test.json @@ -0,0 +1,845 @@ +{ + "purpose": "Full published synthetic test; no fitting, filtering, resampling or test-based tuning", + "created_utc": "2026-09-19T21:10:18.821257+00:00", + "dataset": { + "dataset_repository": "cua-ai/cua-s1-forms", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "path": "artifacts/test.jsonl", + "url": "https://huggingface.co/datasets/cua-ai/cua-s1-forms/resolve/8273f34778b99ac2e12d9f6e7d57dad99ae20845/test.jsonl", + "sha256": "d63a7e0db195d4d20154a40b2f8dd09ce3bb65487a158c638da5c609d4475e7c", + "rows": 24370, + "bytes": 18710924, + "upstream_split_claim": "Synthetic test, form-signature-disjoint from train/validation. Train and validation signatures have not been independently re-audited here.", + "model_card_claim": { + "synthetic_test_top1": 0.9995, + "approximate_decisions": 15000, + "note": "The released file has 24370 rows. No matching full result ledger or exact 15000-row manifest is supplied by the linked model card." + } + }, + "manifest_sha256": "e582b951ecc07809a0b06a2bc0790827b90a86ce57c70d5b6177ef04f0b4f1ca", + "input_census": { + "rows": 24370, + "unique_episode_seeds": 1040, + "maxima": { + "context_bytes": 217, + "option_bytes": 92, + "max_options": 32 + }, + "per_action": { + "check": 816, + "click": 1040, + "fill": 9802, + "skip": 12712 + }, + "option_count_histogram": { + "7": 17, + "8": 46, + "9": 80, + "10": 227, + "11": 361, + "12": 589, + "13": 888, + "14": 1254, + "15": 1513, + "16": 1761, + "17": 2373, + "18": 2014, + "19": 2277, + "20": 2506, + "21": 2396, + "22": 1818, + "23": 1422, + "24": 907, + "25": 688, + "26": 524, + "27": 483, + "28": 89, + "29": 33, + "30": 29, + "31": 35, + "32": 40 + }, + "excluded_rows": 0, + "truncated_contexts": 0, + "truncated_options": 0 + }, + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "conversions": { + "baseline": { + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + } + }, + "ane-gather": { + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "optimization": "ane-gather", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.6771400420111604, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "de18e313c3b625e35d008ed8b6b24108edc6df7bf2fae9533af03519eca11b63", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "38f812a04eb2322080634ba788c61df362336168466ca67e5549058d346ac793" + } + } + }, + "environment": { + "machine": "Apple M5 Pro", + "memory_bytes": 25769803776, + "os": "27.0", + "os_build": "26A428", + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "torch_cpu_threads": 2, + "torch_interop_threads": 1, + "torch_mha_fastpath": false + }, + "protocol": { + "batch_size": 1, + "warmups_per_model": 3, + "timed_passes": 1, + "coreml_compute_units": "CPU_AND_NE", + "coreml_order": "AB on even rows, BA on odd rows", + "timing": "PyTorch CPU forward+softmax / synchronous Core ML predict; excludes encoding, validation, GUI and network", + "preprocessing": "Upstream variable-length collator; independent fixed-shape Core ML encoder shared by both exports", + "load_note": "Process/model creation can use existing system caches; not a cold-start experiment", + "output_audit": "Imperfect finite probability sums are retained unchanged and fail the original normalization gate", + "calibration_note": "NLL/ECE use raw emitted scores; no renormalization or calibration was applied" + }, + "thresholds": { + "argmax_agreement": 1.0, + "max_abs_probability_error": 0.005, + "allow_accuracy_loss": false, + "probability_sum_atol": 0.001, + "probability_sum_rtol": 1e-05 + }, + "host_preprocessing_ms": { + "upstream_collator_ms": { + "count": 24370, + "median_ms": 0.09608300752006471, + "p95_ms": 0.14412500750040635, + "min_ms": 0.04137499490752816, + "max_ms": 0.96245898748748 + }, + "shared_coreml_encoding_ms": { + "count": 24370, + "median_ms": 0.028999987989664078, + "p95_ms": 0.044917018385604024, + "min_ms": 0.012375006917864084, + "max_ms": 0.3988750104326755 + } + }, + "results": { + "upstream_pytorch": { + "correct": 24359, + "rows": 24370, + "accuracy": 0.999548625359048, + "model_call_latency_ms": { + "count": 24370, + "median_ms": 1.786978988093324, + "p95_ms": 3.103799853124656, + "min_ms": 1.019332994474098, + "max_ms": 15.003458975115791 + }, + "load_ms": 8.14687501406297, + "first_call_ms": 5.291624984238297, + "calibration": { + "nll": 0.0021302032038990836, + "ece": 0.00035450826133425167, + "equal_width_bins": 15, + "bins": [ + { + "bin": 7, + "count": 2, + "accuracy": 0.5, + "mean_confidence": 0.5216051042079926 + }, + { + "bin": 8, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.575071656703949 + }, + { + "bin": 9, + "count": 7, + "accuracy": 0.7142857142857143, + "mean_confidence": 0.6404258779117039 + }, + { + "bin": 10, + "count": 4, + "accuracy": 1.0, + "mean_confidence": 0.6963506639003754 + }, + { + "bin": 11, + "count": 11, + "accuracy": 0.9090909090909091, + "mean_confidence": 0.7664791941642761 + }, + { + "bin": 12, + "count": 21, + "accuracy": 0.9523809523809523, + "mean_confidence": 0.8310647464933849 + }, + { + "bin": 13, + "count": 22, + "accuracy": 0.9545454545454546, + "mean_confidence": 0.9173552746122534 + }, + { + "bin": 14, + "count": 24298, + "accuracy": 0.9998353773973166, + "mean_confidence": 0.999802232897422 + } + ], + "nll_probability_floor": 1.1754943508222875e-38 + }, + "upstream_action_metrics": { + "examples": 24370, + "accuracy": 0.999548625359048, + "coverage": 0.4788264259335248, + "abstention_rate": 0.5211735740664751, + "selective_accuracy": 0.9990573313908647, + "wrong_action_rate": 0.00045137464095199015, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.00045137464095199015, + "counts": { + "total": 24370, + "correct": 24359, + "abstained": 12701, + "acted": 11669, + "acted_correct": 11658, + "wrong_action": 11, + "unsafe_action": 11 + }, + "per_action": { + "abstain": { + "accuracy": 0.9991346758967904, + "abstention_rate": 0.9991346758967904, + "examples": 12712 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 816 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 1040 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 9802 + } + } + }, + "macro_action_accuracy": 0.9997836689741976, + "wrong_rows": [ + 137, + 3216, + 6631, + 6748, + 7280, + 10703, + 12255, + 13480, + 15214, + 21022, + 23671 + ] + }, + "baseline": { + "correct": 24359, + "rows": 24370, + "accuracy": 0.999548625359048, + "model_call_latency_ms": { + "count": 24370, + "median_ms": 1.0031874990090728, + "p95_ms": 1.1327079846523702, + "min_ms": 0.8830419974401593, + "max_ms": 3.726124996319413 + }, + "load_ms": 612.4412920034956, + "first_call_ms": 3.2538340019527823, + "calibration": { + "nll": 0.002108235388127147, + "ece": 0.000331678613561794, + "equal_width_bins": 15, + "bins": [ + { + "bin": 7, + "count": 2, + "accuracy": 0.5, + "mean_confidence": 0.5224609375 + }, + { + "bin": 8, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.57607421875 + }, + { + "bin": 9, + "count": 6, + "accuracy": 0.8333333333333334, + "mean_confidence": 0.637451171875 + }, + { + "bin": 10, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.69248046875 + }, + { + "bin": 11, + "count": 12, + "accuracy": 0.9166666666666666, + "mean_confidence": 0.7687581380208334 + }, + { + "bin": 12, + "count": 20, + "accuracy": 0.95, + "mean_confidence": 0.8322265625 + }, + { + "bin": 13, + "count": 22, + "accuracy": 0.9545454545454546, + "mean_confidence": 0.9176136363636364 + }, + { + "bin": 14, + "count": 24298, + "accuracy": 0.9998353773973166, + "mean_confidence": 0.9998245660008025 + } + ], + "nll_probability_floor": 1.1754943508222875e-38 + }, + "upstream_action_metrics": { + "examples": 24370, + "accuracy": 0.999548625359048, + "coverage": 0.4788264259335248, + "abstention_rate": 0.5211735740664751, + "selective_accuracy": 0.9990573313908647, + "wrong_action_rate": 0.00045137464095199015, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.00045137464095199015, + "counts": { + "total": 24370, + "correct": 24359, + "abstained": 12701, + "acted": 11669, + "acted_correct": 11658, + "wrong_action": 11, + "unsafe_action": 11 + }, + "per_action": { + "abstain": { + "accuracy": 0.9991346758967904, + "abstention_rate": 0.9991346758967904, + "examples": 12712 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 816 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 1040 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 9802 + } + } + }, + "macro_action_accuracy": 0.9997836689741976, + "wrong_rows": [ + 137, + 3216, + 6631, + 6748, + 7280, + 10703, + 12255, + 13480, + 15214, + 21022, + 23671 + ], + "paired_with_upstream": { + "rows": 24370, + "argmax_agreement": 24370, + "agreement_rate": 1.0, + "reference_correct_candidate_wrong": 0, + "reference_wrong_candidate_correct": 0, + "both_wrong": 11, + "disagreement_rows": [] + }, + "max_abs_probability_error": 0.020487427711486816, + "mean_row_max_abs_probability_error": 4.673297645567538e-05, + "probability_tolerance_violation_rows": [ + 137, + 884, + 2945, + 3216, + 8706, + 9193, + 12725, + 13480, + 13768, + 15698, + 23992 + ], + "conversion_parity_passed": false, + "output_validation_failures": { + "19270": [ + "Live option probabilities do not sum to one" + ] + } + }, + "ane-gather": { + "correct": 24359, + "rows": 24370, + "accuracy": 0.999548625359048, + "model_call_latency_ms": { + "count": 24370, + "median_ms": 1.0515624890103936, + "p95_ms": 1.1766475494368933, + "min_ms": 0.9340409887954593, + "max_ms": 2.8469589888118207 + }, + "load_ms": 570.0647499761544, + "first_call_ms": 1.2669999850913882, + "calibration": { + "nll": 0.002108235388127147, + "ece": 0.000331678613561794, + "equal_width_bins": 15, + "bins": [ + { + "bin": 7, + "count": 2, + "accuracy": 0.5, + "mean_confidence": 0.5224609375 + }, + { + "bin": 8, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.57607421875 + }, + { + "bin": 9, + "count": 6, + "accuracy": 0.8333333333333334, + "mean_confidence": 0.637451171875 + }, + { + "bin": 10, + "count": 5, + "accuracy": 0.8, + "mean_confidence": 0.69248046875 + }, + { + "bin": 11, + "count": 12, + "accuracy": 0.9166666666666666, + "mean_confidence": 0.7687581380208334 + }, + { + "bin": 12, + "count": 20, + "accuracy": 0.95, + "mean_confidence": 0.8322265625 + }, + { + "bin": 13, + "count": 22, + "accuracy": 0.9545454545454546, + "mean_confidence": 0.9176136363636364 + }, + { + "bin": 14, + "count": 24298, + "accuracy": 0.9998353773973166, + "mean_confidence": 0.9998245660008025 + } + ], + "nll_probability_floor": 1.1754943508222875e-38 + }, + "upstream_action_metrics": { + "examples": 24370, + "accuracy": 0.999548625359048, + "coverage": 0.4788264259335248, + "abstention_rate": 0.5211735740664751, + "selective_accuracy": 0.9990573313908647, + "wrong_action_rate": 0.00045137464095199015, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.00045137464095199015, + "counts": { + "total": 24370, + "correct": 24359, + "abstained": 12701, + "acted": 11669, + "acted_correct": 11658, + "wrong_action": 11, + "unsafe_action": 11 + }, + "per_action": { + "abstain": { + "accuracy": 0.9991346758967904, + "abstention_rate": 0.9991346758967904, + "examples": 12712 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 816 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 1040 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 9802 + } + } + }, + "macro_action_accuracy": 0.9997836689741976, + "wrong_rows": [ + 137, + 3216, + 6631, + 6748, + 7280, + 10703, + 12255, + 13480, + 15214, + 21022, + 23671 + ], + "paired_with_upstream": { + "rows": 24370, + "argmax_agreement": 24370, + "agreement_rate": 1.0, + "reference_correct_candidate_wrong": 0, + "reference_wrong_candidate_correct": 0, + "both_wrong": 11, + "disagreement_rows": [] + }, + "max_abs_probability_error": 0.020487427711486816, + "mean_row_max_abs_probability_error": 4.673297645567538e-05, + "probability_tolerance_violation_rows": [ + 137, + 884, + 2945, + 3216, + 8706, + 9193, + 12725, + 13480, + 13768, + 15698, + 23992 + ], + "conversion_parity_passed": false, + "output_validation_failures": { + "19270": [ + "Live option probabilities do not sum to one" + ] + } + } + }, + "failures": [ + { + "row": 137, + "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"Full name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Last: Novak", + "baseline": "fill Last: Novak", + "ane-gather": "fill Last: Novak" + } + }, + { + "row": 884, + "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Health plan\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "ane-gather": "skip" + } + }, + { + "row": 2945, + "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Organization\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "ane-gather": "skip" + } + }, + { + "row": 3216, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Google Chrome\nELEMENT Edit \"Email\" value=\"oivanova66@yahoo.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Email address: oivanova66@yahoo.com", + "baseline": "fill Email address: oivanova66@yahoo.com", + "ane-gather": "fill Email address: oivanova66@yahoo.com" + } + }, + { + "row": 6631, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Google Chrome\nELEMENT Edit \"Given name\" value=\"Priya\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Physician: Dr. Fischer", + "baseline": "fill Physician: Dr. Fischer", + "ane-gather": "fill Physician: Dr. Fischer" + } + }, + { + "row": 6748, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"State / Province\" value=\"NE\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill ST: NE", + "baseline": "fill ST: NE", + "ane-gather": "fill ST: NE" + } + }, + { + "row": 7280, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Given name\" value=\"Tomas\" hint=\"First\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill First name: Tomas", + "baseline": "fill First name: Tomas", + "ane-gather": "fill First name: Tomas" + } + }, + { + "row": 8706, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Mozilla Firefox\nELEMENT CheckBox \"I authorize verification of the information provided\" unchecked", + "gold_option": "check", + "selected_options": { + "upstream_pytorch": "check", + "baseline": "check", + "ane-gather": "check" + } + }, + { + "row": 9193, + "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding\nELEMENT Edit \"DL number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "ane-gather": "skip" + } + }, + { + "row": 10703, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - JevBrowser\nELEMENT Edit \"Last\" value=\"Silva\" hint=\"Family name\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Family name: Silva", + "baseline": "fill Family name: Silva", + "ane-gather": "fill Family name: Silva" + } + }, + { + "row": 12255, + "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form - JevBrowser\nELEMENT Edit \"Taxpayer identification number\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill National ID: 813-20-3361", + "baseline": "fill National ID: 813-20-3361", + "ane-gather": "fill National ID: 813-20-3361" + } + }, + { + "row": 12725, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Company\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "ane-gather": "skip" + } + }, + { + "row": 13480, + "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Current employer\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Company reg: REG3878384", + "baseline": "fill Company reg: REG3878384", + "ane-gather": "fill Company reg: REG3878384" + } + }, + { + "row": 13768, + "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Legal name\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "ane-gather": "skip" + } + }, + { + "row": 15214, + "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Relation to applicant\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Reg plate: WN9-8147", + "baseline": "fill Reg plate: WN9-8147", + "ane-gather": "fill Reg plate: WN9-8147" + } + }, + { + "row": 15698, + "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer - Mozilla Firefox\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "ane-gather": "skip" + } + }, + { + "row": 19270, + "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Plate number\" value=\"old\"", + "gold_option": "fill License plate: SJ3-1115", + "selected_options": { + "upstream_pytorch": "fill License plate: SJ3-1115", + "baseline": "fill License plate: SJ3-1115", + "ane-gather": "fill License plate: SJ3-1115" + } + }, + { + "row": 21022, + "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - JevBrowser\nELEMENT Edit \"State\" value=\"NE\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill National ID: 794-03-1212", + "baseline": "fill National ID: 794-03-1212", + "ane-gather": "fill National ID: 794-03-1212" + } + }, + { + "row": 23671, + "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Email\" value=\"lchen6@yahoo.com\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "fill Email: lchen6@yahoo.com", + "baseline": "fill Email: lchen6@yahoo.com", + "ane-gather": "fill Email: lchen6@yahoo.com" + } + }, + { + "row": 23992, + "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Microsoft Edge\nELEMENT Edit \"Total years experience\" value=\"\"", + "gold_option": "skip", + "selected_options": { + "upstream_pytorch": "skip", + "baseline": "skip", + "ane-gather": "skip" + } + } + ], + "trace": { + "file": "synthetic-test-decisions.jsonl.gz", + "sha256": "b6f6edc9de34fb37b10847a1f9568b530ce2b8e1bd241f51bf2e637f489d6554", + "rows": 24370 + }, + "upstream_evaluator_sha256": "49dccdeb4e2b456187cdc4491a6d78fabac6a45384272f8e3ec802a58872aa6e", + "harness_sha256": { + "benchmark-synthetic.py": "96b75133b38735594e7b5483cec0f39a5f9868045d0e7438434df8af698215fa", + "synthetic_test.py": "e2b68725236e6827eb422480c8cc8aa189b2490c1594650d8f2928771d1bc626", + "score-report.py": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e" + }, + "conversion_parity_passed": false, + "limitations": [ + "The release file has 24370 rows; the model card claims approximately 15000. This is not a reconstruction of that unspecified manifest.", + "Upstream describes form-disjoint synthetic data; this is not an unseen real-world GUI or task-completion benchmark.", + "No fresh hosted Jev calls are included. Published hosted-Jev percentages are not a matched baseline for this run.", + "Local timings use CPU for PyTorch and CPU+ANE for Core ML; they are backend/deployment measurements, not equal-hardware algorithm comparisons." + ] +} diff --git a/reports/upstream-metrics.json b/reports/upstream-metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c9cd3f7833a3ac1ea2f0c1bf1a6b14eed82adb27 --- /dev/null +++ b/reports/upstream-metrics.json @@ -0,0 +1,128 @@ +{ + "verification_report_sha256": "9a5d22ec4b447f0710c12e03c69d7ed802158bb94e9a3c6b059731730ced9f30", + "evaluator_source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "evaluator_source_path": "libs/cua-s1/evals/metrics.py", + "evaluator_sha256": "49dccdeb4e2b456187cdc4491a6d78fabac6a45384272f8e3ec802a58872aa6e", + "adapter_sha256": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e", + "scope": "Offline action/target classification, not GUI execution or task completion", + "normalization": "Stable row IDs; fill target is entity option index; upstream treats skip as abstention", + "results": { + "ALL": { + "examples": 196, + "accuracy": 1.0, + "coverage": 0.23469387755102042, + "abstention_rate": 0.7653061224489796, + "selective_accuracy": 1.0, + "wrong_action_rate": 0.0, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.0, + "counts": { + "total": 196, + "correct": 196, + "abstained": 150, + "acted": 46, + "acted_correct": 46 + }, + "per_action": { + "abstain": { + "accuracy": 1.0, + "abstention_rate": 1.0, + "examples": 150 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 4 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 6 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 36 + } + } + }, + "CPU_AND_NE": { + "examples": 196, + "accuracy": 1.0, + "coverage": 0.23469387755102042, + "abstention_rate": 0.7653061224489796, + "selective_accuracy": 1.0, + "wrong_action_rate": 0.0, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.0, + "counts": { + "total": 196, + "correct": 196, + "abstained": 150, + "acted": 46, + "acted_correct": 46 + }, + "per_action": { + "abstain": { + "accuracy": 1.0, + "abstention_rate": 1.0, + "examples": 150 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 4 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 6 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 36 + } + } + }, + "upstream_pytorch": { + "examples": 196, + "accuracy": 1.0, + "coverage": 0.23469387755102042, + "abstention_rate": 0.7653061224489796, + "selective_accuracy": 1.0, + "wrong_action_rate": 0.0, + "wrong_target_rate": 0.0, + "unsafe_action_rate": 0.0, + "counts": { + "total": 196, + "correct": 196, + "abstained": 150, + "acted": 46, + "acted_correct": 46 + }, + "per_action": { + "abstain": { + "accuracy": 1.0, + "abstention_rate": 1.0, + "examples": 150 + }, + "check": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 4 + }, + "click": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 6 + }, + "fill": { + "accuracy": 1.0, + "abstention_rate": 0.0, + "examples": 36 + } + } + } + } +} diff --git a/reports/verification.json b/reports/verification.json new file mode 100644 index 0000000000000000000000000000000000000000..a757c5fa6cc4149b60eef1317e605796a42bc725 --- /dev/null +++ b/reports/verification.json @@ -0,0 +1,3309 @@ +{ + "purpose": "Local conversion parity on the upstream demo; not a generalization or live GUI benchmark", + "created_utc": "2026-09-19T16:49:43.157197+00:00", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "dataset_file": "demo.jsonl", + "dataset_sha256": "4f43b442e79ba2e2ce731e27e9b8e340c2b5dfcaffc92d8ff564c34f115ff1ca", + "conversion": { + "model": "cua_s1_forms_fp16_options32.mlpackage", + "precision": "float16", + "minimum_target": "iOS17/macOS14", + "limits": { + "context_bytes": 224, + "option_bytes": 96, + "max_options": 32 + }, + "model_config": { + "context_tokens": 224, + "encoder": "tinyx", + "heads": 4, + "hf_model": "Qwen/Qwen2.5-0.5B", + "layers": 2, + "option_tokens": 96, + "rank": 128, + "width": 128 + }, + "parameters": 706048, + "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", + "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", + "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", + "trace_row": 0, + "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "export_seconds": 0.5374167499830946, + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "package_files": { + "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", + "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", + "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" + } + }, + "environment": { + "machine": "Apple M5 Pro", + "os": "27.0", + "python": "3.11.11", + "torch": "2.7.0", + "coremltools": "9.0", + "torch_cpu_threads": 2 + }, + "rows": 196, + "upstream": { + "correct": 196, + "accuracy": 1.0, + "inference_latency_ms": { + "count": 196, + "median_ms": 2.4375420034630224, + "p95_ms": 3.0990312443464063, + "min_ms": 1.6581250238232315, + "max_ms": 3.908416983904317 + } + }, + "export_adapter_fp32": { + "argmax_agreement": 196, + "max_abs_probability_error": 1.1324882507324219e-06 + }, + "host_preprocessing_latency_ms": { + "count": 196, + "median_ms": 0.03766651207115501, + "p95_ms": 0.04735450784210116, + "min_ms": 0.026416993932798505, + "max_ms": 0.07812498370185494 + }, + "thresholds": { + "argmax_agreement": 1.0, + "max_abs_probability_error": 0.005, + "export_adapter_max_abs_probability_error": 0.0001, + "allow_accuracy_loss": false + }, + "backends": { + "ALL": { + "passed": true, + "correct": 196, + "accuracy": 1.0, + "argmax_agreement": 196, + "max_abs_probability_error": 0.0030989646911621094, + "mean_row_max_abs_probability_error": 1.6251600213152587e-05, + "max_abs_logit_error": 0.03653383255004883, + "load_seconds": 0.1422372910019476, + "warm_inference_latency_ms": { + "count": 196, + "median_ms": 1.8487705092411488, + "p95_ms": 2.4909165003919043, + "min_ms": 1.7662910104263574, + "max_ms": 2.893916011089459 + }, + "reversed_option_order": { + "rows": 6, + "max_abs_probability_error": 5.26619260199368e-07 + }, + "per_action": { + "fill": { + "rows": 36, + "correct": 36 + }, + "skip": { + "rows": 150, + "correct": 150 + }, + "check": { + "rows": 4, + "correct": 4 + }, + "click": { + "rows": 6, + "correct": 6 + } + }, + "decisions": [ + { + "row": 0, + "label": 22, + "upstream": 22, + "coreml": 22, + "max_abs_probability_error": 5.26619260199368e-07, + "max_abs_logit_error": 0.017948150634765625 + }, + { + "row": 1, + "label": 23, + "upstream": 23, + "coreml": 23, + "max_abs_probability_error": 5.320471245795488e-07, + "max_abs_logit_error": 0.0173797607421875 + }, + { + "row": 2, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 3.134431381113245e-09, + "max_abs_logit_error": 0.02239227294921875 + }, + { + "row": 3, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 2.8436625143513083e-07, + "max_abs_logit_error": 0.014525413513183594 + }, + { + "row": 4, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.013245582580566406 + }, + { + "row": 5, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 3.197783371433616e-08, + "max_abs_logit_error": 0.01342916488647461 + }, + { + "row": 6, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 1.7993625078815967e-07, + "max_abs_logit_error": 0.011149883270263672 + }, + { + "row": 7, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 6.677582859992981e-07, + "max_abs_logit_error": 0.013294458389282227 + }, + { + "row": 8, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 4.183561941317748e-09, + "max_abs_logit_error": 0.013601541519165039 + }, + { + "row": 9, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 4.019384505227208e-07, + "max_abs_logit_error": 0.029033660888671875 + }, + { + "row": 10, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 2.5259510039177258e-09, + "max_abs_logit_error": 0.03653383255004883 + }, + { + "row": 11, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.026096343994140625 + }, + { + "row": 12, + "label": 15, + "upstream": 15, + "coreml": 15, + "max_abs_probability_error": 4.578259904519655e-08, + "max_abs_logit_error": 0.029152870178222656 + }, + { + "row": 13, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 8.465372957289219e-06, + "max_abs_logit_error": 0.01841115951538086 + }, + { + "row": 14, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.020061016082763672 + }, + { + "row": 15, + "label": 24, + "upstream": 24, + "coreml": 24, + "max_abs_probability_error": 8.867027645464987e-08, + "max_abs_logit_error": 0.013807952404022217 + }, + { + "row": 16, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.60369528557203e-10, + "max_abs_logit_error": 0.011376380920410156 + }, + { + "row": 17, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 3.9392489270539954e-11, + "max_abs_logit_error": 0.02391815185546875 + }, + { + "row": 18, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.761169319384862e-11, + "max_abs_logit_error": 0.008268356323242188 + }, + { + "row": 19, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.685274307940745e-12, + "max_abs_logit_error": 0.012112617492675781 + }, + { + "row": 20, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.197389020215269e-10, + "max_abs_logit_error": 0.012657880783081055 + }, + { + "row": 21, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.3100631690576847e-13, + "max_abs_logit_error": 0.02386617660522461 + }, + { + "row": 22, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.716319613682572e-09, + "max_abs_logit_error": 0.027971267700195312 + }, + { + "row": 23, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.720178357179975e-09, + "max_abs_logit_error": 0.026054739952087402 + }, + { + "row": 24, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.5336516778763354e-13, + "max_abs_logit_error": 0.012665033340454102 + }, + { + "row": 25, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 9.446710080851517e-11, + "max_abs_logit_error": 0.014183402061462402 + }, + { + "row": 26, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.373890138964498e-12, + "max_abs_logit_error": 0.009427070617675781 + }, + { + "row": 27, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.2128106558791956e-13, + "max_abs_logit_error": 0.016583919525146484 + }, + { + "row": 28, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.1457235160605705e-10, + "max_abs_logit_error": 0.012380242347717285 + }, + { + "row": 29, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.6885971787505838e-13, + "max_abs_logit_error": 0.01409292221069336 + }, + { + "row": 30, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.038820750589366e-09, + "max_abs_logit_error": 0.023344039916992188 + }, + { + "row": 31, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.535194276731545e-12, + "max_abs_logit_error": 0.019451618194580078 + }, + { + "row": 32, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.1524559084818975e-11, + "max_abs_logit_error": 0.016178369522094727 + }, + { + "row": 33, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.133360568900571e-11, + "max_abs_logit_error": 0.016007661819458008 + }, + { + "row": 34, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.463859558105469e-06, + "max_abs_logit_error": 0.021859407424926758 + }, + { + "row": 35, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.966427434917932e-12, + "max_abs_logit_error": 0.012809276580810547 + }, + { + "row": 36, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.978734008924789e-12, + "max_abs_logit_error": 0.011217594146728516 + }, + { + "row": 37, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.058104880437895e-10, + "max_abs_logit_error": 0.015135049819946289 + }, + { + "row": 38, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.1240661179634515e-12, + "max_abs_logit_error": 0.013502120971679688 + }, + { + "row": 39, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.3960612100258913e-12, + "max_abs_logit_error": 0.015411376953125 + }, + { + "row": 40, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.713369369506836e-05, + "max_abs_logit_error": 0.02988719940185547 + }, + { + "row": 41, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.836255592024827e-10, + "max_abs_logit_error": 0.017060812562704086 + }, + { + "row": 42, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5000002068509275e-12, + "max_abs_logit_error": 0.009338855743408203 + }, + { + "row": 43, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.031253304972779e-08, + "max_abs_logit_error": 0.013645172119140625 + }, + { + "row": 44, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.2869124089174306e-12, + "max_abs_logit_error": 0.014712810516357422 + }, + { + "row": 45, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.582760394442829e-13, + "max_abs_logit_error": 0.01000899076461792 + }, + { + "row": 46, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.987421504092481e-12, + "max_abs_logit_error": 0.008915901184082031 + }, + { + "row": 47, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.5084782301630923e-13, + "max_abs_logit_error": 0.012367963790893555 + }, + { + "row": 48, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.020061016082763672 + }, + { + "row": 49, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.312366881833782e-12, + "max_abs_logit_error": 0.02675628662109375 + }, + { + "row": 50, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.60369528557203e-10, + "max_abs_logit_error": 0.011376380920410156 + }, + { + "row": 51, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 3.9392489270539954e-11, + "max_abs_logit_error": 0.02391815185546875 + }, + { + "row": 52, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.761169319384862e-11, + "max_abs_logit_error": 0.008268356323242188 + }, + { + "row": 53, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.685274307940745e-12, + "max_abs_logit_error": 0.012112617492675781 + }, + { + "row": 54, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.197389020215269e-10, + "max_abs_logit_error": 0.012657880783081055 + }, + { + "row": 55, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.3100631690576847e-13, + "max_abs_logit_error": 0.02386617660522461 + }, + { + "row": 56, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.716319613682572e-09, + "max_abs_logit_error": 0.027971267700195312 + }, + { + "row": 57, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.720178357179975e-09, + "max_abs_logit_error": 0.026054739952087402 + }, + { + "row": 58, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.5336516778763354e-13, + "max_abs_logit_error": 0.012665033340454102 + }, + { + "row": 59, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 9.446710080851517e-11, + "max_abs_logit_error": 0.014183402061462402 + }, + { + "row": 60, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.373890138964498e-12, + "max_abs_logit_error": 0.009427070617675781 + }, + { + "row": 61, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.2128106558791956e-13, + "max_abs_logit_error": 0.016583919525146484 + }, + { + "row": 62, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.1457235160605705e-10, + "max_abs_logit_error": 0.012380242347717285 + }, + { + "row": 63, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.6885971787505838e-13, + "max_abs_logit_error": 0.01409292221069336 + }, + { + "row": 64, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.038820750589366e-09, + "max_abs_logit_error": 0.023344039916992188 + }, + { + "row": 65, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.535194276731545e-12, + "max_abs_logit_error": 0.019451618194580078 + }, + { + "row": 66, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.1524559084818975e-11, + "max_abs_logit_error": 0.016178369522094727 + }, + { + "row": 67, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.133360568900571e-11, + "max_abs_logit_error": 0.016007661819458008 + }, + { + "row": 68, + "label": 0, + "upstream": 0, + "coreml": 0, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.02204132080078125 + }, + { + "row": 69, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 5.992624210193753e-07, + "max_abs_logit_error": 0.01074361801147461 + }, + { + "row": 70, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.6200374375330284e-09, + "max_abs_logit_error": 0.012182235717773438 + }, + { + "row": 71, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 7.649177860002965e-07, + "max_abs_logit_error": 0.021416664123535156 + }, + { + "row": 72, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 1.716807673801668e-08, + "max_abs_logit_error": 0.015079259872436523 + }, + { + "row": 73, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.022822856903076172 + }, + { + "row": 74, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 1.3113021850585938e-06, + "max_abs_logit_error": 0.02725982666015625 + }, + { + "row": 75, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 8.482402336085215e-08, + "max_abs_logit_error": 0.01822662353515625 + }, + { + "row": 76, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 4.19224761571968e-08, + "max_abs_logit_error": 0.027760028839111328 + }, + { + "row": 77, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 6.794929504394531e-06, + "max_abs_logit_error": 0.02509307861328125 + }, + { + "row": 78, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.013262271881103516 + }, + { + "row": 79, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.902155630290508e-06, + "max_abs_logit_error": 0.019384384155273438 + }, + { + "row": 80, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.386720320326276e-09, + "max_abs_logit_error": 0.020812511444091797 + }, + { + "row": 81, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5509287954482716e-08, + "max_abs_logit_error": 0.033596038818359375 + }, + { + "row": 82, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 1.1210374850634253e-09, + "max_abs_logit_error": 0.021764755249023438 + }, + { + "row": 83, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.9104718635153404e-11, + "max_abs_logit_error": 0.017470359802246094 + }, + { + "row": 84, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.615042487581377e-09, + "max_abs_logit_error": 0.011733055114746094 + }, + { + "row": 85, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5568436257117355e-12, + "max_abs_logit_error": 0.018376827239990234 + }, + { + "row": 86, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.652814939414384e-09, + "max_abs_logit_error": 0.013185739517211914 + }, + { + "row": 87, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 8.637570658720506e-10, + "max_abs_logit_error": 0.010789871215820312 + }, + { + "row": 88, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.6595947666407938e-11, + "max_abs_logit_error": 0.022840499877929688 + }, + { + "row": 89, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.6561418902938385e-11, + "max_abs_logit_error": 0.009737014770507812 + }, + { + "row": 90, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1621037465658901e-11, + "max_abs_logit_error": 0.01908111572265625 + }, + { + "row": 91, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 8.295308884243013e-13, + "max_abs_logit_error": 0.016066551208496094 + }, + { + "row": 92, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.485815372987865e-14, + "max_abs_logit_error": 0.01638507843017578 + }, + { + "row": 93, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.78640222006743e-10, + "max_abs_logit_error": 0.024122238159179688 + }, + { + "row": 94, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.6049052631881295e-11, + "max_abs_logit_error": 0.0227203369140625 + }, + { + "row": 95, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 7.636086207796211e-12, + "max_abs_logit_error": 0.021586894989013672 + }, + { + "row": 96, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5383029012004954e-12, + "max_abs_logit_error": 0.019191741943359375 + }, + { + "row": 97, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.6573055695043877e-09, + "max_abs_logit_error": 0.016190528869628906 + }, + { + "row": 98, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.6824872040597256e-09, + "max_abs_logit_error": 0.016640663146972656 + }, + { + "row": 99, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 7.239464139274787e-09, + "max_abs_logit_error": 0.012330055236816406 + }, + { + "row": 100, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.094208053284092e-09, + "max_abs_logit_error": 0.024295806884765625 + }, + { + "row": 101, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.488498461796553e-11, + "max_abs_logit_error": 0.010948657989501953 + }, + { + "row": 102, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.78730405212491e-11, + "max_abs_logit_error": 0.0258026123046875 + }, + { + "row": 103, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.291534423828125e-06, + "max_abs_logit_error": 0.013744831085205078 + }, + { + "row": 104, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.948583409714047e-08, + "max_abs_logit_error": 0.02979755401611328 + }, + { + "row": 105, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.98279237052202e-12, + "max_abs_logit_error": 0.012142181396484375 + }, + { + "row": 106, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.4030777368011513e-11, + "max_abs_logit_error": 0.013262748718261719 + }, + { + "row": 107, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.511102437097179e-12, + "max_abs_logit_error": 0.018053054809570312 + }, + { + "row": 108, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.8072548730051494e-09, + "max_abs_logit_error": 0.024975299835205078 + }, + { + "row": 109, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.013262271881103516 + }, + { + "row": 110, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.9665898182429373e-08, + "max_abs_logit_error": 0.02109050750732422 + }, + { + "row": 111, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.0117645263671875 + }, + { + "row": 112, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5509287954482716e-08, + "max_abs_logit_error": 0.033596038818359375 + }, + { + "row": 113, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 1.1210374850634253e-09, + "max_abs_logit_error": 0.021764755249023438 + }, + { + "row": 114, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.9104718635153404e-11, + "max_abs_logit_error": 0.017470359802246094 + }, + { + "row": 115, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.615042487581377e-09, + "max_abs_logit_error": 0.011733055114746094 + }, + { + "row": 116, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5568436257117355e-12, + "max_abs_logit_error": 0.018376827239990234 + }, + { + "row": 117, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.652814939414384e-09, + "max_abs_logit_error": 0.013185739517211914 + }, + { + "row": 118, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 8.637570658720506e-10, + "max_abs_logit_error": 0.010789871215820312 + }, + { + "row": 119, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.6595947666407938e-11, + "max_abs_logit_error": 0.022840499877929688 + }, + { + "row": 120, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.6561418902938385e-11, + "max_abs_logit_error": 0.009737014770507812 + }, + { + "row": 121, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1621037465658901e-11, + "max_abs_logit_error": 0.01908111572265625 + }, + { + "row": 122, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 8.295308884243013e-13, + "max_abs_logit_error": 0.016066551208496094 + }, + { + "row": 123, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.485815372987865e-14, + "max_abs_logit_error": 0.01638507843017578 + }, + { + "row": 124, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.78640222006743e-10, + "max_abs_logit_error": 0.024122238159179688 + }, + { + "row": 125, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.6049052631881295e-11, + "max_abs_logit_error": 0.0227203369140625 + }, + { + "row": 126, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 7.636086207796211e-12, + "max_abs_logit_error": 0.021586894989013672 + }, + { + "row": 127, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5383029012004954e-12, + "max_abs_logit_error": 0.019191741943359375 + }, + { + "row": 128, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.6573055695043877e-09, + "max_abs_logit_error": 0.016190528869628906 + }, + { + "row": 129, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.6824872040597256e-09, + "max_abs_logit_error": 0.016640663146972656 + }, + { + "row": 130, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.027278900146484375 + }, + { + "row": 131, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.3382816632656613e-08, + "max_abs_logit_error": 0.016567230224609375 + }, + { + "row": 132, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 4.5742126530967653e-07, + "max_abs_logit_error": 0.025929927825927734 + }, + { + "row": 133, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.017045140266418457 + }, + { + "row": 134, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.017194747924804688 + }, + { + "row": 135, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 3.5762786865234375e-07, + "max_abs_logit_error": 0.026395797729492188 + }, + { + "row": 136, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.020420074462890625 + }, + { + "row": 137, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.0030989646911621094, + "max_abs_logit_error": 0.021968841552734375 + }, + { + "row": 138, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 1.7425772966817021e-06, + "max_abs_logit_error": 0.010594367980957031 + }, + { + "row": 139, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.015566825866699219 + }, + { + "row": 140, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 2.733497694862308e-08, + "max_abs_logit_error": 0.020360946655273438 + }, + { + "row": 141, + "label": 12, + "upstream": 12, + "coreml": 12, + "max_abs_probability_error": 8.882998372428119e-07, + "max_abs_logit_error": 0.01713848114013672 + }, + { + "row": 142, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.821654435247183e-11, + "max_abs_logit_error": 0.016475677490234375 + }, + { + "row": 143, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 6.903178473294247e-09, + "max_abs_logit_error": 0.02091217041015625 + }, + { + "row": 144, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9354274627403356e-08, + "max_abs_logit_error": 0.014428257942199707 + }, + { + "row": 145, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.937020665034652e-08, + "max_abs_logit_error": 0.013593673706054688 + }, + { + "row": 146, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.295774154641549e-09, + "max_abs_logit_error": 0.03515338897705078 + }, + { + "row": 147, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.638334078459593e-11, + "max_abs_logit_error": 0.016902923583984375 + }, + { + "row": 148, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.354898415684147e-10, + "max_abs_logit_error": 0.007921218872070312 + }, + { + "row": 149, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.4544090638254374e-12, + "max_abs_logit_error": 0.018396615982055664 + }, + { + "row": 150, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.2619059403261588e-14, + "max_abs_logit_error": 0.019584178924560547 + }, + { + "row": 151, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.77121683775772e-12, + "max_abs_logit_error": 0.01009368896484375 + }, + { + "row": 152, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.4402368186949843e-12, + "max_abs_logit_error": 0.016597270965576172 + }, + { + "row": 153, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.3069191001923173e-10, + "max_abs_logit_error": 0.023260116577148438 + }, + { + "row": 154, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.5346018606728933e-13, + "max_abs_logit_error": 0.023482322692871094 + }, + { + "row": 155, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2430889650971721e-12, + "max_abs_logit_error": 0.013761520385742188 + }, + { + "row": 156, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.12792326762068e-14, + "max_abs_logit_error": 0.013568401336669922 + }, + { + "row": 157, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.5832196664788967e-12, + "max_abs_logit_error": 0.013137876987457275 + }, + { + "row": 158, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.681888127715638e-12, + "max_abs_logit_error": 0.026842832565307617 + }, + { + "row": 159, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.731503829205394e-11, + "max_abs_logit_error": 0.015740394592285156 + }, + { + "row": 160, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0416112417033219e-10, + "max_abs_logit_error": 0.01055908203125 + }, + { + "row": 161, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.944752163282828e-09, + "max_abs_logit_error": 0.01996326446533203 + }, + { + "row": 162, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.288525007860699e-11, + "max_abs_logit_error": 0.011535167694091797 + }, + { + "row": 163, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.579597977562912e-13, + "max_abs_logit_error": 0.010066986083984375 + }, + { + "row": 164, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.8768099841490766e-11, + "max_abs_logit_error": 0.014286994934082031 + }, + { + "row": 165, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6796633528493032e-13, + "max_abs_logit_error": 0.014059066772460938 + }, + { + "row": 166, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0951239914902544e-11, + "max_abs_logit_error": 0.012742996215820312 + }, + { + "row": 167, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.0552340091769565e-12, + "max_abs_logit_error": 0.017006397247314453 + }, + { + "row": 168, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.349431037553586e-09, + "max_abs_logit_error": 0.02781963348388672 + }, + { + "row": 169, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.7824977605052084e-13, + "max_abs_logit_error": 0.022431373596191406 + }, + { + "row": 170, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.2377225900326323e-13, + "max_abs_logit_error": 0.006749153137207031 + }, + { + "row": 171, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.13179953081999e-08, + "max_abs_logit_error": 0.015783309936523438 + }, + { + "row": 172, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.681899312686255e-11, + "max_abs_logit_error": 0.013919830322265625 + }, + { + "row": 173, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.2486568163060383e-11, + "max_abs_logit_error": 0.016487598419189453 + }, + { + "row": 174, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.743627630115043e-11, + "max_abs_logit_error": 0.02295398712158203 + }, + { + "row": 175, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.821654435247183e-11, + "max_abs_logit_error": 0.016475677490234375 + }, + { + "row": 176, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1420979717513546e-09, + "max_abs_logit_error": 0.028388500213623047 + }, + { + "row": 177, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.5013057463875157e-10, + "max_abs_logit_error": 0.015703678131103516 + }, + { + "row": 178, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.937020665034652e-08, + "max_abs_logit_error": 0.013593673706054688 + }, + { + "row": 179, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.295774154641549e-09, + "max_abs_logit_error": 0.03515338897705078 + }, + { + "row": 180, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.638334078459593e-11, + "max_abs_logit_error": 0.016902923583984375 + }, + { + "row": 181, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.354898415684147e-10, + "max_abs_logit_error": 0.007921218872070312 + }, + { + "row": 182, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.4544090638254374e-12, + "max_abs_logit_error": 0.018396615982055664 + }, + { + "row": 183, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.2619059403261588e-14, + "max_abs_logit_error": 0.019584178924560547 + }, + { + "row": 184, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.77121683775772e-12, + "max_abs_logit_error": 0.01009368896484375 + }, + { + "row": 185, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.4402368186949843e-12, + "max_abs_logit_error": 0.016597270965576172 + }, + { + "row": 186, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.3069191001923173e-10, + "max_abs_logit_error": 0.023260116577148438 + }, + { + "row": 187, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.5346018606728933e-13, + "max_abs_logit_error": 0.023482322692871094 + }, + { + "row": 188, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2430889650971721e-12, + "max_abs_logit_error": 0.013761520385742188 + }, + { + "row": 189, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.12792326762068e-14, + "max_abs_logit_error": 0.013568401336669922 + }, + { + "row": 190, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.5832196664788967e-12, + "max_abs_logit_error": 0.013137876987457275 + }, + { + "row": 191, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.681888127715638e-12, + "max_abs_logit_error": 0.026842832565307617 + }, + { + "row": 192, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.731503829205394e-11, + "max_abs_logit_error": 0.015740394592285156 + }, + { + "row": 193, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0416112417033219e-10, + "max_abs_logit_error": 0.01055908203125 + }, + { + "row": 194, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.944752163282828e-09, + "max_abs_logit_error": 0.01996326446533203 + }, + { + "row": 195, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.288525007860699e-11, + "max_abs_logit_error": 0.011535167694091797 + } + ] + }, + "CPU_AND_NE": { + "passed": true, + "correct": 196, + "accuracy": 1.0, + "argmax_agreement": 196, + "max_abs_probability_error": 0.00233614444732666, + "mean_row_max_abs_probability_error": 2.934475840582507e-05, + "max_abs_logit_error": 0.07230281829833984, + "load_seconds": 0.578660333994776, + "warm_inference_latency_ms": { + "count": 196, + "median_ms": 0.9031249937834218, + "p95_ms": 0.9396872410434298, + "min_ms": 0.8796670008450747, + "max_ms": 0.9663329983595759 + }, + "reversed_option_order": { + "rows": 6, + "max_abs_probability_error": 3.1828880310058594e-05 + }, + "per_action": { + "fill": { + "rows": 36, + "correct": 36 + }, + "skip": { + "rows": 150, + "correct": 150 + }, + "check": { + "rows": 4, + "correct": 4 + }, + "click": { + "rows": 6, + "correct": 6 + } + }, + "decisions": [ + { + "row": 0, + "label": 22, + "upstream": 22, + "coreml": 22, + "max_abs_probability_error": 3.1828880310058594e-05, + "max_abs_logit_error": 0.03086566925048828 + }, + { + "row": 1, + "label": 23, + "upstream": 23, + "coreml": 23, + "max_abs_probability_error": 3.814697265625e-05, + "max_abs_logit_error": 0.020841598510742188 + }, + { + "row": 2, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.024377822875976562 + }, + { + "row": 3, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 3.218650817871094e-05, + "max_abs_logit_error": 0.030775785446166992 + }, + { + "row": 4, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 2.944469451904297e-05, + "max_abs_logit_error": 0.03573417663574219 + }, + { + "row": 5, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2636184692382812e-05, + "max_abs_logit_error": 0.026729822158813477 + }, + { + "row": 6, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 2.3365020751953125e-05, + "max_abs_logit_error": 0.023685455322265625 + }, + { + "row": 7, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.00014638900756835938, + "max_abs_logit_error": 0.04038810729980469 + }, + { + "row": 8, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.02460002899169922 + }, + { + "row": 9, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 2.1457672119140625e-05, + "max_abs_logit_error": 0.018064022064208984 + }, + { + "row": 10, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.047691673040390015 + }, + { + "row": 11, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 3.2186508178710938e-06, + "max_abs_logit_error": 0.03340768814086914 + }, + { + "row": 12, + "label": 15, + "upstream": 15, + "coreml": 15, + "max_abs_probability_error": 2.384185791015625e-06, + "max_abs_logit_error": 0.053191184997558594 + }, + { + "row": 13, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 0.0005360841751098633, + "max_abs_logit_error": 0.03362560272216797 + }, + { + "row": 14, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.024669170379638672 + }, + { + "row": 15, + "label": 24, + "upstream": 24, + "coreml": 24, + "max_abs_probability_error": 1.6689300537109375e-05, + "max_abs_logit_error": 0.034501075744628906 + }, + { + "row": 16, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.033290743827819824 + }, + { + "row": 17, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03323173522949219 + }, + { + "row": 18, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.022179126739501953 + }, + { + "row": 19, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.029880523681640625 + }, + { + "row": 20, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.031145095825195312 + }, + { + "row": 21, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.03861522674560547 + }, + { + "row": 22, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.027164459228515625 + }, + { + "row": 23, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.058716535568237305 + }, + { + "row": 24, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.035381317138671875 + }, + { + "row": 25, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.04862499237060547 + }, + { + "row": 26, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.029034733772277832 + }, + { + "row": 27, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.03808116912841797 + }, + { + "row": 28, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.03319191932678223 + }, + { + "row": 29, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.030045509338378906 + }, + { + "row": 30, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026231765747070312 + }, + { + "row": 31, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.033112093806266785 + }, + { + "row": 32, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.022128582000732422 + }, + { + "row": 33, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.038306236267089844 + }, + { + "row": 34, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.0004820823669433594, + "max_abs_logit_error": 0.032683372497558594 + }, + { + "row": 35, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.462750809366753e-09, + "max_abs_logit_error": 0.029730796813964844 + }, + { + "row": 36, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.7510341227277024e-10, + "max_abs_logit_error": 0.02333354949951172 + }, + { + "row": 37, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.657592190757896e-08, + "max_abs_logit_error": 0.022855758666992188 + }, + { + "row": 38, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 9.552543478452691e-11, + "max_abs_logit_error": 0.023164749145507812 + }, + { + "row": 39, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 4.04716482549361e-10, + "max_abs_logit_error": 0.02880859375 + }, + { + "row": 40, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 0.00021719932556152344, + "max_abs_logit_error": 0.04735088348388672 + }, + { + "row": 41, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.146233744175333e-09, + "max_abs_logit_error": 0.03671276569366455 + }, + { + "row": 42, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8819494851385343e-09, + "max_abs_logit_error": 0.023815155029296875 + }, + { + "row": 43, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.02039623260498047 + }, + { + "row": 44, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.780525865635596e-10, + "max_abs_logit_error": 0.022927284240722656 + }, + { + "row": 45, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.414331317674879e-11, + "max_abs_logit_error": 0.030417919158935547 + }, + { + "row": 46, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.599257724142603e-10, + "max_abs_logit_error": 0.025608062744140625 + }, + { + "row": 47, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.212314607705437e-11, + "max_abs_logit_error": 0.022733211517333984 + }, + { + "row": 48, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.8596649169921875e-05, + "max_abs_logit_error": 0.024669170379638672 + }, + { + "row": 49, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.932842080831776e-10, + "max_abs_logit_error": 0.030317306518554688 + }, + { + "row": 50, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.440092489701783e-08, + "max_abs_logit_error": 0.033290743827819824 + }, + { + "row": 51, + "label": 25, + "upstream": 25, + "coreml": 25, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03323173522949219 + }, + { + "row": 52, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 8.430758313693332e-09, + "max_abs_logit_error": 0.022179126739501953 + }, + { + "row": 53, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.0266471551243512e-09, + "max_abs_logit_error": 0.029880523681640625 + }, + { + "row": 54, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4500966588902884e-08, + "max_abs_logit_error": 0.031145095825195312 + }, + { + "row": 55, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 1.647379281599637e-11, + "max_abs_logit_error": 0.03861522674560547 + }, + { + "row": 56, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.027164459228515625 + }, + { + "row": 57, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.960464477539062e-07, + "max_abs_logit_error": 0.058716535568237305 + }, + { + "row": 58, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.5055844551924444e-11, + "max_abs_logit_error": 0.035381317138671875 + }, + { + "row": 59, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.995705792434137e-09, + "max_abs_logit_error": 0.04862499237060547 + }, + { + "row": 60, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.470584068551943e-10, + "max_abs_logit_error": 0.029034733772277832 + }, + { + "row": 61, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.973024935241767e-11, + "max_abs_logit_error": 0.03808116912841797 + }, + { + "row": 62, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 5.994321927715873e-09, + "max_abs_logit_error": 0.03319191932678223 + }, + { + "row": 63, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.4426039316183257e-11, + "max_abs_logit_error": 0.030045509338378906 + }, + { + "row": 64, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026231765747070312 + }, + { + "row": 65, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 7.78299757975276e-10, + "max_abs_logit_error": 0.033112093806266785 + }, + { + "row": 66, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 6.185003176284454e-09, + "max_abs_logit_error": 0.022128582000732422 + }, + { + "row": 67, + "label": 26, + "upstream": 26, + "coreml": 26, + "max_abs_probability_error": 3.4556297823229443e-09, + "max_abs_logit_error": 0.038306236267089844 + }, + { + "row": 68, + "label": 0, + "upstream": 0, + "coreml": 0, + "max_abs_probability_error": 3.147125244140625e-05, + "max_abs_logit_error": 0.03265953063964844 + }, + { + "row": 69, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 5.030632019042969e-05, + "max_abs_logit_error": 0.038794517517089844 + }, + { + "row": 70, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.040470123291015625 + }, + { + "row": 71, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 5.2928924560546875e-05, + "max_abs_logit_error": 0.040035247802734375 + }, + { + "row": 72, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 4.649162292480469e-06, + "max_abs_logit_error": 0.042186737060546875 + }, + { + "row": 73, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 2.1576881408691406e-05, + "max_abs_logit_error": 0.033161163330078125 + }, + { + "row": 74, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 4.470348358154297e-05, + "max_abs_logit_error": 0.0293731689453125 + }, + { + "row": 75, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 1.1563301086425781e-05, + "max_abs_logit_error": 0.03249359130859375 + }, + { + "row": 76, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 2.86102294921875e-06, + "max_abs_logit_error": 0.03498554229736328 + }, + { + "row": 77, + "label": 13, + "upstream": 13, + "coreml": 13, + "max_abs_probability_error": 0.0003635883331298828, + "max_abs_logit_error": 0.023473739624023438 + }, + { + "row": 78, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.034421443939208984 + }, + { + "row": 79, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 0.0003609657287597656, + "max_abs_logit_error": 0.03937721252441406 + }, + { + "row": 80, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.03752422332763672 + }, + { + "row": 81, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.025417327880859375 + }, + { + "row": 82, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.04453086853027344 + }, + { + "row": 83, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.023950576782226562 + }, + { + "row": 84, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.022706031799316406 + }, + { + "row": 85, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.035375118255615234 + }, + { + "row": 86, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.03717994689941406 + }, + { + "row": 87, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.04058122634887695 + }, + { + "row": 88, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.032048702239990234 + }, + { + "row": 89, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.030427932739257812 + }, + { + "row": 90, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.0281219482421875 + }, + { + "row": 91, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.030226707458496094 + }, + { + "row": 92, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.04007530212402344 + }, + { + "row": 93, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.02728891372680664 + }, + { + "row": 94, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.0217437744140625 + }, + { + "row": 95, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.03214454650878906 + }, + { + "row": 96, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.03669023513793945 + }, + { + "row": 97, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.019012451171875 + }, + { + "row": 98, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026651382446289062 + }, + { + "row": 99, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.1457672119140625e-06, + "max_abs_logit_error": 0.04161381721496582 + }, + { + "row": 100, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.04680633544921875 + }, + { + "row": 101, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.789079107084035e-08, + "max_abs_logit_error": 0.029542922973632812 + }, + { + "row": 102, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.5862225694339713e-09, + "max_abs_logit_error": 0.023795127868652344 + }, + { + "row": 103, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 0.00024586915969848633, + "max_abs_logit_error": 0.030111312866210938 + }, + { + "row": 104, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.841255187988281e-06, + "max_abs_logit_error": 0.03126239776611328 + }, + { + "row": 105, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.928799035575082e-10, + "max_abs_logit_error": 0.026149272918701172 + }, + { + "row": 106, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.4136979082101107e-09, + "max_abs_logit_error": 0.024585723876953125 + }, + { + "row": 107, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.302839361538304e-10, + "max_abs_logit_error": 0.024765491485595703 + }, + { + "row": 108, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03492164611816406 + }, + { + "row": 109, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.034421443939208984 + }, + { + "row": 110, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 9.5367431640625e-07, + "max_abs_logit_error": 0.0320439338684082 + }, + { + "row": 111, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.811981201171875e-05, + "max_abs_logit_error": 0.032814979553222656 + }, + { + "row": 112, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.025417327880859375 + }, + { + "row": 113, + "label": 19, + "upstream": 19, + "coreml": 19, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.04453086853027344 + }, + { + "row": 114, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 5.0942090545902374e-09, + "max_abs_logit_error": 0.023950576782226562 + }, + { + "row": 115, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.022706031799316406 + }, + { + "row": 116, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.5405053888659666e-10, + "max_abs_logit_error": 0.035375118255615234 + }, + { + "row": 117, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 4.76837158203125e-07, + "max_abs_logit_error": 0.03717994689941406 + }, + { + "row": 118, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.04058122634887695 + }, + { + "row": 119, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3188037328859537e-09, + "max_abs_logit_error": 0.032048702239990234 + }, + { + "row": 120, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9970605169561395e-09, + "max_abs_logit_error": 0.030427932739257812 + }, + { + "row": 121, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.7617375336342889e-09, + "max_abs_logit_error": 0.0281219482421875 + }, + { + "row": 122, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.612549260787688e-10, + "max_abs_logit_error": 0.030226707458496094 + }, + { + "row": 123, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 6.506330630512425e-12, + "max_abs_logit_error": 0.04007530212402344 + }, + { + "row": 124, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.5150824594911683e-08, + "max_abs_logit_error": 0.02728891372680664 + }, + { + "row": 125, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.3666674458789885e-09, + "max_abs_logit_error": 0.0217437744140625 + }, + { + "row": 126, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 3.969476136678196e-10, + "max_abs_logit_error": 0.03214454650878906 + }, + { + "row": 127, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.657726538846106e-10, + "max_abs_logit_error": 0.03669023513793945 + }, + { + "row": 128, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.019012451171875 + }, + { + "row": 129, + "label": 20, + "upstream": 20, + "coreml": 20, + "max_abs_probability_error": 2.384185791015625e-07, + "max_abs_logit_error": 0.026651382446289062 + }, + { + "row": 130, + "label": 1, + "upstream": 1, + "coreml": 1, + "max_abs_probability_error": 3.170967102050781e-05, + "max_abs_logit_error": 0.071075439453125 + }, + { + "row": 131, + "label": 2, + "upstream": 2, + "coreml": 2, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.051357269287109375 + }, + { + "row": 132, + "label": 3, + "upstream": 3, + "coreml": 3, + "max_abs_probability_error": 2.7894973754882812e-05, + "max_abs_logit_error": 0.033801641315221786 + }, + { + "row": 133, + "label": 4, + "upstream": 4, + "coreml": 4, + "max_abs_probability_error": 1.0967254638671875e-05, + "max_abs_logit_error": 0.039325714111328125 + }, + { + "row": 134, + "label": 5, + "upstream": 5, + "coreml": 5, + "max_abs_probability_error": 6.258487701416016e-05, + "max_abs_logit_error": 0.033056676387786865 + }, + { + "row": 135, + "label": 6, + "upstream": 6, + "coreml": 6, + "max_abs_probability_error": 1.2993812561035156e-05, + "max_abs_logit_error": 0.02561807632446289 + }, + { + "row": 136, + "label": 7, + "upstream": 7, + "coreml": 7, + "max_abs_probability_error": 6.198883056640625e-06, + "max_abs_logit_error": 0.03448677062988281 + }, + { + "row": 137, + "label": 8, + "upstream": 8, + "coreml": 8, + "max_abs_probability_error": 0.00233614444732666, + "max_abs_logit_error": 0.04230833053588867 + }, + { + "row": 138, + "label": 9, + "upstream": 9, + "coreml": 9, + "max_abs_probability_error": 0.00022917985916137695, + "max_abs_logit_error": 0.05339241027832031 + }, + { + "row": 139, + "label": 10, + "upstream": 10, + "coreml": 10, + "max_abs_probability_error": 4.851818084716797e-05, + "max_abs_logit_error": 0.032756805419921875 + }, + { + "row": 140, + "label": 11, + "upstream": 11, + "coreml": 11, + "max_abs_probability_error": 3.0994415283203125e-06, + "max_abs_logit_error": 0.029809951782226562 + }, + { + "row": 141, + "label": 12, + "upstream": 12, + "coreml": 12, + "max_abs_probability_error": 9.34600830078125e-05, + "max_abs_logit_error": 0.026587963104248047 + }, + { + "row": 142, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.03671073913574219 + }, + { + "row": 143, + "label": 16, + "upstream": 16, + "coreml": 16, + "max_abs_probability_error": 7.152557373046875e-07, + "max_abs_logit_error": 0.05281543731689453 + }, + { + "row": 144, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1920928955078125e-06, + "max_abs_logit_error": 0.04511451721191406 + }, + { + "row": 145, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.07230281829833984 + }, + { + "row": 146, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03114461898803711 + }, + { + "row": 147, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.020772457122802734 + }, + { + "row": 148, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.03475606441497803 + }, + { + "row": 149, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.020298004150390625 + }, + { + "row": 150, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.04283332824707031 + }, + { + "row": 151, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.028450965881347656 + }, + { + "row": 152, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.02250051498413086 + }, + { + "row": 153, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.06335592269897461 + }, + { + "row": 154, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.02735137939453125 + }, + { + "row": 155, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.015047073364257812 + }, + { + "row": 156, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.04868888854980469 + }, + { + "row": 157, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.026497364044189453 + }, + { + "row": 158, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.022236347198486328 + }, + { + "row": 159, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.020044326782226562 + }, + { + "row": 160, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.05533123016357422 + }, + { + "row": 161, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.0247042179107666 + }, + { + "row": 162, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.028873443603515625 + }, + { + "row": 163, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9131388140358752e-10, + "max_abs_logit_error": 0.015494346618652344 + }, + { + "row": 164, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0254025184508464e-08, + "max_abs_logit_error": 0.02497100830078125 + }, + { + "row": 165, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.15840287151903e-11, + "max_abs_logit_error": 0.02679443359375 + }, + { + "row": 166, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.6797479790729994e-09, + "max_abs_logit_error": 0.028203964233398438 + }, + { + "row": 167, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.54509399403247e-10, + "max_abs_logit_error": 0.03371429443359375 + }, + { + "row": 168, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.028280355036258698 + }, + { + "row": 169, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 2.4169340140378637e-11, + "max_abs_logit_error": 0.038245439529418945 + }, + { + "row": 170, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.0329603412093675e-11, + "max_abs_logit_error": 0.030043363571166992 + }, + { + "row": 171, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.7881393432617188e-06, + "max_abs_logit_error": 0.02963542938232422 + }, + { + "row": 172, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 4.66337235494052e-09, + "max_abs_logit_error": 0.03510093688964844 + }, + { + "row": 173, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2885730260592254e-09, + "max_abs_logit_error": 0.04009199142456055 + }, + { + "row": 174, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.1279766010119374e-09, + "max_abs_logit_error": 0.031810760498046875 + }, + { + "row": 175, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.9073486328125e-06, + "max_abs_logit_error": 0.03671073913574219 + }, + { + "row": 176, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.0728836059570312e-06, + "max_abs_logit_error": 0.05123615264892578 + }, + { + "row": 177, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.2799782567185503e-08, + "max_abs_logit_error": 0.03157186508178711 + }, + { + "row": 178, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.6954879760742188e-06, + "max_abs_logit_error": 0.07230281829833984 + }, + { + "row": 179, + "label": 17, + "upstream": 17, + "coreml": 17, + "max_abs_probability_error": 1.1920928955078125e-07, + "max_abs_logit_error": 0.03114461898803711 + }, + { + "row": 180, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.429776876539563e-09, + "max_abs_logit_error": 0.020772457122802734 + }, + { + "row": 181, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 9.215698426601193e-09, + "max_abs_logit_error": 0.03475606441497803 + }, + { + "row": 182, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.84208304060607e-10, + "max_abs_logit_error": 0.020298004150390625 + }, + { + "row": 183, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.947568818333917e-12, + "max_abs_logit_error": 0.04283332824707031 + }, + { + "row": 184, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 5.291168614363073e-10, + "max_abs_logit_error": 0.028450965881347656 + }, + { + "row": 185, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 7.924417788629512e-10, + "max_abs_logit_error": 0.02250051498413086 + }, + { + "row": 186, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.837478791344438e-08, + "max_abs_logit_error": 0.06335592269897461 + }, + { + "row": 187, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 3.3051329034750054e-11, + "max_abs_logit_error": 0.02735137939453125 + }, + { + "row": 188, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.650468511860126e-10, + "max_abs_logit_error": 0.015047073364257812 + }, + { + "row": 189, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.181078218703284e-12, + "max_abs_logit_error": 0.04868888854980469 + }, + { + "row": 190, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.6455586893115992e-10, + "max_abs_logit_error": 0.026497364044189453 + }, + { + "row": 191, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.109900363426846e-10, + "max_abs_logit_error": 0.022236347198486328 + }, + { + "row": 192, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 6.693580800742893e-09, + "max_abs_logit_error": 0.020044326782226562 + }, + { + "row": 193, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.3718365998727222e-08, + "max_abs_logit_error": 0.05533123016357422 + }, + { + "row": 194, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 8.344650268554688e-07, + "max_abs_logit_error": 0.0247042179107666 + }, + { + "row": 195, + "label": 18, + "upstream": 18, + "coreml": 18, + "max_abs_probability_error": 1.042114572413766e-08, + "max_abs_logit_error": 0.028873443603515625 + } + ] + } + }, + "passed": true, + "code_sha256": { + "assets.py": "f7a5bc931075237e00c73bf21fee3575be35531f934f23b5e22627d50a79f92f", + "convert-coreml.py": "e1e9cc20277515fc750940e97d75f9db7c15f0443e23c26fe65815cef4e23bb7", + "export_model.py": "803b909577fe7c38968c5438b27fff7c07bb095a7c341c1b0430d91256649113", + "preprocessing.py": "18c74ede43f95f91cd638c7e9631923a353e3a24d3b522ddb8b18b480f9c2846", + "verify.py": "31425c5f0081b07725ce94c611664151ed02678f25ca93774497cfca8f46ceeb" + } +} diff --git a/synthetic-test.lock.json b/synthetic-test.lock.json new file mode 100644 index 0000000000000000000000000000000000000000..4bbb3a1b81d09223dc516c1b89e44827d400b304 --- /dev/null +++ b/synthetic-test.lock.json @@ -0,0 +1,15 @@ +{ + "dataset_repository": "cua-ai/cua-s1-forms", + "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", + "path": "artifacts/test.jsonl", + "url": "https://huggingface.co/datasets/cua-ai/cua-s1-forms/resolve/8273f34778b99ac2e12d9f6e7d57dad99ae20845/test.jsonl", + "sha256": "d63a7e0db195d4d20154a40b2f8dd09ce3bb65487a158c638da5c609d4475e7c", + "rows": 24370, + "bytes": 18710924, + "upstream_split_claim": "Synthetic test, form-signature-disjoint from train/validation. Train and validation signatures have not been independently re-audited here.", + "model_card_claim": { + "synthetic_test_top1": 0.9995, + "approximate_decisions": 15000, + "note": "The released file has 24370 rows. No matching full result ledger or exact 15000-row manifest is supplied by the linked model card." + } +}