Add files using upload-large-folder tool

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,12 @@
+cua-s1-4b-coreml
+
+Core ML conversion of Cua-S1-4B-0.2 merged into Qwen3.5-4B.
+
+- Qwen/Qwen3.5-4B, Copyright Alibaba Cloud (Qwen team), Apache License 2.0
+  https://huggingface.co/Qwen/Qwen3.5-4B
+- cua-ai/cua-s1-4b-0.2 LoRA adapters, Copyright Cua, Apache License 2.0
+  https://huggingface.co/cua-ai/cua-s1-4b-0.2
+
+Modifications: each LoRA adapter merged into the base weights in fp32 and stored in fp16;
+converted to Core ML (four decoder parts, host-side embedding table, vision tower);
+optional int8 and GPTQ int4/int8 weight compression.

README.md

@@ -0,0 +1,94 @@
+---
+license: apache-2.0
+library_name: coreml
+pipeline_tag: image-text-to-text
+base_model:
+  - cua-ai/cua-s1-4b-0.2
+  - Qwen/Qwen3.5-4B
+tags:
+  - coreml
+  - computer-use
+  - decision-model
+  - apple-silicon
+  - qwen3.5
+  - fluiduse
+---
+
+# cua-s1-4b-coreml
+
+Core ML conversion of [**Cua-S1-4B-0.2**](https://huggingface.co/cua-ai/cua-s1-4b-0.2) (Cua, Apache-2.0):
+LoRA adapters on [`Qwen/Qwen3.5-4B`](https://huggingface.co/Qwen/Qwen3.5-4B) (Apache-2.0) that pick one
+`(element, action)` option for a GUI screen state. Both adapters are converted, each merged into its own
+copy of the base: `text/` (accessibility tree) and `multimodal/` (screenshot).
+
+The model is a one-pass chooser, not a generator: one forward pass over the prompt, then a softmax over
+the answer-letter logits `A..Z` at the last position (Cua's `FourBModel` readout). The Core ML graphs keep
+exactly that: prefill only, no KV cache, no vocabulary head.
+
+Runs through [FluidUse](https://github.com/FluidInference/FluidUse) (`CuaS1FourBManager`) on macOS 14+.
+
+```swift
+let cua = try await CuaS1FourBManager.load(configuration: .init(modality: .text, variant: "gptq"))
+let decision = try await cua.decide(CuaS1FourBState(
+    app: "portal", taskFamily: "login_auth", goal: "Log in as the demo user",
+    accessibilityTree: "- [el_0] Edit \"Username\" value=\"\"\n- [el_1] Button \"Log in\"",
+    options: [
+        .init(elementId: "el_0", role: "Edit", label: "Username", action: "fill", entityId: "ent_0"),
+        .init(elementId: "el_0", role: "Edit", label: "Username", action: "skip"),
+        .init(elementId: "el_1", role: "Button", label: "Log in", action: "click"),
+    ]))
+print(decision.best?.option, decision.bestPerElement())
+```
+
+## Files
+
+| Path | What | Size |
+| --- | --- | ---: |
+| `tokenizer.json` | Qwen3.5 tokenizer | 12 MB |
+| `embeddings.f16` | tied embedding table, row-major `[248320, 2560]` fp16, gathered on the host | 1.27 GB |
+| `text/L1024/` | text decoder, fp16, 4 parts x 8 layers, prompts up to 1024 tokens | 6.8 GB |
+| `text/L1024-w8/` | int8 linears | 3.4 GB |
+| `text/L1024-gptq/` | GPTQ: MLP int4 (block 16) + other linears int8 | 2.6 GB |
+| `multimodal/L2048/` | multimodal decoder, fp16, prompts up to 2048 tokens | 6.8 GB |
+| `multimodal/L2048-w8/` | int8 linears | 3.4 GB |
+| `multimodal/vision/` | vision tower + merger (up to 4096 patches), position table, config | 0.66 GB |
+
+Each decoder part takes `hidden [1, L, 2560]`, `cos`/`sin [L, 64]` (interleaved M-RoPE, host computed)
+and returns the next hidden state; the last part also takes `last_onehot [1, L]` and returns
+`letter_logits [1, 26]`. Prompts are right-padded; the model is causal, so padding is exact.
+
+## Results
+
+Apple M5 Pro (24 GB), macOS 27, GPU (`cpuAndGPU`). Parity reference: fp32 transformers decoder layers on
+the merged weights and the unpadded prompt, over 38 tasks from Cua's own `cua_bench_s1` generator.
+Accuracy: GUI-360 test split (613 text tasks) rebuilt with Cua's `gui360` converter; Cua's own frozen
+615-task split is published by hash only, so this is protocol-adjacent, not the same task set.
+
+| Build | Size | Argmax vs fp32 (38) | Max \|Δp\| | GUI-360 text (613) |
+| --- | ---: | ---: | ---: | ---: |
+| text fp16 | 6.8 GB | 38/38 | 0.005 | 85.5% |
+| text gptq | 2.6 GB | 38/38 | 0.133 | 85.5% |
+| text w8 | 3.4 GB | 37/38 | 0.019 | — |
+| multimodal fp16 (+ vision) | 7.4 GB | 38/38 | 0.153 | — |
+| multimodal w8 (+ vision) | 4.0 GB | 38/38 | 0.163 | — |
+
+One decision on the GPU: text ~0.7 s (1024-token bucket) and multimodal ~1.65 s (vision 0.2 s +
+2048-token decoder) on an idle machine; about 1.1 s and 3 s while other GPU work was running.
+
+On the same first 100 GUI-360 tasks, Cua's PyTorch runtime (bf16) scores 84%, Core ML fp16 88% and gptq
+87%; the Core ML task outcome matches PyTorch on 96-97 of 100. Post-training int4 without calibration and 4/3/2-bit palettes are not
+published: they flip 10-32 of the 38 decisions.
+
+## Limits
+
+- GPU only: the Neural Engine path falls back to the CPU for most of the graph (19 s / decision).
+- At most 26 options per decision (one letter each). Longer prompts than the loaded bucket are rejected.
+- Screenshots are smart-resized as in the reference processor, capped at 4096 patches (about 1280x800
+  px); larger screenshots are downscaled further than the PyTorch reference would.
+- fp16 vision features move multimodal probabilities by up to 0.15 (argmax unchanged on the fixtures).
+
+## Provenance
+
+Conversion code, parity and benchmark scripts: [mobius](https://github.com/FluidInference/mobius)
+`models/computer-use/cua-s1-4b/coreml`. Base weights: `Qwen/Qwen3.5-4B`; adapters: `cua-ai/cua-s1-4b-0.2`
+(`text/`, `multimodal/`). Both Apache-2.0; see `NOTICE`.

multimodal/L2048-w8/config.json

@@ -0,0 +1,69 @@
+{
+  "modality": "multimodal",
+  "seq_len": 2048,
+  "hidden_size": 2560,
+  "rotary_dim": 64,
+  "rope_theta": 10000000,
+  "mrope_section": [
+    11,
+    11,
+    10
+  ],
+  "vocab_size": 248320,
+  "delta_chunk_size": 64,
+  "parts": [
+    {
+      "layers": [
+        0,
+        8
+      ]
+    },
+    {
+      "layers": [
+        8,
+        16
+      ]
+    },
+    {
+      "layers": [
+        16,
+        24
+      ]
+    },
+    {
+      "layers": [
+        24,
+        32
+      ]
+    }
+  ],
+  "letters": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+  "letter_token_ids": [
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47,
+    48,
+    49,
+    50,
+    51,
+    52,
+    53,
+    54,
+    55,
+    56,
+    57
+  ]
+}

multimodal/L2048/config.json

@@ -0,0 +1,69 @@
+{
+  "modality": "multimodal",
+  "seq_len": 2048,
+  "hidden_size": 2560,
+  "rotary_dim": 64,
+  "rope_theta": 10000000,
+  "mrope_section": [
+    11,
+    11,
+    10
+  ],
+  "vocab_size": 248320,
+  "delta_chunk_size": 64,
+  "parts": [
+    {
+      "layers": [
+        0,
+        8
+      ]
+    },
+    {
+      "layers": [
+        8,
+        16
+      ]
+    },
+    {
+      "layers": [
+        16,
+        24
+      ]
+    },
+    {
+      "layers": [
+        24,
+        32
+      ]
+    }
+  ],
+  "letters": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+  "letter_token_ids": [
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47,
+    48,
+    49,
+    50,
+    51,
+    52,
+    53,
+    54,
+    55,
+    56,
+    57
+  ]
+}

multimodal/vision/vision_config.json

@@ -0,0 +1,16 @@
+{
+  "deepstack_visual_indexes": [],
+  "depth": 24,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_size": 1024,
+  "in_channels": 3,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "model_type": "qwen3_5",
+  "num_heads": 16,
+  "num_position_embeddings": 2304,
+  "out_hidden_size": 2560,
+  "patch_size": 16,
+  "spatial_merge_size": 2,
+  "temporal_patch_size": 2
+}

reports/gui360-text-torch-bf16-first100.json

@@ -0,0 +1 @@
+{"summary": {"backend": "torch", "variant": "bf16-peft", "compute_units": "mps", "tasks": 100, "skipped_too_long": 0, "task_accuracy": 0.84, "by_family": {"form_filling": {"correct": 3, "n": 4, "acc": 0.75}, "multi_step_submit": {"correct": 69, "n": 80, "acc": 0.8625}, "pagination": {"correct": 2, "n": 4, "acc": 0.5}, "search_filter": {"correct": 10, "n": 12, "acc": 0.8333333333333334}}, "median_ms": 47014.42110398784, "p95_ms": 155484.55471454072, "max_tokens": 893}, "rows": [{"id": "gui360_excel_1_101_1", "family": "multi_step_submit", "tokens": 793, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.375, 16.625, 17.5, 17.375, 10.6875, 15.0625, 17.75, 16.625, 12.1875, 20.875, 17.25, 18.625, 17.375, 16.125, 16.625, 16.875, 19.375, 14.875]}, {"id": "gui360_excel_1_101_2", "family": "multi_step_submit", "tokens": 779, "correct": false, "elements_ok": 14, "elements": 16, "logits": [18.875, 14.375, 13.1875, 19.0, 17.875, 18.25, 16.0, 17.25, 14.75, 16.625, 12.625, 21.0, 17.625, 15.125, 18.125, 16.625, 18.0, 17.25]}, {"id": "gui360_excel_1_101_3", "family": "multi_step_submit", "tokens": 780, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 17.875, 14.875, 17.375, 18.5, 19.625, 17.5, 18.875, 15.625, 14.625, 16.625, 13.1875, 17.75, 15.625, 15.375, 19.875, 20.5, 16.375]}, {"id": "gui360_excel_1_101_4", "family": "form_filling", "tokens": 779, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.25, 15.25, 14.875, 15.4375, 15.1875, 15.625, 15.125, 15.75, 13.3125, 16.0, 14.6875, 15.375, 15.0, 14.125, 14.25, 13.875, 15.125, 16.0]}, {"id": "gui360_excel_1_101_5", "family": "multi_step_submit", "tokens": 767, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 16.75, 16.5, 17.375, 13.4375, 21.25, 18.125, 18.75, 15.75, 17.5, 14.8125, 18.5, 18.625, 16.75, 18.25, 15.875, 19.0, 11.0]}, {"id": "gui360_excel_1_116_1", "family": "multi_step_submit", "tokens": 769, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.875, 18.0, 16.125, 18.375, 18.125, 18.75, 17.0, 12.8125, 15.5, 15.9375, 16.875, 18.75, 15.8125, 20.125, 20.0, 17.375, 19.125, 17.375]}, {"id": "gui360_excel_1_2_1", "family": "multi_step_submit", "tokens": 771, "correct": true, "elements_ok": 16, "elements": 16, "logits": [15.3125, 20.625, 17.0, 12.75, 15.75, 18.875, 17.875, 18.75, 15.6875, 17.0, 15.3125, 18.0, 18.0, 15.75, 17.75, 15.875, 17.5, 16.125]}, {"id": "gui360_excel_1_2_2", "family": "multi_step_submit", "tokens": 749, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 17.25, 16.5, 18.5, 18.75, 19.0, 19.625, 12.5625, 12.0625, 21.75, 16.375, 19.0, 18.5, 15.3125, 17.75, 18.25, 19.75, 16.875]}, {"id": "gui360_excel_1_2_3", "family": "multi_step_submit", "tokens": 753, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.0, 16.875, 11.625, 16.375, 21.625, 18.5, 15.875, 19.625, 16.25, 14.3125, 14.75, 18.0, 18.25, 17.375, 18.25, 16.25, 16.625, 15.5625]}, {"id": "gui360_excel_1_2_4", "family": "multi_step_submit", "tokens": 754, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.125, 17.25, 15.875, 18.375, 18.25, 19.125, 16.25, 18.625, 17.0, 15.25, 11.25, 17.375, 17.25, 13.875, 21.25, 16.125, 17.75, 17.625]}, {"id": "gui360_excel_1_2_5", "family": "multi_step_submit", "tokens": 760, "correct": false, "elements_ok": 14, "elements": 16, "logits": [18.875, 17.125, 15.75, 17.625, 15.5, 19.0, 15.0625, 20.5, 16.125, 16.25, 17.25, 18.25, 18.375, 17.375, 19.0, 12.125, 16.75, 16.25]}, {"id": "gui360_excel_1_98_2", "family": "multi_step_submit", "tokens": 786, "correct": true, "elements_ok": 16, "elements": 16, "logits": [15.0625, 20.0, 17.0, 17.0, 16.25, 13.6875, 17.125, 18.125, 15.375, 16.0, 16.25, 18.625, 17.25, 15.4375, 17.875, 15.3125, 18.0, 17.125]}, {"id": "gui360_excel_1_98_3", "family": "multi_step_submit", "tokens": 783, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.375, 13.625, 12.5, 19.25, 17.5, 18.125, 15.5, 19.25, 16.75, 14.875, 13.25, 20.75, 17.875, 15.0, 18.0, 15.3125, 17.75, 17.75]}, {"id": "gui360_excel_1_98_4", "family": "multi_step_submit", "tokens": 783, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.375, 16.75, 15.5625, 14.5625, 13.75, 19.625, 15.9375, 18.125, 16.125, 15.875, 16.625, 17.25, 17.5, 14.625, 17.375, 16.5, 17.25, 17.5]}, {"id": "gui360_excel_1_98_5", "family": "multi_step_submit", "tokens": 777, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.875, 13.625, 14.9375, 16.875, 16.125, 17.0, 14.9375, 16.5, 18.875, 15.9375, 16.25, 16.875, 16.0, 14.8125, 17.375, 13.9375, 17.125, 16.0]}, {"id": "gui360_excel_3_103_1", "family": "multi_step_submit", "tokens": 877, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.75, 18.0, 15.375, 18.5, 17.125, 18.5, 17.625, 16.5, 21.25, 18.625, 15.4375, 19.25, 13.0, 13.8125, 19.0, 16.75, 19.375, 17.0]}, {"id": "gui360_excel_3_103_2", "family": "multi_step_submit", "tokens": 823, "correct": true, "elements_ok": 16, "elements": 16, "logits": [15.4375, 20.75, 17.625, 18.75, 18.5, 13.25, 16.875, 18.625, 15.375, 18.0, 16.875, 18.875, 18.75, 17.625, 19.125, 16.75, 19.5, 15.75]}, {"id": "gui360_excel_3_1193_1", "family": "multi_step_submit", "tokens": 798, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.25, 16.75, 15.5625, 19.75, 17.375, 19.25, 15.5625, 20.125, 16.125, 19.625, 14.0625, 22.0, 19.0, 18.875, 11.5, 14.6875, 18.25, 18.25]}, {"id": "gui360_excel_3_1214_1", "family": "multi_step_submit", "tokens": 797, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.25, 17.375, 17.125, 18.625, 11.9375, 15.5625, 15.8125, 18.75, 15.9375, 15.375, 20.125, 18.25, 17.375, 14.625, 17.625, 17.0, 18.0, 15.6875]}, {"id": "gui360_excel_3_1214_2", "family": "multi_step_submit", "tokens": 786, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.875, 19.75, 12.375, 15.125, 18.25, 19.0, 17.875, 19.5, 16.875, 14.8125, 15.4375, 18.875, 15.75, 20.625, 17.25, 18.375, 18.875, 17.0]}, {"id": "gui360_excel_3_1214_3", "family": "multi_step_submit", "tokens": 790, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.5, 16.25, 17.75, 17.875, 17.875, 17.875, 17.0, 20.0, 10.125, 13.25, 14.9375, 15.375, 20.75, 15.5625, 17.375, 15.1875, 18.25, 17.75]}, {"id": "gui360_excel_3_1254_1", "family": "multi_step_submit", "tokens": 793, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.25, 17.0, 12.0625, 17.875, 17.0, 18.0, 16.125, 18.0, 16.0, 15.375, 16.375, 18.25, 17.75, 15.125, 17.375, 14.5, 20.0, 17.25]}, {"id": "gui360_excel_3_1254_4", "family": "multi_step_submit", "tokens": 775, "correct": false, "elements_ok": 14, "elements": 16, "logits": [17.75, 16.625, 15.8125, 17.0, 17.0, 17.625, 16.5, 17.375, 16.125, 16.875, 15.875, 17.25, 17.125, 16.5, 17.375, 15.25, 18.625, 16.75]}, {"id": "gui360_excel_3_1254_6", "family": "form_filling", "tokens": 822, "correct": false, "elements_ok": 14, "elements": 16, "logits": [17.75, 15.25, 14.0, 16.5, 16.75, 17.25, 14.9375, 16.875, 15.125, 15.75, 14.875, 16.25, 17.875, 16.5, 16.75, 15.125, 16.125, 16.125]}, {"id": "gui360_excel_3_1334_1", "family": "multi_step_submit", "tokens": 792, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.375, 15.9375, 14.0, 19.375, 17.25, 17.5, 15.8125, 17.375, 15.75, 16.625, 15.5625, 16.875, 16.625, 14.9375, 17.5, 15.6875, 17.0, 12.4375]}, {"id": "gui360_excel_3_1334_2", "family": "multi_step_submit", "tokens": 814, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.625, 15.875, 14.875, 16.375, 15.4375, 17.0, 15.5, 16.75, 15.0625, 16.625, 12.9375, 15.625, 16.25, 14.8125, 19.125, 15.8125, 16.125, 16.625]}, {"id": "gui360_excel_3_1334_3", "family": "multi_step_submit", "tokens": 756, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.875, 17.125, 15.9375, 18.875, 12.6875, 22.125, 18.5, 12.375, 16.125, 16.375, 17.0, 19.25, 19.125, 15.875, 19.0, 16.875, 18.375, 14.8125]}, {"id": "gui360_excel_3_1334_4", "family": "multi_step_submit", "tokens": 777, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.25, 16.75, 17.125, 18.0, 13.125, 21.375, 18.875, 19.0, 15.75, 17.125, 16.125, 18.625, 19.25, 15.3125, 18.75, 10.875, 17.625, 15.9375]}, {"id": "gui360_excel_3_135_1", "family": "search_filter", "tokens": 799, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.5, 16.875, 16.5, 17.375, 17.375, 18.125, 15.5, 17.125, 20.875, 18.25, 16.25, 13.6875, 18.375, 14.75, 20.375, 16.0, 19.5, 16.125]}, {"id": "gui360_excel_3_135_2", "family": "pagination", "tokens": 840, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.625, 15.75, 15.125, 15.8125, 15.0625, 15.9375, 15.625, 16.125, 12.1875, 15.9375, 14.375, 15.5, 15.125, 14.3125, 15.4375, 13.9375, 17.25, 16.25]}, {"id": "gui360_excel_3_135_3", "family": "pagination", "tokens": 804, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.0, 15.4375, 14.125, 15.875, 13.6875, 14.9375, 15.1875, 15.5625, 16.375, 15.9375, 15.125, 15.3125, 15.0, 14.4375, 16.0, 13.9375, 15.5, 15.8125]}, {"id": "gui360_excel_3_135_4", "family": "pagination", "tokens": 828, "correct": false, "elements_ok": 14, "elements": 16, "logits": [17.0, 15.9375, 13.75, 14.8125, 15.75, 16.125, 17.0, 16.625, 15.6875, 16.0, 14.875, 16.25, 15.8125, 14.8125, 15.9375, 14.5625, 16.0, 15.625]}, {"id": "gui360_excel_3_135_5", "family": "pagination", "tokens": 811, "correct": false, "elements_ok": 15, "elements": 16, "logits": [16.875, 15.9375, 13.375, 15.6875, 15.75, 15.75, 15.75, 16.0, 15.5, 16.375, 14.9375, 15.1875, 14.875, 14.3125, 15.0, 13.8125, 15.4375, 15.8125]}, {"id": "gui360_excel_3_137_1", "family": "multi_step_submit", "tokens": 806, "correct": true, "elements_ok": 16, "elements": 16, "logits": [14.25, 20.875, 15.625, 18.125, 16.5, 12.8125, 16.5, 18.125, 15.1875, 15.6875, 17.5, 19.0, 17.375, 13.5, 18.25, 14.4375, 18.625, 16.375]}, {"id": "gui360_excel_3_137_2", "family": "multi_step_submit", "tokens": 844, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 16.625, 19.125, 12.5, 15.625, 18.5, 15.8125, 19.625, 15.75, 15.125, 20.5, 18.75, 18.375, 15.875, 17.75, 16.375, 19.625, 16.625]}, {"id": "gui360_excel_3_137_3", "family": "multi_step_submit", "tokens": 805, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.25, 16.75, 17.25, 15.0625, 17.125, 18.875, 18.0, 18.625, 17.875, 17.875, 18.625, 17.5, 18.0, 16.75, 18.125, 12.5, 21.0, 18.0]}, {"id": "gui360_excel_3_137_4", "family": "multi_step_submit", "tokens": 817, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.5, 16.375, 18.375, 17.125, 15.875, 18.5, 16.375, 18.75, 15.9375, 14.8125, 19.5, 17.875, 17.25, 16.75, 18.25, 11.5625, 18.5, 17.25]}, {"id": "gui360_excel_3_137_5", "family": "multi_step_submit", "tokens": 851, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.25, 17.375, 11.8125, 17.0, 16.125, 21.375, 16.75, 17.875, 15.75, 18.375, 14.875, 18.375, 18.5, 17.5, 18.5, 15.5625, 16.0, 17.75]}, {"id": "gui360_excel_3_1410_1", "family": "multi_step_submit", "tokens": 803, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.125, 14.5, 12.5, 15.0625, 14.75, 14.9375, 14.1875, 15.125, 16.25, 15.125, 14.0625, 14.75, 14.125, 13.875, 15.0, 13.625, 14.5625, 15.125]}, {"id": "gui360_excel_3_1410_2", "family": "search_filter", "tokens": 812, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 17.5, 16.375, 18.25, 13.75, 22.0, 16.5, 19.5, 16.625, 12.75, 15.9375, 19.0, 19.25, 16.625, 19.125, 16.375, 18.25, 16.125]}, {"id": "gui360_excel_3_1410_3", "family": "multi_step_submit", "tokens": 803, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.875, 16.875, 14.9375, 17.125, 21.375, 18.375, 16.375, 19.5, 16.875, 17.25, 14.5, 18.125, 17.625, 16.0, 19.125, 10.6875, 18.0, 16.5]}, {"id": "gui360_excel_3_1410_4", "family": "form_filling", "tokens": 814, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.5, 15.625, 18.0, 17.5, 17.25, 17.5, 15.75, 17.25, 15.875, 16.5, 16.75, 14.0625, 15.625, 15.5, 17.0, 15.1875, 17.375, 16.0]}, {"id": "gui360_excel_3_1410_5", "family": "multi_step_submit", "tokens": 806, "correct": true, "elements_ok": 16, "elements": 16, "logits": [15.25, 19.75, 15.75, 17.625, 16.125, 18.125, 16.75, 18.25, 16.375, 14.75, 14.625, 18.25, 17.625, 16.125, 16.75, 14.8125, 10.75, 16.125]}, {"id": "gui360_excel_3_1413_1", "family": "search_filter", "tokens": 762, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 15.4375, 15.875, 16.5, 16.625, 17.75, 15.625, 17.5, 15.875, 16.375, 18.625, 17.625, 16.5, 16.25, 17.0, 15.5, 17.5, 16.875]}, {"id": "gui360_excel_3_1434_1", "family": "multi_step_submit", "tokens": 810, "correct": false, "elements_ok": 15, "elements": 16, "logits": [16.625, 16.75, 14.625, 16.375, 17.375, 17.125, 15.9375, 16.75, 15.75, 16.5, 15.5625, 16.5, 16.375, 15.875, 16.75, 14.5625, 16.125, 16.0]}, {"id": "gui360_excel_3_1434_2", "family": "multi_step_submit", "tokens": 799, "correct": false, "elements_ok": 15, "elements": 16, "logits": [16.625, 15.75, 14.6875, 15.9375, 15.8125, 16.125, 16.25, 16.125, 15.1875, 16.25, 15.25, 16.25, 15.5625, 14.625, 16.25, 14.75, 15.875, 15.5]}, {"id": "gui360_excel_3_1434_3", "family": "multi_step_submit", "tokens": 814, "correct": false, "elements_ok": 14, "elements": 16, "logits": [17.875, 15.5, 14.0625, 17.375, 15.3125, 19.125, 16.0, 17.5, 16.125, 16.5, 14.8125, 17.0, 17.0, 15.8125, 17.25, 15.375, 16.75, 16.125]}, {"id": "gui360_excel_3_1434_4", "family": "form_filling", "tokens": 816, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.0, 17.5, 12.9375, 16.0, 16.875, 19.125, 18.625, 18.25, 16.875, 16.75, 17.25, 18.75, 17.875, 13.1875, 18.625, 12.75, 21.0, 17.625]}, {"id": "gui360_excel_3_1434_5", "family": "search_filter", "tokens": 816, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.25, 14.0625, 12.875, 18.875, 18.125, 19.0, 14.25, 19.125, 14.3125, 15.8125, 14.3125, 21.125, 18.625, 16.0, 18.5, 14.75, 17.625, 15.1875]}, {"id": "gui360_excel_3_1441_1", "family": "multi_step_submit", "tokens": 798, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.125, 17.125, 16.5, 17.75, 15.0, 17.5, 15.75, 19.0, 15.6875, 17.875, 12.4375, 20.25, 17.375, 14.75, 18.75, 10.75, 17.0, 17.0]}, {"id": "gui360_excel_3_1441_2", "family": "search_filter", "tokens": 793, "correct": false, "elements_ok": 15, "elements": 16, "logits": [15.1875, 17.875, 15.625, 16.25, 15.8125, 17.875, 16.625, 17.375, 15.75, 15.375, 16.375, 16.875, 17.125, 16.0, 15.5, 15.5625, 17.125, 16.875]}, {"id": "gui360_excel_3_1441_3", "family": "multi_step_submit", "tokens": 806, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.125, 17.75, 16.875, 18.125, 19.5, 13.4375, 14.125, 20.875, 18.625, 15.875, 16.0, 19.625, 17.5, 15.0625, 16.75, 16.0, 18.625, 16.125]}, {"id": "gui360_excel_3_1441_4", "family": "multi_step_submit", "tokens": 800, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.5, 14.0625, 20.875, 18.375, 14.75, 16.875, 15.4375, 19.25, 16.625, 16.375, 18.375, 18.375, 18.5, 16.5, 19.125, 11.0625, 17.625, 16.5]}, {"id": "gui360_excel_3_1441_5", "family": "multi_step_submit", "tokens": 793, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.125, 16.125, 16.75, 17.25, 17.125, 19.375, 11.125, 14.8125, 15.5, 16.625, 15.625, 18.875, 16.5, 15.25, 13.1875, 19.5, 17.0, 17.625]}, {"id": "gui360_excel_3_1512_1", "family": "multi_step_submit", "tokens": 788, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.875, 16.875, 18.75, 18.375, 11.5, 16.75, 18.125, 17.75, 15.375, 16.875, 16.875, 18.5, 17.375, 13.1875, 18.125, 13.0625, 21.875, 16.875]}, {"id": "gui360_excel_3_1512_2", "family": "multi_step_submit", "tokens": 799, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.375, 16.25, 18.25, 17.25, 16.75, 17.5, 17.875, 14.0, 16.375, 15.375, 16.875, 17.625, 16.625, 15.625, 17.125, 15.0, 17.375, 15.25]}, {"id": "gui360_excel_3_1512_3", "family": "multi_step_submit", "tokens": 799, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.375, 16.75, 16.875, 18.25, 18.25, 20.375, 17.25, 19.875, 15.8125, 18.0, 17.125, 19.625, 12.875, 14.4375, 16.625, 20.75, 20.0, 17.5]}, {"id": "gui360_excel_3_1524_1", "family": "search_filter", "tokens": 800, "correct": false, "elements_ok": 14, "elements": 16, "logits": [17.25, 17.0, 15.1875, 17.0, 15.0, 19.25, 15.4375, 18.125, 11.875, 14.875, 15.625, 16.875, 17.125, 16.125, 16.375, 15.9375, 17.125, 15.5]}, {"id": "gui360_excel_3_1524_2", "family": "search_filter", "tokens": 847, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.875, 16.0, 15.6875, 16.125, 15.25, 16.5, 15.9375, 17.125, 14.3125, 16.625, 18.125, 16.75, 16.5, 16.0, 16.75, 12.1875, 17.125, 16.25]}, {"id": "gui360_excel_3_1524_3", "family": "search_filter", "tokens": 789, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.375, 16.75, 16.5, 18.125, 17.625, 18.25, 16.875, 18.875, 15.125, 11.6875, 13.625, 17.75, 18.375, 12.6875, 21.625, 17.5, 17.75, 16.25]}, {"id": "gui360_excel_3_1524_4", "family": "search_filter", "tokens": 853, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.625, 17.375, 13.75, 21.5, 18.75, 19.25, 19.0, 13.5625, 16.25, 16.125, 16.75, 19.0, 17.875, 16.375, 18.25, 14.1875, 18.125, 16.75]}, {"id": "gui360_excel_3_1527_1", "family": "multi_step_submit", "tokens": 796, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 16.875, 15.5, 17.5, 15.6875, 18.375, 15.8125, 18.25, 16.25, 15.6875, 15.1875, 17.875, 14.75, 18.0, 17.75, 15.6875, 11.625, 16.75]}, {"id": "gui360_excel_3_1527_2", "family": "multi_step_submit", "tokens": 815, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.875, 17.25, 15.375, 17.625, 16.25, 17.75, 15.6875, 20.25, 18.5, 16.375, 15.75, 18.125, 17.125, 17.0, 17.875, 11.25, 15.6875, 17.5]}, {"id": "gui360_excel_3_1527_3", "family": "multi_step_submit", "tokens": 863, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.0, 16.0, 15.9375, 16.875, 18.125, 19.75, 11.3125, 16.875, 17.25, 14.625, 15.875, 18.75, 17.875, 16.375, 17.5, 15.8125, 16.625, 20.25]}, {"id": "gui360_excel_3_1527_4", "family": "multi_step_submit", "tokens": 860, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.75, 14.0, 16.125, 17.75, 17.375, 17.125, 20.875, 18.0, 17.375, 16.0, 16.125, 18.25, 18.625, 16.75, 18.75, 17.125, 18.625, 15.5]}, {"id": "gui360_excel_3_1527_5", "family": "multi_step_submit", "tokens": 816, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.0, 17.0, 16.0, 18.375, 16.75, 19.625, 17.375, 18.125, 15.125, 18.875, 16.75, 19.125, 17.75, 16.25, 11.125, 17.5, 16.5, 20.625]}, {"id": "gui360_excel_3_1532_1", "family": "multi_step_submit", "tokens": 893, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.25, 17.375, 14.1875, 16.0, 18.75, 18.25, 18.25, 19.125, 16.5, 17.75, 15.375, 18.375, 17.75, 13.375, 21.125, 16.5, 18.375, 17.25]}, {"id": "gui360_excel_3_1532_2", "family": "multi_step_submit", "tokens": 848, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.375, 16.875, 15.75, 17.875, 18.875, 13.375, 16.75, 18.75, 15.9375, 15.125, 14.5625, 17.875, 20.625, 14.75, 19.875, 14.75, 18.875, 16.25]}, {"id": "gui360_excel_3_1532_3", "family": "multi_step_submit", "tokens": 840, "correct": false, "elements_ok": 15, "elements": 16, "logits": [17.875, 16.75, 15.8125, 17.875, 17.375, 18.0, 16.125, 18.125, 15.125, 18.0, 14.8125, 17.625, 18.5, 17.0, 16.625, 16.75, 17.5, 16.0]}, {"id": "gui360_excel_3_1579_1", "family": "multi_step_submit", "tokens": 781, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.5, 15.6875, 18.125, 17.125, 16.625, 17.375, 16.25, 17.625, 16.5, 15.875, 16.25, 17.25, 16.375, 16.625, 16.375, 16.125, 17.0, 12.875]}, {"id": "gui360_excel_3_1579_2", "family": "multi_step_submit", "tokens": 821, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.375, 17.5, 17.125, 18.625, 18.625, 20.125, 17.625, 15.5, 16.75, 18.125, 17.0, 18.75, 20.5, 14.25, 19.375, 17.75, 19.625, 17.0]}, {"id": "gui360_excel_3_1584_1", "family": "multi_step_submit", "tokens": 782, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 17.125, 15.75, 16.125, 21.625, 19.25, 18.375, 19.375, 16.125, 17.125, 16.125, 13.5625, 18.25, 16.125, 18.625, 16.75, 18.0, 17.375]}, {"id": "gui360_excel_3_1795_1", "family": "multi_step_submit", "tokens": 774, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.0, 15.3125, 19.0, 17.125, 17.0, 18.0, 16.25, 18.25, 11.4375, 15.0, 17.0, 16.875, 17.0, 14.875, 17.125, 15.25, 17.375, 17.0]}, {"id": "gui360_excel_3_1795_2", "family": "multi_step_submit", "tokens": 774, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.25, 16.875, 16.5, 15.5, 21.125, 18.875, 17.375, 19.0, 16.625, 17.25, 14.5, 17.875, 18.25, 10.6875, 17.0, 16.875, 17.125, 16.25]}, {"id": "gui360_excel_3_1795_3", "family": "multi_step_submit", "tokens": 752, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.625, 17.125, 13.4375, 17.5, 16.875, 19.0, 17.875, 19.125, 17.125, 15.4375, 17.25, 18.625, 18.5, 13.6875, 21.75, 15.9375, 18.625, 18.25]}, {"id": "gui360_excel_3_1795_4", "family": "multi_step_submit", "tokens": 770, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.5, 17.125, 16.25, 19.0, 16.625, 20.125, 18.875, 12.375, 13.1875, 15.25, 14.625, 19.25, 18.0, 13.125, 17.25, 20.375, 20.625, 18.25]}, {"id": "gui360_excel_3_1795_5", "family": "search_filter", "tokens": 772, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 17.5, 15.25, 17.875, 17.625, 18.5, 17.0, 18.125, 16.25, 17.625, 15.1875, 19.75, 17.75, 12.8125, 17.75, 16.5, 17.875, 17.75]}, {"id": "gui360_excel_3_1815_1", "family": "multi_step_submit", "tokens": 768, "correct": false, "elements_ok": 14, "elements": 16, "logits": [18.375, 16.5, 15.75, 17.25, 16.125, 19.75, 16.875, 18.0, 17.125, 16.375, 13.375, 17.625, 17.375, 16.25, 17.125, 16.25, 16.625, 15.8125]}, {"id": "gui360_excel_3_1815_2", "family": "multi_step_submit", "tokens": 767, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.25, 17.75, 17.625, 13.8125, 15.3125, 19.125, 15.75, 21.125, 17.375, 16.5, 16.125, 19.125, 18.375, 17.125, 18.875, 17.125, 18.875, 16.375]}, {"id": "gui360_excel_3_1815_3", "family": "multi_step_submit", "tokens": 776, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.125, 15.3125, 14.625, 18.375, 15.9375, 18.625, 16.375, 17.125, 20.625, 16.75, 16.875, 18.0, 17.375, 15.0, 19.0, 15.875, 18.0, 15.8125]}, {"id": "gui360_excel_3_1815_4", "family": "multi_step_submit", "tokens": 745, "correct": false, "elements_ok": 15, "elements": 16, "logits": [17.5, 18.125, 15.375, 18.25, 17.375, 18.0, 17.125, 18.75, 16.5, 17.875, 16.125, 18.125, 17.375, 14.9375, 18.0, 15.0625, 18.625, 17.25]}, {"id": "gui360_excel_3_1834_1", "family": "multi_step_submit", "tokens": 762, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.5, 16.0, 18.625, 17.125, 15.75, 18.0, 17.25, 17.625, 16.625, 13.5, 16.375, 17.375, 17.625, 16.125, 17.25, 16.375, 17.875, 16.625]}, {"id": "gui360_excel_3_1834_2", "family": "multi_step_submit", "tokens": 759, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.125, 17.25, 16.875, 17.625, 17.75, 14.25, 15.5, 17.5, 16.75, 16.375, 15.9375, 18.125, 17.25, 15.5625, 19.875, 15.375, 17.375, 16.75]}, {"id": "gui360_excel_3_1834_3", "family": "multi_step_submit", "tokens": 762, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.25, 17.125, 17.0, 19.0, 11.1875, 18.625, 17.25, 18.0, 15.9375, 15.0, 21.25, 19.875, 18.75, 13.625, 19.125, 18.0, 19.625, 15.9375]}, {"id": "gui360_excel_3_1834_4", "family": "multi_step_submit", "tokens": 764, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.625, 16.375, 19.0, 17.25, 17.75, 18.0, 16.75, 17.875, 16.75, 15.8125, 13.625, 16.875, 17.625, 16.75, 17.875, 13.9375, 17.5, 15.0625]}, {"id": "gui360_excel_3_1834_5", "family": "multi_step_submit", "tokens": 767, "correct": false, "elements_ok": 15, "elements": 16, "logits": [17.625, 16.75, 15.4375, 16.75, 16.75, 17.375, 17.125, 17.25, 16.0, 16.625, 16.625, 17.375, 16.875, 16.375, 16.625, 16.375, 16.875, 16.875]}, {"id": "gui360_excel_3_19_1", "family": "multi_step_submit", "tokens": 751, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.625, 17.5, 11.5625, 18.125, 17.25, 17.875, 14.0625, 20.25, 17.75, 18.375, 14.5625, 18.0, 17.25, 16.5, 16.5, 14.5, 16.375, 16.625]}, {"id": "gui360_excel_3_19_2", "family": "multi_step_submit", "tokens": 855, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.75, 16.875, 15.125, 15.75, 20.25, 18.75, 17.5, 18.375, 16.375, 15.125, 16.0, 17.375, 17.5, 17.5, 11.375, 14.6875, 16.75, 15.5625]}, {"id": "gui360_excel_3_19_3", "family": "multi_step_submit", "tokens": 767, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.25, 15.75, 15.3125, 15.5625, 18.75, 18.625, 15.625, 17.375, 15.9375, 16.0, 15.6875, 14.125, 16.875, 15.9375, 17.875, 15.1875, 15.75, 16.875]}, {"id": "gui360_excel_3_19_4", "family": "multi_step_submit", "tokens": 843, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.0, 17.75, 17.0, 18.125, 16.5, 17.5, 19.25, 17.0, 16.125, 15.1875, 16.625, 12.375, 17.75, 15.8125, 19.75, 11.75, 22.0, 17.25]}, {"id": "gui360_excel_3_19_5", "family": "multi_step_submit", "tokens": 737, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.25, 13.3125, 14.1875, 18.25, 16.25, 18.125, 17.125, 18.75, 11.6875, 20.5, 15.0, 17.75, 17.625, 16.25, 18.25, 15.9375, 18.375, 16.125]}, {"id": "gui360_excel_3_1941_1", "family": "multi_step_submit", "tokens": 805, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.875, 14.9375, 15.8125, 17.625, 17.75, 18.5, 15.9375, 17.5, 14.9375, 16.5, 15.8125, 18.125, 17.0, 16.625, 16.0, 13.8125, 17.25, 19.125]}, {"id": "gui360_excel_3_2214_1", "family": "multi_step_submit", "tokens": 843, "correct": false, "elements_ok": 15, "elements": 16, "logits": [17.0, 15.8125, 14.375, 17.25, 15.875, 17.125, 15.5, 15.9375, 16.125, 16.125, 15.25, 16.5, 16.0, 15.375, 15.8125, 14.25, 16.125, 16.5]}, {"id": "gui360_excel_3_2267_1", "family": "multi_step_submit", "tokens": 803, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.875, 16.375, 16.5, 18.125, 17.0, 18.375, 16.5, 19.0, 10.5625, 15.6875, 14.625, 18.625, 18.75, 13.25, 21.25, 16.5, 19.0, 17.75]}, {"id": "gui360_excel_3_2306_1", "family": "multi_step_submit", "tokens": 776, "correct": true, "elements_ok": 16, "elements": 16, "logits": [19.375, 15.0625, 16.375, 19.125, 17.5, 19.375, 17.375, 19.5, 16.625, 13.8125, 20.875, 19.5, 18.875, 17.0, 17.5, 15.5, 21.125, 15.5]}, {"id": "gui360_excel_3_2306_2", "family": "multi_step_submit", "tokens": 783, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.875, 17.875, 17.375, 18.25, 15.25, 12.75, 16.75, 17.75, 16.25, 15.75, 16.75, 19.5, 17.625, 18.0, 19.75, 12.25, 22.5, 19.0]}, {"id": "gui360_excel_3_2322_1", "family": "multi_step_submit", "tokens": 774, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.25, 15.125, 13.75, 15.6875, 15.125, 15.75, 14.6875, 15.5625, 14.8125, 15.375, 14.5, 15.5, 16.25, 14.75, 15.8125, 14.5625, 15.4375, 14.75]}, {"id": "gui360_excel_3_2334_1", "family": "search_filter", "tokens": 773, "correct": true, "elements_ok": 16, "elements": 16, "logits": [17.375, 15.75, 20.125, 16.625, 17.25, 18.375, 18.375, 18.375, 11.1875, 15.3125, 16.75, 17.875, 17.625, 15.75, 18.0, 14.9375, 18.25, 16.75]}, {"id": "gui360_excel_3_2334_2", "family": "search_filter", "tokens": 771, "correct": true, "elements_ok": 16, "elements": 16, "logits": [18.0, 18.5, 17.125, 18.625, 14.5, 21.375, 15.8125, 19.25, 13.1875, 17.125, 14.3125, 19.25, 18.75, 17.75, 17.625, 17.75, 19.0, 17.0]}, {"id": "gui360_excel_3_333_1", "family": "multi_step_submit", "tokens": 826, "correct": true, "elements_ok": 16, "elements": 16, "logits": [16.625, 16.0, 14.4375, 16.5, 14.8125, 18.125, 15.75, 17.0, 15.1875, 16.625, 15.0625, 16.875, 16.375, 13.0, 15.625, 14.875, 16.125, 16.5]}]}

reports/parity-coreml-ane-L1024.json

@@ -0,0 +1,249 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_NE",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 37,
+    "max_abs_dprob": 0.01262714508821583,
+    "part_median_ms": [
+      4694.265874975827,
+      4976.397874997929,
+      4808.282167010475,
+      4682.269708020613
+    ],
+    "total_median_ms": 19161.215625004843
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 0.00531286762903288
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": true,
+      "max_dp": 0.00877083609054441
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": true,
+      "max_dp": 0.011253043340287983
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.0071123911676588825
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": true,
+      "max_dp": 0.0026195282081213356
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": true,
+      "max_dp": 0.0061810331949350705
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.0006518362488069744
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.002401381988070217
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.0009043610829074789
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": true,
+      "max_dp": 0.005947872003960963
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.004455633245262547
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.004455633245262547
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.005297656599844713
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.006645744990900371
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.010476979014173882
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.010476979014173882
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 0.0076694974487977186
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.0013854870746224865
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.002329073421382666
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": true,
+      "max_dp": 0.0027069654570323065
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.0032629894319312625
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.0024082371576906736
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 2.2834881480676e-05
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.00231729497433919
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.01262714508821583
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.01262714508821583
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.008531133654261391
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": false,
+      "max_dp": 0.006517987308923168
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 0.005557319607039979
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": true,
+      "max_dp": 0.000922757422737841
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.007420058274930419
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 0.005022253393216047
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": true,
+      "max_dp": 0.002518322299700476
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 0.0013028041298765863
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.0026858984192211777
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.0035410568615769356
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.003952219240703103
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.00398607029141862
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024-gptq.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "g4b16",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 38,
+    "max_abs_dprob": 0.13323538629120957,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      369.5851249503903,
+      376.38858397258446,
+      376.62462500156835,
+      397.07383303903043
+    ],
+    "total_median_ms": 1519.6721669635735
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 0.0521445067535824
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": true,
+      "max_dp": 0.10537974307487363
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": true,
+      "max_dp": 0.04631714455946523
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.011605328373833934
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": true,
+      "max_dp": 0.025813332762375796
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": true,
+      "max_dp": 0.01787707137476413
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.001985891081563041
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.0025523749403736917
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.06589960090935632
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": true,
+      "max_dp": 0.019750520983931236
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.0028298661071747278
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.0028298661071747278
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.05230399543460729
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.004738455840428535
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.06748921207142838
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.06748921207142838
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 0.02494147717805123
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.01576799293933695
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.02275910270986581
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": true,
+      "max_dp": 0.022449204672743694
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.0062459384550479224
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.01293340236104712
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 0.0005595574782154382
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.006104033835984679
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.009771307743092539
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.009771307743092539
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.03369714784369393
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": true,
+      "max_dp": 0.013120551051672535
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 0.05377023723091326
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": true,
+      "max_dp": 0.03507135894791569
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.0218997252232489
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 0.005766793534148509
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": true,
+      "max_dp": 0.13323538629120957
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 0.06105530403621043
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.026457959290345445
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.02896909122675409
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.008374957803187852
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.0404845379429869
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024-m4.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "m4",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 32,
+    "max_abs_dprob": 0.1547224382549044,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      386.18429203052074,
+      334.0645409771241,
+      327.17808400047943,
+      352.87820803932846
+    ],
+    "total_median_ms": 1400.3051250474527
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 0.08465873283038672
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": false,
+      "max_dp": 0.11579835784063336
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": true,
+      "max_dp": 0.07128904692991811
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.08900532858945759
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": true,
+      "max_dp": 0.07661821986784559
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": false,
+      "max_dp": 0.03452593045773966
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.050620813681985344
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.07069490824268082
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.022971635387543876
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": false,
+      "max_dp": 0.1547224382549044
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.030771546282020168
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.030771546282020168
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.03569791143718837
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.050610279988840434
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.03790762782405244
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.03790762782405244
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 0.07661710854521908
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.03125244518695658
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.09443449186506248
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": true,
+      "max_dp": 0.047154536496232136
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.09040636548368763
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.09119614869512305
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 0.00036896487445747805
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.09214161003456256
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.06645669774896314
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.06645669774896314
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.024164417838566654
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": false,
+      "max_dp": 0.07390417213800843
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 0.11710583474601946
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": false,
+      "max_dp": 0.028757822478003736
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": false,
+      "max_dp": 0.10470854761666626
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 0.0897703596823165
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": true,
+      "max_dp": 0.04225986329621445
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 0.020843787565161337
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.028500831182009814
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.037580509533581635
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.07447325811717409
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.060112451417921514
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024-m4b16.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "m4b16",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 35,
+    "max_abs_dprob": 0.0810904416233934,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      429.38845802564174,
+      390.2062079869211,
+      346.49291599635035,
+      362.993374990765
+    ],
+    "total_median_ms": 1529.0809569996782
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 0.042032292305331787
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": true,
+      "max_dp": 0.07487331007382357
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": true,
+      "max_dp": 0.037787123845659076
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.06599705810077544
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": true,
+      "max_dp": 0.03136065308699984
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": false,
+      "max_dp": 0.021745173393810047
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.015442807986642326
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.03531771316090293
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.01069171677678138
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": true,
+      "max_dp": 0.02874578634793723
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.01746805160827336
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.01746805160827336
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.01857119318901379
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.02381343240697703
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.032981189486280815
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.032981189486280815
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 0.026600328089282754
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.0033254041331846063
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.07992863933560669
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": true,
+      "max_dp": 0.0663956798518282
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.03275198468534779
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.02783384829915747
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 0.00016438118628401763
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.01181934440956256
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.05013848082906415
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.05013848082906415
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.019738678404291186
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": false,
+      "max_dp": 0.0630227779901355
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 0.027299724343995302
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": false,
+      "max_dp": 0.03400085564870381
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.029740699759693706
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 0.0481620174655486
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": true,
+      "max_dp": 0.0810904416233934
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 0.008621309620581177
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.015406458134657977
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.03209474389761438
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.031713660815690936
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.025339220865248174
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024-m4gu.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "m4gu",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 35,
+    "max_abs_dprob": 0.11393320929632184,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      398.25599995674565,
+      417.4119159579277,
+      360.41520797880366,
+      317.00962502509356
+    ],
+    "total_median_ms": 1493.0927489185706
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 0.07423367184291418
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": false,
+      "max_dp": 0.11393320929632184
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": true,
+      "max_dp": 0.034915014432512104
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.04979448164232131
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": true,
+      "max_dp": 0.029414872780939155
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": false,
+      "max_dp": 0.028822491676468137
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.03173019440403124
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.053034737449147495
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.06502178840458794
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": true,
+      "max_dp": 0.06925277511174643
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.03959378748721776
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.03959378748721776
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.03122654980007289
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.05031619067151866
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.022659311378469382
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.022659311378469382
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 0.06509421940795102
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.025420309509618932
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.029090351931476355
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": true,
+      "max_dp": 0.026135034295975357
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.05961716899244618
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.06407758504308447
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 0.00012077113361275149
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.06620208586082599
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.042839363085095186
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.042839363085095186
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.02558610782574927
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": false,
+      "max_dp": 0.04379938451184043
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 0.05872126528193622
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": true,
+      "max_dp": 0.02140930945915795
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.07109879973000854
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 0.06369565217422934
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": true,
+      "max_dp": 0.031120112815562595
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 0.022082565798689474
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.03065849725280187
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.028059829821286103
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.028449193484745505
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.05851506715185559
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024-p2.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "p2",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 6,
+    "max_abs_dprob": 0.8135473763051018,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      429.2460420401767,
+      353.15525002079085,
+      387.53008301137015,
+      343.0498329689726
+    ],
+    "total_median_ms": 1512.9812080413103
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": false,
+      "max_dp": 0.5687082525132482
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": false,
+      "max_dp": 0.41328433929777286
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": false,
+      "max_dp": 0.43647201292379934
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": false,
+      "max_dp": 0.4272092319579229
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": false,
+      "max_dp": 0.32375322767290843
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": false,
+      "max_dp": 0.4142483259050501
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.3743470505022063
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": false,
+      "max_dp": 0.627433065688875
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": false,
+      "max_dp": 0.6718467618092826
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": false,
+      "max_dp": 0.4665919790234511
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": false,
+      "max_dp": 0.5326642917768627
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": false,
+      "max_dp": 0.5326642917768627
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": false,
+      "max_dp": 0.5260452676111571
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": false,
+      "max_dp": 0.4804816858684223
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": false,
+      "max_dp": 0.6212987453273393
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": false,
+      "max_dp": 0.6212987453273393
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": false,
+      "max_dp": 0.6830398362099261
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.3965768232005674
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": false,
+      "max_dp": 0.6566645424807097
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": false,
+      "max_dp": 0.42693404110832034
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.39543221124975514
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.3838160625115751
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 0.2745732786294929
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.4088121763358963
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": false,
+      "max_dp": 0.505101010481765
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": false,
+      "max_dp": 0.505101010481765
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": false,
+      "max_dp": 0.5148372020364335
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": false,
+      "max_dp": 0.2981218394768944
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": false,
+      "max_dp": 0.30424236523749426
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": false,
+      "max_dp": 0.5546622944993145
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": false,
+      "max_dp": 0.5048356461813215
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": false,
+      "max_dp": 0.47214688871146926
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": false,
+      "max_dp": 0.6898286069721616
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": false,
+      "max_dp": 0.7098385652992397
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": false,
+      "max_dp": 0.5942935100456078
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": false,
+      "max_dp": 0.8135473763051018
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": false,
+      "max_dp": 0.5241194479260196
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": false,
+      "max_dp": 0.5121825452757068
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024-p3.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "p3",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 24,
+    "max_abs_dprob": 0.6565813498870998,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      399.0579579840414,
+      392.0657499693334,
+      422.44550003670156,
+      404.485291976016
+    ],
+    "total_median_ms": 1618.0544999660924
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 0.20477278141428873
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": true,
+      "max_dp": 0.32182462284515256
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": false,
+      "max_dp": 0.6383872513241502
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": false,
+      "max_dp": 0.5428712577151502
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": false,
+      "max_dp": 0.28012111790100114
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": true,
+      "max_dp": 0.2730227348848564
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.014035262313910463
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.10959428574516555
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.25442104337856575
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": true,
+      "max_dp": 0.37126076908743927
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": false,
+      "max_dp": 0.6565813498870998
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": false,
+      "max_dp": 0.6565813498870998
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.2037210422199619
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.25803973679582753
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": false,
+      "max_dp": 0.6149383815069106
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": false,
+      "max_dp": 0.6149383815069106
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 0.1543348712111362
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.01641404768405741
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.1382605280721213
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": false,
+      "max_dp": 0.15710202827101766
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.11378741988508534
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.09928432017720923
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 0.12112929394216453
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.12896977032051227
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": false,
+      "max_dp": 0.6155217201915915
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": false,
+      "max_dp": 0.6155217201915915
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.19272326872300422
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": true,
+      "max_dp": 0.27443434709506853
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 0.17319810995032459
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": true,
+      "max_dp": 0.1581731006713062
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": false,
+      "max_dp": 0.39554862045404493
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 0.08059553298355264
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": false,
+      "max_dp": 0.6526351707965768
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": false,
+      "max_dp": 0.6155887319240242
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.1968655525406201
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.2238166292610153
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.1914340518507709
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": false,
+      "max_dp": 0.44901466591777284
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024-p4.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "p4",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 28,
+    "max_abs_dprob": 0.6404271157522109,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      466.56716597499326,
+      369.7054999647662,
+      367.6899579586461,
+      377.8666249709204
+    ],
+    "total_median_ms": 1581.829248869326
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 0.14065110666801378
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": false,
+      "max_dp": 0.14352393924093387
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": false,
+      "max_dp": 0.30558581990668504
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": false,
+      "max_dp": 0.2112153083609784
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": true,
+      "max_dp": 0.10345709330955521
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": true,
+      "max_dp": 0.05338048761996056
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.018446724102591072
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.09670670294139116
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.1618101452723214
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": true,
+      "max_dp": 0.34202689619696686
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.10629037171191869
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.10629037171191869
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.16344160908874394
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.20356192593614736
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": false,
+      "max_dp": 0.6404271157522109
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": false,
+      "max_dp": 0.6404271157522109
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 0.13772354099997364
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.04155307767785487
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.06693811751961454
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": false,
+      "max_dp": 0.13617644681578694
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.07300072184236217
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.07632032126032129
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 0.0006130533453932822
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.0945708337940202
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": false,
+      "max_dp": 0.18486173895959152
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": false,
+      "max_dp": 0.18486173895959152
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.20820405528497016
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": true,
+      "max_dp": 0.10358996634122436
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 0.11984177661297724
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": false,
+      "max_dp": 0.1701387399907627
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.13243372105714854
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 0.07808010490012673
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": false,
+      "max_dp": 0.5079165014416612
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 0.32018961743446883
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.10430520164569212
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.07063125525921335
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.011290196320081769
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.1626319683951326
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024-w4.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "w4",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 27,
+    "max_abs_dprob": 0.28687486630569037,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      348.68966601789,
+      319.1623340244405,
+      306.13275000359863,
+      287.22800000105053
+    ],
+    "total_median_ms": 1261.2127500469796
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": false,
+      "max_dp": 0.28687486630569037
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": false,
+      "max_dp": 0.215117218399545
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": true,
+      "max_dp": 0.037370487378679096
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.19235730903722675
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": false,
+      "max_dp": 0.15989540065874677
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": false,
+      "max_dp": 0.17292840293135095
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.06673534662952563
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.15637128029391983
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.1212776024650523
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": false,
+      "max_dp": 0.26838432351644004
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.059219531931129055
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.059219531931129055
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.10129386609378932
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.06449920530279002
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.08438565365029344
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.08438565365029344
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": false,
+      "max_dp": 0.1747322845076672
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.06288963695941752
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.17496832417669272
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": true,
+      "max_dp": 0.07336913610560714
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.1006785107549889
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.101714639963281
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 0.001041388620894295
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.1414300617061629
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.11013718994494187
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.11013718994494187
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.02300337896298682
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": false,
+      "max_dp": 0.09130431766870911
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": false,
+      "max_dp": 0.2012260189291402
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": false,
+      "max_dp": 0.17342324253339314
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": false,
+      "max_dp": 0.1216750202120585
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": false,
+      "max_dp": 0.14403943507599326
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": true,
+      "max_dp": 0.046155205646220554
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 0.06283949173920308
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.12453941885505102
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.023434904267487427
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.02325415239665407
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.13598703370867293
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024-w8.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "w8",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 37,
+    "max_abs_dprob": 0.01866540732834758,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      271.13725000526756,
+      292.92625002563,
+      284.199166984763,
+      265.8617499982938
+    ],
+    "total_median_ms": 1114.1244170139544
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 0.0028505089814712115
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": true,
+      "max_dp": 0.00791437973043406
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": true,
+      "max_dp": 0.006882472978350429
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.01866540732834758
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": true,
+      "max_dp": 0.007077057703832573
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": true,
+      "max_dp": 0.0109383117267077
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.0009742396350422115
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.008292989571482612
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.01332707174002723
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": true,
+      "max_dp": 0.006407588323357888
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.0034236239568858906
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.0034236239568858906
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.006168479890216183
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.005418544837315742
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.004931235993375693
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.004931235993375693
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 0.011089760977621571
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.001662613357885534
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.00539558473048031
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": true,
+      "max_dp": 0.0034671557666123703
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.00021170577901871712
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.0058859875813840334
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 3.196021289741946e-05
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.0024280961833967885
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.005742868221302988
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.005742868221302988
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.009220490312092655
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": false,
+      "max_dp": 0.016968474344047857
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 0.008312426332215517
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": true,
+      "max_dp": 0.010419121933408743
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.0048458512937742115
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 0.007844154239611711
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": true,
+      "max_dp": 0.0010591658741868626
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 0.006739167942930807
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.009507948036988512
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.0018403724333619087
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.011589106181596698
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.010570456488607854
+    }
+  ]
+}

reports/parity-coreml-gpu-L1024.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "fp16",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 38,
+    "max_abs_dprob": 0.005332277217459325,
+    "vision_median_ms": null,
+    "part_median_ms": [
+      252.89000000339,
+      325.01487497938797,
+      295.79125001328066,
+      295.77445797622204
+    ],
+    "total_median_ms": 1169.4705829722807
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 0.0013557465683406988
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": true,
+      "max_dp": 0.0012831014504418592
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": true,
+      "max_dp": 0.0013940747313764823
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.0014271662893113324
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": true,
+      "max_dp": 0.002453201446875608
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": true,
+      "max_dp": 0.0014488863357660642
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.00024320633640428913
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 0.0034050796094206026
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.0007597900170047689
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": true,
+      "max_dp": 0.005332277217459325
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.0009478613775283362
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 0.0009478613775283362
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 0.002025311022411991
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.0009282289231616891
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.0010418617694534227
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 0.0010418617694534227
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 0.0023362010732152494
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 0.0008742227214414422
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 0.0053213682019735975
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": true,
+      "max_dp": 0.003383377541472332
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.0013452840742332883
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 0.0041590026036618655
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 5.078518103718288e-06
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 0.002319079288102599
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.001192676348024821
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 0.001192676348024821
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 0.0015123887087352683
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": true,
+      "max_dp": 0.003312126718628322
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 0.0006214747193888648
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": true,
+      "max_dp": 0.0007672765542694471
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.0007899162710947594
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 0.003828678728060808
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": true,
+      "max_dp": 0.0010450781801469022
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 0.00028261683947886085
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 0.0012716590126354843
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 0.0002130302268968723
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 0.0026736222300280943
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 0.0011167920698399725
+    }
+  ]
+}

reports/parity-coreml-gpu-multimodal-L2048-torchvision.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "fp16",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 2048,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 38,
+    "max_abs_dprob": 0.0358718125040583,
+    "vision_median_ms": 7293.4595420374535,
+    "part_median_ms": [
+      340.32324998406693,
+      347.7547919610515,
+      348.23950001737103,
+      346.817999961786
+    ],
+    "total_median_ms": 8676.595083961729
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 1002,
+      "argmax_ok": true,
+      "max_dp": 0.002222874956873677
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 1026,
+      "argmax_ok": true,
+      "max_dp": 0.001123447421586321
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 781,
+      "argmax_ok": true,
+      "max_dp": 0.001868124573074996
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 817,
+      "argmax_ok": true,
+      "max_dp": 0.0008109904381745567
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 863,
+      "argmax_ok": true,
+      "max_dp": 0.012290941324490934
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 928,
+      "argmax_ok": true,
+      "max_dp": 0.0004926059891846579
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 538,
+      "argmax_ok": true,
+      "max_dp": 0.0008507799427087681
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 548,
+      "argmax_ok": true,
+      "max_dp": 0.02000998734575321
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.002846705866439314
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.005597217626309048
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.0020823003170948873
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.0020823003170948873
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 572,
+      "argmax_ok": true,
+      "max_dp": 0.0006630318313437522
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 620,
+      "argmax_ok": true,
+      "max_dp": 0.0020325439636200704
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.004542320455361293
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.004542320455361293
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 651,
+      "argmax_ok": true,
+      "max_dp": 0.00027697077769795256
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 521,
+      "argmax_ok": true,
+      "max_dp": 0.00045295824500390935
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 549,
+      "argmax_ok": true,
+      "max_dp": 0.0024729572969196534
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 627,
+      "argmax_ok": true,
+      "max_dp": 0.0024866317450186193
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.0009664898006364331
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.0035993701412696533
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 374,
+      "argmax_ok": true,
+      "max_dp": 0.0358718125040583
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 432,
+      "argmax_ok": true,
+      "max_dp": 0.003283712973003783
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.0016774495763075548
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.0016774495763075548
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 840,
+      "argmax_ok": true,
+      "max_dp": 0.001508346342089073
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 900,
+      "argmax_ok": true,
+      "max_dp": 0.0004775361796907074
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 613,
+      "argmax_ok": true,
+      "max_dp": 0.0018847202967135837
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 654,
+      "argmax_ok": true,
+      "max_dp": 0.004349662401258114
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 648,
+      "argmax_ok": true,
+      "max_dp": 0.004430111240639578
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 666,
+      "argmax_ok": true,
+      "max_dp": 0.0006837646831657912
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 576,
+      "argmax_ok": true,
+      "max_dp": 0.0013776390543226436
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 528,
+      "argmax_ok": true,
+      "max_dp": 0.0014011652330296887
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 639,
+      "argmax_ok": true,
+      "max_dp": 0.003835238644366301
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.004137981554018344
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 584,
+      "argmax_ok": true,
+      "max_dp": 0.0009202695252754217
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 626,
+      "argmax_ok": true,
+      "max_dp": 0.0016012722859765072
+    }
+  ]
+}

reports/parity-coreml-gpu-multimodal-L2048-visionfp32.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "fp16",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 2048,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 38,
+    "max_abs_dprob": 0.045551832508339773,
+    "vision_median_ms": 785.1727500092238,
+    "part_median_ms": [
+      345.99254099885,
+      1854.538790998049,
+      622.7105000289157,
+      601.5337919816375
+    ],
+    "total_median_ms": 4209.948374016676
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 1002,
+      "argmax_ok": true,
+      "max_dp": 0.0020506175334727983
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 1026,
+      "argmax_ok": true,
+      "max_dp": 0.0026555252108929617
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 781,
+      "argmax_ok": true,
+      "max_dp": 0.002496370443694329
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 817,
+      "argmax_ok": true,
+      "max_dp": 0.0015541529576972302
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 863,
+      "argmax_ok": true,
+      "max_dp": 0.007324875357573368
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 928,
+      "argmax_ok": true,
+      "max_dp": 0.00016113775471182645
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 538,
+      "argmax_ok": true,
+      "max_dp": 0.00029780156211300923
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 548,
+      "argmax_ok": true,
+      "max_dp": 0.011298172465382117
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.001007906994924701
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.011514368460363883
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.0020824791310292134
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.0020824791310292134
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 572,
+      "argmax_ok": true,
+      "max_dp": 0.0015560286193686546
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 620,
+      "argmax_ok": true,
+      "max_dp": 0.01885322158890068
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.0163651997697839
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.0163651997697839
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 651,
+      "argmax_ok": true,
+      "max_dp": 0.0036545504028639986
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 521,
+      "argmax_ok": true,
+      "max_dp": 0.0025109278151638215
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 549,
+      "argmax_ok": true,
+      "max_dp": 0.00247701041276438
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 627,
+      "argmax_ok": true,
+      "max_dp": 0.008948669308342594
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.0019391658396696876
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.005716856218328403
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 374,
+      "argmax_ok": true,
+      "max_dp": 0.03362193087888282
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 432,
+      "argmax_ok": true,
+      "max_dp": 0.0033055166493513033
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.0017985066098463731
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.0017985066098463731
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 840,
+      "argmax_ok": true,
+      "max_dp": 0.0002134354343437117
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 900,
+      "argmax_ok": true,
+      "max_dp": 0.0020882921001008636
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 613,
+      "argmax_ok": true,
+      "max_dp": 0.0018753809939353397
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 654,
+      "argmax_ok": true,
+      "max_dp": 0.009174002744733456
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 648,
+      "argmax_ok": true,
+      "max_dp": 0.045551832508339773
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 666,
+      "argmax_ok": true,
+      "max_dp": 0.010168928024095525
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 576,
+      "argmax_ok": true,
+      "max_dp": 0.0015190808763746455
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 528,
+      "argmax_ok": true,
+      "max_dp": 0.001173787503128798
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 639,
+      "argmax_ok": true,
+      "max_dp": 0.0018829481093931566
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.00017999163447537242
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 584,
+      "argmax_ok": true,
+      "max_dp": 0.0008989385130264288
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 626,
+      "argmax_ok": true,
+      "max_dp": 0.00024276322225580405
+    }
+  ]
+}

reports/parity-coreml-gpu-multimodal-L2048-visionmixed.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "fp16",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 2048,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 38,
+    "max_abs_dprob": 0.1425908220822863,
+    "vision_median_ms": 882.7606250415556,
+    "part_median_ms": [
+      615.1017500087619,
+      601.3449580059387,
+      624.3066250463016,
+      616.3362910156138
+    ],
+    "total_median_ms": 3339.8502491181716
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 1002,
+      "argmax_ok": true,
+      "max_dp": 0.013243654566553853
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 1026,
+      "argmax_ok": true,
+      "max_dp": 0.008527655604874895
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 781,
+      "argmax_ok": true,
+      "max_dp": 0.0017440406173393486
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 817,
+      "argmax_ok": true,
+      "max_dp": 0.001955666549901647
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 863,
+      "argmax_ok": true,
+      "max_dp": 0.07370303407643375
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 928,
+      "argmax_ok": true,
+      "max_dp": 0.0009452436616089743
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 538,
+      "argmax_ok": true,
+      "max_dp": 0.0003076422427848169
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 548,
+      "argmax_ok": true,
+      "max_dp": 0.09255520225423763
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.016200818219079816
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.022220480249113578
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.00015230869682179328
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.00015230869682179328
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 572,
+      "argmax_ok": true,
+      "max_dp": 0.0038864714950722634
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 620,
+      "argmax_ok": true,
+      "max_dp": 0.009256924933621291
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.038070767606545375
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.038070767606545375
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 651,
+      "argmax_ok": true,
+      "max_dp": 0.006220649569734116
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 521,
+      "argmax_ok": true,
+      "max_dp": 0.0008731258919589813
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 549,
+      "argmax_ok": true,
+      "max_dp": 0.0022493190304799446
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 627,
+      "argmax_ok": true,
+      "max_dp": 0.007809564942040104
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.006592587122146748
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.002637143827428745
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 374,
+      "argmax_ok": true,
+      "max_dp": 0.1425908220822863
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 432,
+      "argmax_ok": true,
+      "max_dp": 0.0033315936814405367
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.0012859741526353163
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.0012859741526353163
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 840,
+      "argmax_ok": true,
+      "max_dp": 0.0030782134761833113
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 900,
+      "argmax_ok": true,
+      "max_dp": 0.001886947610049594
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 613,
+      "argmax_ok": true,
+      "max_dp": 0.006313233727756717
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 654,
+      "argmax_ok": true,
+      "max_dp": 0.00711853656965411
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 648,
+      "argmax_ok": true,
+      "max_dp": 0.013185705732598196
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 666,
+      "argmax_ok": true,
+      "max_dp": 0.021558087948041926
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 576,
+      "argmax_ok": true,
+      "max_dp": 0.005989667653108044
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 528,
+      "argmax_ok": true,
+      "max_dp": 0.01565992155540541
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 639,
+      "argmax_ok": true,
+      "max_dp": 0.002057974132597229
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.011991963134956353
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 584,
+      "argmax_ok": true,
+      "max_dp": 0.005993280710248294
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 626,
+      "argmax_ok": true,
+      "max_dp": 0.006225579441585638
+    }
+  ]
+}

reports/parity-coreml-gpu-multimodal-L2048-w8.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "w8",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 2048,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 38,
+    "max_abs_dprob": 0.16312610859823745,
+    "vision_median_ms": 510.8245830051601,
+    "part_median_ms": [
+      630.2213330054656,
+      638.5109160328284,
+      640.1220000116155,
+      621.254040976055
+    ],
+    "total_median_ms": 3040.9328730311245
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 1002,
+      "argmax_ok": true,
+      "max_dp": 0.022668143910619
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 1026,
+      "argmax_ok": true,
+      "max_dp": 0.00266838639805745
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 781,
+      "argmax_ok": true,
+      "max_dp": 0.002651205888397956
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 817,
+      "argmax_ok": true,
+      "max_dp": 0.0009677791673943226
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 863,
+      "argmax_ok": true,
+      "max_dp": 0.12728227511380252
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 928,
+      "argmax_ok": true,
+      "max_dp": 0.005407598192486485
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 538,
+      "argmax_ok": true,
+      "max_dp": 0.007619874544901095
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 548,
+      "argmax_ok": true,
+      "max_dp": 0.016739540091572302
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.021448813281164325
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.08660003264170216
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.005546105493914155
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.005546105493914155
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 572,
+      "argmax_ok": true,
+      "max_dp": 0.019457031980116946
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 620,
+      "argmax_ok": true,
+      "max_dp": 0.0397565109069854
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.02683040448551266
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.02683040448551266
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 651,
+      "argmax_ok": true,
+      "max_dp": 0.025215934603913803
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 521,
+      "argmax_ok": true,
+      "max_dp": 0.005478776793753415
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 549,
+      "argmax_ok": true,
+      "max_dp": 0.0010590947431804443
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 627,
+      "argmax_ok": true,
+      "max_dp": 0.020568027621589047
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.00947780584383251
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.021806124948511196
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 374,
+      "argmax_ok": true,
+      "max_dp": 0.16312610859823745
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 432,
+      "argmax_ok": true,
+      "max_dp": 0.012304995169048705
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.015915397770951967
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.015915397770951967
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 840,
+      "argmax_ok": true,
+      "max_dp": 0.005218795084001915
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 900,
+      "argmax_ok": true,
+      "max_dp": 0.006459875561862338
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 613,
+      "argmax_ok": true,
+      "max_dp": 0.024495034570041874
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 654,
+      "argmax_ok": true,
+      "max_dp": 0.037513481042803165
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 648,
+      "argmax_ok": true,
+      "max_dp": 0.03224467832972039
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 666,
+      "argmax_ok": true,
+      "max_dp": 0.01564536679096795
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 576,
+      "argmax_ok": true,
+      "max_dp": 0.01485067916009386
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 528,
+      "argmax_ok": true,
+      "max_dp": 0.026978185667620336
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 639,
+      "argmax_ok": true,
+      "max_dp": 0.023508791717588684
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.03001575531576761
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 584,
+      "argmax_ok": true,
+      "max_dp": 0.0068833417060234076
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 626,
+      "argmax_ok": true,
+      "max_dp": 0.02101945227483759
+    }
+  ]
+}

reports/parity-coreml-gpu-multimodal-L2048.json

@@ -0,0 +1,251 @@
+{
+  "summary": {
+    "backend": "coreml",
+    "variant": "fp16",
+    "reference": "fp32",
+    "compute_units": "CPU_AND_GPU",
+    "length": 2048,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 38,
+    "max_abs_dprob": 0.1525075821096949,
+    "vision_median_ms": 550.3935420420021,
+    "part_median_ms": [
+      652.348417090252,
+      658.6461249971762,
+      698.5457079717889,
+      634.9123340332881
+    ],
+    "total_median_ms": 3194.8461261345074
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 1002,
+      "argmax_ok": true,
+      "max_dp": 0.004830977124425151
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 1026,
+      "argmax_ok": true,
+      "max_dp": 0.0033957171406390207
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 781,
+      "argmax_ok": true,
+      "max_dp": 0.00543923057018969
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 817,
+      "argmax_ok": true,
+      "max_dp": 0.0004166579168036755
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 863,
+      "argmax_ok": true,
+      "max_dp": 0.07890622154210147
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 928,
+      "argmax_ok": true,
+      "max_dp": 0.0003646858871921288
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 538,
+      "argmax_ok": true,
+      "max_dp": 0.0037272740085546596
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 548,
+      "argmax_ok": true,
+      "max_dp": 0.01900564908128788
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.014496040391342668
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 598,
+      "argmax_ok": true,
+      "max_dp": 0.008862909867578012
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.0017876899141183178
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 0.0017876899141183178
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 572,
+      "argmax_ok": true,
+      "max_dp": 0.0007830755859213889
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 620,
+      "argmax_ok": true,
+      "max_dp": 0.015055085963436965
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.037008195604134486
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 641,
+      "argmax_ok": true,
+      "max_dp": 0.037008195604134486
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 651,
+      "argmax_ok": true,
+      "max_dp": 0.008628140776856918
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 521,
+      "argmax_ok": true,
+      "max_dp": 0.0030559075882785613
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 549,
+      "argmax_ok": true,
+      "max_dp": 0.004329463167738745
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 627,
+      "argmax_ok": true,
+      "max_dp": 0.009905443066277164
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.004768892637069561
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 429,
+      "argmax_ok": true,
+      "max_dp": 0.0006574433078861963
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 374,
+      "argmax_ok": true,
+      "max_dp": 0.1525075821096949
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 432,
+      "argmax_ok": true,
+      "max_dp": 0.0003508250008680269
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.010556880600999574
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 615,
+      "argmax_ok": true,
+      "max_dp": 0.010556880600999574
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 840,
+      "argmax_ok": true,
+      "max_dp": 0.0036029727687858504
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 900,
+      "argmax_ok": true,
+      "max_dp": 0.0015813545962861664
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 613,
+      "argmax_ok": true,
+      "max_dp": 0.0027808240597879674
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 654,
+      "argmax_ok": true,
+      "max_dp": 0.014192594625531796
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 648,
+      "argmax_ok": true,
+      "max_dp": 0.043673302721276175
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 666,
+      "argmax_ok": true,
+      "max_dp": 0.028177183750349055
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 576,
+      "argmax_ok": true,
+      "max_dp": 0.0067089231023546025
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 528,
+      "argmax_ok": true,
+      "max_dp": 0.1333355971289818
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 639,
+      "argmax_ok": true,
+      "max_dp": 0.0065935895767805786
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 0.009584952016817416
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 584,
+      "argmax_ok": true,
+      "max_dp": 0.0014312604084294467
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 626,
+      "argmax_ok": true,
+      "max_dp": 0.004074090183773138
+    }
+  ]
+}

reports/parity-torch-L1024.json

@@ -0,0 +1,249 @@
+{
+  "summary": {
+    "backend": "torch",
+    "reference": "fp32",
+    "compute_units": "cpu-fp32",
+    "length": 1024,
+    "tasks": 38,
+    "skipped_too_long": 0,
+    "argmax_agree": 38,
+    "max_abs_dprob": 4.702754095753381e-06,
+    "part_median_ms": [
+      2811.394124990329,
+      3006.3704589847475,
+      3003.8133329944685,
+      2851.077416969929
+    ],
+    "total_median_ms": 11672.655333939474
+  },
+  "rows": [
+    {
+      "id": "clinic_intake-20283883-bc689231",
+      "tokens": 768,
+      "argmax_ok": true,
+      "max_dp": 9.147513919671724e-07
+    },
+    {
+      "id": "clinic_intake-20400778-53a51f75",
+      "tokens": 795,
+      "argmax_ok": true,
+      "max_dp": 4.6987792932939865e-07
+    },
+    {
+      "id": "job_application-21044850-e0fefe99",
+      "tokens": 579,
+      "argmax_ok": true,
+      "max_dp": 1.2672676221736623e-06
+    },
+    {
+      "id": "job_application-21012745-f7d6406e",
+      "tokens": 602,
+      "argmax_ok": true,
+      "max_dp": 4.702754095753381e-06
+    },
+    {
+      "id": "vendor_onboarding-20329340-836e7d71",
+      "tokens": 664,
+      "argmax_ok": true,
+      "max_dp": 1.9994469729056608e-06
+    },
+    {
+      "id": "vendor_onboarding-20943483-12390c86",
+      "tokens": 709,
+      "argmax_ok": true,
+      "max_dp": 1.3906649167505503e-06
+    },
+    {
+      "id": "saucedemo_login-21182467-8bf83912",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 3.9784679273235923e-07
+    },
+    {
+      "id": "saucedemo_login-20893662-ce098616",
+      "tokens": 450,
+      "argmax_ok": true,
+      "max_dp": 1.29057923026199e-06
+    },
+    {
+      "id": "portal_login-20685703-bc0d7f51",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 1.5045270255198417e-06
+    },
+    {
+      "id": "portal_login-21146867-6d21d521",
+      "tokens": 481,
+      "argmax_ok": true,
+      "max_dp": 4.837099368271147e-07
+    },
+    {
+      "id": "clinic_consent-21227628-e458c37c",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 1.959718909816388e-06
+    },
+    {
+      "id": "clinic_consent-20673985-696bdc40",
+      "tokens": 404,
+      "argmax_ok": true,
+      "max_dp": 1.959718909816388e-06
+    },
+    {
+      "id": "signup_consent-20620985-aa3f2d4b",
+      "tokens": 452,
+      "argmax_ok": true,
+      "max_dp": 2.5574530382321825e-08
+    },
+    {
+      "id": "signup_consent-21012904-a193c88c",
+      "tokens": 485,
+      "argmax_ok": true,
+      "max_dp": 5.423541816207411e-07
+    },
+    {
+      "id": "cookie_consent-21194772-58cc8067",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 1.5989046097397708e-06
+    },
+    {
+      "id": "cookie_consent-20840184-c06eda3c",
+      "tokens": 493,
+      "argmax_ok": true,
+      "max_dp": 1.5989046097397708e-06
+    },
+    {
+      "id": "patient_registration-20585731-862b3203",
+      "tokens": 558,
+      "argmax_ok": true,
+      "max_dp": 2.1427884557279064e-06
+    },
+    {
+      "id": "patient_registration-20432990-75a864b7",
+      "tokens": 462,
+      "argmax_ok": true,
+      "max_dp": 1.1069526583210032e-06
+    },
+    {
+      "id": "auto_claim-20910192-1e258e12",
+      "tokens": 445,
+      "argmax_ok": true,
+      "max_dp": 2.4285161520642973e-06
+    },
+    {
+      "id": "auto_claim-20612919-4f1b18b4",
+      "tokens": 518,
+      "argmax_ok": true,
+      "max_dp": 1.857031752483529e-06
+    },
+    {
+      "id": "results_pager-21213339-d5041ea7",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 5.2362992053467394e-08
+    },
+    {
+      "id": "results_pager-20362478-e4b57d39",
+      "tokens": 358,
+      "argmax_ok": true,
+      "max_dp": 5.953822818233201e-07
+    },
+    {
+      "id": "directory_pager-20783164-d32388b2",
+      "tokens": 340,
+      "argmax_ok": true,
+      "max_dp": 4.7481346965305704e-08
+    },
+    {
+      "id": "directory_pager-20674764-f028ed82",
+      "tokens": 362,
+      "argmax_ok": true,
+      "max_dp": 2.240133406550271e-07
+    },
+    {
+      "id": "inbox_pager-21188522-314b8677",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 1.1816418577503818e-06
+    },
+    {
+      "id": "inbox_pager-20523952-90dcae07",
+      "tokens": 491,
+      "argmax_ok": true,
+      "max_dp": 1.1816418577503818e-06
+    },
+    {
+      "id": "flight_search-20992474-8c3b227d",
+      "tokens": 638,
+      "argmax_ok": true,
+      "max_dp": 8.140559128455571e-07
+    },
+    {
+      "id": "flight_search-20404065-fccc21db",
+      "tokens": 673,
+      "argmax_ok": true,
+      "max_dp": 1.6499142088122198e-06
+    },
+    {
+      "id": "shop_filter-20481321-b40d9384",
+      "tokens": 479,
+      "argmax_ok": true,
+      "max_dp": 3.3471618451907936e-07
+    },
+    {
+      "id": "shop_filter-20318615-7c05f11e",
+      "tokens": 506,
+      "argmax_ok": true,
+      "max_dp": 1.382792182202941e-06
+    },
+    {
+      "id": "account_settings-20548707-93883a27",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 6.60394960938504e-07
+    },
+    {
+      "id": "account_settings-21050921-39b3271c",
+      "tokens": 511,
+      "argmax_ok": true,
+      "max_dp": 1.6945591401196225e-06
+    },
+    {
+      "id": "order_lookup-20370971-3108be55",
+      "tokens": 488,
+      "argmax_ok": true,
+      "max_dp": 2.273639973293662e-06
+    },
+    {
+      "id": "order_lookup-20862290-73301ee7",
+      "tokens": 433,
+      "argmax_ok": true,
+      "max_dp": 1.4381018013098767e-06
+    },
+    {
+      "id": "dev_portal_profile-21060759-7a38b335",
+      "tokens": 520,
+      "argmax_ok": true,
+      "max_dp": 1.9681068859356277e-06
+    },
+    {
+      "id": "dev_portal_profile-20848041-32c736b7",
+      "tokens": 499,
+      "argmax_ok": true,
+      "max_dp": 2.1845971324985225e-07
+    },
+    {
+      "id": "subscription_manage-20772973-bd02f49c",
+      "tokens": 475,
+      "argmax_ok": true,
+      "max_dp": 1.2955594439478801e-06
+    },
+    {
+      "id": "subscription_manage-20657376-2e7dfb6f",
+      "tokens": 500,
+      "argmax_ok": true,
+      "max_dp": 6.754236206718556e-07
+    }
+  ]
+}

text/L1024-gptq/config.json

@@ -0,0 +1,69 @@
+{
+  "modality": "text",
+  "seq_len": 1024,
+  "hidden_size": 2560,
+  "rotary_dim": 64,
+  "rope_theta": 10000000,
+  "mrope_section": [
+    11,
+    11,
+    10
+  ],
+  "vocab_size": 248320,
+  "delta_chunk_size": 64,
+  "parts": [
+    {
+      "layers": [
+        0,
+        8
+      ]
+    },
+    {
+      "layers": [
+        8,
+        16
+      ]
+    },
+    {
+      "layers": [
+        16,
+        24
+      ]
+    },
+    {
+      "layers": [
+        24,
+        32
+      ]
+    }
+  ],
+  "letters": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+  "letter_token_ids": [
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47,
+    48,
+    49,
+    50,
+    51,
+    52,
+    53,
+    54,
+    55,
+    56,
+    57
+  ]
+}

text/L1024-w8/config.json

@@ -0,0 +1,69 @@
+{
+  "modality": "text",
+  "seq_len": 1024,
+  "hidden_size": 2560,
+  "rotary_dim": 64,
+  "rope_theta": 10000000,
+  "mrope_section": [
+    11,
+    11,
+    10
+  ],
+  "vocab_size": 248320,
+  "delta_chunk_size": 64,
+  "parts": [
+    {
+      "layers": [
+        0,
+        8
+      ]
+    },
+    {
+      "layers": [
+        8,
+        16
+      ]
+    },
+    {
+      "layers": [
+        16,
+        24
+      ]
+    },
+    {
+      "layers": [
+        24,
+        32
+      ]
+    }
+  ],
+  "letters": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+  "letter_token_ids": [
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47,
+    48,
+    49,
+    50,
+    51,
+    52,
+    53,
+    54,
+    55,
+    56,
+    57
+  ]
+}

text/L1024/config.json

@@ -0,0 +1,69 @@
+{
+  "modality": "text",
+  "seq_len": 1024,
+  "hidden_size": 2560,
+  "rotary_dim": 64,
+  "rope_theta": 10000000,
+  "mrope_section": [
+    11,
+    11,
+    10
+  ],
+  "vocab_size": 248320,
+  "delta_chunk_size": 64,
+  "parts": [
+    {
+      "layers": [
+        0,
+        8
+      ]
+    },
+    {
+      "layers": [
+        8,
+        16
+      ]
+    },
+    {
+      "layers": [
+        16,
+        24
+      ]
+    },
+    {
+      "layers": [
+        24,
+        32
+      ]
+    }
+  ],
+  "letters": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+  "letter_token_ids": [
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47,
+    48,
+    49,
+    50,
+    51,
+    52,
+    53,
+    54,
+    55,
+    56,
+    57
+  ]
+}