Add OmniJev 4B MLX 5-bit model card, head, and inference wrapper

LICENSE

@@ -0,0 +1,208 @@
+OmniJev - inference code and model weights
+Copyright 2026 Beijing Zhongguancun Academy, Institute of Automation of the Chinese Academy of Sciences, and Zevo
+(北京中关村学院、中国科学院自动化研究所、智进化)
+
+Licensed under the Apache License, Version 2.0 (full text below). The backbone model (Qwen3-VL-4B-Instruct) keeps its own license.
+
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE.md

@@ -0,0 +1,8 @@
+# Attribution
+
+This repository is an independent Apple Silicon MLX conversion of the official [OmniJev v1.1 4B](https://huggingface.co/tinnel123/OmniJev) checkpoint.
+
+- Upstream decision model and head: Beijing Zhongguancun Academy, Institute of Automation of the Chinese Academy of Sciences, and Zevo.
+- Backbone: [Qwen/Qwen3.5-4B](https://huggingface.co/Qwen/Qwen3.5-4B).
+- Upstream project and license: [tinnel123666888/OmniJev](https://github.com/tinnel123666888/OmniJev), Apache-2.0. The Qwen backbone keeps its own upstream terms.
+- This repository contains a fused and quantized derivative plus an MLX inference wrapper. It is not an official release of the upstream projects.

README.md

@@ -0,0 +1,111 @@
+---
+license: apache-2.0
+base_model:
+- tinnel123/OmniJev
+- Qwen/Qwen3.5-4B
+library_name: mlx
+pipeline_tag: image-text-to-text
+tags:
+- mlx
+- apple-silicon
+- multimodal
+- qwen3.5
+- omnijev
+- decision-making
+- quantized
+language:
+- en
+- zh
+---
+
+# OmniJev-MLX-5bit
+
+An independent Apple Silicon MLX conversion of the official [OmniJev v1.1 4B](https://huggingface.co/tinnel123/OmniJev) checkpoint.
+
+The model uses the official `Qwen/Qwen3.5-4B` backbone and OmniJev decision head. The official LoRA adapter was merged into the backbone in bfloat16, and the fused backbone was then converted to **5-bit affine quantization with group size 64**. The decision head remains a separate FP32 file and is loaded at inference time.
+
+This repository is a conversion and packaging release. **It is not a new fine-tune and does not contain additional training of the 4B model or decision head.** It is not an official release of the upstream OmniJev or Qwen projects.
+
+## What is included
+
+- `model.safetensors`: fused Qwen3.5/OmniJev backbone in MLX 5-bit format.
+- `decision_head/head.npz`: the official OmniJev 4B decision head exported from the released `head.pt`.
+- `decision_head/head_meta.json`: upstream calibration metadata, including the choice temperature.
+- `omni_mlx/classifier.py`: a small MLX inference wrapper for typed visual decisions.
+- `conversion.json`: source, quantization and validation provenance.
+- `head_provenance.json`: checksum and export provenance for the decision head.
+- `benchmarks/`: the local fixed-subset ScienceQA measurements used for this conversion.
+
+The head is separate from the quantized backbone. The runtime first obtains option/question representations from the backbone and then applies the decision head. This is different from merging the head into the Qwen weight tensors.
+
+## Conversion provenance
+
+```text
+Qwen/Qwen3.5-4B
+    + official OmniJev v1.1 LoRA adapter
+    └─ merge in bfloat16
+       fused OmniJev backbone
+    └─ MLX affine quantization, 5 bits, group size 64
+       model.safetensors
+
+official OmniJev head.pt
+    └─ format export only
+       decision_head/head.npz (float32)
+```
+
+The source OmniJev release reports a rank-32 LoRA adapter and an independently calibrated decision head. This repository preserves those learned weights; it does not retrain them.
+
+## Local validation
+
+These measurements use one fixed 50-image ScienceQA visual subset, the same subset for both resolutions. They are local conversion checks, not the official OmniJev benchmark and not a universal accuracy claim.
+
+Hardware: Mac mini M4, 16 GB unified memory. Batch size 1.
+
+| Input setting | Correct | Accuracy | Warm median | P95 |
+|---|---:|---:|---:|---:|
+| `max_pixels=87808` (112-level input) | 44/50 | 88.0% | ~737 ms | ~1,013 ms |
+| `max_pixels=175616` (224-level input) | 44/50 | 88.0% | ~1,050 ms | ~1,354 ms |
+
+The complete per-item outputs are in [`benchmarks/scienceqa-50-112.json`](benchmarks/scienceqa-50-112.json) and [`benchmarks/scienceqa-50-224.json`](benchmarks/scienceqa-50-224.json). The text-only JevBench and the upstream OmniJev score tables should not be inferred from this 50-image check.
+
+## Installation
+
+This package targets Apple Silicon and MLX.
+
+```bash
+hf download Ruiruiz30/OmniJev-MLX-5bit \
+  --local-dir OmniJev-MLX-5bit
+
+cd OmniJev-MLX-5bit
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+The model is about 3.3 GB. Keep the repository's directory structure intact so that `decision_head/head.npz` and `decision_head/head_meta.json` remain next to each other.
+
+## Image decision
+
+The included wrapper returns probabilities for the supplied options and selects the highest-probability option. It does not generate a free-form explanation.
+
+```bash
+python -m omni_mlx.classifier \
+  --model . \
+  --image /path/to/frame.png \
+  --state "A kart is approaching a right turn." \
+  --question "Which steering action is best?" \
+  --options "Turn left" "Hold center" "Turn right" \
+  --max-pixels 87808
+```
+
+The output contains `prediction`, option `probabilities`, `abstain_probability`, elapsed time and the quantization setting. The supplied options must be distinct; the head supports typed choice, yes/no and score-style candidate sets through the same candidate-scoring path.
+
+The default `max_pixels=87808` is the setting used for the faster local validation above. Increase it when small visual details matter, and validate the resulting accuracy and latency on your own frames.
+
+## Upstream sources and license
+
+- [Official OmniJev repository](https://github.com/tinnel123666888/OmniJev)
+- [Official OmniJev v1.1 checkpoint](https://huggingface.co/tinnel123/OmniJev)
+- [Qwen3.5-4B backbone](https://huggingface.co/Qwen/Qwen3.5-4B)
+
+The OmniJev project and this conversion wrapper are released under Apache-2.0. The Qwen backbone retains its own upstream terms. See [`LICENSE`](LICENSE) and [`NOTICE.md`](NOTICE.md).

benchmarks/scienceqa-50-112.json

@@ -0,0 +1,619 @@
+{
+  "summary": {
+    "questions": 50,
+    "correct": 44,
+    "accuracy": 0.88,
+    "latency_ms": {
+      "median": 737.4855829984881,
+      "p95": 1012.7517079818062,
+      "mean": 784.4947082991712
+    },
+    "model": "third_party/OmniJev/models/Qwen3.5-4B-OmniJev-MLX-5bit",
+    "note": "Visual ScienceQA subset; not text-only JevBench."
+  },
+  "results": [
+    {
+      "id": "scienceqa/val/21023",
+      "target_index": 2,
+      "prediction_index": 3,
+      "probabilities": [
+        0.0006199186900630593,
+        0.0036421569529920816,
+        0.16167014837265015,
+        0.7949264645576477,
+        0.039141345769166946
+      ],
+      "elapsed_ms": 1012.7517079818062
+    },
+    {
+      "id": "scienceqa/val/7180",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        0.0006342903943732381,
+        0.001218338031321764,
+        0.998112678527832,
+        3.469285002211109e-05
+      ],
+      "elapsed_ms": 684.1524579795077
+    },
+    {
+      "id": "scienceqa/val/2642",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        1.4533699868479744e-05,
+        0.0008359018247574568,
+        0.9989796280860901,
+        0.00016994107863865793
+      ],
+      "elapsed_ms": 800.3734169760719
+    },
+    {
+      "id": "scienceqa/val/16468",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9725253582000732,
+        0.027463596314191818,
+        5.486343184202269e-07,
+        1.6595530496488209e-06,
+        8.804287972452585e-06
+      ],
+      "elapsed_ms": 699.9967499868944
+    },
+    {
+      "id": "scienceqa/val/19506",
+      "target_index": 1,
+      "prediction_index": 0,
+      "probabilities": [
+        0.4955705404281616,
+        0.17129716277122498,
+        0.3331323564052582
+      ],
+      "elapsed_ms": 712.0736669749022
+    },
+    {
+      "id": "scienceqa/val/8767",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.005915054120123386,
+        0.9934005737304688,
+        0.0006844155723229051
+      ],
+      "elapsed_ms": 633.8063329458237
+    },
+    {
+      "id": "scienceqa/val/16515",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        2.5153735805361066e-07,
+        1.2259709913919892e-09,
+        0.9999985694885254,
+        6.42186108734677e-08,
+        1.0174338740398525e-06
+      ],
+      "elapsed_ms": 677.4182500084862
+    },
+    {
+      "id": "scienceqa/val/16938",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.20825311541557312,
+        0.7906606793403625,
+        0.0010861753253266215
+      ],
+      "elapsed_ms": 1000.156500027515
+    },
+    {
+      "id": "scienceqa/val/13212",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9983437061309814,
+        8.884436101652682e-05,
+        0.000598268467001617,
+        0.0009691402083262801
+      ],
+      "elapsed_ms": 669.4296670029871
+    },
+    {
+      "id": "scienceqa/val/4539",
+      "target_index": 3,
+      "prediction_index": 0,
+      "probabilities": [
+        0.4982156753540039,
+        0.328609824180603,
+        0.06777261197566986,
+        0.10534052550792694,
+        6.127937376732007e-05
+      ],
+      "elapsed_ms": 689.8489589802921
+    },
+    {
+      "id": "scienceqa/val/11884",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.00019638118101283908,
+        0.9997889399528503,
+        1.4690514944959432e-05
+      ],
+      "elapsed_ms": 1089.2853750265203
+    },
+    {
+      "id": "scienceqa/val/15622",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        1.2257983144081663e-06,
+        0.9998711347579956,
+        2.1552657926804386e-05,
+        0.00010605484567349777
+      ],
+      "elapsed_ms": 884.3026669928804
+    },
+    {
+      "id": "scienceqa/val/17192",
+      "target_index": 3,
+      "prediction_index": 3,
+      "probabilities": [
+        3.1174602099781623e-07,
+        1.1572042524221615e-07,
+        3.46970040254746e-07,
+        0.999992847442627,
+        6.40239386484609e-06
+      ],
+      "elapsed_ms": 700.0882920110598
+    },
+    {
+      "id": "scienceqa/val/2461",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        0.00027162706828676164,
+        1.1542095307959244e-05,
+        0.9994391798973083,
+        0.0002776358451228589
+      ],
+      "elapsed_ms": 689.2803750233725
+    },
+    {
+      "id": "scienceqa/val/8688",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9999678134918213,
+        5.3547882998827845e-06,
+        2.685382787603885e-05
+      ],
+      "elapsed_ms": 881.5387500217184
+    },
+    {
+      "id": "scienceqa/val/15419",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        2.8274806496142446e-08,
+        1.1287261969528117e-07,
+        0.9999901056289673,
+        1.562869300641978e-08,
+        9.724893970997073e-06
+      ],
+      "elapsed_ms": 722.5853750132956
+    },
+    {
+      "id": "scienceqa/val/18512",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        3.031034534473065e-09,
+        0.9999986886978149,
+        1.2731197784887627e-06
+      ],
+      "elapsed_ms": 735.6334999785759
+    },
+    {
+      "id": "scienceqa/val/921",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.0015609971014782786,
+        0.994932234287262,
+        0.003506765468046069
+      ],
+      "elapsed_ms": 723.512375028804
+    },
+    {
+      "id": "scienceqa/val/17574",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        4.925988469040021e-05,
+        0.9972556233406067,
+        0.00015376514056697488,
+        0.0025415101554244757
+      ],
+      "elapsed_ms": 738.4067500242963
+    },
+    {
+      "id": "scienceqa/val/4517",
+      "target_index": 1,
+      "prediction_index": 0,
+      "probabilities": [
+        0.8720545768737793,
+        0.1277339905500412,
+        0.00021148081577848643
+      ],
+      "elapsed_ms": 1234.5357919693924
+    },
+    {
+      "id": "scienceqa/val/12959",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.999830961227417,
+        3.3016458473866805e-05,
+        1.574484258526354e-06,
+        0.00013444400974549353
+      ],
+      "elapsed_ms": 897.1697079832666
+    },
+    {
+      "id": "scienceqa/val/4018",
+      "target_index": 3,
+      "prediction_index": 3,
+      "probabilities": [
+        0.0005697154556401074,
+        0.0002200113667640835,
+        0.0010174753842875361,
+        0.9911803603172302,
+        0.007012455724179745
+      ],
+      "elapsed_ms": 730.3262079949491
+    },
+    {
+      "id": "scienceqa/val/17681",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        1.1745694417086838e-09,
+        1.627603296583402e-07,
+        0.9999995231628418,
+        3.7544154452007206e-07
+      ],
+      "elapsed_ms": 725.4358750069514
+    },
+    {
+      "id": "scienceqa/val/9964",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        0.00019135519687552005,
+        0.00010769777145469561,
+        0.9994826316833496,
+        0.00021833789651282132
+      ],
+      "elapsed_ms": 928.818374988623
+    },
+    {
+      "id": "scienceqa/val/340",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.010052929632365704,
+        0.9549948573112488,
+        0.0005331069114618003,
+        0.03441905975341797
+      ],
+      "elapsed_ms": 736.3460840424523
+    },
+    {
+      "id": "scienceqa/val/20723",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9954797029495239,
+        0.0012700710212811828,
+        0.0032501465175300837
+      ],
+      "elapsed_ms": 839.5821249578148
+    },
+    {
+      "id": "scienceqa/val/13835",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9985985159873962,
+        0.00014829420251771808,
+        7.677712710574269e-05,
+        0.0011763935908675194
+      ],
+      "elapsed_ms": 684.5203749835491
+    },
+    {
+      "id": "scienceqa/val/8404",
+      "target_index": 3,
+      "prediction_index": 3,
+      "probabilities": [
+        0.0001755337871145457,
+        0.0001640830741962418,
+        0.0021861197892576456,
+        0.9700518846511841,
+        0.027422383427619934
+      ],
+      "elapsed_ms": 744.3796250154264
+    },
+    {
+      "id": "scienceqa/val/326",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.00031515376758761704,
+        0.9986709356307983,
+        0.0010140076046809554
+      ],
+      "elapsed_ms": 855.3466669982299
+    },
+    {
+      "id": "scienceqa/val/7684",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        0.261148601770401,
+        0.06522262096405029,
+        0.67287677526474,
+        0.0007520208018831909
+      ],
+      "elapsed_ms": 939.7157080238685
+    },
+    {
+      "id": "scienceqa/val/6621",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        9.295739289427729e-09,
+        0.9999979734420776,
+        5.027599669915617e-08,
+        1.4151556797514786e-06,
+        5.762834689448937e-07
+      ],
+      "elapsed_ms": 737.8222920233384
+    },
+    {
+      "id": "scienceqa/val/11466",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.4861493408679962,
+        0.045885149389505386,
+        0.43684297800064087,
+        0.031122520565986633
+      ],
+      "elapsed_ms": 733.1671669962816
+    },
+    {
+      "id": "scienceqa/val/4791",
+      "target_index": 0,
+      "prediction_index": 1,
+      "probabilities": [
+        0.4698958396911621,
+        0.5242437720298767,
+        0.005860346369445324
+      ],
+      "elapsed_ms": 772.0198750030249
+    },
+    {
+      "id": "scienceqa/val/4550",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9999977350234985,
+        1.7362581274937838e-06,
+        3.809745052762992e-08,
+        1.2352370237778132e-09,
+        5.158440217201132e-07
+      ],
+      "elapsed_ms": 743.6611659941263
+    },
+    {
+      "id": "scienceqa/val/8558",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9053918123245239,
+        0.07601229846477509,
+        0.017282212153077126,
+        0.0013136324705556035
+      ],
+      "elapsed_ms": 716.0367500036955
+    },
+    {
+      "id": "scienceqa/val/19945",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.7528671622276306,
+        0.24187956750392914,
+        0.005253252573311329
+      ],
+      "elapsed_ms": 864.9832080118358
+    },
+    {
+      "id": "scienceqa/val/8871",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9999978542327881,
+        1.39499331908155e-06,
+        7.841769900096551e-08,
+        7.269268298415454e-09,
+        5.693436264664342e-07
+      ],
+      "elapsed_ms": 739.8256249725819
+    },
+    {
+      "id": "scienceqa/val/4884",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9666830897331238,
+        0.0034012277610599995,
+        0.029622506350278854,
+        0.0002932194038294256
+      ],
+      "elapsed_ms": 713.5235409950837
+    },
+    {
+      "id": "scienceqa/val/8378",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.0013747070915997028,
+        0.9961742758750916,
+        4.492511652642861e-05,
+        0.002406216226518154
+      ],
+      "elapsed_ms": 737.4855829984881
+    },
+    {
+      "id": "scienceqa/val/20597",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.997296154499054,
+        0.002703010104596615,
+        1.2892381562323862e-07,
+        5.3921453968541755e-08,
+        7.135407713576569e-07
+      ],
+      "elapsed_ms": 749.1085410001688
+    },
+    {
+      "id": "scienceqa/val/20663",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        0.0019003677880391479,
+        0.0003613665176089853,
+        0.9974960684776306,
+        0.00024219938495662063
+      ],
+      "elapsed_ms": 714.918542013038
+    },
+    {
+      "id": "scienceqa/val/8601",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9890605211257935,
+        0.0009218206396326423,
+        0.0004352020041551441,
+        0.0007956480840221047,
+        0.008786870166659355
+      ],
+      "elapsed_ms": 809.6686659846455
+    },
+    {
+      "id": "scienceqa/val/15468",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9999960660934448,
+        1.3521396340365754e-06,
+        5.3496968632771313e-08,
+        4.779113851327565e-07,
+        2.1499056401808048e-06
+      ],
+      "elapsed_ms": 745.8568339934573
+    },
+    {
+      "id": "scienceqa/val/10026",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        1.3849937658960698e-06,
+        0.9994989633560181,
+        0.0004951194860041142,
+        4.56709494756069e-06
+      ],
+      "elapsed_ms": 742.6960830343887
+    },
+    {
+      "id": "scienceqa/val/626",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9972666501998901,
+        0.0027228447142988443,
+        1.0538949936744757e-05
+      ],
+      "elapsed_ms": 837.4422079650685
+    },
+    {
+      "id": "scienceqa/val/8306",
+      "target_index": 2,
+      "prediction_index": 1,
+      "probabilities": [
+        8.581038855481893e-06,
+        0.9967631101608276,
+        0.002022934379056096,
+        0.0011565418681129813,
+        4.877926767221652e-05
+      ],
+      "elapsed_ms": 718.9321659971029
+    },
+    {
+      "id": "scienceqa/val/11428",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.25946858525276184,
+        0.6668545007705688,
+        0.030808407813310623,
+        0.04286855459213257
+      ],
+      "elapsed_ms": 725.4270830308087
+    },
+    {
+      "id": "scienceqa/val/4036",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.01569034904241562,
+        0.9813077449798584,
+        0.0019826041534543037,
+        0.0010193256894126534
+      ],
+      "elapsed_ms": 918.4377079945989
+    },
+    {
+      "id": "scienceqa/val/21080",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.00036287258262746036,
+        0.8422839045524597,
+        0.15620771050453186,
+        9.737018990563229e-05,
+        0.0010480741038918495
+      ],
+      "elapsed_ms": 727.5516249937937
+    },
+    {
+      "id": "scienceqa/val/15237",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.00011872701725224033,
+        0.9992451667785645,
+        0.0004771235981024802,
+        0.000158965282025747
+      ],
+      "elapsed_ms": 715.3530420036986
+    }
+  ]
+}

benchmarks/scienceqa-50-224.json

@@ -0,0 +1,619 @@
+{
+  "summary": {
+    "questions": 50,
+    "correct": 44,
+    "accuracy": 0.88,
+    "latency_ms": {
+      "median": 1050.1905410201289,
+      "p95": 1353.7280409946106,
+      "mean": 1018.8930649985559
+    },
+    "model": "third_party/OmniJev/models/Qwen3.5-4B-OmniJev-MLX-5bit",
+    "note": "Visual ScienceQA subset; not text-only JevBench."
+  },
+  "results": [
+    {
+      "id": "scienceqa/val/21023",
+      "target_index": 2,
+      "prediction_index": 3,
+      "probabilities": [
+        9.572258022672031e-06,
+        8.922968117985874e-05,
+        0.011354859918355942,
+        0.9875626564025879,
+        0.000983729842118919
+      ],
+      "elapsed_ms": 1216.320250008721
+    },
+    {
+      "id": "scienceqa/val/7180",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        0.0029119839891791344,
+        0.006107375491410494,
+        0.9909207820892334,
+        5.9882444475078955e-05
+      ],
+      "elapsed_ms": 1024.4643749902025
+    },
+    {
+      "id": "scienceqa/val/2642",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        1.4533699868479744e-05,
+        0.0008359018247574568,
+        0.9989796280860901,
+        0.00016994107863865793
+      ],
+      "elapsed_ms": 801.7690839478746
+    },
+    {
+      "id": "scienceqa/val/16468",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9368453621864319,
+        0.06314510107040405,
+        3.946696836010233e-07,
+        1.953322907866095e-06,
+        7.181197815953055e-06
+      ],
+      "elapsed_ms": 1021.7106660129502
+    },
+    {
+      "id": "scienceqa/val/19506",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.09320443868637085,
+        0.3865593373775482,
+        0.5202361941337585
+      ],
+      "elapsed_ms": 1179.1406670236029
+    },
+    {
+      "id": "scienceqa/val/8767",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.005915054120123386,
+        0.9934005737304688,
+        0.0006844155723229051
+      ],
+      "elapsed_ms": 656.6533750155941
+    },
+    {
+      "id": "scienceqa/val/16515",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        1.431024969633654e-07,
+        1.834305263237468e-09,
+        0.999997615814209,
+        9.705598813525285e-07,
+        1.2522122005975689e-06
+      ],
+      "elapsed_ms": 1056.747082970105
+    },
+    {
+      "id": "scienceqa/val/16938",
+      "target_index": 1,
+      "prediction_index": 0,
+      "probabilities": [
+        0.7385404706001282,
+        0.2605304718017578,
+        0.0009290423477068543
+      ],
+      "elapsed_ms": 1102.3421249701641
+    },
+    {
+      "id": "scienceqa/val/13212",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9983437061309814,
+        8.884436101652682e-05,
+        0.000598268467001617,
+        0.0009691402083262801
+      ],
+      "elapsed_ms": 675.5916670081206
+    },
+    {
+      "id": "scienceqa/val/4539",
+      "target_index": 3,
+      "prediction_index": 1,
+      "probabilities": [
+        0.26230472326278687,
+        0.3358997404575348,
+        0.115623340010643,
+        0.2861471474170685,
+        2.5058159735635854e-05
+      ],
+      "elapsed_ms": 1027.891166973859
+    },
+    {
+      "id": "scienceqa/val/11884",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        9.703077375888824e-05,
+        0.9998834133148193,
+        1.9517850887496024e-05
+      ],
+      "elapsed_ms": 1107.7883340185508
+    },
+    {
+      "id": "scienceqa/val/15622",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        1.4583752090402413e-06,
+        0.9997345805168152,
+        5.991348734823987e-05,
+        0.00020402560767251998
+      ],
+      "elapsed_ms": 1318.7873339629732
+    },
+    {
+      "id": "scienceqa/val/17192",
+      "target_index": 3,
+      "prediction_index": 3,
+      "probabilities": [
+        2.8217283443154884e-07,
+        1.4603470788188133e-07,
+        1.4119949298674328e-07,
+        0.9999963045120239,
+        3.146950348309474e-06
+      ],
+      "elapsed_ms": 1033.0449999892153
+    },
+    {
+      "id": "scienceqa/val/2461",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        9.025831968756393e-05,
+        2.9825587262166664e-06,
+        0.999823272228241,
+        8.357201295439154e-05
+      ],
+      "elapsed_ms": 824.592458026018
+    },
+    {
+      "id": "scienceqa/val/8688",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9999678134918213,
+        5.3547882998827845e-06,
+        2.685382787603885e-05
+      ],
+      "elapsed_ms": 859.0977909625508
+    },
+    {
+      "id": "scienceqa/val/15419",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        7.925441991574189e-08,
+        2.789672919334407e-07,
+        0.9999805688858032,
+        3.70468811183855e-08,
+        1.913079722726252e-05
+      ],
+      "elapsed_ms": 1026.9342919928022
+    },
+    {
+      "id": "scienceqa/val/18512",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        3.031034534473065e-09,
+        0.9999986886978149,
+        1.2731197784887627e-06
+      ],
+      "elapsed_ms": 714.639707992319
+    },
+    {
+      "id": "scienceqa/val/921",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.0015609971014782786,
+        0.994932234287262,
+        0.003506765468046069
+      ],
+      "elapsed_ms": 714.470416016411
+    },
+    {
+      "id": "scienceqa/val/17574",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        4.704803723143414e-05,
+        0.9971519708633423,
+        4.187240483588539e-05,
+        0.002759092953056097
+      ],
+      "elapsed_ms": 1056.6815420170315
+    },
+    {
+      "id": "scienceqa/val/4517",
+      "target_index": 1,
+      "prediction_index": 0,
+      "probabilities": [
+        0.817062258720398,
+        0.1827545017004013,
+        0.00018327795260120183
+      ],
+      "elapsed_ms": 1234.0599590097554
+    },
+    {
+      "id": "scienceqa/val/12959",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9998112320899963,
+        5.039972165832296e-05,
+        1.4609129266318632e-06,
+        0.00013698486145585775
+      ],
+      "elapsed_ms": 1353.7280409946106
+    },
+    {
+      "id": "scienceqa/val/4018",
+      "target_index": 3,
+      "prediction_index": 3,
+      "probabilities": [
+        0.0001855503796832636,
+        9.175106970360503e-05,
+        0.0003109597892034799,
+        0.9957041144371033,
+        0.0037075893487781286
+      ],
+      "elapsed_ms": 1039.2040419974364
+    },
+    {
+      "id": "scienceqa/val/17681",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        4.3628803347850464e-10,
+        3.0636066128408856e-08,
+        0.9999998807907104,
+        1.403442695391277e-07
+      ],
+      "elapsed_ms": 738.9204169739969
+    },
+    {
+      "id": "scienceqa/val/9964",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        1.625367258384358e-05,
+        7.471736807929119e-06,
+        0.9999535083770752,
+        2.2715945306117646e-05
+      ],
+      "elapsed_ms": 1065.4092500335537
+    },
+    {
+      "id": "scienceqa/val/340",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.23235492408275604,
+        0.7047974467277527,
+        0.0010436835000291467,
+        0.0618039146065712
+      ],
+      "elapsed_ms": 1027.7538750087842
+    },
+    {
+      "id": "scienceqa/val/20723",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9699394702911377,
+        0.02400338463485241,
+        0.006057215388864279
+      ],
+      "elapsed_ms": 1119.317915989086
+    },
+    {
+      "id": "scienceqa/val/13835",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9985985159873962,
+        0.00014829420251771808,
+        7.677712710574269e-05,
+        0.0011763935908675194
+      ],
+      "elapsed_ms": 662.2005829703994
+    },
+    {
+      "id": "scienceqa/val/8404",
+      "target_index": 3,
+      "prediction_index": 3,
+      "probabilities": [
+        2.1427009414765052e-05,
+        1.6831480024848133e-05,
+        5.751501157646999e-05,
+        0.9970983266830444,
+        0.0028059149626642466
+      ],
+      "elapsed_ms": 1038.3789160405286
+    },
+    {
+      "id": "scienceqa/val/326",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.002556850202381611,
+        0.9972189664840698,
+        0.00022422110487241298
+      ],
+      "elapsed_ms": 1279.2820409522392
+    },
+    {
+      "id": "scienceqa/val/7684",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        0.0012439613929018378,
+        0.0003342817071825266,
+        0.995692789554596,
+        0.002728895051404834
+      ],
+      "elapsed_ms": 1391.0568749997765
+    },
+    {
+      "id": "scienceqa/val/6621",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        5.970364647112092e-09,
+        0.999998927116394,
+        1.8963916659231472e-08,
+        6.708435762448062e-07,
+        3.7174092426539573e-07
+      ],
+      "elapsed_ms": 1050.1905410201289
+    },
+    {
+      "id": "scienceqa/val/11466",
+      "target_index": 0,
+      "prediction_index": 2,
+      "probabilities": [
+        0.3485657870769501,
+        0.08763457834720612,
+        0.5546107888221741,
+        0.009188840165734291
+      ],
+      "elapsed_ms": 1019.3432919913903
+    },
+    {
+      "id": "scienceqa/val/4791",
+      "target_index": 0,
+      "prediction_index": 1,
+      "probabilities": [
+        0.4698958396911621,
+        0.5242437720298767,
+        0.005860346369445324
+      ],
+      "elapsed_ms": 752.7294590254314
+    },
+    {
+      "id": "scienceqa/val/4550",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9999986886978149,
+        6.811756634306221e-07,
+        8.064888135095316e-08,
+        1.194044196850541e-09,
+        4.1733542843758187e-07
+      ],
+      "elapsed_ms": 1049.7832499677315
+    },
+    {
+      "id": "scienceqa/val/8558",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.8947974443435669,
+        0.04916587099432945,
+        0.05512939393520355,
+        0.0009072012035176158
+      ],
+      "elapsed_ms": 1072.1199999679811
+    },
+    {
+      "id": "scienceqa/val/19945",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.7528671622276306,
+        0.24187956750392914,
+        0.005253252573311329
+      ],
+      "elapsed_ms": 838.6542919906788
+    },
+    {
+      "id": "scienceqa/val/8871",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9999986886978149,
+        7.551004159722652e-07,
+        8.145114094304517e-08,
+        7.321462991427552e-09,
+        4.373882518393657e-07
+      ],
+      "elapsed_ms": 1065.9168339916505
+    },
+    {
+      "id": "scienceqa/val/4884",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9833201766014099,
+        0.0013428564416244626,
+        0.015262079425156116,
+        7.485473179258406e-05
+      ],
+      "elapsed_ms": 1076.3937499723397
+    },
+    {
+      "id": "scienceqa/val/8378",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.00018541481404099613,
+        0.9995972514152527,
+        1.6776668417151086e-05,
+        0.0002005332353292033
+      ],
+      "elapsed_ms": 878.3530420041643
+    },
+    {
+      "id": "scienceqa/val/20597",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9999960660934448,
+        3.515009439070127e-06,
+        6.72070328278096e-08,
+        4.936427799862031e-08,
+        3.3825261880338076e-07
+      ],
+      "elapsed_ms": 1068.625667016022
+    },
+    {
+      "id": "scienceqa/val/20663",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        0.006027692463248968,
+        0.00041820149635896087,
+        0.993385374546051,
+        0.00016875665460247546
+      ],
+      "elapsed_ms": 1086.184083018452
+    },
+    {
+      "id": "scienceqa/val/8601",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9890605211257935,
+        0.0009218206396326423,
+        0.0004352020041551441,
+        0.0007956480840221047,
+        0.008786870166659355
+      ],
+      "elapsed_ms": 803.3093330450356
+    },
+    {
+      "id": "scienceqa/val/15468",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.999998927116394,
+        4.696783548752137e-07,
+        1.6910020050886487e-08,
+        8.057025269181395e-08,
+        5.363776836020406e-07
+      ],
+      "elapsed_ms": 1068.2248329976574
+    },
+    {
+      "id": "scienceqa/val/10026",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        5.717474209632201e-07,
+        0.9995086193084717,
+        0.00048735918244346976,
+        3.4935785606649006e-06
+      ],
+      "elapsed_ms": 1082.4098750017583
+    },
+    {
+      "id": "scienceqa/val/626",
+      "target_index": 0,
+      "prediction_index": 0,
+      "probabilities": [
+        0.9972666501998901,
+        0.0027228447142988443,
+        1.0538949936744757e-05
+      ],
+      "elapsed_ms": 842.4672919791192
+    },
+    {
+      "id": "scienceqa/val/8306",
+      "target_index": 2,
+      "prediction_index": 2,
+      "probabilities": [
+        0.003358687274158001,
+        0.00698996102437377,
+        0.951853334903717,
+        0.006501566618680954,
+        0.03129644691944122
+      ],
+      "elapsed_ms": 1093.388750043232
+    },
+    {
+      "id": "scienceqa/val/11428",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.2032264918088913,
+        0.716314971446991,
+        0.025267310440540314,
+        0.05519119277596474
+      ],
+      "elapsed_ms": 1104.4623340130784
+    },
+    {
+      "id": "scienceqa/val/4036",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        0.0006258749053813517,
+        0.9989481568336487,
+        0.00010744511382654309,
+        0.0003185884852427989
+      ],
+      "elapsed_ms": 1434.7477499977686
+    },
+    {
+      "id": "scienceqa/val/21080",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        6.803419978496095e-07,
+        0.9944728016853333,
+        0.005440711509436369,
+        1.4968701407269691e-06,
+        8.4117149526719e-05
+      ],
+      "elapsed_ms": 1093.0374580202624
+    },
+    {
+      "id": "scienceqa/val/15237",
+      "target_index": 1,
+      "prediction_index": 1,
+      "probabilities": [
+        3.9620841562282294e-05,
+        0.9990529417991638,
+        0.0008513267966918647,
+        5.614504334516823e-05
+      ],
+      "elapsed_ms": 1096.3321659946814
+    }
+  ]
+}

conversion.json

@@ -0,0 +1,24 @@
+{
+  "source_model": "tinnel123/OmniJev",
+  "source_revision": "v1.1",
+  "source_release": "OmniJev-4B-ckpt",
+  "backbone": "Qwen/Qwen3.5-4B",
+  "operation": "merge official LoRA adapter into the backbone in bfloat16, then convert the fused backbone to MLX affine quantization",
+  "bits": 5,
+  "group_size": 64,
+  "quantization_mode": "affine",
+  "vision_precision": "bfloat16",
+  "decision_head": "official head.pt exported to decision_head/head.npz",
+  "decision_head_precision": "float32",
+  "new_training": false,
+  "local_validation": {
+    "benchmark": "ScienceQA visual subset",
+    "questions": 50,
+    "max_pixels": 87808,
+    "correct": 44,
+    "accuracy": 0.88,
+    "warm_median_ms": 737.4855829984881,
+    "warm_p95_ms": 1012.7517079818062,
+    "hardware": "Mac mini M4, 16 GB unified memory"
+  }
+}

decision_head/head.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4de95c6c6efa73243af8b220003ebda14e252892b6273fa25f6dec362e5f7b42
+size 29417820

decision_head/head_meta.json

@@ -0,0 +1 @@
+{"norm": "softmax", "lm_feats": true, "ordinal": true, "packed_train": true, "rlcd": false, "branch": true, "base_model": "Qwen3.5-4B", "lora_targets": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "in_proj_qkv", "in_proj_z", "out_proj"], "temperatures": {"noul": 1.1991, "choice": 1.1199, "score": 1.1909}, "calibration": {"n_dev": 7767, "n_test": 17754, "raw": {"noul": {"n": 5566, "acc": 0.8917, "ece": 0.0235, "brier": 0.1545, "nll": 0.2524, "e_aurc": 0.0171}, "choice": {"n": 9226, "acc": 0.5633, "ece": 0.022, "brier": 0.5155, "nll": 1.2547, "e_aurc": 0.0557}, "score": {"n": 2962, "acc": 0.6691, "ece": 0.0524, "brier": 0.426, "nll": 0.7504, "e_aurc": 0.0842}}, "calibrated": {"noul": {"n": 5566, "acc": 0.8917, "ece": 0.01, "brier": 0.153, "nll": 0.2484, "e_aurc": 0.0171}, "choice": {"n": 9226, "acc": 0.5633, "ece": 0.0144, "brier": 0.5153, "nll": 1.2517, "e_aurc": 0.0557}, "score": {"n": 2962, "acc": 0.6691, "ece": 0.0342, "brier": 0.4221, "nll": 0.7445, "e_aurc": 0.0837}}}, "biases": {"noul": 0.05310856608843584}}

head_provenance.json

@@ -0,0 +1,7 @@
+{
+  "source_file": "tinnel123/OmniJev@v1.1:head.pt",
+  "source_sha256": "7b801a1901040e70559deb74eadb11709540ff65ef926213ecf881e4f4070320",
+  "export_file": "decision_head/head.npz",
+  "export_script": "training/export_omnijev_head.py",
+  "note": "The decision head was not retrained for this repository; it was converted from the released official checkpoint."
+}

omni_mlx/__init__.py

@@ -0,0 +1 @@
+"""MLX inference wrapper for the OmniJev 5-bit decision model."""

omni_mlx/classifier.py

@@ -0,0 +1,270 @@
+"""Run the fused Qwen3.5/OmniJev 5-bit model with its decision head."""
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import time
+from pathlib import Path
+
+import mlx.core as mx
+import numpy as np
+from PIL import Image
+
+
+def _load_runtime(model_path: Path):
+    from mlx_vlm import load
+
+    mx.set_cache_limit(256 * 1024**2)
+    model, processor = load(str(model_path))
+    model.set_dtype(mx.float32)
+    if model.config.model_type != "qwen3_5":
+        raise ValueError(f"Expected qwen3_5 model, got {model.config.model_type}")
+    return Qwen35Adapter(model, processor)
+
+
+class Qwen35Adapter:
+    def __init__(self, model, processor):
+        self.model = model
+        self.processor = processor
+        self.tokenizer = processor.tokenizer
+
+    def execution_context(self):
+        from mlx_vlm.generate.common import wired_limit
+
+        return wired_limit(self.model)
+
+    def prepare(self, prompt: str, image: Image.Image):
+        from mlx_vlm.utils import prepare_inputs
+
+        inputs = prepare_inputs(self.processor, images=[image], prompts=prompt)
+        if inputs["input_ids"].shape[-1] > 6000:
+            raise ValueError("image and prompt exceed 6000 input tokens")
+        return inputs
+
+    def prefill(self, inputs):
+        ids = inputs["input_ids"]
+        features = self.model.get_input_embeddings(
+            ids,
+            pixel_values=inputs.get("pixel_values"),
+            image_grid_thw=inputs.get("image_grid_thw"),
+            mask=inputs.get("attention_mask"),
+        )
+        mx.eval(features.inputs_embeds, features.position_ids)
+        cache = self.model.language_model.make_cache()
+        out = self.model.language_model(
+            ids,
+            inputs_embeds=features.inputs_embeds,
+            cache=cache,
+            position_ids=features.position_ids,
+            rope_deltas=features.rope_deltas,
+            skip_logits=True,
+            return_hidden=True,
+        )
+        mx.eval(out.hidden_states[-1], *[v for c in cache for v in c.state if v is not None])
+        return cache, int(features.position_ids.max().item()) + 1
+
+    def fork(self, cache, count):
+        from mlx_vlm.models.cache import ArraysCache, KVCache
+
+        result = []
+        for entry in cache:
+            if type(entry) not in (ArraysCache, KVCache):
+                raise TypeError(f"unsupported cache: {type(entry).__name__}")
+            state = [None if x is None else mx.repeat(x, count, axis=0) for x in entry.state]
+            result.append(type(entry).from_state(state, entry.meta_state))
+        mx.eval(*[v for c in result for v in c.state if v is not None])
+        return result
+
+    def suffix_hidden(self, cache, next_position, rows, picks):
+        branches = self.fork(cache, len(rows))
+        length = max(map(len, rows))
+        pad = self.tokenizer.pad_token_id or self.tokenizer.eos_token_id
+        ids = mx.array([row + [pad] * (length - len(row)) for row in rows])
+        positions = mx.broadcast_to(
+            (mx.arange(length) + next_position)[None, None, :],
+            (3, len(rows), length),
+        )
+        out = self.model.language_model(
+            ids,
+            cache=branches,
+            position_ids=positions,
+            skip_logits=True,
+            return_hidden=True,
+        )
+        hidden = out.hidden_states[-1]
+        return [hidden[i, mx.array(row)] for i, row in enumerate(picks)]
+
+
+class OmniHead:
+    def __init__(self, path: Path):
+        with np.load(path) as values:
+            self.weights = {key: value for key, value in values.items()}
+        metadata_path = path.with_name("head_meta.json")
+        metadata = json.loads(metadata_path.read_text()) if metadata_path.exists() else {}
+        self.choice_temperature = float(metadata.get("temperatures", {}).get("choice", 1.0))
+
+    @staticmethod
+    def _linear(x, weight, bias):
+        return x @ mx.array(weight.T) + mx.array(bias)
+
+    @staticmethod
+    def _gelu(x):
+        return 0.5 * x * (1 + mx.tanh(0.7978845608 * (x + 0.044715 * x * x * x)))
+
+    def logits(self, option_hidden, question_hidden, features):
+        w = self.weights
+        option = self._linear(
+            self._gelu(self._linear(option_hidden, w["opt__0__weight"], w["opt__0__bias"])),
+            w["opt__2__weight"], w["opt__2__bias"],
+        )
+        question = self._linear(
+            self._gelu(self._linear(question_hidden, w["que__0__weight"], w["que__0__bias"])),
+            w["que__2__weight"], w["que__2__bias"],
+        )
+        gate = mx.tanh(question)
+        fused = option * gate[None, :] + self._linear(
+            features, w["feat__weight"], w["feat__bias"]
+        )
+        tau = mx.exp(mx.array(w["log_tau"])[1])
+        score = self._linear(fused, w["score__weight"], w["score__bias"]).squeeze(-1)
+        score = score + self._linear(
+            features, w["feat_lin__weight"], w["feat_lin__bias"]
+        ).squeeze(-1)
+        score = score / tau
+        abstain = self._linear(
+            gate, w["abstain__weight"], w["abstain__bias"]
+        ).squeeze(-1) / tau
+        return mx.concatenate([score, mx.reshape(abstain, (1,))])
+
+
+def _marker_positions(row, opening, closing):
+    def find(pattern):
+        width = len(pattern)
+        return [i for i in range(len(row) - width + 1) if row[i : i + width] == pattern]
+
+    opens, closes = find(opening), find(closing)
+    if len(opens) != 1 or len(closes) != 1:
+        raise ValueError(f"option marker count mismatch: opens={opens}, closes={closes}")
+    option_start, close_start = opens[0], closes[0]
+    close_end = close_start + len(closing) - 1
+    inner = list(range(option_start + len(opening), close_start))
+    return option_start, close_end, inner
+
+
+def predict_image(adapter, head, image: Image.Image, question: str, options: list[str]):
+    if not 2 <= len(options) <= 256 or len(set(options)) != len(options):
+        raise ValueError("supply 2–256 distinct options")
+    started = time.perf_counter()
+    tokenizer = adapter.tokenizer
+    opening = tokenizer("<|opt|>", add_special_tokens=False)["input_ids"]
+    closing = tokenizer("<|/opt|>", add_special_tokens=False)["input_ids"]
+    prompt = adapter.processor.apply_chat_template(
+        [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": question}]}],
+        tokenize=False,
+        add_generation_prompt=True,
+    )
+    full_prompt = prompt + "".join(f"<|opt|>{option}<|/opt|>" for option in options)
+    inputs = adapter.prepare(full_prompt, image)
+    full_ids = np.asarray(inputs["input_ids"])[0].tolist()
+    open_at = [
+        i for i in range(len(full_ids) - len(opening) + 1)
+        if full_ids[i : i + len(opening)] == opening
+    ]
+    if len(open_at) != len(options):
+        raise ValueError(f"expanded option markers not found: {open_at}")
+    prefix_end = open_at[0]
+    prefix = dict(inputs)
+    prefix["input_ids"] = inputs["input_ids"][:, :prefix_end]
+    prefix["attention_mask"] = mx.ones_like(prefix["input_ids"])
+    cache, next_position = adapter.prefill(prefix)
+
+    all_closes = [
+        i for i in range(len(full_ids) - len(closing) + 1)
+        if full_ids[i : i + len(closing)] == closing
+    ]
+    rows, picks, option_inner = [], [], []
+    for index, option_start in enumerate(open_at):
+        close_start = all_closes[index]
+        close_end = close_start + len(closing) - 1
+        row = full_ids[prefix_end:open_at[0]] + full_ids[option_start : close_end + 1]
+        marker_start, close_pos, inner = _marker_positions(row, opening, closing)
+        rows.append(row)
+        picks.append([close_pos, max(0, marker_start - 1)] + [position - 1 for position in inner])
+        option_inner.append(inner)
+
+    hidden_rows = adapter.suffix_hidden(cache, next_position, rows, picks)
+    all_hidden = mx.concatenate(hidden_rows, axis=0)
+    language_model = adapter.model.language_model
+    logits = (
+        language_model.model.embed_tokens.as_linear(all_hidden)
+        if language_model.args.tie_word_embeddings
+        else language_model.lm_head(all_hidden)
+    ).astype(mx.float32)
+    scored, offset = [], 0
+    for row_tokens, row_hidden, inner in zip(rows, hidden_rows, option_inner):
+        count = len(inner)
+        token_logits = logits[offset + 2 : offset + 2 + count]
+        targets = mx.array([row_tokens[position] for position in inner])
+        log_probs = token_logits - mx.logsumexp(token_logits, axis=-1, keepdims=True)
+        values = log_probs[mx.arange(count), targets] if count else mx.array([])
+        scored.append((row_hidden[0], row_hidden[1], values, count))
+        offset += len(row_hidden)
+    sums = mx.array([float(mx.sum(row[2]).item()) for row in scored])
+    counts = mx.array([row[3] for row in scored])
+    first = mx.array([float((row[2][0] if row[3] else 0).item()) for row in scored])
+    normalized = sums - mx.logsumexp(sums)
+    features = mx.stack(
+        [sums / 10, sums / mx.maximum(counts, 1), counts / 10, normalized, first, mx.ones_like(sums)],
+        axis=1,
+    )
+    option_hidden = mx.stack([row[0] for row in scored]).astype(mx.float32)
+    question_hidden = scored[0][1].astype(mx.float32)
+    scores = head.logits(option_hidden, question_hidden, features)
+    probabilities = mx.softmax(scores / head.choice_temperature)
+    mx.eval(probabilities)
+    values = probabilities.tolist()
+    choice_index = int(mx.argmax(probabilities[:-1]).item())
+    return {
+        "prediction": options[choice_index],
+        "prediction_index": choice_index,
+        "probabilities": values[:-1],
+        "abstain_probability": values[-1],
+        "elapsed_ms": (time.perf_counter() - started) * 1000,
+        "quantization_bits": 5,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model", type=Path, default=Path("."))
+    parser.add_argument("--head", type=Path, default=Path("decision_head/head.npz"))
+    parser.add_argument("--image", type=Path, required=True)
+    parser.add_argument("--state", default="")
+    parser.add_argument("--question", required=True)
+    parser.add_argument("--options", nargs="+", required=True)
+    parser.add_argument("--max-pixels", type=int, default=87808)
+    parser.add_argument("--repeat", type=int, default=1)
+    args = parser.parse_args()
+    if args.repeat < 1:
+        parser.error("repeat must be positive")
+    model_path = args.model.resolve()
+    head_path = args.head if args.head.is_absolute() else model_path / args.head
+    adapter = _load_runtime(model_path)
+    adapter.processor.image_processor.max_pixels = args.max_pixels
+    head = OmniHead(head_path)
+    with Image.open(args.image) as opened:
+        image = opened.convert("RGB")
+    question = args.question if not args.state else f"{args.question}\nContext: {args.state}"
+    results = []
+    with adapter.execution_context():
+        for _ in range(args.repeat):
+            results.append(predict_image(adapter, head, image, question, args.options))
+    output = {"model": str(model_path), "requests": results}
+    if len(results) > 1:
+        output["warm_median_ms"] = statistics.median(result["elapsed_ms"] for result in results[1:])
+    print(json.dumps(output, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,4 @@
+mlx==0.32.2
+mlx-vlm==0.7.1
+numpy>=2.0
+Pillow>=10.0