Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Alibaba Cloud
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,79 @@
+---
+license: apache-2.0
+base_model: Qwen/Qwen3.6-35B-A3B
+library_name: mlx
+pipeline_tag: text-generation
+tags:
+- mlx
+- quantized
+- vq
+- aqlm
+- codebook
+- mixed-bit
+- moe
+- qwen3_5_moe
+- custom-code
+---
+
+# Qwen3.6-35B-A3B — MLX **VQ** (trained-codebook) @2.6 bpw, custom Metal kernel
+
+**Vector-quantized** (AQLM-style, trained codebooks + GPTQ compensation) MLX build of
+[Qwen/Qwen3.6-35B-A3B](https://huggingface.co/Qwen/Qwen3.6-35B-A3B) — to our knowledge the first
+trained-codebook VQ model served on Apple Silicon, via the bundled `mx.fast.metal_kernel`
+implementation (`code/`). **11.53 GB**, measurably better than scalar quantization at the same size.
+
+⚠️ **Not loadable by stock mlx-lm / LM Studio / Ollama / oMLX** — the experts use a VQ format that
+needs the bundled loader + Metal kernel (~400 lines of Python, `mlx>=0.31` only, no compilation).
+
+## The scalar-vs-VQ tradeoff this build wins
+
+| build | size | think-ja PPL vs bf16 | multilingual |
+|---|---|---|---|
+| [GPTQ-3.5bpw](https://huggingface.co/aquaman164/Qwen3.6-35B-A3B-MLX-GPTQ-3.5bpw) (scalar) | 15.35 GB | +6.6% | +6.9% |
+| [GPTQ-2.7bpw](https://huggingface.co/aquaman164/Qwen3.6-35B-A3B-MLX-GPTQ-2.7bpw) (scalar) | 12.03 GB | +17.5% | +15.4% |
+| **this (VQ)** | **11.53 GB** | **+12.4%** | **+12.2%** |
+
+Smaller *and* significantly better than the scalar 2.7bpw build — the point of codebook
+quantization at low bits.
+
+## Recipe
+
+- **Experts (91.7% of params)**: per-(layer,tensor) bits {2bit: 69, 3bit: 11 tensors} chosen by
+  Fisher/gxw MCKP; **Qwen-native codebooks** (d=4, K=256 / K=4096; k-means on group-normalized
+  subvectors — GLM-fit books measurably transfer worse, so refit); **GPTQ error compensation**
+  (pooled per-layer input Hessians, ja-centric calibration). Expert avg 2.38 bpw + fp16 group scales.
+- **Spine**: GPTQ 4bit gs64 (linear-attention qkv/z/out, attention q/k/v/o, shared experts),
+  `lm_head` 6bit, routers 8bit, embeddings 4bit — inherited from the 2.7bpw scalar build.
+- **Vision tower**: unquantized bf16.
+- Format: `switch_mlp.*.vq_codes` (LSB-first packed 8/12-bit indices, int32) +
+  `vq_scales` (fp16 per-128 group) + `vq_codebooks.safetensors`; metadata in `config.json["vq"]`.
+
+## Speed (M-series, measured on 48 GB)
+
+- **decode ~66 tok/s**, **prefill ~213 tok/s** — fused Metal kernels
+  (gate+up+SiLU in one dispatch; simdgroup-per-row GEMV with half4 loads).
+
+## Run (Apple Silicon, ≥16 GB unified memory)
+
+```bash
+pip install "mlx>=0.31" "mlx-lm>=0.31"
+hf download aquaman164/Qwen3.6-35B-A3B-MLX-VQ-2.6bpw --local-dir qwen-vq
+
+# OpenAI-compatible server
+python qwen-vq/code/vq_serve.py --model qwen-vq --port 8090
+# then: curl http://127.0.0.1:8090/v1/chat/completions -d '{"model":"qwen-vq", ...}'
+#       (use the model id returned by /v1/models)
+
+# or a one-shot generation test
+python qwen-vq/code/vq_generate.py qwen-vq 100
+```
+
+`code/vq_switch.py` exposes `load_vq_model(path) -> (model, config)` if you want to embed it.
+Thinking mode is on by default (Qwen3.6 native); pass `enable_thinking=False` via the chat
+template for direct answers.
+
+## Provenance / license
+
+Base model © Alibaba Cloud, Apache-2.0. Quantization pipeline: loss-aware allocation +
+GPTQ-VQ encode built on [OneCompression](https://github.com/mmzz164/OneCompression);
+serving adapter published as model #3 of [vqmoe](https://github.com/mmzz164/vqmoe).

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

code/vq_generate.py

@@ -0,0 +1,39 @@
+"""End-to-end VQ integration test on the Mac: load_vq_model + real generation.
+
+Run:  ~/work/llm/omlx/.venv/bin/python vq_generate.py <artifact_dir> [max_tokens]
+Loads the full qwen3_5_moe model with VQSwitchLinear experts (level-0 pure-MLX
+decode), runs a short Japanese generation, reports text + tok/s.
+"""
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from vq_switch import load_vq_model
+
+ART = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/models/Qwen3.6-35B-A3B-MLX-VQ-map12")
+MAXTOK = int(sys.argv[2]) if len(sys.argv) > 2 else 40
+
+t0 = time.time()
+def log(m): print(f"[vqgen +{time.time()-t0:.0f}s] {m}", flush=True)
+
+log(f"loading VQ model from {ART} ...")
+model, config = load_vq_model(ART)
+log("model loaded (strict) — VQ modules live")
+
+from mlx_lm.utils import load_tokenizer
+from pathlib import Path
+tokenizer = load_tokenizer(Path(ART))
+msgs = [{"role": "user", "content": "東京の観光名所を3つ、名前だけ簡潔に教えてください。"}]
+prompt = tokenizer.apply_chat_template(msgs, add_generation_prompt=True)
+log(f"prompt tokens: {len(prompt)}")
+
+from mlx_lm import generate
+t1 = time.time()
+out = generate(model, tokenizer, prompt=prompt, max_tokens=MAXTOK, verbose=False)
+dt = time.time() - t1
+log(f"generated {MAXTOK} tokens in {dt:.1f}s ({MAXTOK/dt:.2f} tok/s, level-0)")
+print("---- OUTPUT ----")
+print(out)
+print("----------------")
+print("VQ_GENERATE_DONE")

code/vq_kernel.py

@@ -0,0 +1,207 @@
+"""Fused VQ gather-GEMV Metal kernel (v3 level-1) for map12 (d=4, nbits 8|12).
+
+One thread per (pair n, output row r):
+  y[n,r] = sum_g scale[e,r,g] * sum_{sub in g} dot(cb[idx(e,r,sub)], x[n, sub*4 .. +4])
+codes are LSB-first packed (8-bit idx for K256 / 12-bit for K4096) in uint32 words.
+Correctness reference = VQSwitchLinear._decode (level-0); see vq_test kernel section.
+"""
+import mlx.core as mx
+
+_SRC = """
+    uint r = thread_position_in_grid.x;
+    uint n = thread_position_in_grid.y;
+    const uint C     = (uint)params[0];
+    const uint R     = (uint)params[1];
+    const uint nbits = (uint)params[3];
+    const uint PC    = (uint)params[4];
+    const uint NG    = (uint)params[5];
+    if (r >= R) return;
+    uint e = (uint)eidx[n];
+    device const uint* crow  = codes  + ((ulong)e * R + r) * PC;
+    device const half* srow  = scales + ((ulong)e * R + r) * NG;
+    device const half* xr    = x + (ulong)n * C;
+    float acc = 0.0f;
+    const uint SPG = 32u;                       // subvectors (d=4) per 128-group
+    for (uint g = 0; g < NG; ++g) {
+        float s = (float)srow[g];
+        float gacc = 0.0f;
+        uint base = g * SPG;
+        for (uint j = 0; j < SPG; ++j) {
+            uint k = base + j;
+            uint idx;
+            if (nbits == 8u) {
+                idx = (crow[k >> 2] >> ((k & 3u) * 8u)) & 255u;
+            } else {
+                uint pos = k * 12u; uint w = pos >> 5; uint off = pos & 31u;
+                uint v = crow[w] >> off;
+                if (off > 20u) { v |= crow[w + 1u] << (32u - off); }
+                idx = v & 4095u;
+            }
+            device const half* cv = cb + (ulong)idx * 4u;
+            uint c0 = k * 4u;
+            gacc += (float)cv[0] * (float)xr[c0]
+                  + (float)cv[1] * (float)xr[c0 + 1u]
+                  + (float)cv[2] * (float)xr[c0 + 2u]
+                  + (float)cv[3] * (float)xr[c0 + 3u];
+        }
+        acc += s * gacc;
+    }
+    y[(ulong)n * R + r] = (half)acc;
+"""
+
+_KERNEL = mx.fast.metal_kernel(
+    name="vq_gemv_d4",
+    input_names=["x", "codes", "scales", "cb", "eidx", "params"],
+    output_names=["y"],
+    source=_SRC,
+)
+
+# ---- v2: one simdgroup (32 lanes) per output row, half4 vector loads ----
+_SRC2 = """
+    uint tid  = thread_position_in_grid.x;
+    uint r    = tid / 32u;
+    uint lane = tid % 32u;
+    uint n    = thread_position_in_grid.y;
+    const uint C     = (uint)params[0];
+    const uint R     = (uint)params[1];
+    const uint nbits = (uint)params[3];
+    const uint PC    = (uint)params[4];
+    if (r >= R) return;
+    uint e = (uint)eidx[n];
+    device const uint*  crow = codes + ((ulong)e * R + r) * PC;
+    device const half*  srow = scales + ((ulong)e * R + r) * (C >> 7);
+    device const half4* xr4  = (device const half4*)(x + (ulong)n * C);
+    device const half4* cb4  = (device const half4*)cb;
+    const uint nsub = C >> 2;
+    float acc = 0.0f;
+    for (uint k = lane; k < nsub; k += 32u) {
+        uint idx;
+        if (nbits == 8u) {
+            idx = (crow[k >> 2] >> ((k & 3u) * 8u)) & 255u;
+        } else {
+            uint pos = k * 12u; uint w = pos >> 5; uint off = pos & 31u;
+            uint v = crow[w] >> off;
+            if (off > 20u) { v |= crow[w + 1u] << (32u - off); }
+            idx = v & 4095u;
+        }
+        half4 cv = cb4[idx];
+        half4 xv = xr4[k];
+        float s  = (float)srow[k >> 5];
+        acc += s * ((float)cv.x * (float)xv.x + (float)cv.y * (float)xv.y
+                  + (float)cv.z * (float)xv.z + (float)cv.w * (float)xv.w);
+    }
+    acc = simd_sum(acc);
+    if (lane == 0u) { y[(ulong)n * R + r] = (half)acc; }
+"""
+
+_KERNEL2 = mx.fast.metal_kernel(
+    name="vq_gemv_d4_sg",
+    input_names=["x", "codes", "scales", "cb", "eidx", "params"],
+    output_names=["y"],
+    source=_SRC2,
+)
+
+
+def vq_gemv2(x_f16, codes_u32, scales_f16, cb_f16, eidx_i32, nbits):
+    """simdgroup-per-row variant. Same contract as vq_gemv."""
+    N, C = x_f16.shape
+    E, R, PC = codes_u32.shape
+    NG = scales_f16.shape[-1]
+    params = mx.array([C, R, 4, nbits, PC, NG], dtype=mx.int32)
+    (y,) = _KERNEL2(
+        inputs=[x_f16, codes_u32, scales_f16, cb_f16, eidx_i32, params],
+        output_shapes=[(N, R)],
+        output_dtypes=[mx.float16],
+        grid=(R * 32, N, 1),
+        threadgroup=(256, 1, 1),
+    )
+    return y
+
+
+# ---- v3: fused gate+up GEMV + SiLU (one dispatch per layer instead of two + eltwise) ----
+_SRC3 = """
+    uint tid  = thread_position_in_grid.x;
+    uint r    = tid / 32u;
+    uint lane = tid % 32u;
+    uint n    = thread_position_in_grid.y;
+    const uint C     = (uint)params[0];
+    const uint R     = (uint)params[1];
+    const uint nbits = (uint)params[3];
+    const uint PC    = (uint)params[4];
+    if (r >= R) return;
+    uint e = (uint)eidx[n];
+    ulong row = (ulong)e * R + r;
+    device const uint*  cg   = codes_g  + row * PC;
+    device const uint*  cu   = codes_u  + row * PC;
+    device const half*  sg   = scales_g + row * (C >> 7);
+    device const half*  su   = scales_u + row * (C >> 7);
+    device const half4* xr4  = (device const half4*)(x + (ulong)n * C);
+    device const half4* cb4  = (device const half4*)cb;
+    const uint nsub = C >> 2;
+    float ag = 0.0f, au = 0.0f;
+    for (uint k = lane; k < nsub; k += 32u) {
+        uint ig, iu;
+        if (nbits == 8u) {
+            ig = (cg[k >> 2] >> ((k & 3u) * 8u)) & 255u;
+            iu = (cu[k >> 2] >> ((k & 3u) * 8u)) & 255u;
+        } else {
+            uint pos = k * 12u; uint w = pos >> 5; uint off = pos & 31u;
+            uint vg = cg[w] >> off; uint vu = cu[w] >> off;
+            if (off > 20u) { vg |= cg[w + 1u] << (32u - off); vu |= cu[w + 1u] << (32u - off); }
+            ig = vg & 4095u; iu = vu & 4095u;
+        }
+        half4 xv = xr4[k];
+        half4 g4 = cb4[ig];
+        half4 u4 = cb4[iu];
+        float dx = (float)g4.x * (float)xv.x + (float)g4.y * (float)xv.y
+                 + (float)g4.z * (float)xv.z + (float)g4.w * (float)xv.w;
+        float du = (float)u4.x * (float)xv.x + (float)u4.y * (float)xv.y
+                 + (float)u4.z * (float)xv.z + (float)u4.w * (float)xv.w;
+        ag += (float)sg[k >> 5] * dx;
+        au += (float)su[k >> 5] * du;
+    }
+    ag = simd_sum(ag);
+    au = simd_sum(au);
+    if (lane == 0u) {
+        float act = ag / (1.0f + metal::exp(-ag));    // silu(gate)
+        y[(ulong)n * R + r] = (half)(act * au);
+    }
+"""
+
+_KERNEL3 = mx.fast.metal_kernel(
+    name="vq_swiglu_d4",
+    input_names=["x", "codes_g", "scales_g", "codes_u", "scales_u", "cb", "eidx", "params"],
+    output_names=["y"],
+    source=_SRC3,
+)
+
+
+def vq_swiglu(x_f16, cg_u32, sg_f16, cu_u32, su_f16, cb_f16, eidx_i32, nbits):
+    """Fused silu(gate(x)) * up(x) over VQ experts -> intermediate [N, R] f16."""
+    N, C = x_f16.shape
+    E, R, PC = cg_u32.shape
+    params = mx.array([C, R, 4, nbits, PC, C // 128], dtype=mx.int32)
+    (y,) = _KERNEL3(
+        inputs=[x_f16, cg_u32, sg_f16, cu_u32, su_f16, cb_f16, eidx_i32, params],
+        output_shapes=[(N, R)],
+        output_dtypes=[mx.float16],
+        grid=(R * 32, N, 1),
+        threadgroup=(256, 1, 1),
+    )
+    return y
+
+
+def vq_gemv(x_f16, codes_u32, scales_f16, cb_f16, eidx_i32, nbits):
+    """x [N,C] f16, codes [E,R,PC] uint32, scales [E,R,NG] f16, cb [K,4] f16, eidx [N] int32 -> y [N,R] f16."""
+    N, C = x_f16.shape
+    E, R, PC = codes_u32.shape
+    NG = scales_f16.shape[-1]
+    params = mx.array([C, R, 4, nbits, PC, NG], dtype=mx.int32)
+    (y,) = _KERNEL(
+        inputs=[x_f16, codes_u32, scales_f16, cb_f16, eidx_i32, params],
+        output_shapes=[(N, R)],
+        output_dtypes=[mx.float16],
+        grid=(R, N, 1),
+        threadgroup=(min(R, 256), 1, 1),
+    )
+    return y

code/vq_serve.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""OpenAI-compatible server for the VQ (map12) model — mlx_lm.server with the VQ loader.
+
+Usage:  python vq_serve.py --model ~/models/Qwen3.6-35B-A3B-MLX-VQ-map12 --port 8090
+Then:   curl http://127.0.0.1:8090/v1/chat/completions -d '{"model":"vq","messages":[...]}'
+
+Any model dir whose config.json has a "vq" section loads through vq_switch.load_vq_model
+(VQSwitchLinear experts + Metal kernel); everything else falls back to stock mlx_lm.
+"""
+import json
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+os.environ.setdefault("VQ_KERNEL", "2")
+os.environ.setdefault("VQ_FUSED", "1")
+
+import mlx_lm.server as S
+from mlx_lm.utils import load_tokenizer
+from vq_switch import load_vq_model
+
+_stock_load = S.load
+
+
+def _vq_load(model_path, *args, **kw):
+    p = str(model_path)
+    cfg_path = os.path.join(p, "config.json")
+    if os.path.isdir(p) and os.path.exists(cfg_path):
+        with open(cfg_path) as f:
+            if "vq" in json.load(f):
+                model, _config = load_vq_model(p)
+                tok = load_tokenizer(Path(p), kw.get("tokenizer_config") or {})
+                return model, tok
+    return _stock_load(model_path, *args, **kw)
+
+
+S.load = _vq_load
+
+if __name__ == "__main__":
+    S.main()

code/vq_switch.py

@@ -0,0 +1,225 @@
+"""VQSwitchLinear — pure-MLX VQ (codebook) switch linear for Qwen3.6 map12 (v3, level 0).
+
+Drop-in replacement for mlx_lm's QuantizedSwitchLinear on the switch_mlp projections.
+Weights per module (loaded via model.load_weights):
+  vq_codes  int32  [E, R, PC]   LSB-first packed nbits-bit codebook indices
+  vq_scales float16[E, R, C/128] per-128-group std scales
+Shared codebook cb [K, d] float16 is set post-load via set_codebook().
+
+Call contract mirrors QuantizedSwitchLinear.__call__(x, indices, sorted_indices):
+  x [..., 1, C] broadcast against indices [..., k] -> y [..., k, R] (matching gather_qmm).
+Level 0 = decode active experts on the fly (correctness first; Metal kernel later).
+
+Also provides load_vq_model(path): builds the mlx_lm qwen3_5_moe model, quantizes the
+spine per config, swaps switch_mlp projections for VQSwitchLinear, strict-loads the
+artifact, installs codebooks. Needs mlx_lm >= 0.31 (qwen3_5_moe).
+"""
+import glob
+import json
+import math
+import os
+
+import mlx.core as mx
+import mlx.nn as nn
+
+
+def _unpack_plan(nbits, nsub):
+    """Per-subvector (word, off, need_hi) for LSB-first nbits packing into 32-bit words."""
+    words, offs, hi = [], [], []
+    for k in range(nsub):
+        pos = k * nbits
+        words.append(pos // 32)
+        offs.append(pos % 32)
+        hi.append(1 if (pos % 32) + nbits > 32 else 0)
+    return words, offs, hi
+
+
+class VQSwitchLinear(nn.Module):
+    def __init__(self, num_experts, output_dims, input_dims, nbits, d, norm_group=128):
+        super().__init__()
+        nsub = input_dims // d
+        pc = (nsub * nbits + 31) // 32
+        self.vq_codes = mx.zeros((num_experts, output_dims, pc), dtype=mx.int32)
+        self.vq_scales = mx.zeros((num_experts, output_dims, input_dims // norm_group),
+                                  dtype=mx.float16)
+        self._nbits, self._d, self._C, self._ng = nbits, d, input_dims, norm_group
+        w, o, h = _unpack_plan(nbits, nsub)
+        self._uw = mx.array(w, dtype=mx.int32)          # [nsub] word index
+        self._uo = mx.array(o, dtype=mx.uint32)         # [nsub] bit offset
+        self._uh = mx.array(h, dtype=mx.uint32)         # [nsub] straddles word boundary
+        self._cb = None
+        self.freeze()
+
+    def set_codebook(self, cb):
+        self._cb = cb.astype(mx.float16)                # [K, d]
+        self._codes_u32 = mx.view(self.vq_codes, mx.uint32)
+        self._sc16 = self.vq_scales.astype(mx.float16)
+        kv = os.environ.get("VQ_KERNEL", "2")
+        if kv in ("1", "2") and self._d == 4:
+            import vq_kernel
+            self._gemv = vq_kernel.vq_gemv2 if kv == "2" else vq_kernel.vq_gemv
+        else:
+            self._gemv = None
+
+    @property
+    def input_dims(self):
+        return self._C
+
+    @property
+    def output_dims(self):
+        return self.vq_codes.shape[1]
+
+    @property
+    def num_experts(self):
+        return self.vq_codes.shape[0]
+
+    def _unpack(self, packed):
+        """packed [..., PC] int32 -> codes [..., nsub] int32 (logical shifts via uint32 view)."""
+        u = mx.view(packed, mx.uint32)
+        lo = mx.take(u, self._uw, axis=-1)              # [..., nsub]
+        lo = mx.right_shift(lo, self._uo)
+        pc = u.shape[-1]
+        wnext = mx.minimum(self._uw + 1, pc - 1)
+        hi = mx.take(u, wnext, axis=-1)
+        hi = mx.left_shift(hi, 32 - self._uo) * self._uh
+        codes = mx.bitwise_or(lo, hi) & ((1 << self._nbits) - 1)
+        return codes.astype(mx.int32)
+
+    def _decode(self, eidx):
+        """eidx [N] int -> dequantized weights [N, R, C] float16."""
+        codes = self._unpack(mx.take(self.vq_codes, eidx, axis=0))      # [N, R, nsub]
+        sub = mx.take(self._cb, codes.reshape(-1), axis=0)              # [N*R*nsub, d]
+        N = eidx.shape[0]
+        R, nsub = codes.shape[1], codes.shape[2]
+        sub = sub.reshape(N, R, nsub, self._d)
+        sc = mx.take(self.vq_scales, eidx, axis=0)                      # [N, R, C/ng]
+        sc = mx.repeat(sc, self._ng // self._d, axis=-1)                # [N, R, nsub]
+        return (sub * sc[..., None]).reshape(N, R, self._C)
+
+    def __call__(self, x, indices, sorted_indices=False):
+        # x [..., 1, C]; indices [...]; returns [..., 1, R] per index position
+        if getattr(self, "_gemv", None) is not None:
+            flat = indices.reshape(-1).astype(mx.int32)
+            xk = mx.broadcast_to(x, (*indices.shape, 1, self._C)).reshape(-1, self._C)
+            y = self._gemv(xk.astype(mx.float16), self._codes_u32, self._sc16,
+                           self._cb, flat, self._nbits)
+            return y.reshape(*indices.shape, 1, -1).astype(x.dtype)
+        flat = indices.reshape(-1)
+        W = self._decode(flat)                                          # [N, R, C]
+        W = W.reshape(*indices.shape, *W.shape[1:])                     # [..., R, C]
+        y = mx.matmul(x.astype(W.dtype), mx.swapaxes(W, -1, -2))        # [..., 1|k, R]-broadcast
+        return y
+
+
+class VQSwitchGLU(nn.Module):
+    """Fused replacement for mlx_lm SwitchGLU on VQ layers: one Metal dispatch for
+    silu(gate)*up + one for down. Children keep SwitchGLU's names so load_weights
+    resolves ...switch_mlp.{gate,up,down}_proj.vq_codes unchanged."""
+
+    def __init__(self, gate_proj, up_proj, down_proj):
+        super().__init__()
+        self.gate_proj = gate_proj
+        self.up_proj = up_proj
+        self.down_proj = down_proj
+
+    def __call__(self, x, indices):
+        from mlx_lm.models.switch_layers import _gather_sort, _scatter_unsort
+        from vq_kernel import vq_swiglu
+        g, u, dn = self.gate_proj, self.up_proj, self.down_proj
+        if getattr(dn, "_gemv", None) is None or getattr(g, "_cb", None) is None:
+            # fallback: reference SwitchGLU flow through the modules (level-0)
+            x = mx.expand_dims(x, (-2, -3))
+            do_sort = indices.size >= 64
+            idx, inv_order = indices, None
+            if do_sort:
+                x, idx, inv_order = _gather_sort(x, indices)
+            xu = u(x, idx, sorted_indices=do_sort)
+            xg = g(x, idx, sorted_indices=do_sort)
+            y = dn(mx.sigmoid(xg) * xg * xu, idx, sorted_indices=do_sort)
+            if do_sort:
+                y = _scatter_unsort(y, inv_order, indices.shape)
+            return y.squeeze(-2)
+        dtype = x.dtype
+        x = mx.expand_dims(x, (-2, -3))
+        do_sort = indices.size >= 64
+        idx, inv_order = indices, None
+        if do_sort:
+            x, idx, inv_order = _gather_sort(x, indices)
+        flat = idx.reshape(-1).astype(mx.int32)
+        xk = mx.broadcast_to(x, (*idx.shape, 1, g._C)).reshape(-1, g._C).astype(mx.float16)
+        inter = vq_swiglu(xk, g._codes_u32, g._sc16, u._codes_u32, u._sc16,
+                          g._cb, flat, g._nbits)                      # [N, R]
+        y = dn._gemv(inter, dn._codes_u32, dn._sc16, dn._cb, flat, dn._nbits)  # [N, C]
+        y = y.reshape(*idx.shape, 1, -1).astype(dtype)
+        if do_sort:
+            y = _scatter_unsort(y, inv_order, indices.shape)
+        return y.squeeze(-2)
+
+
+def load_vq_model(path):
+    """Build qwen3_5_moe, quantize spine per config, swap switch_mlp -> VQ, load weights."""
+    import importlib
+
+    with open(os.path.join(path, "config.json")) as f:
+        config = json.load(f)
+    arch = importlib.import_module(f"mlx_lm.models.{config['model_type']}")
+    model = arch.Model(arch.ModelArgs.from_dict(config))
+
+    vq_meta = config["vq"]["modules"]
+    quant = config.get("quantization", {})
+
+    # 1) spine quantization exactly like mlx_lm.load_model
+    def class_predicate(p, m):
+        if p in vq_meta:
+            return False                                  # VQ modules: skip nn.quantize
+        if p in quant:
+            return quant[p]
+        if not hasattr(m, "to_quantized"):
+            return False
+        return True
+
+    nn.quantize(model, group_size=quant.get("group_size", 64), bits=quant.get("bits", 4),
+                mode=quant.get("mode", "affine"), class_predicate=class_predicate)
+
+    # 2) swap switch_mlp projections, then the whole switch_mlp for the fused GLU
+    glu_parents = {}
+    for mpath, meta in vq_meta.items():
+        parts = mpath.split(".")
+        parent = model
+        for q in parts[:-1]:
+            parent = parent[int(q)] if q.isdigit() else getattr(parent, q)
+        old = getattr(parent, parts[-1])
+        vql = VQSwitchLinear(num_experts=256, output_dims=old.output_dims if hasattr(old, "output_dims") else old.weight.shape[1],
+                             input_dims=meta["in_dims"], nbits=meta["nbits"], d=meta["d"],
+                             norm_group=meta["norm_group"])
+        setattr(parent, parts[-1], vql)
+        glu_parents[".".join(parts[:-1])] = parent
+    if os.environ.get("VQ_FUSED", "1") == "1":
+        for gpath, sw in glu_parents.items():
+            parts = gpath.split(".")
+            gp = model
+            for q in parts[:-1]:
+                gp = gp[int(q)] if q.isdigit() else getattr(gp, q)
+            setattr(gp, parts[-1], VQSwitchGLU(sw.gate_proj, sw.up_proj, sw.down_proj))
+
+    # 3) strict load
+    shards = sorted(glob.glob(os.path.join(path, "model-*.safetensors")))
+    weights = {}
+    for s in shards:
+        weights.update(mx.load(s))
+    if hasattr(model, "sanitize"):
+        weights = model.sanitize(weights)
+    model.load_weights(list(weights.items()), strict=True)
+
+    # 4) codebooks
+    cbs = mx.load(os.path.join(path, config["vq"]["codebooks_file"]))
+    for mpath, meta in vq_meta.items():
+        parts = mpath.split(".")
+        parent = model
+        for q in parts[:-1]:
+            parent = parent[int(q)] if q.isdigit() else getattr(parent, q)
+        getattr(parent, parts[-1]).set_codebook(cbs[f"cb{meta['vq_bits']}"])
+
+    mx.eval(model.parameters())
+    model.eval()
+    return model, config

code/vq_test.py

@@ -0,0 +1,99 @@
+"""Mac-side VQ shim test (module level): golden check + micro-benchmark.
+
+Run on the Mac:  python vq_test.py <artifact_dir> <golden.npz>
+1. Metal smoke: trivial mx.fast.metal_kernel JIT + run (proves SSH+Metal+no-keychain).
+2. Golden: build VQSwitchLinear for the two reference modules straight from artifact
+   tensors, decode, y = x @ W_e.T, compare vs CUDA golden (fp16 tolerance).
+3. Micro-bench: decode+matmul timing for a [1 tok, 8 experts] call (level-0 speed).
+"""
+import json
+import os
+import sys
+import time
+
+import mlx.core as mx
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from vq_switch import VQSwitchLinear
+
+ART = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/models/Qwen3.6-35B-A3B-MLX-VQ-map12")
+GOLD = sys.argv[2] if len(sys.argv) > 2 else os.path.expanduser("~/models/vq_golden.npz")
+
+print(f"mlx {mx.__version__} device={mx.default_device()}")
+
+# ---- 1. Metal custom-kernel smoke ----
+src = """
+    uint i = thread_position_in_grid.x;
+    if (i < inp_shape[0]) { out[i] = inp[i] * 2.0f + 1.0f; }
+"""
+try:
+    k = mx.fast.metal_kernel(name="smoke", input_names=["inp"], output_names=["out"], source=src)
+    a = mx.arange(8, dtype=mx.float32)
+    (o,) = k(inputs=[a], output_shapes=[a.shape], output_dtypes=[mx.float32],
+             grid=(8, 1, 1), threadgroup=(8, 1, 1))
+    mx.eval(o)
+    ok = bool(mx.all(o == a * 2 + 1))
+    print(f"[1] metal_kernel JIT smoke: {'PASS' if ok else 'FAIL'} ({o.tolist()[:4]}...)")
+except Exception as e:
+    print(f"[1] metal_kernel smoke FAILED: {e!r}")
+
+# ---- 2. golden check ----
+cfg = json.load(open(os.path.join(ART, "config.json")))
+meta = cfg["vq"]["modules"]
+idx = json.load(open(os.path.join(ART, "model.safetensors.index.json")))["weight_map"]
+cbs = mx.load(os.path.join(ART, cfg["vq"]["codebooks_file"]))
+g = np.load(GOLD)
+mods = sorted(set(k.rsplit("__", 1)[0].replace("__", ".") for k in g.files))
+worst = 0.0
+for m in mods:
+    info = meta[m]
+    key = m.replace(".", "__")
+    x = mx.array(g[key + "__x"])                       # [4, C]
+    y_ref = g[key + "__y"]                             # [4exp, 4tok, R]
+    experts = [int(e) for e in g[key + "__experts"]]
+    shard = mx.load(os.path.join(ART, idx[m + ".vq_codes"]))
+    codes = shard[m + ".vq_codes"]
+    scales = mx.load(os.path.join(ART, idx[m + ".vq_scales"]))[m + ".vq_scales"]
+    R = codes.shape[1]
+    vq = VQSwitchLinear(num_experts=codes.shape[0], output_dims=R,
+                        input_dims=info["in_dims"], nbits=info["nbits"], d=info["d"],
+                        norm_group=info["norm_group"])
+    vq.vq_codes = codes
+    vq.vq_scales = scales
+    vq.set_codebook(cbs[f"cb{info['vq_bits']}"])
+    W = vq._decode(mx.array(experts))                  # [4exp, R, C]
+    y = mx.matmul(x[None].astype(W.dtype), mx.swapaxes(W, -1, -2))   # [4exp, 4tok, R]
+    mx.eval(y)
+    err = float(mx.abs(y - mx.array(y_ref)).max())
+    rel = err / (abs(y_ref).max() + 1e-9)
+    worst = max(worst, rel)
+    print(f"[2] {m.split('layers.')[1]}: max|dy|={err:.4e} rel={rel:.2e} {'PASS' if rel < 2e-2 else 'FAIL'}")
+print(f"[2] golden worst rel = {worst:.2e} -> {'PASS' if worst < 2e-2 else 'FAIL'}")
+
+# ---- 3. kernel vs level-0: correctness then speed ----
+os.environ["VQ_KERNEL"] = "1"
+x1 = mx.random.normal((1, 1, 1, vq._C)).astype(mx.float16)
+idx8 = mx.array([[0, 5, 17, 42, 99, 123, 200, 255]])
+vq.set_codebook(vq._cb)                                  # rebind with kernel enabled
+yk = vq(x1, idx8); mx.eval(yk)
+vq._gemv = None                                          # force level-0
+y0 = vq(x1, idx8).astype(mx.float16); mx.eval(y0)
+kerr = float(mx.abs(yk.astype(mx.float32) - y0.astype(mx.float32)).max())
+kref = float(mx.abs(y0).max())
+print(f"[3] kernel-vs-level0: max|dy|={kerr:.4e} rel={kerr/(kref+1e-9):.2e} "
+      f"{'PASS' if kerr/(kref+1e-9) < 2e-2 else 'FAIL'}")
+
+def bench(fn, n=50):
+    for _ in range(5): mx.eval(fn())
+    t = time.time()
+    for _ in range(n): mx.eval(fn())
+    return (time.time() - t) / n * 1000
+
+vq.set_codebook(vq._cb)                                  # kernel on
+tk = bench(lambda: vq(x1, idx8))
+vq._gemv = None
+t0b = bench(lambda: vq(x1, idx8), n=20)
+print(f"[4] 1tok x 8exp: kernel {tk:.3f} ms vs level-0 {t0b:.2f} ms  (x{t0b/max(tk,1e-6):.1f} speedup)"
+      f"  -> layer-stack est ≈ {tk*120:.1f} ms/token ≈ {1000/max(tk*120,1e-6):.1f} tok/s bound")
+print("VQ_TEST_DONE")

config.json

@@ -0,0 +1,1748 @@
+{
+ "architectures": [
+  "Qwen3_5MoeForConditionalGeneration"
+ ],
+ "eos_token_id": [
+  248046,
+  248044
+ ],
+ "image_token_id": 248056,
+ "model_type": "qwen3_5_moe",
+ "quantization": {
+  "group_size": 64,
+  "bits": 4,
+  "mode": "affine",
+  "language_model.model.layers.0.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.0.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.1.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.1.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.2.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.2.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.3.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.3.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.4.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.4.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.5.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.5.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.6.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.6.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.7.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.7.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.8.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.8.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.9.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.9.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.10.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.10.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.11.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.11.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.12.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.12.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.13.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.13.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.14.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.14.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.15.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.15.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.16.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.16.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.17.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.17.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.18.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.18.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.19.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.19.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.20.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.20.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.21.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.21.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.22.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.22.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.23.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.23.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.24.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.24.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.25.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.25.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.26.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.26.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.27.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.27.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.28.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.28.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.29.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.29.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.30.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.30.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.31.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.31.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.32.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.32.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.33.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.33.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.34.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.34.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.35.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.35.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.36.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.36.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.37.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.37.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.38.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.38.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.39.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.39.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.lm_head": {
+   "group_size": 64,
+   "bits": 6
+  }
+ },
+ "quantization_config": {
+  "group_size": 64,
+  "bits": 4,
+  "mode": "affine",
+  "language_model.model.layers.0.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.0.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.1.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.1.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.2.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.2.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.3.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.3.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.4.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.4.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.5.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.5.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.6.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.6.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.7.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.7.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.8.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.8.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.9.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.9.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.10.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.10.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.11.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.11.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.12.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.12.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.13.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.13.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.14.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.14.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.15.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.15.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.16.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.16.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.17.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.17.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.18.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.18.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.19.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.19.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.20.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.20.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.21.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.21.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.22.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.22.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.23.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.23.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.24.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.24.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.25.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.25.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.26.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.26.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.27.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.27.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.28.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.28.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.29.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.29.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.30.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.30.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.31.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.31.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.32.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.32.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.33.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.33.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.34.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.34.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.35.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.35.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.36.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.36.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.37.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.37.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.38.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.38.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.39.mlp.gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.model.layers.39.mlp.shared_expert_gate": {
+   "group_size": 64,
+   "bits": 8
+  },
+  "language_model.lm_head": {
+   "group_size": 64,
+   "bits": 6
+  }
+ },
+ "text_config": {
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_output_gate": true,
+  "bos_token_id": 248044,
+  "dtype": "bfloat16",
+  "eos_token_id": 248044,
+  "full_attention_interval": 4,
+  "head_dim": 256,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "layer_types": [
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention",
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention",
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention",
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention",
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention",
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention",
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention",
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention",
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention",
+   "linear_attention",
+   "linear_attention",
+   "linear_attention",
+   "full_attention"
+  ],
+  "linear_conv_kernel_dim": 4,
+  "linear_key_head_dim": 128,
+  "linear_num_key_heads": 16,
+  "linear_num_value_heads": 32,
+  "linear_value_head_dim": 128,
+  "mamba_ssm_dtype": "float32",
+  "max_position_embeddings": 262144,
+  "model_type": "qwen3_5_moe_text",
+  "moe_intermediate_size": 512,
+  "mtp_num_hidden_layers": 1,
+  "mtp_use_dedicated_embeddings": false,
+  "num_attention_heads": 16,
+  "num_experts": 256,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 2,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "partial_rotary_factor": 0.25,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+   "mrope_interleaved": true,
+   "mrope_section": [
+    11,
+    11,
+    10
+   ],
+   "partial_rotary_factor": 0.25,
+   "rope_theta": 10000000,
+   "rope_type": "default"
+  },
+  "router_aux_loss_coef": 0.001,
+  "shared_expert_intermediate_size": 512,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 248320
+ },
+ "tie_word_embeddings": false,
+ "transformers_version": "4.57.1",
+ "video_token_id": 248057,
+ "vision_config": {
+  "deepstack_visual_indexes": [],
+  "depth": 27,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_size": 1152,
+  "in_channels": 3,
+  "initializer_range": 0.02,
+  "intermediate_size": 4304,
+  "model_type": "qwen3_5_moe",
+  "num_heads": 16,
+  "num_position_embeddings": 2304,
+  "out_hidden_size": 2048,
+  "patch_size": 16,
+  "spatial_merge_size": 2,
+  "temporal_patch_size": 2
+ },
+ "vision_end_token_id": 248054,
+ "vision_start_token_id": 248053,
+ "vq": {
+  "modules": {
+   "language_model.model.layers.0.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.0.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.0.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.1.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.1.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.1.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.10.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.10.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.10.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.11.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.11.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.11.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.12.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.12.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.12.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.13.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.13.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.13.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.14.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.14.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.14.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.15.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.15.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.15.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.16.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.16.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.16.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.17.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.17.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.17.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.18.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.18.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.18.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.19.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.19.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.19.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.2.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.2.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.2.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.20.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.20.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.20.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.21.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.21.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.21.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.22.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.22.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.22.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.23.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.23.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.23.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.24.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.24.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.24.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.25.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.25.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.25.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.26.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.26.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.26.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.27.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.27.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.27.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.28.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.28.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.28.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.29.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.29.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.29.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.3.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.3.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.3.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.30.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.30.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.30.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.31.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.31.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.31.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.32.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.32.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.32.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.33.mlp.switch_mlp.down_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.33.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.33.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.34.mlp.switch_mlp.down_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.34.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.34.mlp.switch_mlp.up_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.35.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.35.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.35.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.36.mlp.switch_mlp.down_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.36.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.36.mlp.switch_mlp.up_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.37.mlp.switch_mlp.down_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.37.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.37.mlp.switch_mlp.up_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.38.mlp.switch_mlp.down_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.38.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.38.mlp.switch_mlp.up_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.39.mlp.switch_mlp.down_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.39.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.39.mlp.switch_mlp.up_proj": {
+    "vq_bits": 3,
+    "d": 4,
+    "K": 4096,
+    "nbits": 12,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.4.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.4.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.4.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.5.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.5.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.5.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.6.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.6.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.6.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.7.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.7.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.7.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.8.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.8.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.8.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.9.mlp.switch_mlp.down_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 512,
+    "norm_group": 128
+   },
+   "language_model.model.layers.9.mlp.switch_mlp.gate_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   },
+   "language_model.model.layers.9.mlp.switch_mlp.up_proj": {
+    "vq_bits": 2,
+    "d": 4,
+    "K": 256,
+    "nbits": 8,
+    "in_dims": 2048,
+    "norm_group": 128
+   }
+  },
+  "codebooks_file": "vq_codebooks.safetensors",
+  "method": "gptq-vq",
+  "map": "map12"
+ }
+}

configuration.json

@@ -0,0 +1 @@
+{"framework":"Pytorch","task":"visual-question-answering"}

generation_config.json

@@ -0,0 +1,12 @@
+{
+    "bos_token_id": 248044,
+    "do_sample": true,
+    "eos_token_id": [
+        248046,
+        248044
+    ],
+    "pad_token_id": 248044,
+    "temperature": 1.0,
+    "top_k": 20,
+    "top_p": 0.95
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3435fef45265f23bde60a970328181e35d0d9caa101532e1f90b120435e97b57
+size 5318570592

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3ae798d27788ab595009987850f42b6586cc08ec0fdbbabfe3d1d2a08c0958b
+size 5363030360

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be688405bf7c9d68e09d0665cd486ca32cb852b929221e386cb904fe63ade5f6
+size 821583656

preprocessor_config.json

@@ -0,0 +1,21 @@
+{
+    "size": {
+        "longest_edge": 16777216,
+        "shortest_edge": 65536
+    },
+    "patch_size": 16,
+    "temporal_patch_size": 2,
+    "merge_size": 2,
+    "image_mean": [
+        0.5,
+        0.5,
+        0.5
+    ],
+    "image_std": [
+        0.5,
+        0.5,
+        0.5
+    ],
+    "processor_class": "Qwen3VLProcessor",
+    "image_processor_type": "Qwen2VLImageProcessorFast"
+}

processor_config.json

@@ -0,0 +1,64 @@
+{
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Qwen2VLImageProcessor",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_pixels": 16777216,
+    "merge_size": 2,
+    "min_pixels": 65536,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 16777216,
+      "shortest_edge": 65536
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen3VLProcessor",
+  "video_processor": {
+    "data_format": "channels_first",
+    "default_to_square": true,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen3VLVideoProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4
+size 19989343

tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "processor_class": "Qwen3VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

video_preprocessor_config.json

@@ -0,0 +1,21 @@
+{
+    "size": {
+        "longest_edge": 25165824,
+        "shortest_edge": 4096
+    },
+    "patch_size": 16,
+    "temporal_patch_size": 2,
+    "merge_size": 2,
+    "image_mean": [
+        0.5,
+        0.5,
+        0.5
+    ],
+    "image_std": [
+        0.5,
+        0.5,
+        0.5
+    ],
+    "processor_class": "Qwen3VLProcessor",
+    "video_processor_type": "Qwen3VLVideoProcessor"
+}

vq_codebooks.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c86f9c377f106bad8de38f5b45b2e6c8c092c237d47896600c612971856f9d1
+size 39120