Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.ms_upload_cache

@@ -0,0 +1 @@
+{"version": 3, "repo_id": "TaichuAI/ZDTaichu5.0-9B-NVFP4", "files": {"README.md|1789460761.0|23258": {"hash": "b11f22ed8c86f05feb2e7ce747b69ee89f42b28b407d4b158cc299cd1ad489c1", "size": 23258, "status": "c"}, ".vscode/settings.json|1789460406.0|580": {"hash": "1d1a47d4cdda5f67a9302d196d91f6ed8d209972d1fb7d2d7618a9d8e49aee14", "size": 580, "status": "c"}, "assets/logos/gemini.png|1789460699.0|19333": {"hash": "10c628f55d22a9725b9f9fccce7cf062b9fb68da5f7736e87010e0594d7ba6db", "size": 19333, "status": "c"}, "__init__.py|1789455331.0|487": {"hash": "854a695a9b3732f6a4841bacfa83463f6b0e51ffbba692bb3ada8e647e3e37aa", "size": 487, "status": "c"}, "assets/logos/gemma.png|1789460698.0|16826": {"hash": "5f64463f9e00b29595c3ce7222100e263b67c3f29035c304d1249f30c99d2913", "size": 16826, "status": "c"}, "assets/logos/grok.png|1789460699.0|6883": {"hash": "f10ec0b2322f9d65a973d3d48fcdc41270d8b123a5d36e2826dabf9dcefb0762", "size": 6883, "status": "c"}, "assets/logos/qwen.png|1789460699.0|10324": {"hash": "de9bc7e285164e0d284a7b9555511c7f7767af7699d6272b3b67b376eeddbfb3", "size": 10324, "status": "c"}, "config.json|1789455731.0|24594": {"hash": "699af1e41b2054269f2b698f11950c5f35f46d02679a7b9d55c0301e8a52e823", "size": 24594, "status": "c"}, "assets/logos/taichu.png|1789460699.0|46894": {"hash": "90da889f0994e50a92677bf4141e6d58bc24c221547e0828d91e2889fc1bc47c", "size": 46894, "status": "c"}, "cradio_config.py|1789455331.0|2049": {"hash": "4971a6d6c0abb55eaddd0cb94753f4941f50928804b1766c09d7b9bb98f51dfe", "size": 2049, "status": "c"}, "assets/logos/stepfun.png|1789460699.0|6937": {"hash": "49e54bcda77c23f9a8ceacc0d3c1c07da09be4c2534d453ebcad7da201b25b96", "size": 6937, "status": "c"}, "chat_template.jinja|1789455319.0|7756": {"hash": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "size": 7756, "status": "c"}, "assets/taichu-vs-closed-models.svg|1789460699.0|403989": {"hash": "d80d5e9c375d397d268bfba0db7000cdb9ee558a890ee3fe6b6c38a49b3cffed", "size": 403989, "status": "c"}, "cradio_model.py|1789455331.0|26514": {"hash": "4c4d3bafb13cd47203ffacb78e1932c809cb48cc254b0f042cbb90121f8827e8", "size": 26514, "status": "c"}, "assets/taichu-release-benchmark-comparison.svg|1789460698.0|569738": {"hash": "7109868f2f09641ee86a0b12dba6f56f56ffa3b7c71bbb8fa0db655b21876fe0", "size": 569738, "status": "c"}, "configuration.py|1789455331.0|9699": {"hash": "57a04f680bcbaede0791bfc8d2653a7f82c11cbf8e38c68b00452b315a4ce0f0", "size": 9699, "status": "c"}, "generation_config.json|1789455327.0|284": {"hash": "fc84e407226adee495fc06cb354223d4912fea7ba6e8cee1b0b5f2c08e04badc", "size": 284, "status": "c"}, "image_processing.py|1789455331.0|10095": {"hash": "a68a34f5426b1886f5904f0b75940737a67caec2006a3ddf9387a199b6b10303", "size": 10095, "status": "c"}, "modeling.py|1789455331.0|50552": {"hash": "b3f46b9f1635663778eb30bff8b90c474d45b3186f469367df058a5f6a7117fa", "size": 50552, "status": "c"}, "processor_config.json|1789455327.0|724": {"hash": "ca2ea029533bcad96c9e8650e1c32d7412af9d51d934f1358153d20c4fdf5c01", "size": 724, "status": "c"}, "preprocessor_config.json|1789455327.0|528": {"hash": "a9bd6d2e6a32ed98335351904c6f129613f7c14f22bd556d4bf660b461112b9b", "size": 528, "status": "c"}, "processing.py|1789455331.0|27232": {"hash": "0e1fe5d7706886a4520604f3cdb0d7402866158db3636997ee7bb798587ad5f6", "size": 27232, "status": "c"}, "recipe.yaml|1789455224.0|3205": {"hash": "a396a0d93e140200410815c55411335b3a901f11dbe68672526dc7561641af98", "size": 3205, "status": "c"}, "tokenizer_config.json|1789455327.0|930": {"hash": "78311f543bf15a7153eff52ca8d692dbaf5bdc094f265d082ec3909807cc3db7", "size": 930, "status": "c"}, "vision_utils.py|1789455331.0|22068": {"hash": "f4eb3b4b28886f358691976d7910833c811c92f594a3c247985d1040f3bac6b6", "size": 22068, "status": "c"}, "tokenizer.json|1789455327.0|19989343": {"hash": "87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4", "size": 19989343, "status": "c"}, "model.safetensors|1789455210.0|9811274448": {"hash": "d26e50e66812bdb9563368991b4ad9be1eeaab8aa59b5220df5efbc89e202f3b", "size": 9811274448, "status": "c"}}}

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "workbench.colorCustomizations": {
+        "activityBar.background": "#1E1E5C",
+        "titleBar.activeBackground": "#2A2A81",
+        "titleBar.activeForeground": "#FCFCFE",
+        "titleBar.inactiveBackground": "#1E1E5C",
+        "titleBar.inactiveForeground": "#FCFCFE",
+        "statusBar.background": "#1E1E5C",
+        "statusBar.foreground": "#FCFCFE",
+        "statusBar.debuggingBackground": "#1E1E5C",
+        "statusBar.debuggingForeground": "#FCFCFE",
+        "statusBar.noFolderBackground": "#1E1E5C",
+        "statusBar.noFolderForeground": "#FCFCFE"
+    }
+}

README.md

@@ -1,3 +1,655 @@
 ---
-license: apache-2.0
+language:
+  - en
+  - zh
+pipeline_tag: image-text-to-text
+tags:
+  - multimodal
+  - vision-language-model
+  - spatial-reasoning
+  - agent
+  - video-understanding
 ---
+
+# ZDTaichu5.0-9B-NVFP4
+
+> [!Note]
+> This repository contains NVFP4-quantized model weights and configuration files for the post-trained model in the compressed-tensors format. 
+>
+> These artifacts are compatible with Hugging Face Transformers, vLLM, etc.
+>
+> The quantization method is mixed NVFP4/FP8 quantization, with NVFP4 (group size 16, dynamic per-group activations) for selected MLP projections and FP8 (static per-channel weights, dynamic per-token activations) for the other linear layers, and its performance metrics are nearly identical to those of the original model.
+
+[Project Page](https://taichu-ai.github.io/ZDTaichu5.0-9B/) | [GitHub](https://github.com/Taichu-AI/ZDTaichu5.0-9B) | [ModelScope](https://www.modelscope.cn/models/TaichuAI/ZDTaichu5.0-9B)
+
+ZDTaichu5.0-9B is a multimodal foundation model for general visual understanding, spatial reasoning, agentic tool use, and embodied-AI research. It combines a Qwen3.5-9B language backbone with a C-RADIOv4-H vision encoder, supports text, images and videos with any-resolution visual input.
+
+Within the 10B-scale general-purpose VLMs compared in this release blog, ZDTaichu5.0-9B retains first-tier general visual understanding while supporting spatial reasoning, high-level embodied VLM reasoning, and agent tasks under the reported evaluation settings. Rather than trading broad visual competence for specialization, it layers a more comprehensive spatial, embodied, and agent capability profile on top of a strong general-vision foundation.
+
+The model accepts text, one or more images, and video. It is designed for:
+
+- general image, document, chart, diagram, and OCR understanding;
+- visual mathematics and knowledge-grounded visual question answering;
+- fine-grained 2D relations, multi-view association, 3D scene understanding, perspective taking, and mental transformation;
+- multi-step and multi-turn tool use;
+- spatial perception, affordance understanding, and planning for VLA and embodied-AI adaptation.
+
+More demos and showcases are provided at [Project Page](https://taichu-ai.github.io/ZDTaichu5.0-9B/).
+
+## Highlights
+
+- **Strong general vision and broad capabilities:** remains in the leading group of 10B-scale general-purpose VLMs across images, documents, charts, diagrams, OCR, visual mathematics, multiple images and video, while extending to spatial reasoning, high-level embodied understanding and multi-step agent tasks.
+- **Leading spatial reasoning and embodied understanding:** leads spatial capability among the compared 10B-scale general-purpose VLMs, with strong results on SparBench, ViewSpatial, MMSI-Bench and MindCube-tiny. Scores of 48 on ERQA and 56 on RoboSpatial cover scene reasoning, affordances and interaction-oriented understanding.
+- **Strongest agent capability among the compared 10B-scale general-purpose VLMs:** leads the reported TAU2-Bench (87.7) and Claw-Eval (71.4) comparisons, and reaches 93.7 on IFEval.
+- **Entropy-Gated Adaptive Recurrent Reasoning:** Dynamically allocates additional recurrent refinement steps in latent space to more challenging tokens, enabling greater computational depth where needed and improving reasoning performance on complex tasks.
+
+## Model Overview
+
+| Item | Specification |
+|---|---|
+| Model type | Multimodal causal language model with vision encoder |
+| Language backbone | Qwen3.5-9B LLM Decoder|
+| Vision backbone | C-RADIOv4-H |
+| Context length | Up to 128K tokens |
+| Vision resolution | Any-resolution visual input |
+| Input modalities | Text, single image, multiple images, and video |
+
+## Capabilities
+
+### General visual understanding
+
+The model can recognize objects, attributes, and scenes; read text in natural images and documents; interpret tables, forms, plots, and diagrams; and answer questions that combine visual evidence with language and world knowledge.
+
+### Spatial perception and reasoning
+
+Spatial training covers:
+
+- left/right, above/below, front/behind, occlusion, containment, and relative distance;
+- dense counting, fine-grained localization, points, coordinates, and bounding boxes;
+- association across images and viewpoints;
+- camera motion, relative pose, depth ordering, and room-scale layout;
+- egocentric and allocentric perspective taking;
+- 2D/3D rotation, paper folding, three-view projection, cross-sections, and part-motion reasoning;
+- embodied affordances, manipulation semantics, and high-level action planning.
+
+### Multiple images and video
+
+ZDTaichu5.0-9B compares and reasons across multiple images and supports video understanding, including event tracking and detail retrieval from long footage within its 128K-token context window.
+
+### Agentic tool use
+
+The model is designed for multi-step and multi-turn tool-use tasks. Tool execution must be implemented, validated, and secured by the surrounding application; the model does not execute tools by itself.
+
+## Benchmark Results
+
+
+The two figures compare ZDTaichu5.0-9B with open and closed models across general visual understanding, spatial and embodied capabilities, and agent and text capabilities.
+
+**Comparison with open models**
+
+![ZDTaichu5.0-9B benchmark comparison with open models](assets/taichu-release-benchmark-comparison.svg)
+
+**Comparison with closed models**
+
+![ZDTaichu5.0-9B benchmark comparison with closed models](assets/taichu-vs-closed-models.svg)
+
+
+
+### Spatial and embodied reasoning
+
+<table>
+  <thead>
+    <tr>
+      <th align="left">Area</th>
+      <th align="left">Benchmark</th>
+      <th align="right">ZDTaichu5.0-9B</th>
+      <th align="right">Qwen3.5-9B</th>
+      <th align="right">STEP3-VL-10B</th>
+      <th align="right">gemma4-8B-E4B</th>
+      <th align="right">Gemini 3 Pro</th>
+      <th align="right">Grok 4</th>
+      <th align="right">GPT-5.2</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td rowspan="3" align="left" valign="middle">Basic spatial perception</td>
+      <td align="left">CV-Bench</td>
+      <td align="right">86.82</td>
+      <td align="right"><strong>87.19</strong></td>
+      <td align="right">83.49</td>
+      <td align="right">68.10</td>
+      <td align="right"><ins>90.07</ins></td>
+      <td align="right">—</td>
+      <td align="right">86.84</td>
+    </tr>
+    <tr>
+      <td align="left">3DSRBench</td>
+      <td align="right"><strong>60.96</strong></td>
+      <td align="right">56.78</td>
+      <td align="right">55.01</td>
+      <td align="right">53.62</td>
+      <td align="right"><ins>68.92</ins></td>
+      <td align="right">54.93</td>
+      <td align="right">60.20</td>
+    </tr>
+    <tr>
+      <td align="left">SparBench</td>
+      <td align="right"><strong>51.82</strong></td>
+      <td align="right">50.79</td>
+      <td align="right">45.68</td>
+      <td align="right">28.50</td>
+      <td align="right">48.74</td>
+      <td align="right">44.76</td>
+      <td align="right"><ins>55.07</ins></td>
+    </tr>
+    <tr>
+      <td rowspan="3" align="left" valign="middle">Complex spatial reasoning</td>
+      <td align="left">ViewSpatial</td>
+      <td align="right"><strong><ins>62.50</ins></strong></td>
+      <td align="right">48.20</td>
+      <td align="right">46.14</td>
+      <td align="right">41.68</td>
+      <td align="right">50.36</td>
+      <td align="right">43.23</td>
+      <td align="right">47.30</td>
+    </tr>
+    <tr>
+      <td align="left">MMSI-Bench</td>
+      <td align="right"><strong><ins>47.20</ins></strong></td>
+      <td align="right">38.70</td>
+      <td align="right">32.18</td>
+      <td align="right">29.20</td>
+      <td align="right">45.20</td>
+      <td align="right">37.80</td>
+      <td align="right">41.30</td>
+    </tr>
+    <tr>
+      <td align="left">MindCube-tiny</td>
+      <td align="right"><strong><ins>78.27</ins></strong></td>
+      <td align="right">57.60</td>
+      <td align="right">62.81</td>
+      <td align="right">48.85</td>
+      <td align="right">70.87</td>
+      <td align="right">63.56</td>
+      <td align="right">60.38</td>
+    </tr>
+    <tr>
+      <td rowspan="3" align="left" valign="middle">Embodied interaction</td>
+      <td align="left">ERQA</td>
+      <td align="right"><strong>48.00</strong></td>
+      <td align="right">41.50</td>
+      <td align="right">47.75</td>
+      <td align="right">30.20</td>
+      <td align="right"><ins>66.00</ins></td>
+      <td align="right">—</td>
+      <td align="right">59.80</td>
+    </tr>
+    <tr>
+      <td align="left">RoboSpatial</td>
+      <td align="right"><strong>56.00</strong></td>
+      <td align="right">54.10</td>
+      <td align="right">52.86</td>
+      <td align="right">49.43</td>
+      <td align="right"><ins>57.40</ins></td>
+      <td align="right">—</td>
+      <td align="right">43.78</td>
+    </tr>
+    <tr>
+      <td align="left">VSI-Bench</td>
+      <td align="right"><strong><ins>59.69</ins></strong></td>
+      <td align="right">55.68</td>
+      <td align="right">42.42</td>
+      <td align="right">32.91</td>
+      <td align="right">52.51</td>
+      <td align="right">47.92</td>
+      <td align="right">54.49</td>
+    </tr>
+  </tbody>
+</table>
+
+### General visual understanding
+
+<table>
+  <thead>
+    <tr>
+      <th align="left">Area</th>
+      <th align="left">Benchmark</th>
+      <th align="right">ZDTaichu5.0-9B</th>
+      <th align="right">Qwen3.5-9B</th>
+      <th align="right">STEP3-VL-10B</th>
+      <th align="right">gemma4-8B-E4B</th>
+      <th align="right">Gemini 3 Pro</th>
+      <th align="right">Grok 4</th>
+      <th align="right">GPT-5.2</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td align="left" rowspan="3" valign="middle">Multi modal Reasoning</td>
+      <td align="left">MathVista Mini</td>
+      <td align="right">84.50</td>
+      <td align="right"><strong>85.70</strong></td>
+      <td align="right">83.97</td>
+      <td align="right">65.30</td>
+      <td align="right"><ins>87.90</ins></td>
+      <td align="right">72.50</td>
+      <td align="right">83.10</td>
+    </tr>
+    <tr>
+      <td align="left">WeMath</td>
+      <td align="right"><strong>75.90</strong></td>
+      <td align="right">75.20</td>
+      <td align="right">73.03</td>
+      <td align="right">50.19</td>
+      <td align="right"><ins>86.90</ins></td>
+      <td align="right">—</td>
+      <td align="right">79.00</td>
+    </tr>
+    <tr>
+      <td align="left">MathVerse Mini Vision Only</td>
+      <td align="right">76.40</td>
+      <td align="right"><strong><ins>84.14</ins></strong></td>
+      <td align="right">74.60</td>
+      <td align="right">53.55</td>
+      <td align="right">—</td>
+      <td align="right">—</td>
+      <td align="right">—</td>
+    </tr>
+    <tr>
+      <td align="left" rowspan="3" valign="middle">General VQA</td>
+      <td align="left">MMStar</td>
+      <td align="right">76.80</td>
+      <td align="right"><strong>79.70</strong></td>
+      <td align="right">77.48</td>
+      <td align="right">62.00</td>
+      <td align="right"><ins>83.10</ins></td>
+      <td align="right">69.60</td>
+      <td align="right">77.10</td>
+    </tr>
+    <tr>
+      <td align="left">AI2D</td>
+      <td align="right"><strong>91.48</strong></td>
+      <td align="right">90.20</td>
+      <td align="right">89.35</td>
+      <td align="right">79.15</td>
+      <td align="right"><ins>94.10</ins></td>
+      <td align="right">—</td>
+      <td align="right">92.20</td>
+    </tr>
+    <tr>
+      <td align="left">RealWorldQA</td>
+      <td align="right">76.99</td>
+      <td align="right"><strong>80.30</strong></td>
+      <td align="right">74.44</td>
+      <td align="right">59.08</td>
+      <td align="right"><ins>83.30</ins></td>
+      <td align="right">—</td>
+      <td align="right"><ins>83.30</ins></td>
+    </tr>
+    <tr>
+      <td align="left" valign="middle">OCR</td>
+      <td align="left">OCRBench</td>
+      <td align="right">85.50</td>
+      <td align="right"><strong>89.20</strong></td>
+      <td align="right">86.75</td>
+      <td align="right">76.90</td>
+      <td align="right"><ins>90.40</ins></td>
+      <td align="right">—</td>
+      <td align="right">80.70</td>
+    </tr>
+  </tbody>
+</table>
+
+
+
+### Language, reasoning, and agents
+
+<table>
+  <thead>
+    <tr>
+      <th align="left">Area</th>
+      <th align="left">Benchmark</th>
+      <th align="right">ZDTaichu5.0-9B</th>
+      <th align="right">Qwen3.5-9B</th>
+      <th align="right">STEP3-VL-10B</th>
+      <th align="right">gemma4-8B-E4B</th>
+      <th align="right">Gemini 3 Pro</th>
+      <th align="right">Grok 4</th>
+      <th align="right">GPT-5.2</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td rowspan="2" align="left" valign="middle">Knowledge</td>
+      <td align="left">MMLU-Pro</td>
+      <td align="right">77.20</td>
+      <td align="right"><strong>82.50</strong></td>
+      <td align="right">76.02</td>
+      <td align="right">69.40</td>
+      <td align="right"><ins>89.80</ins></td>
+      <td align="right">85.90</td>
+      <td align="right">87.40</td>
+    </tr>
+    <tr>
+      <td align="left">MMLU-Redux</td>
+      <td align="right">88.40</td>
+      <td align="right"><strong>91.10</strong></td>
+      <td align="right">86.50</td>
+      <td align="right">85.30</td>
+      <td align="right"><ins>95.90</ins></td>
+      <td align="right">86.22</td>
+      <td align="right">95.00</td>
+    </tr>
+    <tr>
+      <td rowspan="2" align="left" valign="middle">Instruction following</td>
+      <td align="left">IFEval</td>
+      <td align="right"><strong>93.70</strong></td>
+      <td align="right">88.72</td>
+      <td align="right">82.16</td>
+      <td align="right">87.80</td>
+      <td align="right">93.50</td>
+      <td align="right">92.80</td>
+      <td align="right"><ins>94.80</ins></td>
+    </tr>
+    <tr>
+      <td align="left">IFBench</td>
+      <td align="right"><strong>69.00</strong></td>
+      <td align="right">64.50</td>
+      <td align="right">41.49</td>
+      <td align="right">34.70</td>
+      <td align="right">70.40</td>
+      <td align="right">53.70</td>
+      <td align="right"><ins>75.40</ins></td>
+    </tr>
+    <tr>
+      <td rowspan="5" align="left" valign="middle">Reasoning and coding</td>
+      <td align="left">AIME 2025</td>
+      <td align="right">86.70</td>
+      <td align="right">83.75</td>
+      <td align="right"><strong>87.66</strong></td>
+      <td align="right">41.30</td>
+      <td align="right">95.00</td>
+      <td align="right">91.70</td>
+      <td align="right"><ins>100.00</ins></td>
+    </tr>
+    <tr>
+      <td align="left">AIME 2026</td>
+      <td align="right"><strong>89.20</strong></td>
+      <td align="right">87.92</td>
+      <td align="right">88.75</td>
+      <td align="right">42.50</td>
+      <td align="right">90.60</td>
+      <td align="right">—</td>
+      <td align="right"><ins>96.70</ins></td>
+    </tr>
+    <tr>
+      <td align="left">HMMT Feb 2025</td>
+      <td align="right"><strong>84.20</strong></td>
+      <td align="right">83.20</td>
+      <td align="right">78.18</td>
+      <td align="right">26.70</td>
+      <td align="right">97.30</td>
+      <td align="right">90.00</td>
+      <td align="right"><ins>99.40</ins></td>
+    </tr>
+    <tr>
+      <td align="left">HMMT Feb 2026</td>
+      <td align="right">72.70</td>
+      <td align="right"><strong>73.48</strong></td>
+      <td align="right">63.64</td>
+      <td align="right">33.70</td>
+      <td align="right">86.36</td>
+      <td align="right">—</td>
+      <td align="right"><ins>96.97</ins></td>
+    </tr>
+    <tr>
+      <td align="left">LiveCodeBench v6</td>
+      <td align="right"><strong>73.40</strong></td>
+      <td align="right">65.60</td>
+      <td align="right">58.86</td>
+      <td align="right">52.00</td>
+      <td align="right"><ins>90.70</ins></td>
+      <td align="right">—</td>
+      <td align="right">87.70</td>
+    </tr>
+    <tr>
+      <td rowspan="2" align="left" valign="middle">General agent</td>
+      <td align="left">TAU2-Bench†</td>
+      <td align="right"><strong><ins>87.70</ins></strong></td>
+      <td align="right">79.10</td>
+      <td align="right">81.70</td>
+      <td align="right">42.40</td>
+      <td align="right">85.40</td>
+      <td align="right">—</td>
+      <td align="right">87.10</td>
+    </tr>
+    <tr>
+      <td align="left">Claw-Eval<sub>general</sub> Avg†</td>
+      <td align="right"><strong><ins>71.40</ins></strong></td>
+      <td align="right">66.50</td>
+      <td align="right">66.60</td>
+      <td align="right">52.10</td>
+      <td align="right">—</td>
+      <td align="right">—</td>
+      <td align="right">—</td>
+    </tr>
+  </tbody>
+</table>
+
+<sub><strong>Bold</strong> indicates the best score among the listed open-source models; <ins>underlining</ins> indicates the best score among all listed models. Scores leading both comparisons are both bold and underlined. Tied best scores receive the same marking. Missing scores are excluded from the comparison.</sub>
+
+
+<sub>† Local TAU2-Bench and Claw-Eval general evaluations use DeepSeek-V4-Flash-0731 as the simulated user and/or judge; externally reported scores follow the evaluation setup of their cited sources.</sub>
+
+<sub>‡ Publicly reported external score. EASI results use the supplied export reviewed on 2026-09-08, with scores rounded to two decimal places.</sub>
+
+<sub>For multi-image spatial reasoning evaluations such as ViewSpatial, MMSI-Bench, MindCube-tiny, and VSI-Bench, the following output-format requirement was added to the evaluation prompt: You FIRST think about the reasoning process as an internal monologue and then provide the final answer. The reasoning process MUST BE enclosed within <think> </think> tags. The final answer MUST BE put in \boxed{}.</sub>
+
+
+## Quickstart
+
+
+### Installation
+
+Install a recent version of Hugging Face Transformers together with the standard multimodal dependencies:
+
+```bash
+pip install tranformer==5.3.0 torch==2.10.0 torchvision==0.25.0 accelerate timm
+```
+
+### Offline inference
+
+export CUDA_VISIBLE_DEVICES=0
+
+```python
+import os
+
+import torch
+from transformers import AutoModel, AutoProcessor
+
+model_id = os.environ["ZDTAICHU_MODEL_ID"]
+processor = AutoProcessor.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+    use_fast=False,
+)
+model = AutoModel.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+    attn_implementation="sdpa",
+).eval()
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "image", "image": "floorplan.png"},
+            {"type": "text", "text": "Which room is directly to the left of the kitchen?"},
+        ],
+    }
+]
+inputs = processor.from_messages(messages, return_tensors="pt").to(model.device)
+with torch.inference_mode():
+    output_ids = model.generate(**inputs, max_new_tokens=1024, do_sample=False)
+generated_ids = output_ids[:, inputs["input_ids"].shape[1] :]
+print(processor.batch_decode(generated_ids, skip_special_tokens=True)[0])
+```
+
+### Online Serving
+
+We adapted the vLLM v0.26.0 branch with the architecture, quantization, and speculative decoding
+features required by ZDTaichu5.0, supporting both Docker and source deployment:
+
+**Docker (recommended)**
+
+- **Docker image:** `registry-dx.wair.ac.cn/taichu-public/vllm-openai:v0.26.0.zdtaichu_5_0`
+- CUDA ≥ 12.9，Nvidia Driver ≥ 575.51.03
+
+```bash
+docker run -d \
+  -e CUDA_VISIBLE_DEVICES=0 --gpus all \
+  --privileged --ipc=host \
+  -p 18050:8000 \
+  registry-dx.wair.ac.cn/taichu-public/vllm-openai:v0.26.0.zdtaichu_5_0 \
+  TaichuAI/ZDTaichu5.0-9B \
+    --max-model-len 220000 \
+    --served-model-name zdtaichu \
+    --mamba-ssm-cache-dtype float32 \
+    --gdn-prefill-backend triton \
+    --trust-remote-code \
+    --tensor-parallel-size 1 \
+    --generation-config vllm
+```
+
+**Install from source**
+
+- **vLLM source (GitHub):** https://github.com/Taichu-AI/vllm · branch `v0.26.0-zdtaichu`
+
+```bash
+git clone -b v0.26.0-zdtaichu https://github.com/Taichu-AI/vllm.git
+cd vllm
+pip install -e .
+
+vllm serve TaichuAI/ZDTaichu5.0-9B \
+  --max-model-len 220000 \
+  --served-model-name zdtaichu \
+  --mamba-ssm-cache-dtype float32 \
+  --gdn-prefill-backend triton \
+  --trust-remote-code \
+  --tensor-parallel-size 1 \
+  --generation-config vllm
+```
+
+The server exposes an OpenAI-compatible endpoint at `http://<host>:18050/v1`. The examples below use the
+`requests` library (`pip install requests`):
+
+**Setup**
+
+```python
+import base64
+import requests
+
+URL = "http://<host>:18050/v1/chat/completions"
+
+
+def data_url(path: str, mime: str) -> str:
+    """Encode a local file as a base64 data URI."""
+    with open(path, "rb") as f:
+        return f"data:{mime};base64," + base64.b64encode(f.read()).decode()
+
+
+def chat(body: dict) -> str:
+    resp = requests.post(URL, json=body, timeout=600)
+    resp.raise_for_status()
+    return resp.json()["choices"][0]["message"]["content"]
+
+# Text-only input
+
+body = {
+    "model": "zdtaichu",
+    "messages": [{"role": "user", "content": "Hello"}],
+    "temperature": 1.0,
+    "top_p": 0.95,
+    "top_k": 20,
+}
+print(chat(body))
+
+# Image input (local file, base64)
+
+body = {
+    "model": "zdtaichu",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Which room is directly to the left of the kitchen?"},
+                {"type": "image_url", "image_url": {"url": data_url("floorplan.png", "image/png")}},
+            ],
+        }
+    ],
+    "temperature": 0,
+    "top_p": 0.95,
+    "top_k": 20,
+}
+
+# Video input (local file, base64)
+
+body = {
+    "model": "zdtaichu",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Please describe the video."},
+                {"type": "video_url", "video_url": {"url": data_url("example.mp4", "video/mp4")}},
+            ],
+        }
+    ],
+    "media_io_kwargs": {
+        "video": {
+            "num_frames": 8,
+        },
+    },
+}
+print(chat(body))
+```
+
+`media_io_kwargs.video.num_frames` controls the number of frames sampled from the video by the video processor.
+
+**Recommended sampling parameters**
+
+| Task | temperature | top_p | top_k |
+|---|---|---|---|
+| Spatial reasoning and grounding | 0 | 0.95 | 20 |
+| Other tasks | 1.0 | 0.95 | 20 |
+
+**Reasoning and tool-call parsing arguments (optional)**
+
+To enable reasoning output and tool calls, add the following arguments to the launch command:
+
+```bash
+--reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder
+```
+
+
+## License
+
+The model weights in this repository are made available under the NVIDIA Open Model License Agreement, with the Qwen3.5 Apache-2.0 license and all other third-party notices retained. See `LICENSE`, `NOTICE`, and `THIRD_PARTY_LICENSES.md`.
+
+## Acknowledgements
+
+This model builds on the Qwen3.5 language architecture and NVIDIA C-RADIO vision encoder family. Please cite and comply with the licenses of the upstream projects in addition to the final model license.
+
+## Citation
+
+```bibtex
+@misc{zdtaichu_5_0_9b,
+  title  = {ZDTaichu5.0-9B: A Multimodal Foundation Model for Visual and Spatial Reasoning, Agents, and Embodied AI},
+  author = {{ZDTaichu5.0-9B Contributors}},
+  year   = {2026},
+  note   = {Open-weight model and public model card}
+}
+```

__init__.py

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Public Transformers reference implementation for ZDTaichu-5.0."""
+
+from .configuration import ZDTaichu5_0_Config
+from .image_processing import ZDTaichu5_0_ImageProcessor
+from .modeling import ZDTaichu5_0_ForConditionalGeneration
+from .processing import ZDTaichu5_0_Processor
+
+__all__ = [
+    "ZDTaichu5_0_Config",
+    "ZDTaichu5_0_ForConditionalGeneration",
+    "ZDTaichu5_0_ImageProcessor",
+    "ZDTaichu5_0_Processor",
+]
+
+__version__ = "0.1.0"

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,673 @@
+{
+  "_model_size_label": "9B",
+  "architectures": [
+    "ZDTaichu5_0_ForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration.ZDTaichu5_0_Config",
+    "AutoImageProcessor": "image_processing.ZDTaichu5_0_ImageProcessor",
+    "AutoModel": "modeling.ZDTaichu5_0_ForConditionalGeneration",
+    "AutoModelForCausalLM": "modeling.ZDTaichu5_0_ForConditionalGeneration",
+    "AutoProcessor": "processing.ZDTaichu5_0_Processor"
+  },
+  "bos_token_id": null,
+  "downsample_ratio": 0.5,
+  "dtype": "bfloat16",
+  "eos_token_id": 248046,
+  "force_image_size": 512,
+  "hidden_size": 4096,
+  "image_tag_type": "internvl",
+  "img_context_token": "<|image_pad|>",
+  "img_context_token_id": 248056,
+  "img_end_token": "<|vision_end|>",
+  "img_start_token": "<|vision_start|>",
+  "keys_to_ignore_at_inference": [
+    "past_key_values"
+  ],
+  "llm_config": {
+    "architectures": [
+      "Qwen3_5ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 12288,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "linear_value_head_dim": 128,
+    "max_position_embeddings": 262144,
+    "model_type": "qwen3_5_text",
+    "mtp_loss_scaling_factor": 0.1,
+    "mtp_num_layers": 1,
+    "num_attention_heads": 16,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 4,
+    "pad_token_id": 248044,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": false,
+    "use_cache": false,
+    "mamba_ssm_dtype": "float32",
+    "vocab_size": 248320
+  },
+  "max_dynamic_patch": 12,
+  "min_dynamic_patch": 1,
+  "model_type": "zdtaichu5_0",
+  "pad_token_id": 248044,
+  "patch_size": 16,
+  "projector_hidden_size": 20480,
+  "ps_version": "v2",
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "float-quantized",
+        "input_activations": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": true,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": null,
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "token",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        },
+        "output_activations": null,
+        "targets": [
+          "re:.*self_attn\\.(q|k|v|o)_proj$",
+          "re:.*linear_attn\\.(in_proj_qkv|in_proj_z|out_proj)$",
+          "re:.*lm_head$",
+          "re:.*layers\\.(?:28|29|30|31)\\.mlp\\.(?:gate|up|down)_proj$"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "channel",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        }
+      },
+      "group_1": {
+        "format": "nvfp4-pack-quantized",
+        "input_activations": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": "local",
+          "group_size": 16,
+          "num_bits": 4,
+          "observer": "static_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": "torch.float8_e4m3fn",
+          "strategy": "tensor_group",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        },
+        "output_activations": null,
+        "targets": [
+          "re:.*layers\\.(?:0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27)\\.mlp\\.(?:gate|up|down)_proj$"
+        ],
+        "weights": {
+          "actorder": "static",
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 16,
+          "num_bits": 4,
+          "observer": "imatrix_mse",
+          "observer_kwargs": {},
+          "scale_dtype": "torch.float8_e4m3fn",
+          "strategy": "tensor_group",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        }
+      }
+    },
+    "format": "mixed-precision",
+    "global_compression_ratio": null,
+    "ignore": [
+      "language_model.model.layers.0.linear_attn.in_proj_b",
+      "language_model.model.layers.0.linear_attn.in_proj_a",
+      "language_model.model.layers.1.linear_attn.in_proj_b",
+      "language_model.model.layers.1.linear_attn.in_proj_a",
+      "language_model.model.layers.2.linear_attn.in_proj_b",
+      "language_model.model.layers.2.linear_attn.in_proj_a",
+      "language_model.model.layers.4.linear_attn.in_proj_b",
+      "language_model.model.layers.4.linear_attn.in_proj_a",
+      "language_model.model.layers.5.linear_attn.in_proj_b",
+      "language_model.model.layers.5.linear_attn.in_proj_a",
+      "language_model.model.layers.6.linear_attn.in_proj_b",
+      "language_model.model.layers.6.linear_attn.in_proj_a",
+      "language_model.model.layers.8.linear_attn.in_proj_b",
+      "language_model.model.layers.8.linear_attn.in_proj_a",
+      "language_model.model.layers.9.linear_attn.in_proj_b",
+      "language_model.model.layers.9.linear_attn.in_proj_a",
+      "language_model.model.layers.10.linear_attn.in_proj_b",
+      "language_model.model.layers.10.linear_attn.in_proj_a",
+      "language_model.model.layers.12.linear_attn.in_proj_b",
+      "language_model.model.layers.12.linear_attn.in_proj_a",
+      "language_model.model.layers.13.linear_attn.in_proj_b",
+      "language_model.model.layers.13.linear_attn.in_proj_a",
+      "language_model.model.layers.14.linear_attn.in_proj_b",
+      "language_model.model.layers.14.linear_attn.in_proj_a",
+      "language_model.model.layers.16.linear_attn.in_proj_b",
+      "language_model.model.layers.16.linear_attn.in_proj_a",
+      "language_model.model.layers.17.linear_attn.in_proj_b",
+      "language_model.model.layers.17.linear_attn.in_proj_a",
+      "language_model.model.layers.18.linear_attn.in_proj_b",
+      "language_model.model.layers.18.linear_attn.in_proj_a",
+      "language_model.model.layers.20.linear_attn.in_proj_b",
+      "language_model.model.layers.20.linear_attn.in_proj_a",
+      "language_model.model.layers.21.linear_attn.in_proj_b",
+      "language_model.model.layers.21.linear_attn.in_proj_a",
+      "language_model.model.layers.22.linear_attn.in_proj_b",
+      "language_model.model.layers.22.linear_attn.in_proj_a",
+      "language_model.model.layers.24.linear_attn.in_proj_b",
+      "language_model.model.layers.24.linear_attn.in_proj_a",
+      "language_model.model.layers.25.linear_attn.in_proj_b",
+      "language_model.model.layers.25.linear_attn.in_proj_a",
+      "language_model.model.layers.26.linear_attn.in_proj_b",
+      "language_model.model.layers.26.linear_attn.in_proj_a",
+      "language_model.model.layers.28.linear_attn.in_proj_b",
+      "language_model.model.layers.28.linear_attn.in_proj_a",
+      "language_model.model.layers.29.linear_attn.in_proj_b",
+      "language_model.model.layers.29.linear_attn.in_proj_a",
+      "language_model.model.layers.30.linear_attn.in_proj_b",
+      "language_model.model.layers.30.linear_attn.in_proj_a",
+      "vision_model.radio_model.model.blocks.0.attn.qkv",
+      "vision_model.radio_model.model.blocks.0.attn.proj",
+      "vision_model.radio_model.model.blocks.0.mlp.fc1",
+      "vision_model.radio_model.model.blocks.0.mlp.fc2",
+      "vision_model.radio_model.model.blocks.1.attn.qkv",
+      "vision_model.radio_model.model.blocks.1.attn.proj",
+      "vision_model.radio_model.model.blocks.1.mlp.fc1",
+      "vision_model.radio_model.model.blocks.1.mlp.fc2",
+      "vision_model.radio_model.model.blocks.2.attn.qkv",
+      "vision_model.radio_model.model.blocks.2.attn.proj",
+      "vision_model.radio_model.model.blocks.2.mlp.fc1",
+      "vision_model.radio_model.model.blocks.2.mlp.fc2",
+      "vision_model.radio_model.model.blocks.3.attn.qkv",
+      "vision_model.radio_model.model.blocks.3.attn.proj",
+      "vision_model.radio_model.model.blocks.3.mlp.fc1",
+      "vision_model.radio_model.model.blocks.3.mlp.fc2",
+      "vision_model.radio_model.model.blocks.4.attn.qkv",
+      "vision_model.radio_model.model.blocks.4.attn.proj",
+      "vision_model.radio_model.model.blocks.4.mlp.fc1",
+      "vision_model.radio_model.model.blocks.4.mlp.fc2",
+      "vision_model.radio_model.model.blocks.5.attn.qkv",
+      "vision_model.radio_model.model.blocks.5.attn.proj",
+      "vision_model.radio_model.model.blocks.5.mlp.fc1",
+      "vision_model.radio_model.model.blocks.5.mlp.fc2",
+      "vision_model.radio_model.model.blocks.6.attn.qkv",
+      "vision_model.radio_model.model.blocks.6.attn.proj",
+      "vision_model.radio_model.model.blocks.6.mlp.fc1",
+      "vision_model.radio_model.model.blocks.6.mlp.fc2",
+      "vision_model.radio_model.model.blocks.7.attn.qkv",
+      "vision_model.radio_model.model.blocks.7.attn.proj",
+      "vision_model.radio_model.model.blocks.7.mlp.fc1",
+      "vision_model.radio_model.model.blocks.7.mlp.fc2",
+      "vision_model.radio_model.model.blocks.8.attn.qkv",
+      "vision_model.radio_model.model.blocks.8.attn.proj",
+      "vision_model.radio_model.model.blocks.8.mlp.fc1",
+      "vision_model.radio_model.model.blocks.8.mlp.fc2",
+      "vision_model.radio_model.model.blocks.9.attn.qkv",
+      "vision_model.radio_model.model.blocks.9.attn.proj",
+      "vision_model.radio_model.model.blocks.9.mlp.fc1",
+      "vision_model.radio_model.model.blocks.9.mlp.fc2",
+      "vision_model.radio_model.model.blocks.10.attn.qkv",
+      "vision_model.radio_model.model.blocks.10.attn.proj",
+      "vision_model.radio_model.model.blocks.10.mlp.fc1",
+      "vision_model.radio_model.model.blocks.10.mlp.fc2",
+      "vision_model.radio_model.model.blocks.11.attn.qkv",
+      "vision_model.radio_model.model.blocks.11.attn.proj",
+      "vision_model.radio_model.model.blocks.11.mlp.fc1",
+      "vision_model.radio_model.model.blocks.11.mlp.fc2",
+      "vision_model.radio_model.model.blocks.12.attn.qkv",
+      "vision_model.radio_model.model.blocks.12.attn.proj",
+      "vision_model.radio_model.model.blocks.12.mlp.fc1",
+      "vision_model.radio_model.model.blocks.12.mlp.fc2",
+      "vision_model.radio_model.model.blocks.13.attn.qkv",
+      "vision_model.radio_model.model.blocks.13.attn.proj",
+      "vision_model.radio_model.model.blocks.13.mlp.fc1",
+      "vision_model.radio_model.model.blocks.13.mlp.fc2",
+      "vision_model.radio_model.model.blocks.14.attn.qkv",
+      "vision_model.radio_model.model.blocks.14.attn.proj",
+      "vision_model.radio_model.model.blocks.14.mlp.fc1",
+      "vision_model.radio_model.model.blocks.14.mlp.fc2",
+      "vision_model.radio_model.model.blocks.15.attn.qkv",
+      "vision_model.radio_model.model.blocks.15.attn.proj",
+      "vision_model.radio_model.model.blocks.15.mlp.fc1",
+      "vision_model.radio_model.model.blocks.15.mlp.fc2",
+      "vision_model.radio_model.model.blocks.16.attn.qkv",
+      "vision_model.radio_model.model.blocks.16.attn.proj",
+      "vision_model.radio_model.model.blocks.16.mlp.fc1",
+      "vision_model.radio_model.model.blocks.16.mlp.fc2",
+      "vision_model.radio_model.model.blocks.17.attn.qkv",
+      "vision_model.radio_model.model.blocks.17.attn.proj",
+      "vision_model.radio_model.model.blocks.17.mlp.fc1",
+      "vision_model.radio_model.model.blocks.17.mlp.fc2",
+      "vision_model.radio_model.model.blocks.18.attn.qkv",
+      "vision_model.radio_model.model.blocks.18.attn.proj",
+      "vision_model.radio_model.model.blocks.18.mlp.fc1",
+      "vision_model.radio_model.model.blocks.18.mlp.fc2",
+      "vision_model.radio_model.model.blocks.19.attn.qkv",
+      "vision_model.radio_model.model.blocks.19.attn.proj",
+      "vision_model.radio_model.model.blocks.19.mlp.fc1",
+      "vision_model.radio_model.model.blocks.19.mlp.fc2",
+      "vision_model.radio_model.model.blocks.20.attn.qkv",
+      "vision_model.radio_model.model.blocks.20.attn.proj",
+      "vision_model.radio_model.model.blocks.20.mlp.fc1",
+      "vision_model.radio_model.model.blocks.20.mlp.fc2",
+      "vision_model.radio_model.model.blocks.21.attn.qkv",
+      "vision_model.radio_model.model.blocks.21.attn.proj",
+      "vision_model.radio_model.model.blocks.21.mlp.fc1",
+      "vision_model.radio_model.model.blocks.21.mlp.fc2",
+      "vision_model.radio_model.model.blocks.22.attn.qkv",
+      "vision_model.radio_model.model.blocks.22.attn.proj",
+      "vision_model.radio_model.model.blocks.22.mlp.fc1",
+      "vision_model.radio_model.model.blocks.22.mlp.fc2",
+      "vision_model.radio_model.model.blocks.23.attn.qkv",
+      "vision_model.radio_model.model.blocks.23.attn.proj",
+      "vision_model.radio_model.model.blocks.23.mlp.fc1",
+      "vision_model.radio_model.model.blocks.23.mlp.fc2",
+      "vision_model.radio_model.model.blocks.24.attn.qkv",
+      "vision_model.radio_model.model.blocks.24.attn.proj",
+      "vision_model.radio_model.model.blocks.24.mlp.fc1",
+      "vision_model.radio_model.model.blocks.24.mlp.fc2",
+      "vision_model.radio_model.model.blocks.25.attn.qkv",
+      "vision_model.radio_model.model.blocks.25.attn.proj",
+      "vision_model.radio_model.model.blocks.25.mlp.fc1",
+      "vision_model.radio_model.model.blocks.25.mlp.fc2",
+      "vision_model.radio_model.model.blocks.26.attn.qkv",
+      "vision_model.radio_model.model.blocks.26.attn.proj",
+      "vision_model.radio_model.model.blocks.26.mlp.fc1",
+      "vision_model.radio_model.model.blocks.26.mlp.fc2",
+      "vision_model.radio_model.model.blocks.27.attn.qkv",
+      "vision_model.radio_model.model.blocks.27.attn.proj",
+      "vision_model.radio_model.model.blocks.27.mlp.fc1",
+      "vision_model.radio_model.model.blocks.27.mlp.fc2",
+      "vision_model.radio_model.model.blocks.28.attn.qkv",
+      "vision_model.radio_model.model.blocks.28.attn.proj",
+      "vision_model.radio_model.model.blocks.28.mlp.fc1",
+      "vision_model.radio_model.model.blocks.28.mlp.fc2",
+      "vision_model.radio_model.model.blocks.29.attn.qkv",
+      "vision_model.radio_model.model.blocks.29.attn.proj",
+      "vision_model.radio_model.model.blocks.29.mlp.fc1",
+      "vision_model.radio_model.model.blocks.29.mlp.fc2",
+      "vision_model.radio_model.model.blocks.30.attn.qkv",
+      "vision_model.radio_model.model.blocks.30.attn.proj",
+      "vision_model.radio_model.model.blocks.30.mlp.fc1",
+      "vision_model.radio_model.model.blocks.30.mlp.fc2",
+      "vision_model.radio_model.model.blocks.31.attn.qkv",
+      "vision_model.radio_model.model.blocks.31.attn.proj",
+      "vision_model.radio_model.model.blocks.31.mlp.fc1",
+      "vision_model.radio_model.model.blocks.31.mlp.fc2",
+      "mlp1.1",
+      "mlp1.3"
+    ],
+    "kv_cache_scheme": {
+      "actorder": null,
+      "block_structure": null,
+      "dynamic": false,
+      "group_size": null,
+      "num_bits": 8,
+      "observer": "static_minmax",
+      "observer_kwargs": {},
+      "scale_dtype": null,
+      "strategy": "tensor",
+      "symmetric": true,
+      "type": "float",
+      "zp_dtype": null
+    },
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.16.0"
+  },
+  "template": "qwen3_5",
+  "tie_word_embeddings": false,
+  "transformers_version": "5.3.0",
+  "use_thumbnail": true,
+  "video_context_token": "<|video_pad|>",
+  "video_context_token_id": 248057,
+  "vision_config": {
+    "adaptor_configs": {},
+    "adaptor_names": null,
+    "architectures": [
+      "RADIOModel"
+    ],
+    "args": {
+      "aa": null,
+      "amp": true,
+      "amp_dtype": "bfloat16",
+      "amp_impl": "native",
+      "aug_repeats": 0,
+      "aug_splits": 0,
+      "auto_workload_inspector": false,
+      "bn_eps": null,
+      "bn_momentum": null,
+      "cache_dir": null,
+      "channels_last": false,
+      "checkpoint_folder": null,
+      "checkpoint_hist": 10,
+      "chk_keep_forever": 100,
+      "class_map": "",
+      "clip_grad": null,
+      "clip_mode": "norm",
+      "cls_token_per_teacher": true,
+      "coco_annotations_file": null,
+      "coco_image_dir": null,
+      "color_jitter": 0.4,
+      "cooldown_epochs": 0,
+      "cpe_max_size": 2048,
+      "cpe_num_registers": null,
+      "crd_loss": false,
+      "crd_loss_weight": 0.8,
+      "crop_pct": null,
+      "cutmix": 0.0,
+      "cutmix_minmax": null,
+      "dataset_download": false,
+      "debug_full_knn": false,
+      "decay_epochs": 90,
+      "decay_milestones": [
+        90,
+        180,
+        270
+      ],
+      "decay_rate": 0.1,
+      "depchain": true,
+      "detect_anomaly": false,
+      "dist_bn": "reduce",
+      "dist_norm_weight": 0.0,
+      "distributed": true,
+      "drop": 0.0,
+      "drop_block": null,
+      "drop_connect": null,
+      "drop_path": null,
+      "dtype": "float32",
+      "epoch": 299,
+      "epoch_repeats": 0.0,
+      "eval": false,
+      "eval_metric": "knn_top1",
+      "eval_teacher": false,
+      "eval_teacher_only": false,
+      "eval_throughput": false,
+      "fast_norm": false,
+      "fd_loss_fn": "MSE",
+      "feature_normalization": "PHI_STANDARDIZE",
+      "feature_summarizer": "cls_token",
+      "feature_upscale_factor": null,
+      "force_disable_damp": false,
+      "force_disable_spectral_reparam": false,
+      "force_new_wandb_id": false,
+      "force_spectral_reparam": false,
+      "freeze_bn": false,
+      "fsdp": true,
+      "full_equivariance": false,
+      "fuser": "",
+      "gp": null,
+      "grad_accum_steps": 1,
+      "grad_checkpointing": false,
+      "head_init_bias": null,
+      "head_init_scale": null,
+      "head_lr": null,
+      "head_warmup": 3,
+      "head_weight_decay": 0.0005,
+      "hflip": 0.5,
+      "img_size": null,
+      "in_chans": null,
+      "initial_checkpoint": null,
+      "input_size": null,
+      "interpolation": "",
+      "layer_decay": null,
+      "local_rank": 0,
+      "log_interval": 50,
+      "log_mlflow": false,
+      "log_teacher_timings": true,
+      "log_train_metrics_per_epoch": true,
+      "log_train_metrics_per_log_interval": true,
+      "log_wandb": true,
+      "loss_auto_balance": false,
+      "lr_base": 0.1,
+      "lr_base_scale": "",
+      "lr_base_size": 256,
+      "lr_cycle_decay": 0.5,
+      "lr_cycle_limit": 1,
+      "lr_cycle_mul": 1.0,
+      "lr_k_decay": 1.0,
+      "lr_noise": null,
+      "lr_noise_pct": 0.67,
+      "lr_noise_std": 1.0,
+      "mean": null,
+      "mesa": false,
+      "min_lr": 1e-05,
+      "mixup": 0.0,
+      "mixup_mode": "batch",
+      "mixup_off_epoch": 0,
+      "mixup_prob": 1.0,
+      "mixup_switch_prob": 0.5,
+      "mlp_hidden_size": 1520,
+      "mlp_num_inner": 2,
+      "mlp_version": "v2",
+      "model": "vit_huge_patch16_224",
+      "model_kwargs": {},
+      "model_norm": false,
+      "momentum": 0.9,
+      "no_custom_validation": false,
+      "no_ddp_bb": true,
+      "no_knn": false,
+      "no_prefetcher": false,
+      "no_resume_opt": false,
+      "no_save_checkpoint": false,
+      "no_val": false,
+      "num_classes": null,
+      "on_demand_workload_inspector": false,
+      "one_logger_app_tag": "",
+      "one_logger_is_baseline": false,
+      "one_logger_run_name": "",
+      "onelogger": null,
+      "opt_betas": null,
+      "opt_eps": null,
+      "overfit": false,
+      "patience_epochs": 10,
+      "perf_test_no_aug": false,
+      "perf_test_no_decode": false,
+      "perf_test_no_io": false,
+      "perf_test_only_dataloader": false,
+      "perf_test_simple_aug": false,
+      "pin_mem": false,
+      "prefetcher": true,
+      "pretrained": false,
+      "processed_neck_outputs": null,
+      "profile_train_exit_after_profiling": false,
+      "profile_train_export_chrome_trace": true,
+      "profile_train_export_csv": false,
+      "profile_train_iterations": 0,
+      "qradio": false,
+      "qradio_max_tokens": 512,
+      "qradio_min_tokens": 32,
+      "qradio_patch_token_mask_initial_ratio": 0.95,
+      "qradio_progressive_2d": false,
+      "qradio_quantizer": null,
+      "qradio_ramp_alpha": 1.5,
+      "rank": 0,
+      "ratio": [
+        0.75,
+        1.3333333333333333
+      ],
+      "recount": 1,
+      "recovery_interval": 0,
+      "register_multiple": 10,
+      "remode": "pixel",
+      "reprob": 0.0,
+      "reset_loss_state": true,
+      "resplit": false,
+      "sample_tracking": false,
+      "save_images": false,
+      "scale": [
+        0.5,
+        1.0
+      ],
+      "sched": "cosine",
+      "seed": 42,
+      "shift_equivariance": false,
+      "smoothing": 0.1,
+      "source_tracking": false,
+      "spectral_heads": false,
+      "spectral_reparam": false,
+      "spectral_weight_decay": null,
+      "split_bn": false,
+      "start_epoch": null,
+      "std": null,
+      "stream_teachers": false,
+      "student_intermediate_indices": null,
+      "student_load_skip_state_dict_keys_regex": null,
+      "student_reinit_model_layers_regex": null,
+      "student_strict_load_ignore_mismatched_shape_keys_regex": null,
+      "student_strict_load_ignore_missing_keys_regex": null,
+      "student_strict_load_ignore_unexpected_keys_regex": null,
+      "student_strict_load_state_dict": false,
+      "sync_bn": false,
+      "sync_resolutions_across_ranks": true,
+      "synchronize_step": false,
+      "teachers": [
+        {
+          "model": "siglip2-g-384",
+          "name": "siglip2-g",
+          "spatial_mlp_version": "attn",
+          "type": "siglip2",
+          "use_summary": true
+        },
+        {
+          "model": "dinov3_vit7b16",
+          "name": "dino_v3_7b",
+          "type": "dino_v3",
+          "use_summary": true
+        },
+        {
+          "model": "default",
+          "name": "sam3",
+          "type": "sam3",
+          "use_summary": false
+        }
+      ],
+      "timing_warmup_iters": 20,
+      "tokenizer_kwargs": {},
+      "tokenizer_type": null,
+      "tome": null,
+      "torchcompile": null,
+      "torchscript": false,
+      "train_interpolation": "random",
+      "train_split": "train",
+      "tta": 0,
+      "untie_neck_weights": false,
+      "use_coco": false,
+      "use_multi_epochs_loader": false,
+      "val_ema_only": false,
+      "val_split": "val",
+      "vflip": 0.0,
+      "vitdet_version": 1,
+      "wandb_entity": "",
+      "wandb_id": "",
+      "wandb_job_type": "",
+      "wandb_name": "",
+      "wandb_project": "",
+      "wandb_tags": null,
+      "warmup_lr": 1e-05,
+      "warmup_prefix": false,
+      "worker_seeding": "all",
+      "workers": 8,
+      "workload_inspector_analyze_nsys_traces": false,
+      "workload_inspector_baseline_start_iter": 1500,
+      "workload_inspector_major_slowdown_p95_factor": 10.0,
+      "workload_inspector_minor_slowdown_p95_factor": 3.0,
+      "workload_inspector_no_slowdown_check": false,
+      "workload_inspector_simulate_slowdown_num_times": 1,
+      "workload_inspector_simulate_slowdown_start_iter": null,
+      "world_size": 256
+    },
+    "auto_map": {
+      "AutoConfig": "cradio_config.RADIOConfig",
+      "AutoModel": "cradio_model.RADIOModel"
+    },
+    "dtype": "bfloat16",
+    "feature_normalizer_config": null,
+    "inter_feature_normalizer_config": null,
+    "max_resolution": 2048,
+    "model_type": "radio",
+    "patch_size": 16,
+    "preferred_resolution": [
+      512,
+      512
+    ],
+    "use_flash_attn": false,
+    "version": "c-radio_v4-h",
+    "vitdet_window_size": null
+  },
+  "vit_hidden_size": 1280
+}

configuration.py

@@ -0,0 +1,231 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ============================================================================
+#  ZDTaichu-5.0 — Top-Level Configuration
+#
+#  Architecture:
+#    - LLM backbone: Qwen3 (pure Transformer) → Qwen3.5 (hybrid DeltaNet/Transformer)
+#      · 3:1 linear-to-full attention ratio (Gated DeltaNet + full attention)
+#      · Custom Qwen3_5DynamicCache for hybrid KV / recurrent states
+#      · head_dim=256 (was 128), partial_rotary_factor=0.25
+#      · Interleaved M-RoPE with 4D position IDs
+#      · Attention output gating (sigmoid gate on q_proj)
+#    - Vision encoder: C-RADIOv4-H (unchanged)
+#    - Token IDs updated for Qwen3.5 vocabulary (vocab_size=248320)
+#      · img_context_token_id: 151655 → 248056  (<|image_pad|>)
+#      · video_context_token_id: 151656 → 248057 (<|video_pad|>)
+#    - Projector output adapts to Qwen3.5 hidden_size (4096 for 9B variant)
+# ============================================================================
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+from .cradio_config import RADIOConfig
+
+logger = logging.get_logger(__name__)
+
+# ---------------------------------------------------------------------------
+# Import Qwen3.5 text config — requires transformers >= 5.3.0
+# ---------------------------------------------------------------------------
+try:
+    from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig
+except ImportError:
+    Qwen3_5TextConfig = None
+    logger.warning(
+        "Could not import Qwen3_5TextConfig from transformers. "
+        "Ensure transformers >= 5.3.0 is installed. "
+        "Falling back to PretrainedConfig with manual attributes."
+    )
+
+
+# ---------------------------------------------------------------------------
+# Default Qwen3.5 text configuration (9B-class variant)
+# ---------------------------------------------------------------------------
+
+_LAYER_TYPES_32 = [
+    "linear_attention" if bool((i + 1) % 4) else "full_attention"
+    for i in range(32)
+]
+# Result: [lin, lin, lin, full, lin, lin, lin, full, ... lin, lin, lin, full]
+# 24 linear + 8 full attention layers
+
+
+def _default_qwen3_5_text_dict() -> dict:
+    """Return a dict of Qwen3.5 text config values (9B-class)."""
+    return dict(
+        vocab_size=248320,
+        hidden_size=4096,
+        intermediate_size=12288,
+        num_hidden_layers=32,
+        num_attention_heads=16,
+        num_key_value_heads=4,
+        head_dim=256,
+        hidden_act="silu",
+        max_position_embeddings=262144,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        tie_word_embeddings=False,
+        attention_bias=False,
+        attention_dropout=0.0,
+        torch_dtype="bfloat16",
+        # --- Hybrid layer architecture ---
+        layer_types=list(_LAYER_TYPES_32),  # copy to avoid mutation
+        full_attention_interval=4,
+        # --- Linear attention (Gated DeltaNet) ---
+        linear_conv_kernel_dim=4,
+        linear_key_head_dim=128,
+        linear_value_head_dim=128,
+        linear_num_key_heads=16,
+        linear_num_value_heads=32,
+        # --- RoPE ---
+        rope_parameters={
+            "rope_type": "default",
+            "rope_theta": 10000000,
+            "partial_rotary_factor": 0.25,
+            "mrope_interleaved": True,
+            "mrope_section": [11, 11, 10],
+        },
+    )
+
+
+def _build_llm_config(cfg_dict: dict = None) -> PretrainedConfig:
+    """
+    Construct the LLM sub-config from a dict or defaults.
+
+    Uses Qwen3_5TextConfig when available (transformers >= 5.3);
+    otherwise falls back to a plain PretrainedConfig with the correct
+    model_type so that AutoModelForCausalLM can still resolve it.
+    """
+    if cfg_dict is None:
+        cfg_dict = _default_qwen3_5_text_dict()
+
+    if Qwen3_5TextConfig is not None:
+        return Qwen3_5TextConfig(**cfg_dict)
+    else:
+        config = PretrainedConfig(**cfg_dict)
+        config.model_type = "qwen3_5_text"
+        return config
+
+
+class ZDTaichu5_0_Config(PretrainedConfig):
+    """
+    Configuration for ZDTaichu-5.0-9B:
+        Vision encoder : C-RADIOv4-H  (ViT-H/16, 653 M params)
+        LLM decoder    : Qwen3.5      (hybrid DeltaNet/Transformer)
+        Projector      : RMSNorm → Linear(5120→20480) → SquaredReLU → Linear(20480→H)
+
+    The projector input side is unchanged (C-RADIOv4-H ViT-H features at 1280,
+    pixel-shuffled to 5120). Only the final projection layer adapts to the
+    target LLM hidden_size (4096 for the 9B variant, vs 5120 for Qwen3-14B).
+
+    Qwen3.5 hybrid architecture
+    ----------------------------
+    The text backbone alternates Gated DeltaNet (linear attention) and standard
+    multi-head attention layers in a 3:1 ratio. Linear layers use a causal 1D
+    convolution + gated delta rule recurrence for O(1) per-token memory during
+    generation, while every 4th layer uses full quadratic attention to preserve
+    global context. A custom DynamicCache handles both attention KV states and
+    recurrent states.
+    """
+
+    model_type = "zdtaichu5_0"
+    is_composition = True
+
+    def __init__(
+        self,
+        vision_config=None,
+        llm_config=None,
+        force_image_size=None,
+        downsample_ratio=0.5,
+        template=None,
+        ps_version="v2",
+        image_tag_type="internvl",
+        projector_hidden_size=20480,    # 4 × pixel_shuffle_dim (5120)
+        vit_hidden_size=1280,           # ViT-H feature dim — same for C-RADIOv4-H
+        attn_implementation="flash_attention_2",
+        # Special token IDs for Qwen3.5 vocabulary (vocab_size=248320)
+        img_context_token_id: int = 248056,   # <|image_pad|>
+        video_context_token_id: int = 248057,  # <|video_pad|>
+        **kwargs,
+    ):
+
+        # ------------------------------------------------------------------
+        # Transformers 5.5.x compatibility:
+        # PretrainedConfig.__init__ may call self.get_text_config()
+        # during token-id validation. Therefore llm_config must exist
+        # before calling super().__init__().
+        # ------------------------------------------------------------------
+
+        # ── Vision encoder ───────────────────────────────────────────────────
+        if vision_config is not None:
+            if isinstance(vision_config, dict):
+                self.vision_config = RADIOConfig(**vision_config)
+            else:
+                self.vision_config = vision_config
+        else:
+            self.vision_config = RADIOConfig(version="c-radio_v4-h")
+
+        # ── Language model (Qwen3.5 hybrid) ──────────────────────────────────
+        if llm_config is not None:
+            if isinstance(llm_config, PretrainedConfig):
+                self.llm_config = llm_config
+            elif isinstance(llm_config, dict):
+                self.llm_config = _build_llm_config(llm_config)
+            else:
+                raise TypeError(
+                    f"llm_config must be a dict or PretrainedConfig, got {type(llm_config)}"
+                )
+        else:
+            self.llm_config = _build_llm_config(None)
+
+
+        # Make tokenizer/generation token ids visible early.
+        # Transformers 5.5.x may validate these during super().__init__().
+        kwargs.setdefault("bos_token_id", getattr(self.llm_config, "bos_token_id", 248040))
+        kwargs.setdefault("eos_token_id", getattr(self.llm_config, "eos_token_id", 248044))
+        kwargs.setdefault("pad_token_id", getattr(self.llm_config, "pad_token_id", 248040))
+        super().__init__(**kwargs)
+
+        self.tie_word_embeddings = getattr(self.llm_config, "tie_word_embeddings", False)
+
+        # ── VL configuration ─────────────────────────────────────────────────
+        self.force_image_size = force_image_size
+        self.downsample_ratio = downsample_ratio
+        self.template = template
+        self.ps_version = ps_version
+        self.image_tag_type = image_tag_type
+        self.projector_hidden_size = projector_hidden_size
+        self.vit_hidden_size = vit_hidden_size
+
+        # Special token IDs
+        self.img_context_token_id = img_context_token_id
+        self.video_context_token_id = video_context_token_id
+
+        # Attention implementation propagation
+        self._attn_implementation = attn_implementation
+        self.vision_config.use_flash_attn = (
+            self._attn_implementation is not None
+            and "flash_attention" in self._attn_implementation
+        )
+        self.llm_config._attn_implementation = self._attn_implementation
+
+    def get_text_config(self, decoder=False):
+        # Robust fallback for Transformers 5.5.x validation.
+        if hasattr(self, "llm_config"):
+            return self.llm_config
+        return _build_llm_config(None)
+
+    @property
+    def text_config(self):
+        return self.get_text_config(decoder=True)

cradio_config.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2026, ZDTaichu-5.0-9B Contributors.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Standalone inference configuration for the C-RADIO vision tower."""
+
+from typing import Dict, List, Optional, Tuple, Union
+from transformers import PretrainedConfig
+
+
+class RADIOConfig(PretrainedConfig):
+    model_type = "radio"
+
+    def __init__(
+        self,
+        args: Optional[dict] = None,
+        version: str = "c-radio_v4-h",
+        patch_size: int = 16,
+        max_resolution: int = 2048,
+        preferred_resolution: Tuple[int, int] = (768, 768),
+        adaptor_names: Union[str, List[str], None] = None,
+        adaptor_configs: Optional[Dict] = None,
+        vitdet_window_size: Optional[int] = None,
+        feature_normalizer_config: Optional[dict] = None,
+        inter_feature_normalizer_config: Optional[dict] = None,
+        **kwargs,
+    ):
+        self.args = args or {}
+        self.version = version
+        self.patch_size = patch_size
+        self.max_resolution = max_resolution
+        self.preferred_resolution = preferred_resolution
+        self.adaptor_names = adaptor_names
+        self.adaptor_configs = adaptor_configs
+        self.vitdet_window_size = vitdet_window_size
+        self.feature_normalizer_config = feature_normalizer_config
+        self.inter_feature_normalizer_config = inter_feature_normalizer_config
+        super().__init__(**kwargs)
+
+
+__all__ = ["RADIOConfig"]

cradio_model.py

@@ -0,0 +1,699 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2026, ZDTaichu-5.0-9B Contributors.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Standalone inference-only C-RADIO ViT vision tower.
+
+This file intentionally contains the small subset of C-RADIO needed by the
+ZDTaichu-5.0-9B checkpoint. It does not depend on the cradio_v4 package.
+"""
+
+from __future__ import annotations
+
+import math
+from contextlib import contextmanager
+from types import MethodType
+from typing import Callable, Iterable, List, NamedTuple, Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+from transformers import PreTrainedModel
+
+try:
+    from timm.models import VisionTransformer, checkpoint_seq
+except ImportError as exc:  # pragma: no cover - import-time dependency guard
+    raise ImportError("cradio_model.py requires timm to build the C-RADIO ViT tower") from exc
+
+from .cradio_config import RADIOConfig
+
+class Resolution(NamedTuple):
+    height: int
+    width: int
+
+
+class RadioOutput(NamedTuple):
+    summary: Optional[torch.Tensor]
+    features: Optional[torch.Tensor]
+
+    def to(self, *args, **kwargs) -> "RadioOutput":
+        return RadioOutput(
+            self.summary.to(*args, **kwargs) if self.summary is not None else None,
+            self.features.to(*args, **kwargs) if self.features is not None else None,
+        )
+
+
+class InputConditioner(nn.Module):
+    def __init__(
+        self,
+        input_scale: float,
+        norm_mean: Union[Tuple[float, float, float], torch.Tensor],
+        norm_std: Union[Tuple[float, float, float], torch.Tensor],
+        dtype: Optional[torch.dtype] = None,
+    ) -> None:
+        super().__init__()
+        self.dtype = dtype
+        self.register_buffer("norm_mean", torch.as_tensor(norm_mean, dtype=torch.float32).view(-1, 1, 1) / input_scale)
+        self.register_buffer("norm_std", torch.as_tensor(norm_std, dtype=torch.float32).view(-1, 1, 1) / input_scale)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        y = (x - self.norm_mean) / self.norm_std
+        if self.dtype is not None:
+            y = y.to(self.dtype)
+        return y
+
+
+def get_default_conditioner() -> InputConditioner:
+    from timm.data.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
+
+    return InputConditioner(1.0, OPENAI_CLIP_MEAN, OPENAI_CLIP_STD)
+
+
+class ClsToken(nn.Module):
+    def __init__(
+        self,
+        ndim: int,
+        num_tokens: int = 1,
+        enabled: bool = True,
+        register_multiple: Optional[int] = None,
+        num_registers: Optional[int] = None,
+    ) -> None:
+        super().__init__()
+        self.ndim = ndim
+        self.enabled = enabled
+        self.num_registers = 0
+        self.num_tokens = num_tokens
+        if enabled:
+            if num_registers:
+                self.num_registers = num_registers
+            elif register_multiple:
+                self.num_registers = register_multiple - (num_tokens % register_multiple)
+            scale = ndim ** -0.5
+            self.token = nn.Parameter(torch.randn(num_tokens + self.num_registers, ndim) * scale)
+        else:
+            self.token = None
+        self.num_patches = self.num_tokens + self.num_registers
+
+    def disable(self) -> None:
+        self.token = None
+        self.enabled = False
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if self.token is None:
+            return x
+        token = self.token.unsqueeze(0).expand(x.shape[0], -1, -1)
+        return torch.cat([token, x], dim=1)
+
+    def no_weight_decay(self) -> List[str]:
+        return ["token"]
+
+
+class Im2Patches(nn.Module):
+    def __init__(self, patch_size: int) -> None:
+        super().__init__()
+        self.patch_size = patch_size
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if self.patch_size == 1:
+            return x.flatten(2).transpose(1, 2)
+        return F.unfold(x, kernel_size=self.patch_size, stride=self.patch_size).transpose(1, 2)
+
+
+class ViTPatchLinear(nn.Linear):
+    def __init__(self, patch_size: int, embed_dim: int, bias: bool = False, **factory) -> None:
+        super().__init__(3 * (patch_size ** 2), embed_dim, bias=bias, **factory)
+        self.patch_size = patch_size
+
+
+class ViTPatchGenerator(nn.Module):
+    def __init__(
+        self,
+        patch_size: int,
+        embed_dim: int,
+        input_dims: Union[int, Tuple[int, int]],
+        abs_pos: bool = True,
+        normalize_patches: bool = False,
+        cls_token: bool = False,
+        max_input_dims: Optional[Union[int, Tuple[int, int]]] = None,
+        pos_dropout: float = 0.0,
+        return_pos_enc: bool = False,
+        num_cls_tokens: int = 1,
+        register_multiple: Optional[int] = None,
+        num_registers: Optional[int] = None,
+        patch_bias: bool = False,
+        device=None,
+        dtype=None,
+    ) -> None:
+        super().__init__()
+        if isinstance(input_dims, int):
+            input_dims = (input_dims, input_dims)
+        if max_input_dims is None:
+            max_input_dims = input_dims
+        if isinstance(max_input_dims, int):
+            max_input_dims = (max_input_dims, max_input_dims)
+
+        max_input_dims = tuple(int(math.ceil(d / patch_size) * patch_size) for d in max_input_dims)
+        factory = dict(device=device, dtype=dtype)
+
+        self.cpe_mode = max_input_dims != input_dims
+        self.pos_dropout = pos_dropout
+        self.return_pos_enc = return_pos_enc
+        self.patch_size = patch_size
+        self.abs_pos = abs_pos
+        self.embed_dim = embed_dim
+        self.num_rows = max_input_dims[0] // patch_size
+        self.num_cols = max_input_dims[1] // patch_size
+        self.input_dims = tuple(d // patch_size for d in input_dims)
+        self.num_patches = self.num_rows * self.num_cols
+        self.max_input_dims = max_input_dims
+        self.im_to_patches = Im2Patches(patch_size)
+        self.embedder = ViTPatchLinear(patch_size, embed_dim, bias=patch_bias, **factory)
+        if abs_pos:
+            scale = embed_dim ** -0.5
+            self.pos_embed = nn.Parameter(torch.randn(1, self.num_patches, embed_dim, **factory) * scale)
+        self.cls_token = ClsToken(
+            embed_dim,
+            num_tokens=num_cls_tokens,
+            enabled=cls_token,
+            register_multiple=register_multiple,
+            num_registers=num_registers,
+        )
+        self.patch_normalizer = nn.LayerNorm(embed_dim) if normalize_patches else nn.Identity()
+        self.num_video_frames = None
+
+    @property
+    def apply_cls_token(self) -> bool:
+        return self.cls_token.enabled
+
+    @property
+    def num_cls_tokens(self) -> int:
+        return self.cls_token.num_tokens
+
+    @property
+    def num_cls_patches(self) -> int:
+        return self.cls_token.num_patches
+
+    @property
+    def num_registers(self) -> int:
+        return self.cls_token.num_registers
+
+    @property
+    def num_skip(self) -> int:
+        return self.num_cls_tokens + self.num_registers
+
+    def no_weight_decay(self) -> List[str]:
+        return ["pos_embed"]
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        patches = self.embedder(self.im_to_patches(x))
+        patches, pos_enc = self.apply_pos_enc(patches, input_size=x.shape[2:])
+        patches = self.cls_token(patches)
+        patches = self.patch_normalizer(patches)
+        if self.return_pos_enc:
+            return patches, pos_enc
+        return patches
+
+    def apply_pos_enc(
+        self,
+        patches: torch.Tensor,
+        patch_idxs: Optional[torch.Tensor] = None,
+        input_size: Optional[Tuple[int, int]] = None,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        if not self.abs_pos:
+            return patches, torch.empty(0, device=patches.device, dtype=patches.dtype)
+        pos_enc = self.get_pos_enc(patches.shape[0], patch_idxs, input_size)
+        if self.training and self.pos_dropout > 0:
+            keeps = torch.rand(patches.shape[0], 1, 1, dtype=pos_enc.dtype, device=pos_enc.device) > self.pos_dropout
+            pos_enc_drop = torch.where(keeps, pos_enc, 0)
+        else:
+            pos_enc_drop = pos_enc
+        return patches + pos_enc_drop, pos_enc
+
+    def get_pos_enc(
+        self,
+        batch_size: int,
+        patch_idxs: Optional[torch.Tensor] = None,
+        input_size: Optional[Tuple[int, int]] = None,
+    ) -> torch.Tensor:
+        input_dims = self.input_dims if input_size is None else tuple(d // self.patch_size for d in input_size)
+        pos_embed = self._get_pos_embeddings(batch_size, input_dims)
+        if patch_idxs is None:
+            return pos_embed
+        exp_patch_idxs = patch_idxs.unsqueeze(-1).expand(-1, -1, pos_embed.shape[-1])
+        return torch.gather(pos_embed.expand(patch_idxs.shape[0], -1, -1), dim=1, index=exp_patch_idxs)
+
+    def _get_pos_embeddings(self, batch_size: int, input_dims: Tuple[int, int]) -> torch.Tensor:
+        if (self.num_rows, self.num_cols) == input_dims:
+            return self.pos_embed
+
+        pos_embed = self.pos_embed.reshape(1, self.num_rows, self.num_cols, -1).permute(0, 3, 1, 2)
+
+        def window_select(pe: torch.Tensor) -> torch.Tensor:
+            if input_dims[0] < pe.shape[-2]:
+                pe = pe[..., :input_dims[0], :]
+            if input_dims[1] < pe.shape[-1]:
+                pe = pe[..., :, :input_dims[1]]
+            return pe
+
+        if self.cpe_mode:
+            if self.training:
+                if self.num_video_frames is not None:
+                    if batch_size % self.num_video_frames != 0:
+                        raise ValueError(
+                            f"Batch size {batch_size} must be divisible by num_video_frames "
+                            f"{self.num_video_frames} for CPE mode."
+                        )
+                    batch_size //= self.num_video_frames
+
+                min_scale = math.sqrt(0.1)
+                scale = torch.rand(batch_size, 1, 1, device=pos_embed.device) * (1 - min_scale) + min_scale
+                aspect_min = math.log(3 / 4)
+                aspect = torch.exp(torch.rand(batch_size, 1, 1, device=pos_embed.device) * (-2 * aspect_min) + aspect_min)
+                scale_xy = torch.stack([scale * aspect, scale / aspect], dim=-1).clamp_(0, 1)
+                pos_xy = torch.rand(batch_size, 1, 1, 2, device=pos_embed.device) * (1 - scale_xy)
+                lin_x = torch.linspace(0, 1, steps=input_dims[1], device=pos_embed.device)[None, None].expand(batch_size, input_dims[0], -1)
+                lin_y = torch.linspace(0, 1, steps=input_dims[0], device=pos_embed.device)[None, :, None].expand(batch_size, -1, input_dims[1])
+                grid_xy = torch.stack([lin_x, lin_y], dim=-1) * scale_xy + pos_xy
+                grid_xy.mul_(2).sub_(1)
+                pos_embed = F.grid_sample(
+                    pos_embed.float().expand(batch_size, -1, -1, -1),
+                    grid=grid_xy,
+                    mode="bilinear",
+                    padding_mode="zeros",
+                    align_corners=True,
+                ).to(pos_embed.dtype)
+                if self.num_video_frames is not None:
+                    pos_embed = torch.repeat_interleave(pos_embed, self.num_video_frames, dim=0)
+            else:
+                max_dim = max(input_dims)
+                pos_embed = F.interpolate(pos_embed.float(), size=(max_dim, max_dim), align_corners=False, mode="bilinear").to(pos_embed.dtype)
+                pos_embed = window_select(pos_embed)
+        else:
+            pos_embed = window_select(pos_embed)
+
+        if pos_embed.shape[-2:] != input_dims:
+            pos_embed = F.interpolate(pos_embed.float(), size=input_dims, align_corners=False, mode="bilinear").to(pos_embed.dtype)
+        return pos_embed.flatten(2).permute(0, 2, 1)
+
+
+def _forward_cpe(self: VisionTransformer, x: torch.Tensor) -> torch.Tensor:
+    x = self.patch_generator(x)
+    if getattr(self, "grad_checkpointing", False) and not torch.jit.is_scripting():
+        x = checkpoint_seq(self.blocks, x)
+    else:
+        x = self.blocks(x)
+    x = self.norm(x)
+    return x
+
+
+@contextmanager
+def _video_mode(self: VisionTransformer, t: int):
+    original_num_frames = self.patch_generator.num_video_frames
+    self.patch_generator.num_video_frames = t
+    try:
+        yield
+    finally:
+        self.patch_generator.num_video_frames = original_num_frames
+
+
+def enable_cpe(
+    model: VisionTransformer,
+    max_img_size: Union[int, Tuple[int, int]] = 1024,
+    num_cls_tokens: int = 1,
+    pos_dropout: float = 0.1,
+    register_multiple: Optional[int] = None,
+    num_registers: Optional[int] = None,
+) -> None:
+    if not isinstance(model, VisionTransformer):
+        raise ValueError(f"CPE only supports timm VisionTransformer models, got {type(model)}")
+
+    patch_size = model.patch_embed.patch_size[0]
+    embed_dim = model.embed_dim
+    input_dims = model.patch_embed.img_size
+    normalize_patches = not isinstance(model.patch_embed.norm, nn.Identity)
+    cls_token = model.cls_token is not None
+    if isinstance(max_img_size, int):
+        max_img_size = int(round(max_img_size / patch_size) * patch_size)
+    else:
+        max_img_size = tuple(int(round(d / patch_size) * patch_size) for d in max_img_size)
+
+    model.patch_generator = ViTPatchGenerator(
+        patch_size=patch_size,
+        embed_dim=embed_dim,
+        input_dims=input_dims,
+        normalize_patches=normalize_patches,
+        cls_token=cls_token,
+        max_input_dims=max_img_size,
+        pos_dropout=pos_dropout,
+        num_cls_tokens=num_cls_tokens,
+        register_multiple=register_multiple,
+        num_registers=num_registers,
+    )
+    model.patch_embed = None
+    model.cls_token = None
+    model.pos_embed = None
+    model.pos_drop = None
+    model.patch_size = patch_size
+    model.num_cls_tokens = num_cls_tokens
+    model.num_registers = model.patch_generator.num_registers
+    model.forward_features = MethodType(_forward_cpe, model)
+    model.cpe_video_mode = MethodType(_video_mode, model)
+
+
+class FeatureNormalizer(nn.Module):
+    def __init__(self, embed_dim: int, dtype: torch.dtype = torch.float32) -> None:
+        super().__init__()
+        self.register_buffer("mean", torch.zeros(embed_dim, dtype=dtype))
+        self.register_buffer("tx", torch.eye(embed_dim, dtype=dtype))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if x.ndim <= 3:
+            return (x - self.mean) @ self.tx.T
+        if x.ndim == 4:
+            kernel = self.tx.reshape(*self.tx.shape, 1, 1)
+            return F.conv2d(x - self.mean.reshape(1, -1, 1, 1), weight=kernel, bias=None, stride=1, padding=0)
+        raise ValueError(f"Unsupported input dimension: {x.ndim}, shape: {x.shape}")
+
+
+class InnerRADIOModel(nn.Module):
+    def __init__(
+        self,
+        model: nn.Module,
+        input_conditioner: nn.Module,
+        patch_size: int,
+        max_resolution: int,
+        preferred_resolution: Resolution,
+        summary_idxs: Optional[torch.Tensor] = None,
+        feature_normalizer: Optional[nn.Module] = None,
+        window_size: Optional[int] = None,
+    ) -> None:
+        super().__init__()
+        self.model = model
+        self.input_conditioner = input_conditioner
+        if summary_idxs is not None:
+            self.register_buffer("summary_idxs", summary_idxs)
+        else:
+            self.summary_idxs = None
+        self._preferred_resolution = preferred_resolution
+        self._patch_size = patch_size
+        self._max_resolution = max_resolution
+        self._window_size = window_size
+        self.feature_normalizer = feature_normalizer if feature_normalizer is not None else nn.Identity()
+
+    @property
+    def num_summary_tokens(self) -> int:
+        patch_gen = getattr(self.model, "patch_generator", None)
+        if patch_gen is not None:
+            return patch_gen.num_skip
+        if getattr(self.model, "global_pool", None) == "avg":
+            return 0
+        return 1
+
+    @property
+    def num_cls_tokens(self) -> int:
+        patch_gen = getattr(self.model, "patch_generator", None)
+        if patch_gen is not None:
+            return patch_gen.num_cls_tokens
+        if getattr(self.model, "global_pool", None) == "avg":
+            return 0
+        return 1
+
+    @property
+    def patch_size(self) -> int:
+        if self._patch_size is not None:
+            return self._patch_size
+        if hasattr(self.model, "patch_size"):
+            return self.model.patch_size
+        patch_gen = getattr(self.model, "patch_generator", None)
+        if patch_gen is not None:
+            return patch_gen.patch_size
+        raise AttributeError("Unable to infer patch_size from RADIO vision model")
+
+    @property
+    def max_resolution(self) -> int:
+        return self._max_resolution
+
+    @property
+    def preferred_resolution(self) -> Resolution:
+        return self._preferred_resolution
+
+    @property
+    def window_size(self) -> Optional[int]:
+        return self._window_size
+
+    @property
+    def min_resolution_step(self) -> int:
+        res = self.patch_size
+        if self.window_size is not None:
+            res *= self.window_size
+        return res
+
+    @property
+    def blocks(self) -> Iterable[nn.Module]:
+        return getattr(self.model, "blocks", None)
+
+    @property
+    def embed_dim(self) -> int:
+        return self.model.embed_dim
+
+    @property
+    def summary_dim(self) -> int:
+        embed_dim = self.embed_dim
+        if self.summary_idxs is not None:
+            embed_dim *= self.summary_idxs.shape[0]
+        return embed_dim
+
+    def make_preprocessor_external(self) -> Callable[[torch.Tensor], torch.Tensor]:
+        ret = self.input_conditioner
+        self.input_conditioner = nn.Identity()
+        return ret
+
+    def get_nearest_supported_resolution(self, height: int, width: int) -> Resolution:
+        height = int(round(height / self.min_resolution_step) * self.min_resolution_step)
+        width = int(round(width / self.min_resolution_step) * self.min_resolution_step)
+        return Resolution(max(height, self.min_resolution_step), max(width, self.min_resolution_step))
+
+    def switch_to_deploy(self) -> None:
+        fn = getattr(self.model, "switch_to_deploy", None)
+        if fn is not None:
+            fn()
+
+    def cpe_video_mode(self, t: int):
+        return self.model.cpe_video_mode(t)
+
+    def forward(self, x: torch.Tensor, feature_fmt: str = "NLC") -> RadioOutput:
+        res_step = self.min_resolution_step
+        if res_step is not None and (x.shape[-2] % res_step != 0 or x.shape[-1] % res_step != 0):
+            raise ValueError(
+                "The input resolution must be a multiple of self.min_resolution_step. "
+                f"Input: {x.shape[-2:]}, Nearest: {self.get_nearest_supported_resolution(*x.shape[-2:])}"
+            )
+        x = self.input_conditioner(x)
+        y = self.model.forward_features(x)
+        return self._extract_final(x, y, feature_fmt=feature_fmt)
+
+    def _extract_final(self, x: torch.Tensor, y: torch.Tensor, feature_fmt: str = "NLC") -> RadioOutput:
+        patch_gen = getattr(self.model, "patch_generator", None)
+        if patch_gen is not None:
+            all_summary = y[:, : patch_gen.num_cls_tokens]
+            bb_summary = all_summary[:, self.summary_idxs] if self.summary_idxs is not None else all_summary
+            all_feat = y[:, patch_gen.num_skip :]
+        elif getattr(self.model, "global_pool", None) == "avg":
+            all_summary = y[:, self.model.num_prefix_tokens :].mean(dim=1)
+            bb_summary = all_summary
+            all_feat = y
+        else:
+            all_summary = y[:, 0]
+            bb_summary = all_summary
+            all_feat = y[:, 1:]
+
+        all_feat = self.feature_normalizer(all_feat)
+        if feature_fmt == "NCHW":
+            fmt_feat = all_feat.reshape(
+                all_feat.shape[0],
+                x.shape[-2] // self.patch_size,
+                x.shape[-1] // self.patch_size,
+                all_feat.shape[2],
+            ).permute(0, 3, 1, 2)
+        elif feature_fmt == "NLC":
+            fmt_feat = all_feat
+        else:
+            raise ValueError(f"Unsupported feature_fmt: {feature_fmt}. Must be one of ['NLC', 'NCHW']")
+        return RadioOutput(bb_summary.flatten(1), fmt_feat)
+
+
+def _as_namespace(value):
+    if value is None:
+        return type("RADIOArgs", (), {})()
+    if isinstance(value, dict):
+        ns = type("RADIOArgs", (), {})()
+        for k, v in value.items():
+            setattr(ns, k, v)
+        return ns
+    return value
+
+
+def _dtype_from_config(config: RADIOConfig) -> torch.dtype:
+    dtype_name = getattr(config, "dtype", None) or getattr(config, "amp_dtype", None)
+    if isinstance(dtype_name, torch.dtype):
+        return dtype_name
+    if isinstance(dtype_name, str) and hasattr(torch, dtype_name):
+        return getattr(torch, dtype_name)
+    return torch.float32
+
+
+def create_vit_from_config(config: RADIOConfig) -> VisionTransformer:
+    args = _as_namespace(getattr(config, "args", {}))
+    model_name = getattr(args, "model", None) or "vit_huge_patch16_224"
+    if model_name != "vit_huge_patch16_224":
+        raise ValueError(
+            "This standalone cradio_model.py keeps only the ZDTaichu ViT-H/16 structure. "
+            f"Unsupported RADIO args.model={model_name!r}."
+        )
+
+    model = VisionTransformer(
+        img_size=224,
+        patch_size=16,
+        embed_dim=1280,
+        depth=32,
+        num_heads=16,
+        mlp_ratio=4.0,
+        qkv_bias=True,
+        num_classes=0,
+        global_pool="",
+    )
+
+    # The ZDTaichu checkpoint was exported after RADIO removed the final ViT norm/head
+    # and replaced patch embedding, cls token, and absolute pos embedding with CPE.
+    if hasattr(model, "norm") and not getattr(args, "model_norm", False):
+        model.norm = nn.Identity()
+    model.head = nn.Identity()
+
+    cpe_max_size = getattr(args, "cpe_max_size", None) or getattr(config, "max_resolution", None)
+    if cpe_max_size is not None:
+        teachers = getattr(args, "teachers", []) or []
+        teacher_names = {t.get("name") for t in teachers if isinstance(t, dict) and t.get("name")}
+        num_cls_tokens = len(teacher_names) if getattr(args, "cls_token_per_teacher", False) and teacher_names else 1
+        enable_cpe(
+            model,
+            cpe_max_size,
+            num_cls_tokens=num_cls_tokens,
+            register_multiple=getattr(args, "register_multiple", None),
+            num_registers=getattr(args, "cpe_num_registers", None),
+        )
+    return model
+
+
+class RADIOModel(PreTrainedModel):
+    """Inference-only HuggingFace wrapper for the ZDTaichu C-RADIO ViT tower."""
+
+    config_class = RADIOConfig
+    base_model_prefix = "radio_model"
+    main_input_name = "pixel_values"
+    supports_gradient_checkpointing = False
+
+    def __init__(self, config: RADIOConfig) -> None:
+        super().__init__(config)
+        args = _as_namespace(getattr(config, "args", {}))
+        dtype = _dtype_from_config(config)
+        vit = create_vit_from_config(config)
+
+        summary_idxs = None
+        if getattr(args, "cls_token_per_teacher", False):
+            teachers = getattr(args, "teachers", []) or []
+            if teachers:
+                summary_idxs = torch.tensor(
+                    [i for i, t in enumerate(teachers) if not isinstance(t, dict) or t.get("use_summary", True)],
+                    dtype=torch.int64,
+                )
+
+        feature_normalizer = None
+        fn_cfg = getattr(config, "feature_normalizer_config", None)
+        if fn_cfg is not None:
+            embed_dim = fn_cfg.get("embed_dim", vit.embed_dim) if isinstance(fn_cfg, dict) else vit.embed_dim
+            feature_normalizer = FeatureNormalizer(embed_dim, dtype=torch.float32)
+
+        pref = getattr(config, "preferred_resolution", (512, 512))
+        self.radio_model = InnerRADIOModel(
+            model=vit,
+            input_conditioner=get_default_conditioner(),
+            patch_size=getattr(config, "patch_size", 16),
+            max_resolution=getattr(config, "max_resolution", 2048),
+            preferred_resolution=Resolution(int(pref[0]), int(pref[1])),
+            summary_idxs=summary_idxs,
+            feature_normalizer=feature_normalizer,
+            window_size=getattr(config, "vitdet_window_size", None),
+        )
+        if dtype is not torch.float32:
+            self.radio_model = self.radio_model.to(dtype=dtype)
+
+    @property
+    def adaptors(self):
+        return nn.ModuleDict()
+
+    @property
+    def model(self) -> nn.Module:
+        return self.radio_model.model
+
+    @property
+    def input_conditioner(self) -> nn.Module:
+        return self.radio_model.input_conditioner
+
+    @property
+    def num_summary_tokens(self) -> int:
+        return self.radio_model.num_summary_tokens
+
+    @property
+    def patch_size(self) -> int:
+        return self.radio_model.patch_size
+
+    @property
+    def max_resolution(self) -> int:
+        return self.radio_model.max_resolution
+
+    @property
+    def preferred_resolution(self) -> Resolution:
+        return self.radio_model.preferred_resolution
+
+    @property
+    def window_size(self) -> Optional[int]:
+        return self.radio_model.window_size
+
+    @property
+    def min_resolution_step(self) -> int:
+        return self.radio_model.min_resolution_step
+
+    def make_preprocessor_external(self) -> Callable[[torch.Tensor], torch.Tensor]:
+        return self.radio_model.make_preprocessor_external()
+
+    def get_nearest_supported_resolution(self, height: int, width: int) -> Resolution:
+        return self.radio_model.get_nearest_supported_resolution(height, width)
+
+    def switch_to_deploy(self) -> None:
+        self.radio_model.switch_to_deploy()
+
+    def forward(self, pixel_values: torch.Tensor, feature_fmt: str = "NLC", **kwargs) -> RadioOutput:
+        return self.radio_model(pixel_values, feature_fmt=feature_fmt)
+
+
+__all__ = [
+    "RADIOModel",
+    "RADIOConfig",
+    "RadioOutput",
+    "Resolution",
+    "InputConditioner",
+    "ViTPatchGenerator",
+]

generation_config.json

@@ -0,0 +1,16 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 248040,
+  "do_sample": true,
+  "eos_token_id": [
+    248044,
+    248040,
+    248046
+  ],
+  "pad_token_id": 248040,
+  "repetition_penalty": 1.0,
+  "temperature": 1.0,
+  "top_k": 20,
+  "top_p": 0.95,
+  "transformers_version": "5.3.0"
+}

image_processing.py

@@ -0,0 +1,268 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List, Optional, Union, Any, Dict, Tuple
+
+from PIL import Image
+import torch
+from transformers.image_processing_base import BatchFeature
+from transformers.image_processing_utils_fast import BaseImageProcessorFast
+from transformers.image_utils import make_list_of_images, get_image_type, ImageInput, ImageType
+from transformers.utils import TensorType
+import torchvision.transforms as T
+
+import math
+
+class ZDTaichu5_0_ImageProcessor(BaseImageProcessorFast):
+    model_input_names = ["pixel_values", "image_grid_thw"]
+
+    def __init__(self, image_size=512, max_num_tiles=12, use_thumbnail=True, norm_mean=None, norm_std=None, do_rescale=True, patch_size=16, downsample_ratio=0.5, merge_size=1, **kwargs):
+        super().__init__(**kwargs)
+        self.image_size = image_size
+        self.max_num_tiles = max_num_tiles
+        self.use_thumbnail = use_thumbnail
+        self.norm_mean = norm_mean
+        self.norm_std = norm_std
+        self.do_rescale = do_rescale
+        self.merge_size = merge_size
+        self.num_image_token = int((image_size // patch_size) ** 2 * (downsample_ratio ** 2))
+
+    def _process_image(
+        self,
+        image: ImageInput,
+        **kwargs,
+    ) -> torch.Tensor:
+        image_type = get_image_type(image)
+        if image_type == ImageType.PIL:
+            if image.mode != 'RGB':
+                image = image.convert('RGB')
+            # Keep PIL input through tiling so resize order matches vLLM.
+        return image
+
+    def _preprocess(
+        self,
+        images: List[torch.Tensor],
+        image_size: int = None,
+        max_num_tiles: int = None,
+        use_thumbnail: bool = None,
+        do_rescale: bool = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        **kwargs,
+    ) -> List[torch.Tensor]:
+        image_size = image_size if image_size is not None else self.image_size
+        max_num_tiles = max_num_tiles if max_num_tiles is not None else self.max_num_tiles
+        use_thumbnail = use_thumbnail if use_thumbnail is not None else self.use_thumbnail
+        do_rescale = do_rescale if do_rescale is not None else self.do_rescale
+
+        images = make_list_of_images(images)
+
+        all_patches = []
+        num_patches = []
+        image_grid_thw = []
+        for image in images:
+            patches, tile_rows, tile_cols = dynamic_preprocess(image, image_size, max_num_tiles, use_thumbnail)
+            all_patches.extend(patches)
+            num_patches.append(len(patches))
+            image_grid_thw.append([1, tile_rows, tile_cols])
+
+        # vLLM converts each already-cropped PIL tile with ToTensor.
+        pixel_values = torch.stack([T.ToTensor()(patch) for patch in all_patches], dim=0)
+        norm_mean = torch.Tensor(self.norm_mean).view(1, 3, 1, 1)
+        norm_std = torch.Tensor(self.norm_std).view(1, 3, 1, 1)
+        pixel_values = (pixel_values - norm_mean) / norm_std
+        pixel_values = pixel_values.to(torch.bfloat16)
+        return BatchFeature(
+            data={
+                "pixel_values": pixel_values,
+                "num_patches": num_patches,
+                "image_grid_thw": image_grid_thw,
+            },
+            tensor_type=return_tensors,
+        )
+
+
+def get_internvl_target_ratios(
+    min_num: int,
+    max_num: int,
+) -> list[tuple[int, int]]:
+    target_ratios = {(i, j)
+                     for n in range(min_num, max_num + 1)
+                     for i in range(1, n + 1)
+                     for j in range(1, n + 1) if min_num <= i * j <= max_num}
+    return sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+
+# From https://github.com/OpenGVLab/InternVL/blob/c62fa4f7c850165d7386bdc48ac6bc5a6fab0864/internvl_chat/internvl/train/dataset.py#L685
+# Copyright (c) 2023 OpenGVLab.
+def find_closest_aspect_ratio(
+    aspect_ratio: float,
+    target_ratios: list[tuple[int, int]],
+    width: int,
+    height: int,
+    image_size: int,
+) -> tuple[int, int]:
+    best_ratio_diff = float("inf")
+    best_ratio = (1, 1)
+    area = width * height
+    for ratio in target_ratios:
+        target_aspect_ratio = ratio[0] / ratio[1]
+        ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+        if ratio_diff < best_ratio_diff:
+            best_ratio_diff = ratio_diff
+            best_ratio = ratio
+        elif ratio_diff == best_ratio_diff:
+            if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
+                best_ratio = ratio
+    return best_ratio
+
+def select_tile_grid(
+    *,
+    orig_width: int,
+    orig_height: int,
+    image_size: int,
+    min_num_tiles: int,
+    max_num_tiles: int,
+) -> Tuple[int, int, int, int]:
+    """
+    Choose (rw, rh) tile grid with small-image and aspect-sanity guards.
+
+    Returns (num_grid_blocks, target_width, target_height, effective_max_tiles).
+    num_grid_blocks = rw * rh (does NOT include the optional thumbnail).
+
+    Guards:
+      1. Area cap: don't create more tiles than the source has pixels for.
+      2. Aspect-sanity: drop candidates whose ratio differs from source by >3x.
+    """
+    # ── Guard 1: area cap ─────────────────────────────────────────────────
+    src_pixels = orig_width * orig_height
+    tile_pixels = image_size * image_size
+    area_max_tiles = max(1, math.ceil(src_pixels / tile_pixels))
+    effective_max = min(max_num_tiles, area_max_tiles)
+    effective_max = max(effective_max, min_num_tiles)
+
+    target_ratios = get_internvl_target_ratios(min_num_tiles, effective_max)
+
+    # ── Guard 2: aspect-sanity ────────────────────────────────────────────
+    src_ar = orig_width / orig_height
+    filtered = [
+        (rw, rh) for (rw, rh) in target_ratios
+        if (1.0 / 3.0) <= (rw / rh) / src_ar <= 3.0
+    ]
+    # Fall back to unfiltered set for extreme panoramas / long strips where
+    # no candidate is within 3x — better to pick *something* than error.
+    if filtered:
+        target_ratios = filtered
+
+    # ── Pick best ratio ───────────────────────────────────────────────────
+    rw, rh = find_closest_aspect_ratio(
+        src_ar, target_ratios,
+        width=orig_width, height=orig_height, image_size=image_size,
+    )
+
+    target_width = image_size * rw
+    target_height = image_size * rh
+    num_grid_blocks = rw * rh
+
+    return num_grid_blocks, target_width, target_height, effective_max
+
+
+def count_tiles(
+    *,
+    orig_width: int,
+    orig_height: int,
+    image_size: int,
+    min_num_tiles: int,
+    max_num_tiles: int,
+    use_thumbnail: bool,
+) -> int:
+    """
+    Total number of tiles (grid blocks + optional thumbnail) for this image.
+    This MUST match what the actual tiling produces, or prompt expansion and
+    embedding count will diverge.
+    """
+    n_grid, _, _, _ = select_tile_grid(
+        orig_width=orig_width, orig_height=orig_height,
+        image_size=image_size,
+        min_num_tiles=min_num_tiles, max_num_tiles=max_num_tiles,
+    )
+    if use_thumbnail and n_grid != 1:
+        return n_grid + 1
+    return n_grid
+
+def calculate_targets(
+    orig_width: int,
+    orig_height: int,
+    target_ratios: list[tuple[int, int]],
+    image_size: int,
+) -> tuple[int, int, int]:
+    aspect_ratio = orig_width / orig_height
+
+    # find the closest aspect ratio to the target
+    target_aspect_ratio = find_closest_aspect_ratio(
+        aspect_ratio,
+        target_ratios,
+        width=orig_width,
+        height=orig_height,
+        image_size=image_size,
+    )
+
+    # calculate the target width and height
+    target_width = image_size * target_aspect_ratio[0]
+    target_height = image_size * target_aspect_ratio[1]
+    blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+
+    return blocks, target_width, target_height
+
+def dynamic_preprocess(image, image_size=512, max_num_tiles=12, use_thumbnail=True, min_num_tiles=1):
+    """Split a PIL image using vLLM's resize/crop order."""
+    if isinstance(image, torch.Tensor):
+        image = T.ToPILImage()(image)
+    elif not isinstance(image, Image.Image):
+        image = Image.fromarray(image)
+    if image.mode != 'RGB':
+        image = image.convert('RGB')
+    orig_width, orig_height = image.size
+
+    n_grid, target_width, target_height, _ = select_tile_grid(
+        orig_width=orig_width,
+        orig_height=orig_height,
+        image_size=image_size,
+        min_num_tiles=min_num_tiles,
+        max_num_tiles=max_num_tiles,
+    )
+    
+    # Tile grid dimensions (rows × cols of the InternVL tiling)
+    tile_rows = target_height // image_size
+    tile_cols = target_width // image_size
+
+    resized_img = image.resize((target_width, target_height), Image.BICUBIC)
+    cols = target_width // image_size
+    patches = []
+    for i in range(n_grid):
+        col = i % cols
+        row = i // cols
+        patches.append(
+            resized_img.crop(
+                (col * image_size, row * image_size,
+                 (col + 1) * image_size, (row + 1) * image_size)
+            )
+        )
+    assert len(patches) == n_grid
+
+    if use_thumbnail and n_grid != 1:
+        thumbnail = image.resize((image_size, image_size), Image.BICUBIC)
+        patches.append(thumbnail)
+        
+    #print(orig_height, orig_width, target_width, target_height, len(patches))
+
+    return patches, tile_rows, tile_cols

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d26e50e66812bdb9563368991b4ad9be1eeaab8aa59b5220df5efbc89e202f3b
+size 9811274448

modeling.py

@@ -0,0 +1,1109 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ============================================================================
+#  ZDTaichu-5.0  —  Main Model
+#
+#  Architecture
+#  ────────────
+#    Vision encoder : C-RADIOv4-H (ViT-H/16, 653 M)
+#    Projector      : RMSNorm → Linear(5120→20480) → SquaredReLU → Linear(20480→H)
+#    LLM decoder    : Qwen3.5 hybrid (Gated DeltaNet + full attention, 3:1 ratio)
+#
+#  Position encoding (M-RoPE)
+#  ──────────────────────────
+#  Vision tokens receive 3D position IDs (temporal, height, width) computed
+#  from the InternVL-style tile grid via ``get_rope_index()``.  Text tokens
+#  receive standard 1D positions (all three M-RoPE channels are identical).
+#
+#  This matches the official Qwen3.5 VL pipeline where ``Qwen3_5Model.forward()``
+#  calls ``compute_3d_position_ids()`` → ``get_rope_index()`` before forwarding
+#  to ``Qwen3_5TextModel``.  The resulting ``position_ids`` of shape ``(3, B, S)``
+#  are consumed directly by ``Qwen3_5TextRotaryEmbedding``, which applies
+#  interleaved M-RoPE across temporal / height / width frequency bands.
+#
+#  Generation
+#  ──────────
+#  This model inherits from ``GenerationMixin``, owning the generation loop
+#  (like ``Qwen3_5ForConditionalGeneration``).  Key overrides:
+#    - ``_prepare_position_ids_for_generation``: computes 3D ``position_ids``
+#      on the prefill step and caches ``rope_deltas``; applies ``rope_deltas``
+#      on subsequent decode steps.
+#    - ``prepare_inputs_for_generation``: clears ``pixel_values`` /
+#      ``pixel_values_videos`` after the first step (vision features are
+#      already embedded in the KV cache).
+#
+#  Cache handling
+#  ──────────────
+#  ``Qwen3_5DynamicCache`` is created internally by ``Qwen3_5TextModel`` when
+#  ``use_cache=True``.  It stores KV states for full-attention layers and
+#  ``conv_states`` + ``recurrent_states`` for Gated DeltaNet layers.
+# ============================================================================
+
+import itertools
+import warnings
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import torch
+import transformers
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from transformers import AutoModel, GenerationConfig
+from transformers.generation import GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import logging
+
+from .configuration import ZDTaichu5_0_Config
+from .cradio_model import RADIOModel
+
+logger = logging.get_logger(__name__)
+
+# ---------------------------------------------------------------------------
+# Import Qwen3.5 model classes — requires transformers >= 5.3.0
+# ---------------------------------------------------------------------------
+
+_MIN_TRANSFORMERS = "5.3.0"
+
+try:
+    from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5ForCausalLM
+    from transformers.cache_utils import DynamicCache as Qwen3_5DynamicCache
+    _HAS_QWEN3_5 = True
+except Exception as e:
+    _HAS_QWEN3_5 = False
+    Qwen3_5ForCausalLM = None
+    Qwen3_5DynamicCache = None
+    logger.warning(
+        f"Could not import Qwen3_5ForCausalLM from transformers. "
+        f"Import error: {e!r}"
+    )
+
+
+def _version_ge(v1, v2):
+    """Check if version v1 >= v2."""
+    from packaging import version
+    return version.parse(v1) >= version.parse(v2)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Projector components
+# ─────────────────────────────────────────────────────────────────────────────
+
+class SquaredReLU(nn.Module):
+    """Squared ReLU activation — same non-linearity used in the projector."""
+    def forward(self, x):
+        return torch.pow(torch.nn.functional.relu(x), 2)
+
+
+class RMSNorm(nn.Module):
+    """
+    Standard RMSNorm for the projector (NOT the Qwen3.5 LLM variant).
+
+    Qwen3.5's internal ``Qwen3_5RMSNorm`` uses zero-initialized weight with
+    ``output * (1 + weight)``.  The projector uses ones-initialized weight
+    with ``output * weight`` — the standard formulation.
+    """
+    def __init__(self, hidden_size: int, eps: float = 1e-5):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.eps = eps
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.eps)
+        return (self.weight.to(torch.float32) * hidden_states).to(input_dtype)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Main model
+# ─────────────────────────────────────────────────────────────────────────────
+
+class ZDTaichu5_0_ForConditionalGeneration(PreTrainedModel, GenerationMixin):
+    """
+    ZDTaichu-5.0: C-RADIOv4-H vision encoder + Qwen3.5 language decoder.
+
+    Architecture overview::
+
+        pixel_values
+          └─► C-RADIOv4-H (ViT-H/16, 653 M)
+                └─► pixel_shuffle(0.5)
+                      └─► mlp1: RMSNorm → Linear → SquaredReLU → Linear
+                            └─► inject into Qwen3.5 embeddings at <image> positions
+                                  └─► Qwen3.5 (hybrid DeltaNet / Transformer)
+    """
+
+    config_class = ZDTaichu5_0_Config
+    main_input_name = "input_ids"
+    _tied_weights_keys = None#["language_model.lm_head.weight"]
+    _keys_to_ignore_on_load_unexpected = [
+        # The RADIO input_conditioner registers norm_mean / norm_std as
+        # buffers, but make_preprocessor_external() removes the conditioner
+        # at init time (normalization is handled by the image processor).
+        # The build script still saves these from the source checkpoint, so
+        # they appear as unexpected keys during loading — safe to ignore.
+        r"vision_model\.radio_model\.input_conditioner\..*",
+        r"^mtp\..*",
+    ]
+
+    _supports_flash_attn_2 = True
+    _supports_flash_attention_2 = True
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _no_split_modules = ["Qwen3_5DecoderLayer"]
+    _is_stateful = True
+    supports_gradient_checkpointing = True
+
+    def __init__(self, config: ZDTaichu5_0_Config):
+        super().__init__(config)
+
+        # Guard for bleeding-edge transformers (>= 4.57.0.dev) where
+        # _finalize_model_loading reads all_tied_weights_keys but
+        # PreTrainedModel.__init__ may not yet initialise it.
+        if not hasattr(self, "all_tied_weights_keys"):
+            self.all_tied_weights_keys = {}
+
+        assert _version_ge(transformers.__version__, _MIN_TRANSFORMERS), (
+            f"Qwen3.5 support requires transformers >= {_MIN_TRANSFORMERS} "
+            f"(found {transformers.__version__})"
+        )
+        assert _HAS_QWEN3_5, (
+            "Qwen3_5ForCausalLM is not available. "
+            f"Ensure transformers >= {_MIN_TRANSFORMERS} is installed."
+        )
+
+        image_size = config.force_image_size
+        patch_size = config.vision_config.patch_size
+        self.patch_size = patch_size
+        self.template = config.template
+        self.num_image_token = int(
+            (image_size // patch_size) ** 2 * (config.downsample_ratio ** 2)
+        )
+        self.downsample_ratio = config.downsample_ratio
+        self.ps_version = config.ps_version
+        self.image_tag_type = config.image_tag_type
+        self.img_context_token_id = config.img_context_token_id
+        self.video_context_token_id = config.video_context_token_id
+
+        # Per-tile token dimensions (e.g. 14×14 for 448px, patch=16, ds=0.5)
+        self.tile_h = int((image_size // patch_size) * config.downsample_ratio)
+        self.tile_w = self.tile_h
+
+        logger.info(f"num_image_token: {self.num_image_token}")
+        logger.info(f"tile_h={self.tile_h}, tile_w={self.tile_w}")
+        logger.info(f"ps_version: {self.ps_version}")
+        logger.info(f"Vision encoder: {config.vision_config.version}")
+        logger.info(
+            f"LLM: Qwen3.5 ({config.llm_config.num_hidden_layers} layers, "
+            f"hidden={config.llm_config.hidden_size}, "
+            f"hybrid="
+            f"{sum(1 for t in config.llm_config.layer_types if t == 'linear_attention')} linear + "
+            f"{sum(1 for t in config.llm_config.layer_types if t == 'full_attention')} full)"
+        )
+
+        # ── Language model ───────────────────────────────────────────────────
+        self.language_model = Qwen3_5ForCausalLM(config.llm_config)
+
+        # ── Vision encoder ───────────────────────────────────────────────────
+        self.vision_model = RADIOModel(config.vision_config)
+        self.vision_model.model._initialize_weights = (
+            self.vision_model.model._init_weights
+        )
+        self.vision_model.radio_model.make_preprocessor_external()
+        self.vision_model = self.vision_model.to(
+            self.language_model.config.torch_dtype
+        )
+
+        self.drop_vision_class_token = True
+
+        # ── MLP projector ────────────────────────────────────────────────────
+        vit_hidden_size = config.vit_hidden_size
+        proj_hidden = config.projector_hidden_size
+        llm_hidden = config.llm_config.hidden_size
+        pixel_shuffle_dim = vit_hidden_size * int(1 / self.downsample_ratio) ** 2
+
+        self.mlp1 = nn.Sequential(
+            RMSNorm(pixel_shuffle_dim, eps=1e-5),
+            nn.Linear(pixel_shuffle_dim, proj_hidden, bias=False),
+            SquaredReLU(),
+            nn.Linear(proj_hidden, llm_hidden, bias=False),
+        )
+        self.mlp1 = self.mlp1.to(self.language_model.config.torch_dtype)
+
+        # Cached rope_deltas for multi-step generation
+        self.rope_deltas = None
+
+    # ── Embedding accessors (required by GenerationMixin) ─────────────────
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.lm_head = new_embeddings
+
+    def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
+        # 大头在 LLM:直接委托给内层 Qwen3.5(它原生支持 GC)
+        self.language_model.gradient_checkpointing_enable(
+            gradient_checkpointing_kwargs=gradient_checkpointing_kwargs
+        )
+        # 视觉塔可选:支持就开,不支持就跳过(不影响主显存)
+        vm = getattr(self, "vision_model", None)
+        if vm is not None and getattr(vm, "supports_gradient_checkpointing", False):
+            try:
+                vm.gradient_checkpointing_enable(
+                    gradient_checkpointing_kwargs=gradient_checkpointing_kwargs
+                )
+            except Exception:
+                pass
+
+    def gradient_checkpointing_disable(self):
+        self.language_model.gradient_checkpointing_disable()
+        vm = getattr(self, "vision_model", None)
+        if vm is not None and hasattr(vm, "gradient_checkpointing_disable"):
+            try:
+                vm.gradient_checkpointing_disable()
+            except Exception:
+                pass
+
+    # ─────────────────────────────────────────────────────────────────────────
+    # Vision helpers
+    # ─────────────────────────────────────────────────────────────────────────
+
+    def pixel_shuffle(
+        self, x: torch.Tensor, scale_factor: float = 0.5
+    ) -> torch.Tensor:
+        """Space-to-depth rearrangement (ps_version='v2' = corrected layout)."""
+        n, w, h, c = x.size()
+        x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
+        x = x.permute(0, 2, 1, 3).contiguous()
+        x = x.view(
+            n, int(h * scale_factor), int(w * scale_factor),
+            int(c / (scale_factor * scale_factor)),
+        )
+        if self.ps_version == "v1":
+            warnings.warn(
+                "ps_version='v1' produces a transposed spatial layout. "
+                "Use ps_version='v2' for correct output."
+            )
+        else:
+            x = x.permute(0, 2, 1, 3).contiguous()
+        return x
+
+    def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        """Run pixels through C-RADIOv4-H → pixel_shuffle → MLP projector."""
+        vit_embeds = self.vision_model(pixel_values).features
+        vit_embeds = vit_embeds.to(dtype=torch.bfloat16)
+
+        h = w = int(vit_embeds.shape[1] ** 0.5)
+        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
+        vit_embeds = self.pixel_shuffle(
+            vit_embeds, scale_factor=self.downsample_ratio
+        )
+        vit_embeds = vit_embeds.reshape(
+            vit_embeds.shape[0], -1, vit_embeds.shape[-1]
+        )
+        vit_embeds = self.mlp1(vit_embeds)
+        return vit_embeds
+
+    # ─────────────────────────────────────────────────────────────────────────
+    # 3D M-RoPE position IDs
+    # ─────────────────────────────────────────────────────────────────────────
+
+    def get_vision_position_ids(
+        self,
+        start_position: int,
+        tile_rows: int,
+        tile_cols: int,
+        has_thumbnail: bool = True,
+        device: torch.device = None,
+    ) -> torch.LongTensor:
+        """
+        Compute 3D (temporal, height, width) position IDs for vision tokens
+        from a single InternVL-style tiled image.
+
+        Token layout (flattened order expected by the model):
+          1. Grid tiles in raster order: tile(0,0), tile(0,1), …, tile(R-1,C-1).
+             Each tile has ``tile_h × tile_w`` tokens in raster order.
+          2. Thumbnail tile (optional): a single tile covering the full image
+             at reduced resolution.
+
+        Args:
+            start_position: Offset added to all positional indices.
+            tile_rows: Number of tile rows in the image grid.
+            tile_cols: Number of tile columns in the image grid.
+            has_thumbnail: Whether a thumbnail tile is appended after grid tiles.
+            device: Target device.
+
+        Returns:
+            ``torch.LongTensor`` of shape ``(3, num_vision_tokens)``.
+        """
+        tile_h, tile_w = self.tile_h, self.tile_w
+        npt = tile_h * tile_w  # num tokens per tile
+
+        # ── Grid tiles ───────────────────────────────────────────────────────
+        num_grid_tiles = tile_rows * tile_cols
+        tile_idx = torch.arange(num_grid_tiles, device=device)
+        tr = tile_idx // tile_cols
+        tc = tile_idx % tile_cols
+
+        local_idx = torch.arange(npt, device=device)
+        lr = local_idx // tile_w
+        lc = local_idx % tile_w
+
+        # (num_grid_tiles, npt) → flatten
+        global_h = (tr[:, None] * tile_h + lr[None, :]).reshape(-1).long()
+        global_w = (tc[:, None] * tile_w + lc[None, :]).reshape(-1).long()
+
+        total_grid = num_grid_tiles * npt
+        pos_t = torch.full(
+            (total_grid,), start_position, device=device, dtype=torch.long
+        )
+        pos_h = start_position + global_h
+        pos_w = start_position + global_w
+
+        # ── Thumbnail tile ───────────────────────────────────────────────────
+        if has_thumbnail:
+            # Map thumbnail local(r, c) → global(r * tile_rows, c * tile_cols)
+            # so its positions overlay the grid at coarser resolution.
+            thumb_h = (lr * tile_rows).long()
+            thumb_w = (lc * tile_cols).long()
+            pos_t = torch.cat([
+                pos_t,
+                torch.full(
+                    (npt,), start_position, device=device, dtype=torch.long
+                ),
+            ])
+            pos_h = torch.cat([pos_h, start_position + thumb_h])
+            pos_w = torch.cat([pos_w, start_position + thumb_w])
+
+        return torch.stack([pos_t, pos_h, pos_w], dim=0)
+
+    def get_rope_index(
+        self,
+        input_ids: torch.LongTensor,
+        mm_token_type_ids: torch.IntTensor,
+        image_grid_thw: Optional[torch.LongTensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        """
+        Compute 3D M-RoPE position IDs for a mixed vision + text sequence.
+
+        Follows the same structure as ``Qwen3_5Model.get_rope_index``:
+        uses ``mm_token_type_ids`` to group tokens by modality
+        (text=0, image=1, video=2) via ``itertools.groupby``.  Vision
+        tokens receive spatial position IDs (temporal, height, width)
+        while text tokens receive standard 1D positions.
+
+        Args:
+            input_ids: ``(B, S)`` token IDs.
+            mm_token_type_ids: ``(B, S)`` modality labels —
+                0 = text, 1 = image, 2 = video.
+            image_grid_thw: ``(num_images, 3)`` — each row
+                ``(T=1, tile_rows, tile_cols)`` for InternVL-style tiled images.
+            video_grid_thw: ``(num_videos, 3)`` — each row
+                ``(num_frames, 1, 1)``.
+            attention_mask: ``(B, S)`` binary mask.
+
+        Returns:
+            ``position_ids``: ``(3, B, S)``
+            ``mrope_position_deltas``: ``(B, 1)``
+        """
+        tile_h, tile_w = self.tile_h, self.tile_w
+        npt = tile_h * tile_w
+
+        B, S = input_ids.shape
+        device = input_ids.device
+
+        position_ids = torch.zeros(3, B, S, dtype=input_ids.dtype, device=device)
+        mrope_position_deltas = []
+
+        # ------------------------------------------------------------------
+        # video-as-image compatibility for verl / vLLM rollout.
+        # ------------------------------------------------------------------
+        if mm_token_type_ids is not None and video_grid_thw is None and torch.any(mm_token_type_ids == 2).item():
+            mm_token_type_ids = mm_token_type_ids.clone()
+
+            if image_grid_thw is not None:
+                # Count contiguous visual groups, because get_rope_index consumes
+                # one grid_thw row per contiguous image/video segment.
+                total_visual_groups = 0
+                for b in range(mm_token_type_ids.shape[0]):
+                    cur_types = mm_token_type_ids[b]
+                    if attention_mask is not None:
+                        cur_types = cur_types[attention_mask[b].bool()]
+
+                    prev_type = None
+                    for t in cur_types.tolist():
+                        if t in (1, 2) and t != prev_type:
+                            total_visual_groups += 1
+                        prev_type = t
+
+                num_image_grids = image_grid_thw.shape[0]
+
+                if total_visual_groups <= num_image_grids:
+                    # True video-as-image case: consume image_grid_thw for both image and video types.
+                    mm_token_type_ids[mm_token_type_ids == 2] = 1
+
+                    if "logger" in globals():
+                        logger.warning_once(
+                            "Converting mm_token_type_ids type 2 to type 1 because "
+                            "video_grid_thw is None and image_grid_thw has enough grids. "
+                            "This matches video-as-image processing."
+                        )
+                else:
+                    # Some type-2 tokens are likely generated orphan <|video_pad|> tokens.
+                    # Treat them as text to avoid consuming non-existent grids.
+                    mm_token_type_ids[mm_token_type_ids == 2] = 0
+
+                    if "logger" in globals():
+                        logger.warning_once(
+                            "mm_token_type_ids contains type 2 but video_grid_thw is None, "
+                            "and image_grid_thw does not have enough grids. Treating type 2 "
+                            "as text. This likely means the model generated orphan <|video_pad|> tokens."
+                        )
+            else:
+                # No visual grid exists, so type 2 cannot represent valid visual tokens.
+                mm_token_type_ids[mm_token_type_ids == 2] = 0
+
+                if "logger" in globals():
+                    logger.warning_once(
+                        "mm_token_type_ids contains type 2, but both video_grid_thw and "
+                        "image_grid_thw are None. Treating type 2 as text."
+                    )
+
+        grid_iters = {
+            1: iter(image_grid_thw) if image_grid_thw is not None else None,
+            2: iter(video_grid_thw) if video_grid_thw is not None else None,
+        }
+
+        for batch_idx, current_input_ids in enumerate(input_ids):
+            input_token_type = mm_token_type_ids[batch_idx]
+            if attention_mask is not None:
+                current_input_ids = current_input_ids[attention_mask[batch_idx].bool()]
+                input_token_type = input_token_type[attention_mask[batch_idx].bool()]
+
+            # Group contiguous runs of the same modality type
+            input_type_group = []
+            for key, group in itertools.groupby(
+                enumerate(input_token_type.tolist()), lambda x: x[1]
+            ):
+                group = list(group)
+                start_index = group[0][0]
+                end_index = group[-1][0] + 1
+                input_type_group.append((key, start_index, end_index))
+
+            current_pos = 0
+            llm_pos_ids_list: List[torch.Tensor] = []
+
+            # ── Per-video state machine ──────────────────────────────────────
+            # Mirrors the Megatron-side implementation in
+            # modeling.py: a single video_grid_thw entry
+            # of [num_frames, 1, 1] is consumed across multiple non-contiguous
+            # type-2 runs (one per <|video_pad|> block, separated by frame
+            # header text).
+            #
+            # Within a video, every frame's tokens use:
+            #   t = vid_spatial_start + frame_idx     (anchored at video start)
+            #   h = vid_spatial_start + local_row     (constant across frames)
+            #   w = vid_spatial_start + local_col     (constant across frames)
+            #
+            # Text between frames advances ``current_pos`` normally — those
+            # text positions live in a different range than the video frame
+            # positions, which is fine for M-RoPE (RoPE requires no
+            # monotonicity, only consistent training/inference).
+            vid_active = False
+            vid_num_frames = 0
+            vid_frame_idx = 0
+            vid_spatial_start = 0
+
+            for modality_type, start_idx, end_idx in input_type_group:
+                # text == 0
+                if modality_type == 0:
+                    text_len = end_idx - start_idx
+                    llm_pos_ids_list.append(
+                        torch.arange(text_len, device=device).view(1, -1).expand(3, -1)
+                        + current_pos
+                    )
+                    current_pos += text_len
+
+                # image == 1
+                elif modality_type == 1:
+                    seg_len = end_idx - start_idx
+                    grid = next(grid_iters[1])
+                    tile_rows = grid[1].item()
+                    tile_cols = grid[2].item()
+                    grid_tokens = tile_rows * tile_cols * npt
+                    has_thumbnail = seg_len > grid_tokens
+
+                    vpos = self.get_vision_position_ids(
+                        start_position=current_pos,
+                        tile_rows=tile_rows,
+                        tile_cols=tile_cols,
+                        has_thumbnail=has_thumbnail,
+                        device=device,
+                    )
+                    assert vpos.shape[1] == seg_len, (
+                        f"Position count ({vpos.shape[1]}) ≠ image token count "
+                        f"({seg_len}) for grid=({tile_rows},{tile_cols}), "
+                        f"thumbnail={has_thumbnail}"
+                    )
+                    llm_pos_ids_list.append(vpos)
+                    current_pos += max(tile_rows * tile_h, tile_cols * tile_w)
+
+                # video == 2
+                elif modality_type == 2:
+                    seg_len = end_idx - start_idx
+
+                    # Activate per-video state on the FIRST type-2 run for
+                    # this video.  Subsequent type-2 runs (one per frame
+                    # block, separated by frame-header text) reuse the same
+                    # vid_spatial_start anchor.
+                    if not vid_active:
+                        grid = next(grid_iters[2])
+                        vid_num_frames = grid[0].item()
+                        vid_active = True
+                        vid_frame_idx = 0
+                        vid_spatial_start = current_pos
+
+                    # Each frame contributes exactly ``npt`` tokens.
+                    if seg_len % npt != 0:
+                        raise ValueError(
+                            f"Video segment length {seg_len} is not a "
+                            f"multiple of npt={npt} (tile_h*tile_w). "
+                            f"Check that the processor produced one "
+                            f"<|video_pad|> block per frame with exactly "
+                            f"npt tokens each."
+                        )
+                    frames_in_run = seg_len // npt
+
+                    # Sanity guard against malformed grids — never consume
+                    # more frames than the grid declared.
+                    if vid_frame_idx + frames_in_run > vid_num_frames:
+                        raise ValueError(
+                            f"Video has {vid_num_frames} frames but "
+                            f"input_ids contain at least "
+                            f"{vid_frame_idx + frames_in_run} frame blocks. "
+                            f"Check the processor's video_grid_thw against "
+                            f"the actual <|video_pad|> count."
+                        )
+
+                    local_idx = torch.arange(npt, device=device)
+                    lr = local_idx // tile_w
+                    lc = local_idx % tile_w
+
+                    all_t, all_h, all_w = [], [], []
+                    for _ in range(frames_in_run):
+                        # Temporal: anchored at video_start, advances by frame_idx.
+                        all_t.append(torch.full(
+                            (npt,),
+                            vid_spatial_start + vid_frame_idx,
+                            device=device, dtype=torch.long,
+                        ))
+                        # Spatial: constant base across frames within this video.
+                        all_h.append((vid_spatial_start + lr).long())
+                        all_w.append((vid_spatial_start + lc).long())
+                        vid_frame_idx += 1
+                        # Advance current_pos by one frame's spatial extent so
+                        # subsequent text positions stay strictly above any
+                        # h/w position used by this video.  After all frames,
+                        # current_pos has advanced by num_frames * max(tile_h, tile_w),
+                        # which always exceeds vid_spatial_start + max(num_frames, tile_h, tile_w)
+                        # for num_frames >= 1 (so text after the video sees
+                        # positions strictly greater than every video token).
+                        current_pos += max(tile_h, tile_w)
+
+                    vpos = torch.stack([
+                        torch.cat(all_t), torch.cat(all_h), torch.cat(all_w),
+                    ], dim=0)
+                    assert vpos.shape[1] == seg_len, (
+                        f"Position count ({vpos.shape[1]}) ≠ video token "
+                        f"count ({seg_len})"
+                    )
+                    llm_pos_ids_list.append(vpos)
+
+                    # End the video once all declared frames have been
+                    # consumed; reset state so the next video (if any) gets
+                    # a fresh grid pull.
+                    if vid_frame_idx >= vid_num_frames:
+                        vid_active = False
+                        vid_num_frames = 0
+                        vid_frame_idx = 0
+                        vid_spatial_start = 0
+
+            # Sanity check: if a video's last frame isn't followed by any text,
+            # the loop ends with vid_active=False (we already reset on the
+            # final frame).  But if the input is malformed and the type-2
+            # runs don't cover all declared frames, surface that loudly
+            # rather than silently advancing the iterator the next time we
+            # see another video.
+            if vid_active:
+                raise ValueError(
+                    f"Reached end of input with video state still active: "
+                    f"consumed {vid_frame_idx}/{vid_num_frames} frames. "
+                    f"video_grid_thw declares more frames than the "
+                    f"<|video_pad|> blocks contain."
+                )
+
+            llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
+            if attention_mask is not None:
+                position_ids[:, batch_idx, attention_mask[batch_idx].bool()] = (
+                    llm_positions.to(position_ids.device)
+                )
+            else:
+                position_ids[:, batch_idx] = llm_positions.to(position_ids.device)
+
+            mrope_position_deltas.append(
+                llm_positions.max() + 1 - len(current_input_ids)
+            )
+
+        mrope_position_deltas = torch.tensor(
+            mrope_position_deltas, device=device
+        ).unsqueeze(1)
+        return position_ids, mrope_position_deltas
+
+    def _build_text_position_ids(
+        self,
+        input_ids: torch.LongTensor,
+        attention_mask: Optional[torch.Tensor] = None,
+    ) -> torch.LongTensor:
+        """
+        Build text position ids of shape (B, S).
+        For padding mask, positions are 0,1,2,... on valid tokens.
+        Padding positions stay 0.
+        """
+        batch_size, seq_len = input_ids.shape
+        device = input_ids.device
+
+        if attention_mask is not None:
+            valid = attention_mask > 0
+            text_position_ids = valid.long().cumsum(-1) - 1
+            text_position_ids = text_position_ids.masked_fill(~valid, 0)
+        else:
+            text_position_ids = torch.arange(
+                seq_len, device=device, dtype=torch.long
+            ).unsqueeze(0).expand(batch_size, -1)
+
+        return text_position_ids.contiguous()
+    def _prepend_text_position_channel(
+        self,
+        input_ids: torch.LongTensor,
+        vision_position_ids: torch.LongTensor,
+        attention_mask: Optional[torch.Tensor] = None,
+    ) -> torch.LongTensor:
+        """
+        Convert vision M-RoPE position ids from (3, B, S) to Qwen3.5-compatible
+        position ids of shape (4, B, S):
+
+        channel 0   : text positions, used for causal mask / FA2 varlen logic
+        channel 1-3 : temporal / height / width vision M-RoPE positions
+        """
+        if vision_position_ids is None:
+            return None
+
+        if vision_position_ids.dim() == 3 and vision_position_ids.shape[0] == 4:
+            return vision_position_ids.contiguous()
+
+        assert vision_position_ids.dim() == 3 and vision_position_ids.shape[0] == 3, (
+            f"Expected vision_position_ids shape (3, B, S), got "
+            f"{tuple(vision_position_ids.shape)}"
+        )
+
+        text_position_ids = self._build_text_position_ids(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+        ).to(device=vision_position_ids.device)
+
+        position_ids = torch.cat(
+            [
+                text_position_ids.unsqueeze(0),   # (1, B, S)
+                vision_position_ids,              # (3, B, S)
+            ],
+            dim=0,
+        )
+        return position_ids.contiguous()
+
+    def _compute_position_ids(
+        self,
+        input_ids: Optional[torch.LongTensor],
+        inputs_embeds: torch.FloatTensor,
+        image_grid_thw: Optional[torch.LongTensor],
+        video_grid_thw: Optional[torch.LongTensor],
+        attention_mask: Optional[torch.Tensor],
+        past_key_values=None,
+        mm_token_type_ids: Optional[torch.IntTensor] = None,
+        use_cache: Optional[bool] = None,
+    ) -> Optional[torch.Tensor]:
+        """
+        Mirror of ``Qwen3_5Model.compute_3d_position_ids``.
+
+        - Vision info available + first forward → ``get_rope_index``, cache
+          ``rope_deltas``.
+        - ``rope_deltas`` cached (decode step) → derive from attention_mask +
+          ``rope_deltas``.
+        - Pure text → return ``None`` (``Qwen3_5TextModel`` auto-generates).
+        """
+        past_length = 0
+        if past_key_values is not None:
+            past_length = past_key_values.get_seq_length()
+
+        can_compute = (
+            input_ids is not None
+            and mm_token_type_ids is not None
+            and (image_grid_thw is not None or video_grid_thw is not None)
+        )
+
+        if can_compute and past_length == 0:
+            vision_position_ids, rope_deltas = self.get_rope_index(
+                input_ids,
+                mm_token_type_ids=mm_token_type_ids,
+                image_grid_thw=image_grid_thw,
+                video_grid_thw=video_grid_thw,
+                attention_mask=attention_mask,
+            )
+
+            # Training / log-prob forward should not keep rope_deltas across batches.
+            # Generation prefill can keep it for decode.
+            if use_cache:
+                self.rope_deltas = rope_deltas
+            else:
+                self.rope_deltas = None
+
+            return self._prepend_text_position_channel(
+                input_ids=input_ids,
+                vision_position_ids=vision_position_ids,
+                attention_mask=attention_mask,
+            )
+
+        elif self.rope_deltas is not None and past_length != 0:
+            batch_size, seq_length = inputs_embeds.shape[:2]
+
+            if attention_mask is not None:
+                text_position_ids = attention_mask.long().cumsum(-1) - 1
+                text_position_ids = text_position_ids.masked_fill(attention_mask == 0, 0)
+                text_position_ids = text_position_ids[:, -seq_length:]
+            else:
+                text_position_ids = torch.arange(
+                    past_length,
+                    past_length + seq_length,
+                    device=inputs_embeds.device,
+                    dtype=torch.long,
+                ).unsqueeze(0).expand(batch_size, -1)
+
+            delta = self.rope_deltas.repeat_interleave(
+                batch_size // self.rope_deltas.shape[0], dim=0
+            ).to(device=inputs_embeds.device)
+
+            # Decode step follows generation convention: (1, B, S)
+            position_ids = text_position_ids.unsqueeze(0) + delta.view(1, batch_size, 1)
+            return position_ids.contiguous()
+
+        return None
+
+    # ─────────────────────────────────────────────────────────────────────────
+    # Forward
+    # ─────────────────────────────────────────────────────────────────────────
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        pixel_values_videos: Optional[torch.FloatTensor] = None,
+        num_patches = None,
+        image_flags: Optional[torch.LongTensor] = None,
+        image_grid_thw: Optional[torch.LongTensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
+        mm_token_type_ids: Optional[torch.IntTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values=None,
+        labels: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        """
+        Forward pass for training and generation steps.
+
+        Args:
+            input_ids: ``(B, S)`` token IDs.
+            pixel_values: ``(total_tiles, C, H, W)`` image tiles from C-RADIOv4-H.
+            pixel_values_videos: ``(total_frames, C, H, W)`` video frames.
+            image_flags: ``(B, max_tiles)`` — 1 for real tiles, 0 for padding.
+            image_grid_thw: ``(num_images, 3)`` — ``(T=1, tile_rows, tile_cols)``
+                per image.  Required for correct M-RoPE spatial positions.
+            video_grid_thw: ``(num_videos, 3)`` — ``(num_frames, 1, 1)`` per video.
+            mm_token_type_ids: ``(B, S)`` modality labels —
+                0 = text, 1 = image, 2 = video.  Required for computing
+                3D M-RoPE position IDs.  Produced by the processor.
+            attention_mask: ``(B, S)`` binary mask.  Must be 2-D; the
+                ``Qwen3_5TextModel`` internally creates the 4-D causal mask
+                for full-attention layers and the 2-D mask for DeltaNet layers.
+            position_ids: ``(3, B, S)`` or ``None``.  If ``None`` and vision
+                tokens are present, computed via ``get_rope_index()``.
+        """
+        return_dict = (
+            return_dict if return_dict is not None
+            else self.config.use_return_dict
+        )
+
+        # ── Embed tokens ─────────────────────────────────────────────────────
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        # ── Inject image features ────────────────────────────────────────────
+        if pixel_values is not None:
+            if image_flags is None:
+                image_flags = torch.ones(
+                    pixel_values.shape[0], dtype=torch.long,
+                    device=pixel_values.device,
+                )
+            image_flags_sq = image_flags.squeeze(-1)
+            vit_embeds = self.extract_feature(pixel_values)
+            vit_embeds = vit_embeds[image_flags_sq == 1]
+            del pixel_values
+
+            B, N, C = inputs_embeds.shape
+            flat = inputs_embeds.reshape(B * N, C)
+            ids_flat = input_ids.reshape(B * N)
+            selected = ids_flat == self.img_context_token_id
+
+            try:
+                flat[selected] = flat[selected] * 0.0 + vit_embeds.reshape(-1, C)
+            except Exception as e:
+                vit_flat = vit_embeds.reshape(-1, C)
+                logger.warning(
+                    f"Image injection shape mismatch: {e}. "
+                    f"selected={selected.sum()}, vit={vit_flat.shape}"
+                )
+                n_tok = selected.sum()
+                flat[selected] = flat[selected] * 0.0 + vit_flat[:n_tok]
+            del vit_embeds
+            inputs_embeds = flat.reshape(B, N, C)
+
+        # ── Inject video features ────────────────────────────────────────────
+        if pixel_values_videos is not None:
+            video_vit = self.extract_feature(pixel_values_videos)
+            del pixel_values_videos
+
+            B, N, C = inputs_embeds.shape
+            flat = inputs_embeds.reshape(B * N, C)
+            ids_flat = input_ids.reshape(B * N)
+            vmask = ids_flat == self.video_context_token_id
+
+            flat[vmask] = (
+                flat[vmask] * 0.0
+                + video_vit.reshape(-1, C).to(flat.device, flat.dtype)
+            )
+            inputs_embeds = flat.reshape(B, N, C)
+
+            del video_vit
+
+        # GRPO actor/ref training and log-prob computation should not use cache.
+        if labels is not None:
+            use_cache = False
+            self.rope_deltas = None
+
+        # ── 3D position IDs ──────────────────────────────────────────────────
+        if position_ids is None:
+            position_ids = self._compute_position_ids(
+                input_ids=input_ids,
+                inputs_embeds=inputs_embeds,
+                image_grid_thw=image_grid_thw,
+                video_grid_thw=video_grid_thw,
+                attention_mask=attention_mask,
+                past_key_values=past_key_values,
+                mm_token_type_ids=mm_token_type_ids,
+                use_cache=use_cache,
+            )
+
+            if position_ids is not None:
+                position_ids = position_ids.contiguous()
+
+        # ── LLM forward ─────────────────────────────────────────────────────
+        outputs = self.language_model(
+            input_ids=None,
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            cache_position=cache_position,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        logits = outputs.logits
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(
+                -1, self.language_model.config.vocab_size
+            )
+            shift_labels = shift_labels.view(-1).to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    # ─────────────────────────────────────────────────────────────────────────
+    # GenerationMixin overrides
+    # ─────────────────────────────────────────────────────────────────────────
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        cache_position=None,
+        position_ids=None,
+        use_cache=True,
+        pixel_values=None,
+        pixel_values_videos=None,
+        num_patches=None,
+        image_flags=None,
+        image_grid_thw=None,
+        video_grid_thw=None,
+        mm_token_type_ids=None,
+        is_first_iteration=False,
+        **kwargs,
+    ):
+        """
+        Prepare inputs for each generation step.
+
+        After the first iteration, ``pixel_values`` / ``pixel_values_videos``
+        are cleared because vision features are already in the KV cache.
+        """
+        model_inputs = super().prepare_inputs_for_generation(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            position_ids=position_ids,
+            pixel_values=pixel_values,
+            pixel_values_videos=pixel_values_videos,
+            num_patches=num_patches,
+            image_flags=image_flags,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+            mm_token_type_ids=mm_token_type_ids,
+            use_cache=use_cache,
+            is_first_iteration=is_first_iteration,
+            **kwargs,
+        )
+
+        if not is_first_iteration and use_cache:
+            model_inputs["pixel_values"] = None
+            model_inputs["pixel_values_videos"] = None
+
+        return model_inputs
+
+    def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs):
+        """
+        Override to compute 3D M-RoPE position IDs during generation.
+
+        Mirrors ``Qwen3_5ForConditionalGeneration._prepare_position_ids_for_generation``:
+        - Prefill step: compute 3D positions via ``get_rope_index``, cache
+          ``rope_deltas``.
+        - Decode steps: apply cached ``rope_deltas`` to sequential text positions.
+
+        Returns position_ids of shape ``(4, B, S)`` on the prefill step
+        (text + 3D vision channels) or ``(1, B, S)`` on decode steps
+        (text + rope_deltas).
+        When ``Qwen3_5TextModel`` receives ``shape[0]==4``, it splits into
+        ``text_position_ids = [0]`` (for causal mask) and
+        ``position_ids = [1:]`` (for rotary embedding).
+        When ``shape[0]!=4``, it sets ``text_position_ids=None``.
+        """
+        text_positions = super()._prepare_position_ids_for_generation(
+            inputs_tensor, model_kwargs
+        )
+
+        # Decode step — apply rope_deltas
+        past_length = 0
+        cache = model_kwargs.get("past_key_values")
+        if cache is not None:
+            past_length = cache.get_seq_length()
+        if past_length != 0 and self.rope_deltas is not None:
+            position_ids = text_positions[None, ...] + self.rope_deltas
+            return position_ids
+
+        # Prefill step — compute 3D vision positions
+        if "input_ids" in model_kwargs and model_kwargs["input_ids"].shape[1] > 0:
+            inputs_tensor = model_kwargs["input_ids"]
+
+        is_input_ids = (
+            len(inputs_tensor.shape) == 2
+            and inputs_tensor.dtype in [torch.int, torch.long]
+        )
+        has_vision = (
+            model_kwargs.get("mm_token_type_ids") is not None
+            and (
+                model_kwargs.get("image_grid_thw") is not None
+                or model_kwargs.get("video_grid_thw") is not None
+            )
+        )
+
+        if is_input_ids and has_vision:
+            vision_positions, rope_deltas = self.get_rope_index(
+                inputs_tensor,
+                mm_token_type_ids=model_kwargs.get("mm_token_type_ids"),
+                image_grid_thw=model_kwargs.get("image_grid_thw"),
+                video_grid_thw=model_kwargs.get("video_grid_thw"),
+                attention_mask=model_kwargs.get("attention_mask"),
+            )
+            self.rope_deltas = rope_deltas
+        else:
+            vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1)
+            self.rope_deltas = torch.zeros(
+                inputs_tensor.shape[0], 1,
+                dtype=torch.long, device=inputs_tensor.device,
+            )
+
+        # Concatenate text + vision → (4, B, S)
+        # Channel 0 = text positions   → used by create_causal_mask
+        # Channels 1-3 = vision positions → used by rotary embedding
+        # This matches Qwen3_5ForConditionalGeneration's convention.
+        text_positions = text_positions[None, ...]  # (1, B, S)
+        position_ids = torch.cat(
+            [text_positions, vision_positions], dim=0
+        )  # (4, B, S)
+        #print(f"{position_ids.permute(1, 2, 0).cpu().tolist()}")
+        return position_ids

preprocessor_config.json

@@ -0,0 +1,25 @@
+{
+  "auto_map": {
+    "AutoImageProcessor": "image_processing.ZDTaichu5_0_ImageProcessor",
+    "AutoProcessor": "processing.ZDTaichu5_0_Processor"
+  },
+  "data_format": "channels_first",
+  "do_rescale": true,
+  "image_processor_type": "ZDTaichu5_0_ImageProcessor",
+  "image_size": 512,
+  "max_num_tiles": 12,
+  "merge_size": 1,
+  "norm_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "norm_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "num_image_token": 256,
+  "rescale_factor": 0.00392156862745098,
+  "use_thumbnail": true
+}

processing.py

@@ -0,0 +1,530 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Optional, Union, List
+
+import numpy as np
+import torch
+
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ImageInput
+from transformers.processing_utils import ImagesKwargs, MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack, VideosKwargs
+from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
+from transformers.video_utils import VideoInput
+
+
+class ZDTaichu5_0_ImagesKwargs(ImagesKwargs):
+    min_pixels: Optional[int]
+    max_pixels: Optional[int]
+    patch_size: Optional[int]
+    temporal_patch_size: Optional[int]
+    merge_size: Optional[int]
+
+
+class ZDTaichu5_0_ProcessorKwargs(ProcessingKwargs, total=False):
+    images_kwargs: ZDTaichu5_0_ImagesKwargs
+    videos_kwargs: VideosKwargs
+    _defaults = {
+        "text_kwargs": {
+            "padding": False,
+        },
+    }
+
+
+class ZDTaichu5_0_Processor(ProcessorMixin):
+    r"""
+    Constructs a ZDTaichu-5.0 processor which wraps an image processor and a tokenizer into a single processor.
+    [`ZDTaichu5_0_Processor`] offers all the functionalities of the image processor and tokenizer. See the
+    [`~ZDTaichu5_0_Processor.__call__`] and [`~ZDTaichu5_0_Processor.decode`] for more information.
+    Args:
+        image_processor ([`AutoImageProcessor`], *optional*):
+            The image processor is a required input.
+        tokenizer ([`AutoTokenizer`], *optional*):
+            The tokenizer is a required input.
+        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
+            in a chat into a tokenizable string.
+    """
+
+    attributes = ["image_processor", "tokenizer"]
+
+    image_processor_class = "AutoImageProcessor"
+    video_processor_class = "AutoVideoProcessor"
+    tokenizer_class = ("AutoTokenizer")
+
+    def __init__(self, image_processor=None, tokenizer=None, chat_template=None, **kwargs):
+        # Defaults to Qwen3's built-in vision tokens; overridden by tokenizer_config.json attributes.
+        self.image_token       = getattr(tokenizer, "image_token",       "<|image_pad|>")
+        self.video_token       = getattr(tokenizer, "video_token",       "<|video_pad|>")
+        self.image_start_token = getattr(tokenizer, "image_start_token", "<|vision_start|>")
+        self.image_end_token   = getattr(tokenizer, "image_end_token",   "<|vision_end|>")
+        self.image_token_id = (
+            tokenizer.image_token_id
+            if getattr(tokenizer, "image_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.image_token)
+        )
+        self.video_token_id = (
+            tokenizer.video_token_id
+            if getattr(tokenizer, "video_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.video_token)
+        )
+        super().__init__(image_processor, tokenizer, chat_template=chat_template)
+
+    def __call__(
+        self,
+        images: ImageInput = None,
+        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
+        videos: VideoInput = None,
+        **kwargs: Unpack[ZDTaichu5_0_ProcessorKwargs],
+    ) -> BatchFeature:
+        """
+        Main method to prepare multimodal inputs (text, images, videos) for the model. This method processes text by
+        replacing image/video tokens with appropriate placeholder sequences, processes images and videos through the
+        image processor, and tokenizes the final text.
+
+        Video-as-multi-image convention
+        ───────────────────────────────
+        Videos are NOT processed as a separate temporal stream. Each frame is
+        passed through the *image* pipeline with `max_num_tiles=1`, producing
+        one 512x512 tile per frame, and the frame tile tensors are then
+        APPENDED to the image stream (`pixel_values` / `num_patches` /
+        `image_grid_thw`). The downstream model therefore sees a single
+        uniform image batch with no separate video path.
+
+        In the rendered prompt every frame is wrapped in
+        `<|vision_start|> ... <|image_pad|> ... <|vision_end|>` — *image*
+        tokens, not video tokens — and prefaced by a per-frame
+        "Frame N sampled at T.TT seconds:" header. After tokenisation
+        `mm_token_type_ids` therefore has no type=2 entries.
+
+        The method performs the following key operations:
+        1. Processes images using the image processor to get pixel values and patch counts
+        2. Processes videos as multi-image (max_num_tiles=1) and appends frame data
+           into the same pixel_values / num_patches / image_grid_thw containers
+        3. Replaces `<|image_pad|>` tokens in text with `<|vision_start|>` + image tokens + `<|vision_end|>` sequences
+        4. Replaces `<|video_pad|>` tokens in text with frame-by-frame descriptions including timestamps (if metadata provided)
+        5. Tokenizes the processed text and combines all outputs
+
+        Args:
+            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
+                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
+                tensor. Both channels-first and channels-last formats are supported.
+            text (`str`, `List[str]`, *optional*):
+                The sequence or batch of sequences to be encoded. Each sequence should be a string. The text can contain
+                special tokens `<|image_pad|>` and `<|video_pad|>` that will be replaced with appropriate token sequences.
+            videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
+                The video or batch of videos to be prepared. Each video should be a 4D NumPy array or PyTorch
+                tensor with shape (num_frames, channels, height, width). Both channels-first and channels-last formats
+                are supported. Note: Currently only supports batch size of 1 for videos.
+            images_kwargs (`Dict`, *optional*):
+                Additional keyword arguments for image processing, including:
+                - `min_pixels` (`int`, *optional*): Minimum number of pixels for image processing
+                - `max_pixels` (`int`, *optional*): Maximum number of pixels for image processing
+                - `patch_size` (`int`, *optional*): Size of patches for image processing
+                - `temporal_patch_size` (`int`, *optional*): Size of temporal patches
+                - `merge_size` (`int`, *optional*): Size for merging patches
+            videos_kwargs (`Dict`, *optional*):
+                Additional keyword arguments for video processing, including:
+                - `video_metadata` (`VideoMetadata`, *optional*): Metadata containing fps information for timestamp calculation
+            text_kwargs (`Dict`, *optional*):
+                Additional keyword arguments for text tokenization, including:
+                - `return_tensors` (`str` or [`~utils.TensorType`], *optional*): Framework for returned tensors ('tf', 'pt', 'np', 'jax')
+                - `padding` (`bool`, *optional*): Whether to pad sequences (defaults to False)
+
+        Returns:
+            [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model.
+            - **pixel_values** -- Concatenated tile pixel values from BOTH real images and video frames,
+              in the order they appear in `input_ids` (real images first, then video frames). Returned
+              when `images` is not `None` or `videos` is not `None`.
+            - **num_patches** -- List of tile counts, one entry per real image followed by one entry per
+              video frame (frame entries are always 1 because max_num_tiles=1).
+            - **image_grid_thw** -- LongTensor[N_images + N_frames, 3] with [1, tile_rows, tile_cols] per
+              real image and [1, 1, 1] per video frame.
+            - **mm_token_type_ids** -- Per-token modality classification (0=text, 1=image incl. frames).
+
+        Raises:
+            AssertionError: If videos are provided with batch size > 1 (not currently supported).
+
+        Note:
+            - Image tokens `<|image_pad|>` in text are replaced with `<|vision_start|>` + repeated image tokens + `<|vision_end|>`
+            - Video tokens `<|video_pad|>` in text are replaced with frame-by-frame descriptions, each frame using `<|image_pad|>` slots
+            - When video metadata with fps is provided, frame descriptions include timestamps
+            - Videos are processed with max_num_tiles=1 regardless of the images setting
+        """
+        output_kwargs = self._merge_kwargs(
+            ZDTaichu5_0_ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        # Initialise as independent dicts so later `**image_inputs` merging
+        # is well-defined whether or not images / videos are provided.
+        image_inputs: dict = {}
+        image_grid_thw = None
+        # Frame counts default to empty so the video-text-expansion loop is a
+        # no-op when `videos` is None.
+        video_num_patches: list = []
+
+        if images is not None:
+            image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])
+            image_num_patches = image_inputs["num_patches"]
+            # image_grid_thw: list of [T=1, tile_rows, tile_cols] per image
+            image_grid_thw = image_inputs.pop("image_grid_thw")
+            image_pixel_values = image_inputs["pixel_values"]
+        else:
+            image_num_patches = []
+
+        if videos is not None:
+            # ── Multi-image treatment of video ─────────────────────────────────
+            # Every video frame is processed by the *image* pipeline with
+            # max_num_tiles=1 so that one frame = one 512x512 tile = num_image_token
+            # (e.g. 256) tokens. Frame tile tensors are then APPENDED to the
+            # real-image stream:
+            #
+            #   pixel_values     : torch.cat([images, frames])         (total_tiles, C, H, W)
+            #   num_patches      : image_num_patches + [1] * N_frames  List[int]
+            #   image_grid_thw   : torch.cat([image_grids, frame_grids], dim=0)
+            #
+            # Order matters: text expansion below replaces image tokens
+            # before video tokens, so frame slots come *after* real-image
+            # slots in input_ids — these tensors must follow the same order.
+            #
+            # In the rendered prompt every frame is wrapped in
+            #   <|vision_start|> ... <|image_pad|> x num_image_token ... <|vision_end|>
+            # (image tokens, NOT video tokens) and prefaced by a per-frame
+            # "Frame N sampled at T.TT seconds:" header. After tokenisation
+            # mm_token_type_ids therefore has *no* type=2 entries — every
+            # visual slot is type=1. The downstream model sees a single
+            # uniform image stream and does not need a separate video path.
+            orig_tiles = self.image_processor.max_num_tiles
+            self.image_processor.max_num_tiles = 1
+            try:
+                frame_inputs = self.image_processor(
+                    images=videos, **output_kwargs["images_kwargs"]
+                )
+            finally:
+                self.image_processor.max_num_tiles = orig_tiles
+
+            frame_pixel_values = frame_inputs["pixel_values"]   # (N_frames, C, H, W)
+            frame_num_patches  = list(frame_inputs["num_patches"])
+            frame_grid_thw     = frame_inputs["image_grid_thw"]  # (N_frames, 3) list/tensor
+            video_num_patches  = frame_num_patches               # for text expansion below
+
+            # Normalise grid containers to LongTensor so torch.cat works
+            # whether the image processor returned lists or tensors.
+            def _to_long_tensor(x):
+                return x if isinstance(x, torch.Tensor) else torch.tensor(x, dtype=torch.long)
+
+            if image_inputs:
+                # Real images + video frames — concat along batch dim.
+                image_inputs["pixel_values"] = torch.cat(
+                    [image_inputs["pixel_values"], frame_pixel_values], dim=0
+                )
+                image_inputs["num_patches"] = (
+                    list(image_inputs["num_patches"]) + frame_num_patches
+                )
+                image_grid_thw = torch.cat(
+                    [_to_long_tensor(image_grid_thw), _to_long_tensor(frame_grid_thw)],
+                    dim=0,
+                )
+                image_num_patches = image_inputs["num_patches"]
+            else:
+                # Video-only — frames become the entire image stream.
+                image_inputs = {
+                    "pixel_values": frame_pixel_values,
+                    "num_patches": frame_num_patches,
+                }
+                image_grid_thw = _to_long_tensor(frame_grid_thw)
+                image_num_patches = frame_num_patches
+
+        if not isinstance(text, list):
+            text = [text]
+        final_image_pixel_values = []
+        final_image_num_patches = []
+        final_image_grid_thw = []
+        text = text.copy()  # below lines change text in-place
+        if images is not None:
+            index = 0
+            wrapped_token = self.image_start_token + self.image_token + self.image_end_token
+
+            for i in range(len(text)):
+                while self.image_token in text[i]:
+                    expansion = (
+                        self.image_start_token
+                        + "<|placeholder|>" * image_num_patches[index] * self.image_processor.num_image_token
+                        + self.image_end_token
+                    )
+                    # If the chat template already wrapped it, replace the whole
+                    # <vision_start><image_pad><vision_end> span — avoids double wrapping.
+                    # Otherwise fall back to replacing the bare <image_pad> token.
+                    search = wrapped_token if wrapped_token in text[i] else self.image_token
+                    text[i] = text[i].replace(search, expansion, 1)
+                    index += 1
+                    #final_image_pixel_values.append(image_pixel_values[index])
+                    #final_image_num_patches.append(i)
+                text[i] = text[i].replace("<|placeholder|>", self.image_token)
+        if videos is not None:
+            assert len(text) == 1, "Video is not supported for batch size > 1"
+            video_metadata = output_kwargs.get("videos_kwargs", {}).get("video_metadata", None)
+            i = 0
+            wrapped_token = self.image_start_token + self.video_token + self.image_end_token
+            if self.video_token in text[i]:
+                each_frame = (
+                    self.image_start_token
+                    + "<|placeholder|>" * self.image_processor.num_image_token
+                    + self.image_end_token
+                )
+                video_prompt = "This is a video:\n"
+                # One iteration per frame. video_num_patches has length N_frames
+                # (always 1 per frame because max_num_tiles=1 was forced above),
+                # so its length is the authoritative frame count even when
+                # `images` is None and `image_num_patches` is unset.
+                n_frames = len(video_num_patches)
+                for j in range(n_frames):
+                    if video_metadata is not None and video_metadata.fps is not None:
+                        timestamp = j / video_metadata.fps
+                        video_prompt += f"Frame {j+1} sampled at {timestamp:.2f} seconds: {each_frame}\n"
+                    else:
+                        # Fallback to original format without timestamps
+                        video_prompt += f"Frame {j+1}: {each_frame}\n"
+                # Strip the chat-template-applied <|vision_start|>...<|vision_end|>
+                # wrapping if present; otherwise replace the bare <|video_pad|>
+                # token. The fallback is video_token (NOT image_token), since by
+                # this point image expansion has already consumed every
+                # <|image_pad|> in the prompt.
+                search = wrapped_token if wrapped_token in text[i] else self.video_token
+                text[i] = text[i].replace(search, video_prompt, 1)
+            text[i] = text[i].replace("<|placeholder|>", self.image_token)
+
+        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
+        text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+
+        # ── Build mm_token_type_ids from tokenized input_ids ─────────────
+        # 0 = text, 1 = image. type=2 (video) is unreachable under the
+        # multi-image-as-video convention because every frame gets expanded
+        # to <|image_pad|> tokens — but we keep the video_token branch as a
+        # defensive fallback for any tokenizer-injected video_pad token.
+        input_ids = text_inputs["input_ids"]
+        if isinstance(input_ids, list):
+            mm_token_type_ids = []
+            for ids in input_ids:
+                tt = [0] * len(ids)
+                for j, tok_id in enumerate(ids):
+                    if tok_id == self.image_token_id:
+                        tt[j] = 1
+                    elif tok_id == self.video_token_id:
+                        tt[j] = 2
+                mm_token_type_ids.append(tt)
+        else:
+            # Already a tensor (when return_tensors is set before tokenizer call)
+            mm_token_type_ids = torch.zeros_like(input_ids)
+            mm_token_type_ids[input_ids == self.image_token_id] = 1
+            mm_token_type_ids[input_ids == self.video_token_id] = 2
+
+        # ── Assemble output ──────────────────────────────────────────────
+        # Note: video frames have already been merged into image_inputs above,
+        # so there are no separate `pixel_values_videos` / `video_grid_thw`
+        # outputs. Downstream code consumes a single image stream.
+        data = {**text_inputs, **image_inputs}
+        data["mm_token_type_ids"] = mm_token_type_ids
+        if image_grid_thw is not None:
+            data["image_grid_thw"] = image_grid_thw
+
+        return BatchFeature(data=data, tensor_type=return_tensors)
+
+    def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
+        """
+        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.
+        Args:
+            image_sizes (`list[list[int]]`, *optional*):
+                The input sizes formatted as (height, width) per each image.
+            video_sizes (`list[list[int]]`, *optional*):
+                The input sizes formatted as (num_frames, height, width) per each video.
+        Returns:
+            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
+            input modalities, along with other useful data.
+        """
+
+        vision_data = {}
+        if image_sizes is not None:
+            images_kwargs = ZDTaichu5_0_ProcessorKwargs._defaults.get("images_kwargs", {})
+            images_kwargs.update(kwargs)
+            merge_size = images_kwargs.get("merge_size", None) or self.image_processor.merge_size
+
+            num_image_patches = [
+                self.image_processor.get_number_of_image_patches(*image_size, images_kwargs)
+                for image_size in image_sizes
+            ]
+            num_image_tokens = [(num_patches // merge_size**2) for num_patches in num_image_patches]
+            vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches})
+        return MultiModalData(**vision_data)
+
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to the tokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to the tokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    def post_process_image_text_to_text(
+        self, generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False, **kwargs
+    ):
+        """
+        Post-process the output of the model to decode the text.
+
+        Args:
+            generated_outputs (`torch.Tensor` or `np.ndarray`):
+                The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
+                or `(sequence_length,)`.
+            skip_special_tokens (`bool`, *optional*, defaults to `True`):
+                Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
+            clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
+                Whether or not to clean up the tokenization spaces. Argument passed to the tokenizer's `batch_decode` method.
+            **kwargs:
+                Additional arguments to be passed to the tokenizer's `batch_decode method`.
+
+        Returns:
+            `list[str]`: The decoded text.
+        """
+        return self.tokenizer.batch_decode(
+            generated_outputs,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+    @property
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        image_processor_input_names = self.image_processor.model_input_names
+        names_from_processor = list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
+        # Note: video_grid_thw is NOT emitted under multi-image-as-video —
+        # frame data lives in image_grid_thw alongside real images.
+        return names_from_processor + ["mm_token_type_ids"]
+
+
+    def from_messages(
+        self,
+        messages: list,
+        return_tensors: str = "pt",
+        add_vision_id: bool = True,
+        **kwargs,
+    ) -> BatchFeature:
+        """
+        Prepare model inputs directly from Qwen-style structured messages.
+
+        This is the high-level entry point that handles the full pipeline:
+        structured messages → vision loading → chat template → tokenization.
+
+        Supports messages with typed content lists::
+
+            messages = [
+                {"role": "user", "content": [
+                    {"type": "image", "image": "photo.jpg"},
+                    {"type": "text", "text": "What's in this image?"},
+                ]},
+            ]
+
+        Video inputs (file path, URL, or list of frame paths)::
+
+            messages = [
+                {"role": "user", "content": [
+                    {"type": "video", "video": "clip.mp4", "fps": 2.0},
+                    {"type": "text", "text": "Describe this video."},
+                ]},
+            ]
+
+        Multi-image with automatic labelling::
+
+            messages = [
+                {"role": "user", "content": [
+                    {"type": "image", "image": "a.jpg"},
+                    {"type": "image", "image": "b.jpg"},
+                    {"type": "text", "text": "Compare them."},
+                ]},
+            ]
+            # With add_vision_id=True (default), the prompt includes:
+            #   Picture 1: <|vision_start|><|image_pad|><|vision_end|>
+            #   Picture 2: <|vision_start|><|image_pad|><|vision_end|>
+            #   Compare them.
+
+        Args:
+            messages: List of message dicts with structured ``content``.
+            return_tensors: Framework for returned tensors (default ``"pt"``).
+            add_vision_id: If ``True`` (default), the chat template prepends
+                ``Picture N:`` / ``Video N:`` labels before each vision token.
+                Set to ``False`` to omit labels.
+            **kwargs: Forwarded to ``self.__call__``.
+
+        Returns:
+            ``BatchFeature`` ready for ``model.generate(**inputs)``.
+        """
+        from .vision_utils import process_vision_info
+
+        # 1. Load images and videos from the structured messages
+        image_inputs, video_inputs, video_kwargs = process_vision_info(messages)
+
+        # 2. Apply chat template — pass structured messages directly so the
+        #    template can iterate typed content dicts, count vision elements,
+        #    and emit "Picture N:" / "Video N:" labels when add_vision_id=True.
+        prompt = self.tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+            add_vision_id=add_vision_id,
+        )
+
+        # 3. Prepare video inputs and metadata for timestamps
+        videos_kwargs = {}
+        flat_videos = None
+        if video_inputs is not None:
+            if len(video_inputs) > 1:
+                raise ValueError(
+                    "Multiple videos in a single message batch are not yet "
+                    "supported. Please use one video per call."
+                )
+            flat_videos = video_inputs[0]  # List[Image.Image]
+
+            if video_kwargs.get("metadata_list"):
+                meta = video_kwargs["metadata_list"][0]
+                fps = meta.get("sample_fps") or meta.get("fps")
+                if fps:
+                    from transformers.video_utils import VideoMetadata
+                    videos_kwargs["video_metadata"] = VideoMetadata(
+                        fps=fps,
+                        total_num_frames=len(flat_videos),
+                    )
+
+        return self(
+            images=image_inputs,
+            text=prompt,
+            videos=flat_videos,
+            return_tensors=return_tensors,
+            videos_kwargs=videos_kwargs,
+            **kwargs,
+        )
+
+    @property
+    def ctx_image_token_id(self) -> int:
+        return self.image_token_id
+
+
+__all__ = ["ZDTaichu5_0_Processor"]

processor_config.json

@@ -0,0 +1,31 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing.ZDTaichu5_0_Processor"
+  },
+  "image_processor": {
+    "auto_map": {
+      "AutoImageProcessor": "image_processing.ZDTaichu5_0_ImageProcessor",
+      "AutoProcessor": "processing.ZDTaichu5_0_Processor"
+    },
+    "data_format": "channels_first",
+    "do_rescale": true,
+    "image_processor_type": "ZDTaichu5_0_ImageProcessor",
+    "image_size": 512,
+    "max_num_tiles": 12,
+    "merge_size": 1,
+    "norm_mean": [
+      0.485,
+      0.456,
+      0.406
+    ],
+    "norm_std": [
+      0.229,
+      0.224,
+      0.225
+    ],
+    "num_image_token": 256,
+    "rescale_factor": 0.00392156862745098,
+    "use_thumbnail": true
+  },
+  "processor_class": "ZDTaichu5_0_Processor"
+}

recipe.yaml

@@ -0,0 +1,94 @@
+default_stage:
+  default_modifiers:
+    IMatrixGatherer:
+      targets: ['re:.*layers\.(?:0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27)\.mlp\.(?:gate|up|down)_proj$']
+      ignore: ['re:.*mtp.*', 're:.*visual.*', 're:.*vision.*', 're:.*linear_attn\.(in_proj_a|in_proj_b)$']
+      weight_observer: imatrix_mse
+    QuantizationModifier:
+      config_groups:
+        fp8_group:
+          targets: ['re:.*self_attn\.(q|k|v|o)_proj$', 're:.*linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$',
+            're:.*lm_head$', 're:.*layers\.(?:28|29|30|31)\.mlp\.(?:gate|up|down)_proj$']
+          weights:
+            num_bits: 8
+            type: float
+            symmetric: true
+            group_size: null
+            strategy: channel
+            block_structure: null
+            dynamic: false
+            actorder: null
+            scale_dtype: null
+            zp_dtype: null
+            observer: memoryless_minmax
+            observer_kwargs: {}
+          input_activations:
+            num_bits: 8
+            type: float
+            symmetric: true
+            group_size: null
+            strategy: token
+            block_structure: null
+            dynamic: true
+            actorder: null
+            scale_dtype: null
+            zp_dtype: null
+            observer: null
+            observer_kwargs: {}
+          output_activations: null
+          format: null
+      targets: [Linear]
+      ignore: ['re:.*mtp.*', 're:.*visual.*', 're:.*vision.*', 're:.*linear_attn\.(in_proj_a|in_proj_b)$']
+      kv_cache_scheme:
+        num_bits: 8
+        type: float
+        symmetric: true
+        group_size: null
+        strategy: tensor
+        block_structure: null
+        dynamic: false
+        actorder: null
+        scale_dtype: null
+        zp_dtype: null
+        observer: static_minmax
+        observer_kwargs: {}
+      bypass_divisibility_checks: false
+    GPTQModifier:
+      config_groups:
+        nvfp4_group:
+          targets: ['re:.*layers\.(?:0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27)\.mlp\.(?:gate|up|down)_proj$']
+          weights:
+            num_bits: 4
+            type: float
+            symmetric: true
+            group_size: 16
+            strategy: tensor_group
+            block_structure: null
+            dynamic: false
+            actorder: static
+            scale_dtype: torch.float8_e4m3fn
+            zp_dtype: null
+            observer: imatrix_mse
+            observer_kwargs: {}
+          input_activations:
+            num_bits: 4
+            type: float
+            symmetric: true
+            group_size: 16
+            strategy: tensor_group
+            block_structure: null
+            dynamic: local
+            actorder: null
+            scale_dtype: torch.float8_e4m3fn
+            zp_dtype: null
+            observer: static_minmax
+            observer_kwargs: {}
+          output_activations: null
+          format: null
+      targets: [Linear]
+      ignore: ['re:.*mtp.*', 're:.*visual.*', 're:.*vision.*', 're:.*linear_attn\.(in_proj_a|in_proj_b)$']
+      bypass_divisibility_checks: false
+      block_size: 128
+      dampening_frac: 0.01
+      actorder: static
+      offload_hessians: false

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4
+size 19989343

tokenizer_config.json

@@ -0,0 +1,27 @@
+{
+  "add_prefix_space": false,
+  "auto_map": {
+    "AutoProcessor": "processing.ZDTaichu5_0_Processor"
+  },
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "image_end_token": "<|vision_end|>",
+  "image_start_token": "<|vision_start|>",
+  "image_token": "<|image_pad|>",
+  "image_token_id": 248056,
+  "is_local": true,
+  "model_max_length": 262144,
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "processor_class": "ZDTaichu5_0_Processor",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "video_token_id": 248057,
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

vision_utils.py

@@ -0,0 +1,583 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ============================================================================
+#  Vision utilities for ZDTaichu-5.0
+#
+#  Provides ``process_vision_info()`` to extract images and videos from
+#  Qwen-style structured messages, following the conventions established
+#  by ``qwen_vl_utils``.  This allows the model to accept messages like:
+#
+#      messages = [
+#          {"role": "user", "content": [
+#              {"type": "video", "video": "path/to/video.mp4", "fps": 2.0},
+#              {"type": "text", "text": "Describe this video."},
+#          ]}
+#      ]
+#
+#  Supported input formats:
+#    - Images: local path, ``file://`` URI, ``http(s)://`` URL, base64 data
+#              URI, ``PIL.Image.Image`` object
+#    - Videos: local path, ``file://`` URI, ``http(s)://`` URL (string),
+#              or a list of image paths/URLs (treated as pre-extracted frames)
+#
+#  Video decoding backends (auto-detected, in priority order):
+#    1. decord      — fastest, recommended
+#    2. torchvision — fallback, always available
+#
+#  Frame sampling follows the same ``smart_nframes`` logic as qwen_vl_utils:
+#    - Default: 2 FPS, clamped to [4, 768] frames, rounded to factor of 2
+#    - Override via ``fps``, ``nframes``, ``min_frames``, ``max_frames``
+#    - Temporal trimming via ``video_start`` / ``video_end`` (seconds)
+# ============================================================================
+
+import base64
+import copy
+import logging
+import math
+import os
+import sys
+import time
+import warnings
+from functools import lru_cache
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import requests
+import torch
+from PIL import Image
+
+logger = logging.getLogger(__name__)
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Constants  (aligned with qwen_vl_utils defaults)
+# ─────────────────────────────────────────────────────────────────────────────
+
+FPS = 2.0                   # default sampling rate
+FRAME_FACTOR = 2            # frame count must be divisible by this
+FPS_MIN_FRAMES = 4          # minimum sampled frames
+FPS_MAX_FRAMES = 768        # maximum sampled frames
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Rounding helpers
+# ─────────────────────────────────────────────────────────────────────────────
+
+def round_by_factor(number: float, factor: int) -> int:
+    """Closest integer to *number* divisible by *factor*."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: float, factor: int) -> int:
+    """Smallest integer ≥ *number* divisible by *factor*."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: float, factor: int) -> int:
+    """Largest integer ≤ *number* divisible by *factor*."""
+    return math.floor(number / factor) * factor
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Image loading
+# ─────────────────────────────────────────────────────────────────────────────
+
+def fetch_image(ele: Dict[str, Any]) -> Image.Image:
+    """
+    Load a single image from various sources.
+
+    Supported formats for ``ele["image"]``:
+      - ``PIL.Image.Image`` instance
+      - Local file path (``/path/to/img.jpg``)
+      - ``file://`` URI
+      - ``http://`` or ``https://`` URL
+      - Base64 data URI (``data:image/...;base64,...``)
+
+    Returns:
+        PIL.Image.Image in RGB mode.
+    """
+    image = ele.get("image") or ele.get("image_url")
+    if image is None:
+        raise ValueError("Element must contain 'image' or 'image_url' key")
+
+    image_obj = None
+    if isinstance(image, Image.Image):
+        image_obj = image
+    elif image.startswith("http://") or image.startswith("https://"):
+        with requests.get(image, stream=True, timeout=30) as resp:
+            resp.raise_for_status()
+            image_obj = copy.deepcopy(Image.open(BytesIO(resp.content)))
+    elif image.startswith("file://"):
+        image_obj = Image.open(image[7:])
+    elif image.startswith("data:image"):
+        if "base64," in image:
+            _, b64 = image.split("base64,", 1)
+            image_obj = copy.deepcopy(Image.open(BytesIO(base64.b64decode(b64))))
+    else:
+        # Treat as local file path
+        image_obj = Image.open(image)
+
+    if image_obj is None:
+        raise ValueError(
+            f"Unrecognised image input. Supported: local path, file:// URI, "
+            f"http(s) URL, base64 data URI, PIL.Image. Got: {image!r:.120}"
+        )
+
+    # Convert to RGB
+    if image_obj.mode == "RGBA":
+        bg = Image.new("RGB", image_obj.size, (255, 255, 255))
+        bg.paste(image_obj, mask=image_obj.split()[3])
+        return bg
+    return image_obj.convert("RGB")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Frame sampling
+# ─────────────────────────────────────────────────────────────────────────────
+
+def smart_nframes(
+    ele: Dict[str, Any],
+    total_frames: int,
+    video_fps: float,
+) -> int:
+    """
+    Compute the number of frames to sample from a video.
+
+    Follows the same logic as ``qwen_vl_utils.smart_nframes``:
+      - If ``ele["nframes"]`` is set, use it directly (rounded to FRAME_FACTOR).
+      - Otherwise, sample at ``ele.get("fps", 2.0)`` FPS, clamped to
+        ``[min_frames, max_frames]`` and rounded down to FRAME_FACTOR.
+
+    Args:
+        ele: Dict with optional keys ``fps``, ``nframes``, ``min_frames``,
+             ``max_frames``.
+        total_frames: Total frames in the (possibly trimmed) video.
+        video_fps: Original video FPS.
+
+    Returns:
+        Number of frames to sample.
+    """
+    assert not ("fps" in ele and "nframes" in ele), (
+        "Only accept either `fps` or `nframes`, not both"
+    )
+
+    if "nframes" in ele:
+        nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
+    else:
+        fps = ele.get("fps", FPS)
+        min_frames = ceil_by_factor(
+            ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR
+        )
+        max_frames = floor_by_factor(
+            ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)), FRAME_FACTOR
+        )
+        nframes = total_frames / video_fps * fps
+        if nframes > total_frames:
+            logger.warning(
+                f"smart_nframes: computed nframes ({nframes:.1f}) > "
+                f"total_frames ({total_frames})"
+            )
+        nframes = min(min(max(nframes, min_frames), max_frames), total_frames)
+        nframes = floor_by_factor(nframes, FRAME_FACTOR)
+
+    if not (FRAME_FACTOR <= nframes <= total_frames):
+        raise ValueError(
+            f"nframes should be in [{FRAME_FACTOR}, {total_frames}], "
+            f"got {nframes}."
+        )
+    return nframes
+
+
+def calculate_video_frame_range(
+    ele: Dict[str, Any],
+    total_frames: int,
+    video_fps: float,
+) -> Tuple[int, int, int]:
+    """
+    Calculate start/end frame indices from optional ``video_start``/``video_end``
+    keys (in seconds).
+
+    Returns:
+        (start_frame, end_frame, frame_count)  — end_frame is inclusive.
+    """
+    if video_fps <= 0:
+        raise ValueError("video_fps must be positive")
+    if total_frames <= 0:
+        raise ValueError("total_frames must be positive")
+
+    video_start = ele.get("video_start")
+    video_end = ele.get("video_end")
+
+    if video_start is None and video_end is None:
+        return 0, total_frames - 1, total_frames
+
+    max_duration = total_frames / video_fps
+
+    if video_start is not None:
+        start_sec = max(0.0, min(video_start, max_duration))
+        start_frame = math.ceil(start_sec * video_fps)
+    else:
+        start_frame = 0
+
+    if video_end is not None:
+        end_sec = max(0.0, min(video_end, max_duration))
+        end_frame = min(math.floor(end_sec * video_fps), total_frames - 1)
+    else:
+        end_frame = total_frames - 1
+
+    if start_frame >= end_frame:
+        raise ValueError(
+            f"Invalid time range: start_frame={start_frame} >= end_frame={end_frame}. "
+            f"Video: {max_duration:.2f}s ({total_frames} frames @ {video_fps:.1f}fps)"
+        )
+
+    return start_frame, end_frame, end_frame - start_frame + 1
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Video decoding backends
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _read_video_decord(
+    ele: Dict[str, Any],
+) -> Tuple[torch.Tensor, dict, float]:
+    """Read video with decord. Returns (video_TCHW, metadata, sample_fps)."""
+    import decord
+
+    video_path = ele["video"]
+    if video_path.startswith("file://"):
+        video_path = video_path[7:]
+
+    st = time.time()
+    vr = decord.VideoReader(video_path)
+    total_frames, video_fps = len(vr), vr.get_avg_fps()
+
+    start_frame, end_frame, total_frames = calculate_video_frame_range(
+        ele, total_frames, video_fps
+    )
+    nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
+    idx = torch.linspace(start_frame, end_frame, nframes).round().long().tolist()
+    sample_fps = nframes / max(total_frames, 1e-6) * video_fps
+
+    video = torch.from_numpy(vr.get_batch(idx).asnumpy()).permute(0, 3, 1, 2)  # TCHW
+    logger.info(
+        f"decord: {video_path}, {total_frames} frames, "
+        f"{video_fps:.1f} fps, sampled {nframes}, "
+        f"time={time.time() - st:.3f}s"
+    )
+
+    metadata = dict(
+        fps=video_fps,
+        sample_fps=sample_fps,
+        frames_indices=idx,
+        total_num_frames=total_frames,
+        video_backend="decord",
+    )
+    return video, metadata, sample_fps
+
+
+def _read_video_torchvision(
+    ele: Dict[str, Any],
+) -> Tuple[torch.Tensor, dict, float]:
+    """Read video with torchvision. Returns (video_TCHW, metadata, sample_fps)."""
+    from torchvision import io as tio
+
+    video_path = ele["video"]
+    if video_path.startswith("file://"):
+        video_path = video_path[7:]
+
+    st = time.time()
+    video, _audio, info = tio.read_video(
+        video_path,
+        start_pts=ele.get("video_start", 0.0),
+        end_pts=ele.get("video_end"),
+        pts_unit="sec",
+        output_format="TCHW",
+    )
+    total_frames, video_fps = video.size(0), info["video_fps"]
+
+    nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
+    idx = torch.linspace(0, total_frames - 1, nframes).round().long()
+    sample_fps = nframes / max(total_frames, 1e-6) * video_fps
+    video = video[idx]
+
+    logger.info(
+        f"torchvision: {video_path}, {total_frames} frames, "
+        f"{video_fps:.1f} fps, sampled {nframes}, "
+        f"time={time.time() - st:.3f}s"
+    )
+
+    metadata = dict(
+        fps=video_fps,
+        sample_fps=sample_fps,
+        frames_indices=idx.tolist(),
+        total_num_frames=total_frames,
+        video_backend="torchvision",
+    )
+    return video, metadata, sample_fps
+
+
+def _is_decord_available() -> bool:
+    import importlib.util
+    return importlib.util.find_spec("decord") is not None
+
+
+@lru_cache(maxsize=1)
+def _get_video_backend() -> str:
+    forced = os.getenv("TAICHU_VIDEO_READER")
+    if forced is not None:
+        backend = forced
+    elif _is_decord_available():
+        backend = "decord"
+    else:
+        backend = "torchvision"
+    print(
+        f"ZDTaichu-5.0 utilities using {backend} to read video.",
+        file=sys.stderr,
+    )
+    return backend
+
+
+_VIDEO_BACKENDS = {
+    "decord": _read_video_decord,
+    "torchvision": _read_video_torchvision,
+}
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# fetch_video — main entry point for video loading
+# ─────────────────────────────────────────────────────────────────────────────
+
+def fetch_video(
+    ele: Dict[str, Any],
+) -> Tuple[List[Image.Image], float, dict]:
+    """
+    Load and sample frames from a video.
+
+    The ``ele["video"]`` value can be:
+      - A string path / URI  → decoded with decord or torchvision
+      - A list of image paths → loaded as pre-extracted frames
+
+    Returns:
+        (frames, sample_fps, metadata)
+          - frames: list of PIL.Image.Image in RGB (one per sampled frame)
+          - sample_fps: effective sampling rate after frame selection
+          - metadata: dict with ``fps``, ``sample_fps``, ``total_num_frames``,
+                      ``frames_indices``, ``video_backend``
+    """
+    if isinstance(ele["video"], str):
+        # ── Decode from video file ──────────────────────────────────���────
+        backend = _get_video_backend()
+        try:
+            video_tensor, metadata, sample_fps = _VIDEO_BACKENDS[backend](ele)
+        except Exception as exc:
+            if backend != "torchvision":
+                logger.warning(
+                    f"{backend} failed ({exc}), falling back to torchvision"
+                )
+                video_tensor, metadata, sample_fps = _read_video_torchvision(ele)
+            else:
+                raise
+
+        # Convert TCHW tensor → list of PIL images
+        frames = []
+        for i in range(video_tensor.size(0)):
+            frame_np = video_tensor[i].permute(1, 2, 0).numpy().astype(np.uint8)  # HWC
+            frames.append(Image.fromarray(frame_np, "RGB"))
+
+    elif isinstance(ele["video"], (list, tuple)):
+        # ── Pre-extracted frames (paths or PIL images) ───────────────────
+        frame_elements = ele["video"]
+        frames = []
+        for item in frame_elements:
+            frames.append(fetch_image({"image": item}))
+
+        # Pad to FRAME_FACTOR multiple
+        nframes = ceil_by_factor(len(frames), FRAME_FACTOR)
+        while len(frames) < nframes:
+            frames.append(frames[-1].copy())
+
+        sample_fps = ele.get("fps", FPS)
+        raw_fps = ele.get("raw_fps", sample_fps)
+        metadata = dict(
+            fps=raw_fps,
+            sample_fps=sample_fps,
+            frames_indices=list(range(len(frames))),
+            total_num_frames=len(frames),
+            video_backend="frames_list",
+        )
+    else:
+        raise TypeError(
+            f"ele['video'] must be a string (path) or list (frames), "
+            f"got {type(ele['video'])}"
+        )
+
+    return frames, sample_fps, metadata
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Message parsing
+# ─────────────────────────────────────────────────────────────────────────────
+
+def extract_vision_info(
+    conversations: Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]],
+) -> List[Dict[str, Any]]:
+    """
+    Extract all vision elements (image / video dicts) from Qwen-style
+    structured messages.
+
+    Args:
+        conversations: Either a single conversation (list of message dicts)
+            or a batch of conversations.
+
+    Returns:
+        Flat list of vision element dicts, in order of appearance.
+    """
+    # Normalise to batch format
+    if isinstance(conversations[0], dict):
+        conversations = [conversations]
+
+    vision_infos = []
+    for conversation in conversations:
+        for message in conversation:
+            content = message.get("content")
+            if not isinstance(content, list):
+                continue
+            for ele in content:
+                if (
+                    "image" in ele
+                    or "image_url" in ele
+                    or "video" in ele
+                    or ele.get("type") in ("image", "image_url", "video")
+                ):
+                    vision_infos.append(ele)
+    return vision_infos
+
+
+def process_vision_info(
+    conversations: Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]],
+) -> Tuple[Optional[List[Image.Image]], Optional[List[List[Image.Image]]], Optional[Dict[str, Any]]]:
+    """
+    Extract and load all images and videos from structured messages.
+
+    This is the main entry point — equivalent to
+    ``qwen_vl_utils.process_vision_info`` — adapted for ZDTaichu-5.0.
+
+    Args:
+        conversations: Qwen-style messages with structured ``content`` lists
+            containing ``{"type": "image", "image": ...}`` and/or
+            ``{"type": "video", "video": ...}`` elements.
+
+    Returns:
+        (image_inputs, video_inputs, video_kwargs)
+          - image_inputs: list of PIL images, or None
+          - video_inputs: list of frame-lists (each is ``List[PIL.Image]``),
+                          or None
+          - video_kwargs: dict with ``sample_fps_list`` and ``metadata_list``
+
+    Example::
+
+        from vision_utils import process_vision_info
+
+        messages = [
+            {"role": "user", "content": [
+                {"type": "video", "video": "clip.mp4", "fps": 2.0},
+                {"type": "text", "text": "Describe this video."},
+            ]}
+        ]
+
+        images, videos, video_kwargs = process_vision_info(messages)
+        # images = None
+        # videos = [[PIL.Image, PIL.Image, ...]]  (one list of frames per video)
+        # video_kwargs = {"sample_fps_list": [2.0], "metadata_list": [...]}
+    """
+    vision_infos = extract_vision_info(conversations)
+
+    image_inputs: List[Image.Image] = []
+    video_inputs: List[List[Image.Image]] = []
+    sample_fps_list: List[float] = []
+    metadata_list: List[dict] = []
+
+    for info in vision_infos:
+        if "image" in info or "image_url" in info:
+            image_inputs.append(fetch_image(info))
+
+        elif "video" in info:
+            frames, sample_fps, metadata = fetch_video(info)
+            video_inputs.append(frames)
+            sample_fps_list.append(sample_fps)
+            metadata_list.append(metadata)
+
+        else:
+            raise ValueError(
+                "Vision element must contain 'image', 'image_url', or 'video' key."
+            )
+
+    video_kwargs = {
+        "sample_fps_list": sample_fps_list,
+        "metadata_list": metadata_list,
+    }
+
+    return (
+        image_inputs if image_inputs else None,
+        video_inputs if video_inputs else None,
+        video_kwargs,
+    )
+
+
+def build_text_from_messages(
+    messages: List[Dict[str, Any]],
+    image_token: str = "<|image_pad|>",
+    video_token: str = "<|video_pad|>",
+) -> List[Dict[str, Any]]:
+    """
+    Convert structured messages (with typed content lists) into plain-text
+    messages that ``apply_chat_template`` can handle.
+
+    Each ``{"type": "image", ...}`` is replaced with ``image_token``.
+    Each ``{"type": "video", ...}`` is replaced with ``video_token``.
+    Text elements are concatenated.
+
+    Returns:
+        New message list with plain string ``content`` fields.
+    """
+    output = []
+    for msg in messages:
+        content = msg.get("content")
+        if isinstance(content, str):
+            output.append(msg)
+            continue
+
+        parts = []
+        for ele in content:
+            typ = ele.get("type", "text")
+            if typ == "text":
+                parts.append(ele.get("text", ""))
+            elif typ in ("image", "image_url"):
+                parts.append(image_token)
+            elif typ == "video":
+                parts.append(video_token)
+        output.append({**msg, "content": "".join(parts)})
+    return output
+
+
+__all__ = [
+    "fetch_image",
+    "fetch_video",
+    "smart_nframes",
+    "extract_vision_info",
+    "process_vision_info",
+    "build_text_from_messages",
+]