Add BF16 multimodal projector and audited Q8 DFlash auxiliary package

.gitattributes

@@ -37,3 +37,5 @@ MiMo-V2.6-Flash-RL-MXFP4-BF16-00001-of-00004.gguf filter=lfs diff=lfs merge=lfs
 MiMo-V2.6-Flash-RL-MXFP4-BF16-00002-of-00004.gguf filter=lfs diff=lfs merge=lfs -text
 MiMo-V2.6-Flash-RL-MXFP4-BF16-00003-of-00004.gguf filter=lfs diff=lfs merge=lfs -text
 MiMo-V2.6-Flash-RL-MXFP4-BF16-00004-of-00004.gguf filter=lfs diff=lfs merge=lfs -text
+MiMo-V2.6-Flash-RL-DFlash-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
+mmproj-MiMo-V2.6-Flash-RL-BF16.gguf filter=lfs diff=lfs merge=lfs -text

MiMo-V2.6-Flash-RL-DFlash-Q8_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06a56d466017739918f2e1c9b771f08ac5cb92d7ea2173bbb5537a82284a7d8c
+size 1565911104

README.md

@@ -29,7 +29,21 @@ The four `MXFP4-BF16` shards preserve the original routed experts through an exa
 | `MiMo-V2.6-Flash-RL-MXFP4-BF16-00004-of-00004.gguf` | 41,422,766,592 |
 | **Total** | **174,908,254,624** |
 
-Keep all shards together and open the first shard. The language artifact includes the checkpoint's three embedded MTP blocks; this is not evidence of validated speculative decoding. Multimodal encoders and the separate DFlash model are separate components and are not supplied by these four shards alone.
+Keep all shards together and open the first shard. The language artifact includes the checkpoint's three embedded MTP blocks; this is not evidence of validated speculative decoding. Multimodal encoders and the separate DFlash model are separate components and are not supplied by these four shards alone. They are included as the auxiliary files listed below.
+
+## Auxiliary artifacts
+
+| File | Role | Validation |
+|---|---|---|
+| `mmproj-MiMo-V2.6-Flash-RL-BF16.gguf` | BF16 image/audio input encoders and projector, about 2.75 GB | One-image native/GGUF numerical comparison; full runtime media validation pending |
+| `MiMo-V2.6-Flash-RL-DFlash-Q8_0.gguf` | Five-layer Q8_0 draft, about 1.56 GB | 63-tensor mapping/shape audit, exact F32 and sampled Q8 error checks; runtime acceptance untested |
+| `dflash/` | Original config, learned mask embedding, upstream example, and explicit runtime contract | Source files retained byte for byte |
+
+The DFlash GGUF explicitly preserves 64 rotary dimensions per 128-dimensional head
+and attention value scale 0.612. Its config, sinks, target-layer mapping, and learned
+mask embedding must be consumed by a compatible runtime. The bundled upstream
+Python example does not implement every MiMo-specific draft setting; the runtime
+contract records the production reference. No stock-runtime compatibility is claimed.
 
 ## Validation status
 

SHA256SUMS

@@ -2,3 +2,5 @@
 caf0e120484239878cbf389914e17f2dd9cb733dd0f9c87c35437b9e649ec551  MiMo-V2.6-Flash-RL-MXFP4-BF16-00002-of-00004.gguf
 c92a8d8ba4eea32c8ff8ca7ba403bc4df90b3fec1b6bf5630d8672aeb2837d16  MiMo-V2.6-Flash-RL-MXFP4-BF16-00003-of-00004.gguf
 5c91e1e8489000fe5e281616b4b13c7710eaddf7ea82a6808458b26eb575c783  MiMo-V2.6-Flash-RL-MXFP4-BF16-00004-of-00004.gguf
+333af1e9f86c4c4be2305c655a9a4fe7684a68e127d513832b8ea27cd89408d6  mmproj-MiMo-V2.6-Flash-RL-BF16.gguf
+06a56d466017739918f2e1c9b771f08ac5cb92d7ea2173bbb5537a82284a7d8c  MiMo-V2.6-Flash-RL-DFlash-Q8_0.gguf

artifact-manifest.json

@@ -22,6 +22,16 @@
       "file": "MiMo-V2.6-Flash-RL-MXFP4-BF16-00004-of-00004.gguf",
       "bytes": 41422766592,
       "sha256": "5c91e1e8489000fe5e281616b4b13c7710eaddf7ea82a6808458b26eb575c783"
+    },
+    {
+      "file": "mmproj-MiMo-V2.6-Flash-RL-BF16.gguf",
+      "bytes": 2748509792,
+      "sha256": "333af1e9f86c4c4be2305c655a9a4fe7684a68e127d513832b8ea27cd89408d6"
+    },
+    {
+      "file": "MiMo-V2.6-Flash-RL-DFlash-Q8_0.gguf",
+      "bytes": 1565911104,
+      "sha256": "06a56d466017739918f2e1c9b771f08ac5cb92d7ea2173bbb5537a82284a7d8c"
     }
   ]
 }

dflash-audit.json

@@ -0,0 +1,419 @@
+{
+  "passed": true,
+  "tensor_count": 63,
+  "source_to_gguf_mapping": "exact set and shape match",
+  "f32_check": "all elements equal",
+  "q8_check": "three deterministic rows per matrix, relative L2 below 0.03",
+  "metadata": {
+    "dflash.block_count": 5,
+    "dflash.context_length": 1048576,
+    "dflash.embedding_length": 4096,
+    "dflash.feed_forward_length": 16384,
+    "dflash.attention.head_count": 64,
+    "dflash.attention.head_count_kv": 8,
+    "dflash.attention.causal": false,
+    "dflash.rope.freq_base": 10000.0,
+    "dflash.attention.layer_norm_rms_epsilon": 9.999999974752427e-07,
+    "dflash.attention.key_length": 128,
+    "dflash.attention.value_length": 128,
+    "dflash.block_size": 8,
+    "dflash.target_layers": [
+      1,
+      12,
+      24,
+      36,
+      48
+    ],
+    "dflash.attention.sliding_window": 1024,
+    "dflash.attention.sliding_window_pattern": [
+      true,
+      true,
+      true,
+      true,
+      true
+    ],
+    "dflash.rope.dimension_count": 64,
+    "dflash.attention.value_scale": 0.6119999885559082
+  },
+  "rows": [
+    {
+      "source": "fc.weight",
+      "gguf": "fc.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005588418338447809
+    },
+    {
+      "source": "hidden_norm.weight",
+      "gguf": "enc.output_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.0.input_layernorm.weight",
+      "gguf": "blk.0.attn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.0.mlp.down_proj.weight",
+      "gguf": "blk.0.ffn_down.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005761090200394392
+    },
+    {
+      "source": "layers.0.mlp.gate_proj.weight",
+      "gguf": "blk.0.ffn_gate.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005506220273673534
+    },
+    {
+      "source": "layers.0.mlp.up_proj.weight",
+      "gguf": "blk.0.ffn_up.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005492477677762508
+    },
+    {
+      "source": "layers.0.post_attention_layernorm.weight",
+      "gguf": "blk.0.ffn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.0.self_attn.attention_sink_bias",
+      "gguf": "blk.0.attn_sinks.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.0.self_attn.k_norm.weight",
+      "gguf": "blk.0.attn_k_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.0.self_attn.k_proj.weight",
+      "gguf": "blk.0.attn_k.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005711852107197046
+    },
+    {
+      "source": "layers.0.self_attn.o_proj.weight",
+      "gguf": "blk.0.attn_output.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.00542857451364398
+    },
+    {
+      "source": "layers.0.self_attn.q_norm.weight",
+      "gguf": "blk.0.attn_q_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.0.self_attn.q_proj.weight",
+      "gguf": "blk.0.attn_q.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.00566287524998188
+    },
+    {
+      "source": "layers.0.self_attn.v_proj.weight",
+      "gguf": "blk.0.attn_v.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005591947585344315
+    },
+    {
+      "source": "layers.1.input_layernorm.weight",
+      "gguf": "blk.1.attn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.1.mlp.down_proj.weight",
+      "gguf": "blk.1.ffn_down.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005702429451048374
+    },
+    {
+      "source": "layers.1.mlp.gate_proj.weight",
+      "gguf": "blk.1.ffn_gate.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.0054396879859268665
+    },
+    {
+      "source": "layers.1.mlp.up_proj.weight",
+      "gguf": "blk.1.ffn_up.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005450685042887926
+    },
+    {
+      "source": "layers.1.post_attention_layernorm.weight",
+      "gguf": "blk.1.ffn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.1.self_attn.attention_sink_bias",
+      "gguf": "blk.1.attn_sinks.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.1.self_attn.k_norm.weight",
+      "gguf": "blk.1.attn_k_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.1.self_attn.k_proj.weight",
+      "gguf": "blk.1.attn_k.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005590083077549934
+    },
+    {
+      "source": "layers.1.self_attn.o_proj.weight",
+      "gguf": "blk.1.attn_output.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005548989400267601
+    },
+    {
+      "source": "layers.1.self_attn.q_norm.weight",
+      "gguf": "blk.1.attn_q_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.1.self_attn.q_proj.weight",
+      "gguf": "blk.1.attn_q.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.00569014111533761
+    },
+    {
+      "source": "layers.1.self_attn.v_proj.weight",
+      "gguf": "blk.1.attn_v.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005482448264956474
+    },
+    {
+      "source": "layers.2.input_layernorm.weight",
+      "gguf": "blk.2.attn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.2.mlp.down_proj.weight",
+      "gguf": "blk.2.ffn_down.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005693534389138222
+    },
+    {
+      "source": "layers.2.mlp.gate_proj.weight",
+      "gguf": "blk.2.ffn_gate.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005409534554928541
+    },
+    {
+      "source": "layers.2.mlp.up_proj.weight",
+      "gguf": "blk.2.ffn_up.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005460306536406279
+    },
+    {
+      "source": "layers.2.post_attention_layernorm.weight",
+      "gguf": "blk.2.ffn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.2.self_attn.attention_sink_bias",
+      "gguf": "blk.2.attn_sinks.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.2.self_attn.k_norm.weight",
+      "gguf": "blk.2.attn_k_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.2.self_attn.k_proj.weight",
+      "gguf": "blk.2.attn_k.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.00555426487699151
+    },
+    {
+      "source": "layers.2.self_attn.o_proj.weight",
+      "gguf": "blk.2.attn_output.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005506662651896477
+    },
+    {
+      "source": "layers.2.self_attn.q_norm.weight",
+      "gguf": "blk.2.attn_q_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.2.self_attn.q_proj.weight",
+      "gguf": "blk.2.attn_q.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.0055401320569217205
+    },
+    {
+      "source": "layers.2.self_attn.v_proj.weight",
+      "gguf": "blk.2.attn_v.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005393540486693382
+    },
+    {
+      "source": "layers.3.input_layernorm.weight",
+      "gguf": "blk.3.attn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.3.mlp.down_proj.weight",
+      "gguf": "blk.3.ffn_down.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005636085756123066
+    },
+    {
+      "source": "layers.3.mlp.gate_proj.weight",
+      "gguf": "blk.3.ffn_gate.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005447070114314556
+    },
+    {
+      "source": "layers.3.mlp.up_proj.weight",
+      "gguf": "blk.3.ffn_up.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005448319483548403
+    },
+    {
+      "source": "layers.3.post_attention_layernorm.weight",
+      "gguf": "blk.3.ffn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.3.self_attn.attention_sink_bias",
+      "gguf": "blk.3.attn_sinks.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.3.self_attn.k_norm.weight",
+      "gguf": "blk.3.attn_k_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.3.self_attn.k_proj.weight",
+      "gguf": "blk.3.attn_k.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005607506725937128
+    },
+    {
+      "source": "layers.3.self_attn.o_proj.weight",
+      "gguf": "blk.3.attn_output.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005487577058374882
+    },
+    {
+      "source": "layers.3.self_attn.q_norm.weight",
+      "gguf": "blk.3.attn_q_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.3.self_attn.q_proj.weight",
+      "gguf": "blk.3.attn_q.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005722770467400551
+    },
+    {
+      "source": "layers.3.self_attn.v_proj.weight",
+      "gguf": "blk.3.attn_v.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005510271061211824
+    },
+    {
+      "source": "layers.4.input_layernorm.weight",
+      "gguf": "blk.4.attn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.4.mlp.down_proj.weight",
+      "gguf": "blk.4.ffn_down.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005580178461968899
+    },
+    {
+      "source": "layers.4.mlp.gate_proj.weight",
+      "gguf": "blk.4.ffn_gate.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005537654273211956
+    },
+    {
+      "source": "layers.4.mlp.up_proj.weight",
+      "gguf": "blk.4.ffn_up.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005511666182428598
+    },
+    {
+      "source": "layers.4.post_attention_layernorm.weight",
+      "gguf": "blk.4.ffn_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.4.self_attn.attention_sink_bias",
+      "gguf": "blk.4.attn_sinks.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.4.self_attn.k_norm.weight",
+      "gguf": "blk.4.attn_k_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.4.self_attn.k_proj.weight",
+      "gguf": "blk.4.attn_k.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005514967255294323
+    },
+    {
+      "source": "layers.4.self_attn.o_proj.weight",
+      "gguf": "blk.4.attn_output.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005578942131251097
+    },
+    {
+      "source": "layers.4.self_attn.q_norm.weight",
+      "gguf": "blk.4.attn_q_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    },
+    {
+      "source": "layers.4.self_attn.q_proj.weight",
+      "gguf": "blk.4.attn_q.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005511818453669548
+    },
+    {
+      "source": "layers.4.self_attn.v_proj.weight",
+      "gguf": "blk.4.attn_v.weight",
+      "type": "Q8_0",
+      "sampled_relative_l2": 0.005557454191148281
+    },
+    {
+      "source": "norm.weight",
+      "gguf": "output_norm.weight",
+      "type": "F32",
+      "sampled_relative_l2": 0.0
+    }
+  ],
+  "scope": "Conversion integrity; speculative decoding and acceptance rate not tested."
+}

dflash/config.json

@@ -0,0 +1,57 @@
+{
+  "architectures": [
+    "DFlashDraftModel"
+  ],
+  "model_type": "qwen3",
+  "auto_map": {
+    "AutoModel": "dflash.DFlashDraftModel"
+  },
+  "hidden_size": 4096,
+  "intermediate_size": 16384,
+  "num_hidden_layers": 5,
+  "num_attention_heads": 64,
+  "num_key_value_heads": 8,
+  "head_dim": 128,
+  "v_head_dim": 128,
+  "partial_rotary_factor": 0.5,
+  "block_size": 8,
+  "dflash_config": {
+    "target_layer_ids": [
+      0,
+      11,
+      23,
+      35,
+      47
+    ],
+    "mask_token_id": 151675,
+    "num_anchors": 4096,
+    "block_size": 8,
+    "loss_decay_gamma": 7.0,
+    "attention_value_scale": 0.612,
+    "attention_sink_bias": true
+  },
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention"
+  ],
+  "sliding_window": 1024,
+  "use_sliding_window": true,
+  "is_causal": false,
+  "num_target_layers": 48,
+  "target_hidden_size": 4096,
+  "vocab_size": 152576,
+  "max_position_embeddings": 1048576,
+  "rope_theta": 10000.0,
+  "rms_norm_eps": 1e-06,
+  "torch_dtype": "bfloat16",
+  "hidden_act": "silu",
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "add_swa_attention_sink_bias": true,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  
+}

dflash/dflash.py

@@ -0,0 +1,379 @@
+from typing import Callable, Optional
+
+import torch
+from torch import nn
+from transformers import DynamicCache
+from transformers.cache_utils import Cache
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.models.qwen3.modeling_qwen3 import (
+    ALL_ATTENTION_FUNCTIONS,
+    FlashAttentionKwargs,
+    GradientCheckpointingLayer,
+    Qwen3Config,
+    Qwen3MLP,
+    Qwen3PreTrainedModel,
+    Qwen3RMSNorm,
+    Qwen3RotaryEmbedding,
+    eager_attention_forward,
+    rotate_half,
+)
+from typing_extensions import Tuple, Unpack
+
+
+def sample(logits: torch.Tensor, temperature: float = 0.0) -> torch.Tensor:
+    if temperature < 1e-5:
+        return torch.argmax(logits, dim=-1)
+    bsz, seq_len, vocab_size = logits.shape
+    logits = logits.view(-1, vocab_size)
+    logits = logits / temperature
+    probs = torch.softmax(logits, dim=-1)
+    return torch.multinomial(probs, num_samples=1).view(bsz, seq_len)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+    q_len = q.size(-2)
+    q_embed = (q * cos[..., -q_len:, :]) + (rotate_half(q) * sin[..., -q_len:, :])
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+class Qwen3DFlashAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: Qwen3Config, layer_idx: int):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = getattr(
+            config, "head_dim", config.hidden_size // config.num_attention_heads
+        )
+        self.num_key_value_groups = (
+            config.num_attention_heads // config.num_key_value_heads
+        )
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = config.attention_dropout
+        self.is_causal = False
+        self.q_proj = nn.Linear(
+            config.hidden_size,
+            config.num_attention_heads * self.head_dim,
+            bias=config.attention_bias,
+        )
+        self.k_proj = nn.Linear(
+            config.hidden_size,
+            config.num_key_value_heads * self.head_dim,
+            bias=config.attention_bias,
+        )
+        self.v_proj = nn.Linear(
+            config.hidden_size,
+            config.num_key_value_heads * self.head_dim,
+            bias=config.attention_bias,
+        )
+        self.o_proj = nn.Linear(
+            config.num_attention_heads * self.head_dim,
+            config.hidden_size,
+            bias=config.attention_bias,
+        )
+        self.q_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+        self.k_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+        self.sliding_window = (
+            config.sliding_window
+            if config.layer_types[layer_idx] == "sliding_attention"
+            else None
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        target_hidden: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: Optional[torch.Tensor],
+        past_key_values: Optional[Cache] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
+        bsz, q_len = hidden_states.shape[:-1]
+        ctx_len = target_hidden.shape[1]
+        q = self.q_proj(hidden_states)
+        q = q.view(bsz, q_len, -1, self.head_dim)
+        q = self.q_norm(q).transpose(1, 2)
+        k_ctx = self.k_proj(target_hidden)
+        k_noise = self.k_proj(hidden_states)
+        v_ctx = self.v_proj(target_hidden)
+        v_noise = self.v_proj(hidden_states)
+        k = torch.cat([k_ctx, k_noise], dim=1).view(
+            bsz, ctx_len + q_len, -1, self.head_dim
+        )
+        v = torch.cat([v_ctx, v_noise], dim=1).view(
+            bsz, ctx_len + q_len, -1, self.head_dim
+        )
+        k = self.k_norm(k).transpose(1, 2)
+        v = v.transpose(1, 2)
+        cos, sin = position_embeddings
+        q, k = apply_rotary_pos_emb(q, k, cos, sin)
+        if past_key_values is not None:
+            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+            k, v = past_key_values.update(k, v, self.layer_idx, cache_kwargs)
+        attn_fn: Callable = eager_attention_forward
+        if self.config._attn_implementation != "eager":
+            attn_fn = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+        attn_output, attn_weights = attn_fn(
+            self,
+            q,
+            k,
+            v,
+            attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            sliding_window=self.sliding_window,
+            **kwargs,
+        )
+        attn_output = attn_output.reshape(bsz, q_len, -1)
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+class Qwen3DFlashDecoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config: Qwen3Config, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.self_attn = Qwen3DFlashAttention(config=config, layer_idx=layer_idx)
+        self.mlp = Qwen3MLP(config)
+        self.input_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = Qwen3RMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps
+        )
+
+    def forward(
+        self,
+        target_hidden: Optional[torch.Tensor] = None,
+        hidden_states: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        cache_position: Optional[torch.LongTensor] = None,
+        position_embeddings: Optional[
+            Tuple[torch.Tensor, torch.Tensor]
+        ] = None,  # necessary, but kept here for BC
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> Tuple[
+        torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
+    ]:
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        hidden_states = self.self_attn(
+            hidden_states=hidden_states,
+            target_hidden=target_hidden,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+            cache_position=cache_position,
+            position_embeddings=position_embeddings,
+            **kwargs,
+        )[0]
+        hidden_states = residual + hidden_states
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        return hidden_states
+
+
+def build_target_layer_ids(num_target_layers: int, num_draft_layers: int):
+    if num_draft_layers == 1:
+        return [(num_target_layers // 2)]
+    start = 1
+    end = num_target_layers - 3
+    span = end - start
+    target_layer_ids = [
+        int(round(start + (i * span) / (num_draft_layers - 1)))
+        for i in range(num_draft_layers)
+    ]
+    return target_layer_ids
+
+
+def extract_context_feature(
+    hidden_states: list[torch.Tensor],
+    layer_ids: Optional[list[int]],
+) -> torch.Tensor:
+    offset = 1
+    selected_states = []
+    for layer_id in layer_ids:
+        selected_states.append(hidden_states[layer_id + offset])
+    target_hidden = torch.cat(selected_states, dim=-1)
+    return target_hidden
+
+
+class DFlashDraftModel(Qwen3PreTrainedModel):
+    config_class = Qwen3Config
+    _no_split_modules = ["Qwen3DFlashDecoderLayer"]
+
+    def __init__(self, config) -> None:
+        super().__init__(config)
+        self.config = config
+        self.layers = nn.ModuleList(
+            [
+                Qwen3DFlashDecoderLayer(config, layer_idx)
+                for layer_idx in range(config.num_hidden_layers)
+            ]
+        )
+        dflash_config = getattr(config, "dflash_config", {}) or {}
+        self.target_layer_ids = dflash_config.get(
+            "target_layer_ids",
+            build_target_layer_ids(config.num_target_layers, config.num_hidden_layers),
+        )
+        self.norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = Qwen3RotaryEmbedding(config)
+        self.fc = nn.Linear(
+            len(self.target_layer_ids) * config.hidden_size,
+            config.hidden_size,
+            bias=False,
+        )
+        self.hidden_norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.block_size = config.block_size
+        self.mask_token_id = dflash_config.get("mask_token_id", None)
+        self.post_init()
+
+    def forward(
+        self,
+        position_ids: torch.LongTensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        noise_embedding: Optional[torch.Tensor] = None,
+        target_hidden: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Cache] = None,
+        use_cache: bool = False,
+        **kwargs,
+    ) -> CausalLMOutputWithPast:
+        hidden_states = noise_embedding
+        target_hidden = self.hidden_norm(self.fc(target_hidden))
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+        for layer in self.layers:
+            hidden_states = layer(
+                hidden_states=hidden_states,
+                target_hidden=target_hidden,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_values,
+                use_cache=use_cache,
+                position_embeddings=position_embeddings,
+                **kwargs,
+            )
+        return self.norm(hidden_states)
+
+    @torch.inference_mode()
+    def spec_generate(
+        self,
+        target: nn.Module,
+        input_ids: torch.LongTensor,
+        max_new_tokens: int,
+        stop_token_ids: list[int],
+        temperature: float,
+    ):
+        self.eval()
+        num_input_tokens = input_ids.shape[1]
+        max_length = num_input_tokens + max_new_tokens
+
+        block_size = self.block_size
+        output_ids = torch.full(
+            (1, max_length + block_size),
+            self.mask_token_id,
+            dtype=torch.long,
+            device=target.device,
+        )
+        position_ids = torch.arange(
+            output_ids.shape[1], device=target.device
+        ).unsqueeze(0)
+
+        past_key_values_target = DynamicCache()
+        past_key_values_draft = DynamicCache()
+
+        # Prefill stage
+        output = target(
+            input_ids,
+            position_ids=position_ids[:, :num_input_tokens],
+            past_key_values=past_key_values_target,
+            use_cache=True,
+            logits_to_keep=1,
+            output_hidden_states=True,
+        )
+
+        output_ids[:, :num_input_tokens] = input_ids
+        output_ids[:, num_input_tokens : num_input_tokens + 1] = sample(
+            output.logits, temperature
+        )
+        target_hidden = extract_context_feature(
+            output.hidden_states, self.target_layer_ids
+        )
+
+        # Decode stage
+        acceptance_lengths = []
+        start = input_ids.shape[1]
+        while start < max_length:
+            block_output_ids = output_ids[:, start : start + block_size].clone()
+            block_position_ids = position_ids[:, start : start + block_size]
+            noise_embedding = target.model.embed_tokens(block_output_ids)
+            draft_logits = target.lm_head(
+                self(
+                    target_hidden=target_hidden,
+                    noise_embedding=noise_embedding,
+                    position_ids=position_ids[
+                        :, past_key_values_draft.get_seq_length() : start + block_size
+                    ],
+                    past_key_values=past_key_values_draft,
+                    use_cache=True,
+                    is_causal=False,
+                )[:, -block_size + 1 :, :]
+            )
+            past_key_values_draft.crop(start)
+            block_output_ids[:, 1:] = sample(draft_logits)
+
+            output = target(
+                block_output_ids,
+                position_ids=block_position_ids,
+                past_key_values=past_key_values_target,
+                use_cache=True,
+                output_hidden_states=True,
+            )
+
+            posterior = sample(output.logits, temperature)
+            acceptance_length = (
+                (block_output_ids[:, 1:] == posterior[:, :-1])
+                .cumprod(dim=1)
+                .sum(dim=1)[0]
+                .item()
+            )
+            output_ids[:, start : start + acceptance_length + 1] = block_output_ids[
+                :, : acceptance_length + 1
+            ]
+            output_ids[:, start + acceptance_length + 1] = posterior[
+                :, acceptance_length
+            ]
+            start += acceptance_length + 1
+            past_key_values_target.crop(start)
+            target_hidden = extract_context_feature(
+                output.hidden_states, self.target_layer_ids
+            )[:, : acceptance_length + 1, :]
+            acceptance_lengths.append(acceptance_length + 1)
+            if stop_token_ids is not None and any(
+                stop_token_id in output_ids[:, num_input_tokens:]
+                for stop_token_id in stop_token_ids
+            ):
+                break
+        output_ids = output_ids[:, :max_length]
+        output_ids = output_ids[:, output_ids[0] != self.mask_token_id]
+        if stop_token_ids is not None:
+            stop_token_ids = torch.tensor(stop_token_ids, device=output_ids.device)
+            stop_token_indices = torch.isin(
+                output_ids[0][num_input_tokens:], stop_token_ids
+            ).nonzero(as_tuple=True)[0]
+            if stop_token_indices.numel() > 0:
+                output_ids = output_ids[
+                    :, : num_input_tokens + stop_token_indices[0] + 1
+                ]
+
+        return output_ids

dflash/mask_embedding.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b35b379fe0497ffdc0d6407f502622237d9346e0d03f457ec574be60f6c2cee0
+size 9882

dflash/runtime-contract.json

@@ -0,0 +1,38 @@
+{
+  "source_revision": "3b38d063180c3e4aed9691fdc735f3d10b266ee4",
+  "gguf": "MiMo-V2.6-Flash-RL-DFlash-Q8_0.gguf",
+  "embedded_mtp_is_separate": true,
+  "block_count": 5,
+  "block_size": 8,
+  "mask_token_id": 151675,
+  "mask_embedding_file": "mask_embedding.pt",
+  "source_target_layer_ids": [
+    0,
+    11,
+    23,
+    35,
+    47
+  ],
+  "gguf_hidden_state_indices": [
+    1,
+    12,
+    24,
+    36,
+    48
+  ],
+  "partial_rotary_factor": 0.5,
+  "rotary_dimensions": 64,
+  "head_dimensions": 128,
+  "attention_value_scale": 0.612,
+  "attention_sinks": true,
+  "is_causal": false,
+  "sliding_window": 1024,
+  "config_note": "Original config retained byte for byte, including its trailing comma; normalized JSON used only in conversion staging.",
+  "runtime_status": "Deferred; no validated acceptance rate or ds4 support claimed.",
+  "source_python_note": "Bundled HF example does not implement all MiMo-specific draft behavior; use production SGLang draft implementation and the checkpoint config as integration references.",
+  "files": {
+    "config.json": "29f18def0d74535771b2364b28107f4914ebb88872abb93189012f7573e10e4b",
+    "mask_embedding.pt": "b35b379fe0497ffdc0d6407f502622237d9346e0d03f457ec574be60f6c2cee0",
+    "dflash.py": "da5ab1738b954800950405131f1d1d97c3345f37e32676d511d3a25dfddd9d75"
+  }
+}

mmproj-MiMo-V2.6-Flash-RL-BF16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:333af1e9f86c4c4be2305c655a9a4fe7684a68e127d513832b8ea27cd89408d6
+size 2748509792

toolchain-pins.json

@@ -0,0 +1,5 @@
+{
+  "llama.cpp": "58367713a6935c0810103378144008df32e3d5db",
+  "ds4-dfm-rs": "d9fa8b65d3cdb505e6fa9060cb9a327bc3b0d432",
+  "source": "3b38d063180c3e4aed9691fdc735f3d10b266ee4"
+}

vision-parity-fp32.json

@@ -0,0 +1,13 @@
+{
+  "native_compute_dtype": "float32",
+  "shape": [
+    88,
+    4096
+  ],
+  "cosine": 0.999983453167901,
+  "relative_l2": 0.005762081450217797,
+  "max_abs": 0.029657483100891113,
+  "native_rms": 0.0693792951089455,
+  "gguf_rms": 0.06940096774910624,
+  "scope": "One real still image; identical production-normalized pixels; BF16 PyTorch versus converted GGUF. Video and audio require separate checks."
+}

vision-parity.json

@@ -0,0 +1,12 @@
+{
+  "shape": [
+    88,
+    4096
+  ],
+  "cosine": 0.9974313942728767,
+  "relative_l2": 0.07171386073721285,
+  "max_abs": 0.45852723717689514,
+  "native_rms": 0.06933624871181736,
+  "gguf_rms": 0.06940096774910624,
+  "scope": "One real still image; identical production-normalized pixels; BF16 PyTorch versus converted GGUF. Video and audio require separate checks."
+}