File size: 1,897 Bytes
17e0f9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | diff --git a/vllm/models/deepseek_v4/quant_config.py b/vllm/models/deepseek_v4/quant_config.py
index 89cf695..5918fa0 100644
--- a/vllm/models/deepseek_v4/quant_config.py
+++ b/vllm/models/deepseek_v4/quant_config.py
@@ -4,6 +4,7 @@
from __future__ import annotations
+import re
from typing import TYPE_CHECKING, cast
from vllm.config import get_current_vllm_config
@@ -175,7 +176,7 @@ class DeepseekV4FP8Config(Fp8Config):
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if self.expert_dtype == "fp4":
- if self.moe_quant_algo == "NVFP4":
+ if self.moe_quant_algo == "NVFP4" and not self._is_draft_layer(prefix):
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4FusedMoE,
)
@@ -189,7 +190,20 @@ class DeepseekV4FP8Config(Fp8Config):
# returns Fp8MoEMethod with block-wise float32 scales.
return super().get_quant_method(layer, prefix)
+ def _is_draft_layer(self, prefix: str) -> bool:
+ # NVFP4 conversions quantize only the main stack; the speculative
+ # module (runtime layer index >= num_hidden_layers) keeps its
+ # original MXFP4 experts and must load through the MXFP4 path.
+ m = re.search(r"layers\.(\d+)\.", prefix)
+ if m is None:
+ return False
+ try:
+ hf_config = get_current_vllm_config().model_config.hf_config
+ except Exception:
+ return False
+ return int(m.group(1)) >= hf_config.num_hidden_layers
+
def is_mxfp4_quant(self, prefix, layer):
if not isinstance(layer, RoutedExperts) or self.expert_dtype != "fp4":
return False
- return self.moe_quant_algo != "NVFP4"
+ return self.moe_quant_algo != "NVFP4" or self._is_draft_layer(prefix)
|