diff --git a/vllm/models/deepseek_v4/quant_config.py b/vllm/models/deepseek_v4/quant_config.py index 89cf695..5918fa0 100644 --- a/vllm/models/deepseek_v4/quant_config.py +++ b/vllm/models/deepseek_v4/quant_config.py @@ -4,6 +4,7 @@ from __future__ import annotations +import re from typing import TYPE_CHECKING, cast from vllm.config import get_current_vllm_config @@ -175,7 +176,7 @@ class DeepseekV4FP8Config(Fp8Config): ): return UnquantizedFusedMoEMethod(layer.moe_config) if self.expert_dtype == "fp4": - if self.moe_quant_algo == "NVFP4": + if self.moe_quant_algo == "NVFP4" and not self._is_draft_layer(prefix): from vllm.model_executor.layers.quantization.modelopt import ( ModelOptNvFp4FusedMoE, ) @@ -189,7 +190,20 @@ class DeepseekV4FP8Config(Fp8Config): # returns Fp8MoEMethod with block-wise float32 scales. return super().get_quant_method(layer, prefix) + def _is_draft_layer(self, prefix: str) -> bool: + # NVFP4 conversions quantize only the main stack; the speculative + # module (runtime layer index >= num_hidden_layers) keeps its + # original MXFP4 experts and must load through the MXFP4 path. + m = re.search(r"layers\.(\d+)\.", prefix) + if m is None: + return False + try: + hf_config = get_current_vllm_config().model_config.hf_config + except Exception: + return False + return int(m.group(1)) >= hf_config.num_hidden_layers + def is_mxfp4_quant(self, prefix, layer): if not isinstance(layer, RoutedExperts) or self.expert_dtype != "fp4": return False - return self.moe_quant_algo != "NVFP4" + return self.moe_quant_algo != "NVFP4" or self._is_draft_layer(prefix)