| |
| |
| |
| |
| @@ -4,6 +4,7 @@ |
| |
| from __future__ import annotations |
| |
| +import re |
| from typing import TYPE_CHECKING, cast |
| |
| from vllm.config import get_current_vllm_config |
| @@ -175,7 +176,7 @@ class DeepseekV4FP8Config(Fp8Config): |
| ): |
| return UnquantizedFusedMoEMethod(layer.moe_config) |
| if self.expert_dtype == "fp4": |
| - if self.moe_quant_algo == "NVFP4": |
| + if self.moe_quant_algo == "NVFP4" and not self._is_draft_layer(prefix): |
| from vllm.model_executor.layers.quantization.modelopt import ( |
| ModelOptNvFp4FusedMoE, |
| ) |
| @@ -189,7 +190,20 @@ class DeepseekV4FP8Config(Fp8Config): |
| # returns Fp8MoEMethod with block-wise float32 scales. |
| return super().get_quant_method(layer, prefix) |
| |
| + def _is_draft_layer(self, prefix: str) -> bool: |
| + # NVFP4 conversions quantize only the main stack; the speculative |
| + # module (runtime layer index >= num_hidden_layers) keeps its |
| + # original MXFP4 experts and must load through the MXFP4 path. |
| + m = re.search(r"layers\.(\d+)\.", prefix) |
| + if m is None: |
| + return False |
| + try: |
| + hf_config = get_current_vllm_config().model_config.hf_config |
| + except Exception: |
| + return False |
| + return int(m.group(1)) >= hf_config.num_hidden_layers |
| + |
| def is_mxfp4_quant(self, prefix, layer): |
| if not isinstance(layer, RoutedExperts) or self.expert_dtype != "fp4": |
| return False |
| - return self.moe_quant_algo != "NVFP4" |
| + return self.moe_quant_algo != "NVFP4" or self._is_draft_layer(prefix) |
|
|