Add files using upload-large-folder tool

lingbot_sdnq_runtime/__init__.py

@@ -0,0 +1,680 @@
+from __future__ import annotations
+
+import json
+import os
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+from types import ModuleType
+from typing import Any, Iterable
+
+import torch
+from torch import nn
+
+
+RUNTIME_VERSION = "1"
+EXPERT_MANIFEST_NAME = "sdnq_experts.json"
+
+
+def _install_sgl_kernel_compat_stub() -> None:
+    """Provide the two helpers needed by SGLang's Triton MoE path on Torch 2.8.
+
+    The upstream sglang-kernel 0.4.4 wheels target the newer Torch/CUDA ABI.
+    LingBot only needs token alignment and top-k reduction from that extension;
+    its matrix multiplies and activations remain SGLang Triton kernels.
+    """
+
+    module = ModuleType("sgl_kernel")
+
+    def moe_sum_reduce(
+        value: torch.Tensor,
+        output: torch.Tensor,
+        routed_scaling_factor: float = 1.0,
+    ) -> None:
+        torch.sum(value, dim=1, out=output)
+        if routed_scaling_factor != 1.0:
+            output.mul_(routed_scaling_factor)
+
+    def moe_align_block_size(
+        topk_ids: torch.Tensor,
+        num_experts_with_sentinel: int,
+        block_size: int,
+        sorted_ids: torch.Tensor,
+        expert_ids: torch.Tensor,
+        num_tokens_post_pad: torch.Tensor,
+        cumsum_buffer: torch.Tensor,
+        _use_int32: bool,
+    ) -> None:
+        flat = topk_ids.reshape(-1).to(torch.int64)
+        num_experts = int(num_experts_with_sentinel) - 1
+        valid = (flat >= 0) & (flat < num_experts)
+        valid_positions = torch.arange(flat.numel(), device=flat.device, dtype=torch.int64)[valid]
+        valid_experts = flat[valid]
+        counts = torch.bincount(valid_experts, minlength=num_experts)
+        padded_counts = ((counts + block_size - 1) // block_size) * block_size
+        padded_offsets = torch.cumsum(padded_counts, dim=0) - padded_counts
+        original_offsets = torch.cumsum(counts, dim=0) - counts
+        order = torch.argsort(valid_experts, stable=True)
+        sorted_experts = valid_experts[order]
+        rank_in_expert = torch.arange(
+            order.numel(),
+            device=flat.device,
+            dtype=torch.int64,
+        ) - original_offsets[sorted_experts]
+        destinations = padded_offsets[sorted_experts] + rank_in_expert
+        total_padded = int(padded_counts.sum().item())
+        sorted_ids.fill_(flat.numel())
+        sorted_ids[destinations] = valid_positions[order].to(sorted_ids.dtype)
+        block_experts = torch.repeat_interleave(
+            torch.arange(num_experts, device=flat.device, dtype=expert_ids.dtype),
+            (padded_counts // block_size).to(torch.int64),
+        )
+        expert_ids.fill_(-1)
+        expert_ids[: block_experts.numel()] = block_experts
+        num_tokens_post_pad.fill_(total_padded)
+        cumsum_buffer.zero_()
+        cumulative = torch.cumsum(counts.to(cumsum_buffer.dtype), dim=0)
+        cumsum_buffer[1 : num_experts + 1] = cumulative
+
+    module.moe_sum_reduce = moe_sum_reduce
+    module.moe_align_block_size = moe_align_block_size
+    sys.modules["sgl_kernel"] = module
+
+
+_install_sgl_kernel_compat_stub()
+
+
+def _dtype_to_name(dtype: torch.dtype) -> str:
+    return str(dtype).removeprefix("torch.")
+
+
+def _dtype_from_name(name: str) -> torch.dtype:
+    dtype = getattr(torch, name.removeprefix("torch."), None)
+    if not isinstance(dtype, torch.dtype):
+        raise ValueError(f"unsupported torch dtype {name!r}")
+    return dtype
+
+
+def _shape(value: torch.Size | Iterable[int] | None) -> list[int] | None:
+    return None if value is None else [int(item) for item in value]
+
+
+def _dequantizer_to_dict(dequantizer: Any) -> dict[str, Any]:
+    return {
+        "result_dtype": _dtype_to_name(dequantizer.result_dtype),
+        "result_shape": _shape(dequantizer.result_shape),
+        "original_shape": _shape(dequantizer.original_shape),
+        "original_stride": [int(item) for item in dequantizer.original_stride],
+        "quantized_weight_shape": _shape(dequantizer.quantized_weight_shape),
+        "weights_dtype": dequantizer.weights_dtype,
+        "quantized_matmul_dtype": dequantizer.quantized_matmul_dtype,
+        "hadamard_group_size": int(dequantizer.hadamard_group_size),
+        "group_size": int(dequantizer.group_size),
+        "svd_rank": int(dequantizer.svd_rank),
+        "svd_steps": int(dequantizer.svd_steps),
+        "use_quantized_matmul": bool(dequantizer.use_quantized_matmul),
+        "re_quantize_for_matmul": bool(dequantizer.re_quantize_for_matmul),
+        "use_stochastic_rounding": bool(dequantizer.use_stochastic_rounding),
+        "use_hadamard": bool(dequantizer.use_hadamard),
+        "layer_class_name": dequantizer.layer_class_name,
+    }
+
+
+def _dequantizer_from_dict(metadata: dict[str, Any]):
+    from sdnq.dequantizer import SDNQDequantizer
+
+    return SDNQDequantizer(
+        result_dtype=_dtype_from_name(metadata["result_dtype"]),
+        result_shape=(
+            None
+            if metadata.get("result_shape") is None
+            else torch.Size(metadata["result_shape"])
+        ),
+        original_shape=torch.Size(metadata["original_shape"]),
+        original_stride=list(metadata["original_stride"]),
+        quantized_weight_shape=torch.Size(metadata["quantized_weight_shape"]),
+        weights_dtype=metadata["weights_dtype"],
+        quantized_matmul_dtype=metadata["quantized_matmul_dtype"],
+        hadamard_group_size=int(metadata["hadamard_group_size"]),
+        group_size=int(metadata["group_size"]),
+        svd_rank=int(metadata["svd_rank"]),
+        svd_steps=int(metadata["svd_steps"]),
+        use_quantized_matmul=bool(metadata["use_quantized_matmul"]),
+        re_quantize_for_matmul=bool(metadata["re_quantize_for_matmul"]),
+        use_stochastic_rounding=bool(metadata["use_stochastic_rounding"]),
+        use_hadamard=bool(metadata["use_hadamard"]),
+        layer_class_name=metadata.get("layer_class_name"),
+    )
+
+
+class SDNQExpertWeight(nn.Module):
+    """Packed SDNQ storage for one 3-D LingBot expert tensor."""
+
+    def __init__(
+        self,
+        metadata: dict[str, Any],
+        *,
+        weight: torch.Tensor,
+        scale: torch.Tensor,
+        zero_point: torch.Tensor | None,
+    ) -> None:
+        super().__init__()
+        self.metadata = dict(metadata)
+        self.weight = nn.Parameter(weight, requires_grad=False)
+        self.scale = nn.Parameter(scale, requires_grad=False)
+        self.register_parameter(
+            "zero_point",
+            None if zero_point is None else nn.Parameter(zero_point, requires_grad=False),
+        )
+        self._dequantizer = _dequantizer_from_dict(self.metadata)
+
+    @classmethod
+    def from_float(
+        cls,
+        weight: torch.Tensor,
+        *,
+        quantization_device: torch.device | str = "cuda",
+        return_device: torch.device | str = "cpu",
+        result_dtype: torch.dtype = torch.bfloat16,
+    ) -> "SDNQExpertWeight":
+        from sdnq.quantizer import sdnq_quantize_layer_weight
+
+        source = weight.detach().to(
+            device=quantization_device,
+            dtype=result_dtype,
+            copy=True,
+        )
+        dequantizer, tensors = sdnq_quantize_layer_weight(
+            source,
+            layer_class_name=None,
+            weights_dtype="uint4",
+            quantized_matmul_dtype=None,
+            group_size=0,
+            hadamard_group_size=256,
+            svd_rank=32,
+            svd_steps=8,
+            use_svd=False,
+            use_hadamard=False,
+            use_quantized_matmul=False,
+            use_stochastic_rounding=False,
+            dequantize_fp32=False,
+            torch_dtype=result_dtype,
+        )
+        del source
+        return cls(
+            _dequantizer_to_dict(dequantizer),
+            weight=tensors["weight"].to(return_device),
+            scale=tensors["scale"].to(return_device),
+            zero_point=(
+                None
+                if tensors["zero_point"] is None
+                else tensors["zero_point"].to(return_device)
+            ),
+        )
+
+    @classmethod
+    def empty(cls, metadata: dict[str, Any]) -> "SDNQExpertWeight":
+        from sdnq.common import dtype_dict
+
+        packed_shape = metadata["stored_weight_shape"]
+        scale_shape = metadata["scale_shape"]
+        zero_point_shape = metadata.get("zero_point_shape")
+        storage_dtype = dtype_dict[metadata["weights_dtype"]]["storage_dtype"]
+        scale_dtype = _dtype_from_name(metadata["scale_dtype"])
+        return cls(
+            metadata,
+            weight=torch.empty(packed_shape, dtype=storage_dtype),
+            scale=torch.empty(scale_shape, dtype=scale_dtype),
+            zero_point=(
+                None
+                if zero_point_shape is None
+                else torch.empty(zero_point_shape, dtype=_dtype_from_name(metadata["zero_point_dtype"]))
+            ),
+        )
+
+    def manifest(self) -> dict[str, Any]:
+        result = dict(self.metadata)
+        result.update(
+            {
+                "stored_weight_shape": _shape(self.weight.shape),
+                "stored_weight_dtype": _dtype_to_name(self.weight.dtype),
+                "scale_shape": _shape(self.scale.shape),
+                "scale_dtype": _dtype_to_name(self.scale.dtype),
+                "zero_point_shape": (
+                    None if self.zero_point is None else _shape(self.zero_point.shape)
+                ),
+                "zero_point_dtype": (
+                    None if self.zero_point is None else _dtype_to_name(self.zero_point.dtype)
+                ),
+                "logical_numel": int(torch.tensor(self.metadata["original_shape"]).prod().item()),
+                "stored_bytes": int(
+                    self.weight.numel() * self.weight.element_size()
+                    + self.scale.numel() * self.scale.element_size()
+                    + (
+                        0
+                        if self.zero_point is None
+                        else self.zero_point.numel() * self.zero_point.element_size()
+                    )
+                ),
+            }
+        )
+        return result
+
+    def forward(self, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
+        return self._dequantizer(
+            self.weight,
+            self.scale,
+            zero_point=self.zero_point,
+            dtype=dtype,
+        )
+
+
+def _is_grouped_experts(module: nn.Module) -> bool:
+    return module.__class__.__name__ == "LingBotVideoGroupedExperts"
+
+
+def _is_packed_experts(module: nn.Module) -> bool:
+    return all(hasattr(module, f"{name}_sdnq") for name in ("w1", "w2", "w3"))
+
+
+def quantize_moe_experts(
+    model: nn.Module,
+    *,
+    quantization_device: torch.device | str = "cuda",
+    return_device: torch.device | str = "cpu",
+) -> dict[str, Any]:
+    entries: list[dict[str, Any]] = []
+    for module_path, module in list(model.named_modules()):
+        if not _is_grouped_experts(module) or _is_packed_experts(module):
+            continue
+        entry: dict[str, Any] = {"module": module_path, "weights": {}}
+        for name in ("w1", "w2", "w3"):
+            original = getattr(module, name)
+            original_numel = int(original.numel())
+            original_bytes = int(original.numel() * original.element_size())
+            packed = SDNQExpertWeight.from_float(
+                original,
+                quantization_device=quantization_device,
+                return_device=return_device,
+            )
+            delattr(module, name)
+            module.add_module(f"{name}_sdnq", packed)
+            weight_manifest = packed.manifest()
+            weight_manifest["original_numel"] = original_numel
+            weight_manifest["original_bytes"] = original_bytes
+            entry["weights"][name] = weight_manifest
+            del original
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+        entries.append(entry)
+    return {
+        "format": "lingbot-video-sdnq-experts",
+        "version": RUNTIME_VERSION,
+        "weights_dtype": "uint4",
+        "group_size": 0,
+        "use_dynamic_quantization": False,
+        "use_svd": False,
+        "quant_conv": False,
+        "quant_embedding": False,
+        "entries": entries,
+    }
+
+
+def install_prequantized_experts(model: nn.Module, manifest: dict[str, Any]) -> None:
+    modules = dict(model.named_modules())
+    for entry in manifest.get("entries", []):
+        module_path = entry["module"]
+        module = modules.get(module_path)
+        if module is None or not _is_grouped_experts(module):
+            raise KeyError(f"LingBot grouped experts module not found: {module_path}")
+        for name in ("w1", "w2", "w3"):
+            if hasattr(module, name):
+                delattr(module, name)
+            module.add_module(
+                f"{name}_sdnq",
+                SDNQExpertWeight.empty(entry["weights"][name]),
+            )
+
+
+_PATCHED = False
+
+
+def install_runtime_patch() -> None:
+    global _PATCHED
+    if _PATCHED:
+        return
+    from lingbot_video.transformer_lingbot_video import (
+        LingBotVideoBlock,
+        LingBotVideoSparseMoeBlock,
+    )
+    from lingbot_video.sglang_moe_shim import (
+        LightSglangMoeRunnerConfig,
+        LightSglangStandardTopKOutput,
+        ensure_sglang_moe_ready,
+        sglang_fused_experts,
+    )
+
+    original_grouped = LingBotVideoSparseMoeBlock._run_grouped_experts
+    original_loop = LingBotVideoSparseMoeBlock._run_experts_for_loop
+    original_sglang = LingBotVideoSparseMoeBlock._run_sglang_triton_experts
+
+    def packed_aware_block_forward(
+        self,
+        x,
+        temb6,
+        rotary_emb,
+        attention_mask=None,
+        moe_padding_mask=None,
+        packed_indices=None,
+        parallel_config=None,
+    ):
+        expected_tokens = x.shape[0] * x.shape[1]
+        if temb6.ndim != 2 or temb6.shape[0] != expected_tokens:
+            raise ValueError(
+                "LingBotVideoBlock expects token-level temb6 with shape "
+                f"(B*S, 6D); got {tuple(temb6.shape)} for hidden states {tuple(x.shape)}."
+            )
+        mod = temb6.view(x.shape[0], x.shape[1], -1) + self.scale_shift_table.unsqueeze(0)
+        shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=-1)
+        gate_msa, gate_mlp = gate_msa.tanh(), gate_mlp.tanh()
+        scale_msa, scale_mlp = 1.0 + scale_msa, 1.0 + scale_mlp
+        to_q_dequantizer = getattr(self.attn.to_q, "sdnq_dequantizer", None)
+        bulk_dtype = (
+            to_q_dequantizer.result_dtype
+            if to_q_dequantizer is not None
+            else self.attn.to_q.weight.dtype
+        )
+        attn_in = (self.norm1(x) * scale_msa + shift_msa).to(bulk_dtype)
+        attn_out = self.attn(
+            attn_in,
+            rotary_emb,
+            attention_mask,
+            packed_indices=packed_indices,
+            parallel_config=parallel_config,
+        )
+        x = x + (gate_msa * self.norm_post_attn(attn_out)).to(x.dtype)
+        ffn_in = (self.norm2(x) * scale_mlp + shift_mlp).to(bulk_dtype)
+        if isinstance(self.ffn, LingBotVideoSparseMoeBlock):
+            ffn_out = self.ffn(ffn_in, padding_mask=moe_padding_mask)
+        else:
+            ffn_out = self.ffn(ffn_in)
+        ffn_normed = self.norm_post_ffn(ffn_out)
+        return x + (gate_mlp * ffn_normed).to(x.dtype)
+
+    def packed_loop(self, tokens: torch.Tensor, counts: torch.Tensor) -> torch.Tensor:
+        if not _is_packed_experts(self.experts):
+            return original_loop(self, tokens, counts)
+        w1 = self.experts.w1_sdnq(torch.bfloat16)
+        w2 = self.experts.w2_sdnq(torch.bfloat16)
+        w3 = self.experts.w3_sdnq(torch.bfloat16)
+        count_list = counts.tolist()
+        splits = torch.split(tokens, count_list, dim=0)
+        outputs = []
+        for expert_idx, expert_tokens in enumerate(splits):
+            if expert_tokens.numel() == 0:
+                continue
+            h = torch.nn.functional.silu(
+                expert_tokens @ w1[expert_idx].transpose(-2, -1)
+            )
+            h = h * (expert_tokens @ w3[expert_idx].transpose(-2, -1))
+            outputs.append(h @ w2[expert_idx].transpose(-2, -1))
+        if not outputs:
+            return tokens.new_zeros(tokens.shape)
+        return torch.cat(outputs, dim=0)
+
+    def packed_grouped(self, tokens: torch.Tensor, counts: torch.Tensor) -> torch.Tensor:
+        if not _is_packed_experts(self.experts):
+            return original_grouped(self, tokens, counts)
+        if not hasattr(torch, "_grouped_mm"):
+            return packed_loop(self, tokens, counts)
+        input_shape, padded_tokens, permuted_indices, aligned_counts = self._pad_grouped_tokens(
+            tokens,
+            counts,
+        )
+        offsets = torch.cumsum(aligned_counts, dim=0, dtype=torch.int32)
+        w1 = self.experts.w1_sdnq(torch.bfloat16)
+        h = torch.nn.functional.silu(
+            torch._grouped_mm(
+                padded_tokens.bfloat16(),
+                w1.transpose(-2, -1),
+                offs=offsets,
+            )
+        )
+        del w1
+        w3 = self.experts.w3_sdnq(torch.bfloat16)
+        h = h * torch._grouped_mm(
+            padded_tokens.bfloat16(),
+            w3.transpose(-2, -1),
+            offs=offsets,
+        )
+        del w3
+        w2 = self.experts.w2_sdnq(torch.bfloat16)
+        output = torch._grouped_mm(
+            h,
+            w2.transpose(-2, -1),
+            offs=offsets,
+        ).type_as(padded_tokens)
+        del w2
+        return self._unpad_grouped_tokens(output, input_shape, permuted_indices)
+
+    def packed_sglang(
+        self,
+        tokens: torch.Tensor,
+        top_scores: torch.Tensor,
+        top_indices: torch.Tensor,
+    ) -> torch.Tensor:
+        if not _is_packed_experts(self.experts):
+            return original_sglang(self, tokens, top_scores, top_indices)
+        ensure_sglang_moe_ready()
+        topk_output = LightSglangStandardTopKOutput(
+            top_scores.float(),
+            top_indices.to(torch.int32),
+            torch.empty(0, device=tokens.device),
+        )
+        runner_config = LightSglangMoeRunnerConfig(
+            num_experts=self.num_experts,
+            num_local_experts=self.num_experts,
+            activation="silu",
+            is_gated=True,
+            inplace=False,
+        )
+        w1 = self.experts.w1_sdnq(torch.bfloat16)
+        w3 = self.experts.w3_sdnq(torch.bfloat16)
+        w13 = torch.cat((w1, w3), dim=1).contiguous()
+        del w1, w3
+        w2 = self.experts.w2_sdnq(torch.bfloat16).contiguous()
+        output = sglang_fused_experts(
+            tokens.contiguous().bfloat16(),
+            w13,
+            w2,
+            topk_output,
+            runner_config,
+        ).type_as(tokens)
+        del w13, w2
+        return output
+
+    LingBotVideoSparseMoeBlock._run_experts_for_loop = packed_loop
+    LingBotVideoSparseMoeBlock._run_grouped_experts = packed_grouped
+    LingBotVideoSparseMoeBlock._run_sglang_triton_experts = packed_sglang
+    LingBotVideoBlock.forward = packed_aware_block_forward
+    _PATCHED = True
+
+
+def save_expert_manifest(manifest: dict[str, Any], transformer_dir: str | Path) -> Path:
+    path = Path(transformer_dir) / EXPERT_MANIFEST_NAME
+    path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return path
+
+
+def load_expert_manifest(transformer_dir: str | Path) -> dict[str, Any]:
+    path = Path(transformer_dir) / EXPERT_MANIFEST_NAME
+    if not path.exists():
+        return {"format": "lingbot-video-sdnq-experts", "version": RUNTIME_VERSION, "entries": []}
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _load_state_dict(transformer_dir: Path, device: torch.device | str = "cpu") -> dict[str, torch.Tensor]:
+    from sdnq.file_loader import load_files
+
+    files = sorted(str(path) for path in transformer_dir.glob("*.safetensors"))
+    if not files:
+        raise FileNotFoundError(f"no safetensors shards in {transformer_dir}")
+    return load_files(files, device=device, method="safetensors")
+
+
+def load_transformer(
+    model_root: str | Path,
+    *,
+    subfolder: str = "transformer",
+    torch_dtype: torch.dtype = torch.bfloat16,
+    state_device: torch.device | str = "cpu",
+) -> nn.Module:
+    from accelerate import init_empty_weights
+    from lingbot_video.transformer_lingbot_video import LingBotVideoTransformer3DModel
+    from sdnq import sdnq_post_load_quant
+    from sdnq.loader import apply_sdnq_options_to_model, post_process_model
+    from sdnq.utils import get_quant_args_from_config
+
+    install_runtime_patch()
+    transformer_dir = Path(model_root) / subfolder
+    config = LingBotVideoTransformer3DModel.load_config(str(transformer_dir))
+    if hasattr(config, "to_dict"):
+        config = config.to_dict()
+    config = dict(config)
+    config.pop("quantization_config", None)
+    quant_config = json.loads(
+        (transformer_dir / "quantization_config.json").read_text(encoding="utf-8")
+    )
+    expert_manifest = load_expert_manifest(transformer_dir)
+
+    with init_empty_weights():
+        model = LingBotVideoTransformer3DModel.from_config(config)
+        install_prequantized_experts(model, expert_manifest)
+        model = sdnq_post_load_quant(
+            model,
+            torch_dtype=torch_dtype,
+            pre_quantized=True,
+            **get_quant_args_from_config(quant_config),
+        )
+
+    state_dict = _load_state_dict(transformer_dir, device=state_device)
+    incompatible = model.load_state_dict(state_dict, strict=True, assign=True)
+    if incompatible.missing_keys or incompatible.unexpected_keys:
+        raise RuntimeError(f"incompatible SDNQ state dict: {incompatible}")
+    del state_dict
+    model = post_process_model(model)
+    model = apply_sdnq_options_to_model(
+        model,
+        dtype=torch_dtype,
+        dequantize_fp32=False,
+        use_quantized_matmul=False,
+    )
+    # LingBot derives autocast/device from its first parameter. Packed SDNQ
+    # weights are uint8, so expose a zero-sized floating anchor ahead of child
+    # parameters without adding any checkpoint storage.
+    model.register_parameter(
+        "_sdnq_dtype_anchor",
+        nn.Parameter(torch.empty(0, dtype=torch_dtype), requires_grad=False),
+    )
+    model.eval()
+    return model
+
+
+@contextmanager
+def _patch_qwen_loader():
+    from transformers import Qwen3VLForConditionalGeneration
+
+    original = Qwen3VLForConditionalGeneration.from_pretrained
+    attn_implementation = os.environ.get("LINGBOT_QWEN_ATTN_IMPLEMENTATION", "sdpa")
+
+    @classmethod
+    def patched(cls, pretrained_model_name_or_path, *args, **kwargs):
+        kwargs.setdefault("attn_implementation", attn_implementation)
+        if "torch_dtype" in kwargs and "dtype" not in kwargs:
+            kwargs["dtype"] = kwargs.pop("torch_dtype")
+        return original(pretrained_model_name_or_path, *args, **kwargs)
+
+    Qwen3VLForConditionalGeneration.from_pretrained = patched
+    try:
+        yield
+    finally:
+        Qwen3VLForConditionalGeneration.from_pretrained = original
+
+
+def resolve_model_root(repo_id_or_path: str | Path, *, revision: str | None = None) -> Path:
+    path = Path(repo_id_or_path)
+    if path.exists():
+        return path.resolve()
+    from huggingface_hub import snapshot_download
+
+    return Path(snapshot_download(str(repo_id_or_path), revision=revision))
+
+
+def load_pipeline(
+    repo_id_or_path: str | Path,
+    *,
+    revision: str | None = None,
+    transformer_subfolder: str = "transformer",
+    device: torch.device | str | None = "cuda",
+    torch_dtype: dict[str, torch.dtype] | None = None,
+):
+    from lingbot_video.pipeline_lingbot_video import LingBotVideoPipeline
+
+    model_root = resolve_model_root(repo_id_or_path, revision=revision)
+    dtype_map = torch_dtype or {
+        "default": torch.bfloat16,
+        "transformer": torch.bfloat16,
+        "text_encoder": torch.bfloat16,
+        "vae": torch.float32,
+    }
+    transformer = load_transformer(
+        model_root,
+        subfolder=transformer_subfolder,
+        torch_dtype=dtype_map["transformer"],
+    )
+    with _patch_qwen_loader():
+        pipe = LingBotVideoPipeline.from_pretrained(
+            str(model_root),
+            transformer=transformer,
+            trust_remote_code=True,
+            torch_dtype=dtype_map,
+        )
+    if device is not None:
+        pipe = pipe.to(device)
+    return pipe
+
+
+def expert_storage_summary(manifest: dict[str, Any]) -> dict[str, int | float]:
+    logical_numel = 0
+    original_bytes = 0
+    stored_bytes = 0
+    tensor_count = 0
+    for entry in manifest.get("entries", []):
+        for weight in entry["weights"].values():
+            logical_numel += int(weight["logical_numel"])
+            original_bytes += int(weight["original_bytes"])
+            stored_bytes += int(weight["stored_bytes"])
+            tensor_count += 1
+    return {
+        "tensor_count": tensor_count,
+        "logical_numel": logical_numel,
+        "original_bytes": original_bytes,
+        "stored_bytes": stored_bytes,
+        "compression_ratio": (original_bytes / stored_bytes if stored_bytes else 0.0),
+    }
+
+
+__all__ = [
+    "EXPERT_MANIFEST_NAME",
+    "RUNTIME_VERSION",
+    "SDNQExpertWeight",
+    "expert_storage_summary",
+    "install_prequantized_experts",
+    "install_runtime_patch",
+    "load_expert_manifest",
+    "load_pipeline",
+    "load_transformer",
+    "quantize_moe_experts",
+    "resolve_model_root",
+    "save_expert_manifest",
+]

refiner/.quantization_complete

@@ -0,0 +1 @@
+ok

refiner/component.SHA256SUMS

@@ -0,0 +1,11 @@
+dc51b8c96c2d745df3bd5590d990230a482fd247123599548e0632fdbf97fc22  ./.quantization_complete
+2effac3d3a8c72dc3938d44b17b25e928b690717cb2e22a061df8aa3ed4a8c9a  ./config.json
+b21bc0b086b03cc9a0000cf7256a712dab2695ffcda7026b076c3abb99dd1eb3  ./coverage.json
+f4fcfa25c57404a23e8da451e107b5533af5b579d907aa38b89f0d6a1a2828e7  ./diffusion_pytorch_model-00001-of-00005.safetensors
+2489df8e6a45a7883fcff10b34faa0209f07d7b97e6c5681439fafe9d4041303  ./diffusion_pytorch_model-00002-of-00005.safetensors
+e0cf002ed56e1e04140afb7137365ea5d8a3497ef7a4c55911b3db2fdf6545f5  ./diffusion_pytorch_model-00003-of-00005.safetensors
+5bd87a652b9d279dbb5290a02c14ae6ce5987fdb0eb3caa30cea2980b2c63131  ./diffusion_pytorch_model-00004-of-00005.safetensors
+0e5e9a038b797c68ea42398bbe425707c7b09414c352c6bfbd2923e7750f25ee  ./diffusion_pytorch_model-00005-of-00005.safetensors
+48d8cb723c47742c0e875083a3e699b8a4541d7fc1036b25a1a160746efe051d  ./diffusion_pytorch_model.safetensors.index.json
+e5ab2ee967920427f6897837cf55b798d10b0bce96635b92b315d747e8095c21  ./quantization_config.json
+b49557484202c5499185e6af092f850d4b69788fddf2ccda57c06f5493887027  ./sdnq_experts.json

refiner/config.json

@@ -0,0 +1,260 @@
+{
+  "_class_name": "LingBotVideoTransformer3DModel",
+  "_diffusers_version": "0.39.0",
+  "_name_or_path": "models/moe",
+  "axes_dims": [
+    32,
+    48,
+    48
+  ],
+  "axes_lens": [
+    4096,
+    512,
+    512
+  ],
+  "decoder_sparse_step": 1,
+  "depth": 48,
+  "freq_dim": 256,
+  "hidden_size": 2048,
+  "in_channels": 16,
+  "intermediate_size": 6144,
+  "mlp_only_layers": [],
+  "moe_intermediate_size": 768,
+  "n_group": 4,
+  "n_shared_experts": 1,
+  "norm_eps": 1e-06,
+  "norm_topk_prob": true,
+  "num_attention_heads": 16,
+  "num_experts": 128,
+  "num_experts_per_tok": 8,
+  "out_bias": true,
+  "out_channels": 16,
+  "patch_embed_bias": true,
+  "patch_size": [
+    1,
+    2,
+    2
+  ],
+  "qkv_bias": false,
+  "quantization_config": {
+    "add_skip_keys": true,
+    "dequantize_fp32": false,
+    "dynamic_loss_threshold": null,
+    "group_size": 0,
+    "hadamard_group_size": 256,
+    "is_integer": true,
+    "is_training": false,
+    "minimum_allowed_numel": 16384,
+    "modules_dtype_dict": {},
+    "modules_quant_config": {},
+    "modules_to_not_convert": [
+      "time_embedder",
+      ".proj_out",
+      "time_modulation",
+      ".context_embedder",
+      ".t_embedder",
+      ".vid_out",
+      "norm_post_attn",
+      "time_text_embed",
+      "norm_out_modulation",
+      ".txt_in",
+      "patch_emb",
+      "patch_embedding",
+      ".condition_embedder",
+      "wte",
+      ".time_embed",
+      "scale_shift_table",
+      ".txt_out",
+      "norm1",
+      ".vid_in",
+      ".final_layer",
+      ".img_in",
+      "norm",
+      "patch_embed",
+      ".x_embedder",
+      "norm2",
+      ".emb_in",
+      "norm_k",
+      "multi_modal_projector",
+      ".img_out",
+      ".emb_out",
+      "norm_out",
+      "router",
+      "lm_head",
+      ".norm_out",
+      "norm_q",
+      ".y_embedder",
+      "norm_post_ffn",
+      "blocks.0.ffn.experts.w1_sdnq.weight",
+      "blocks.0.ffn.experts.w2_sdnq.weight",
+      "blocks.0.ffn.experts.w3_sdnq.weight",
+      "blocks.1.ffn.experts.w1_sdnq.weight",
+      "blocks.1.ffn.experts.w2_sdnq.weight",
+      "blocks.1.ffn.experts.w3_sdnq.weight",
+      "blocks.2.ffn.experts.w1_sdnq.weight",
+      "blocks.2.ffn.experts.w2_sdnq.weight",
+      "blocks.2.ffn.experts.w3_sdnq.weight",
+      "blocks.3.ffn.experts.w1_sdnq.weight",
+      "blocks.3.ffn.experts.w2_sdnq.weight",
+      "blocks.3.ffn.experts.w3_sdnq.weight",
+      "blocks.4.ffn.experts.w1_sdnq.weight",
+      "blocks.4.ffn.experts.w2_sdnq.weight",
+      "blocks.4.ffn.experts.w3_sdnq.weight",
+      "blocks.5.ffn.experts.w1_sdnq.weight",
+      "blocks.5.ffn.experts.w2_sdnq.weight",
+      "blocks.5.ffn.experts.w3_sdnq.weight",
+      "blocks.6.ffn.experts.w1_sdnq.weight",
+      "blocks.6.ffn.experts.w2_sdnq.weight",
+      "blocks.6.ffn.experts.w3_sdnq.weight",
+      "blocks.7.ffn.experts.w1_sdnq.weight",
+      "blocks.7.ffn.experts.w2_sdnq.weight",
+      "blocks.7.ffn.experts.w3_sdnq.weight",
+      "blocks.8.ffn.experts.w1_sdnq.weight",
+      "blocks.8.ffn.experts.w2_sdnq.weight",
+      "blocks.8.ffn.experts.w3_sdnq.weight",
+      "blocks.9.ffn.experts.w1_sdnq.weight",
+      "blocks.9.ffn.experts.w2_sdnq.weight",
+      "blocks.9.ffn.experts.w3_sdnq.weight",
+      "blocks.10.ffn.experts.w1_sdnq.weight",
+      "blocks.10.ffn.experts.w2_sdnq.weight",
+      "blocks.10.ffn.experts.w3_sdnq.weight",
+      "blocks.11.ffn.experts.w1_sdnq.weight",
+      "blocks.11.ffn.experts.w2_sdnq.weight",
+      "blocks.11.ffn.experts.w3_sdnq.weight",
+      "blocks.12.ffn.experts.w1_sdnq.weight",
+      "blocks.12.ffn.experts.w2_sdnq.weight",
+      "blocks.12.ffn.experts.w3_sdnq.weight",
+      "blocks.13.ffn.experts.w1_sdnq.weight",
+      "blocks.13.ffn.experts.w2_sdnq.weight",
+      "blocks.13.ffn.experts.w3_sdnq.weight",
+      "blocks.14.ffn.experts.w1_sdnq.weight",
+      "blocks.14.ffn.experts.w2_sdnq.weight",
+      "blocks.14.ffn.experts.w3_sdnq.weight",
+      "blocks.15.ffn.experts.w1_sdnq.weight",
+      "blocks.15.ffn.experts.w2_sdnq.weight",
+      "blocks.15.ffn.experts.w3_sdnq.weight",
+      "blocks.16.ffn.experts.w1_sdnq.weight",
+      "blocks.16.ffn.experts.w2_sdnq.weight",
+      "blocks.16.ffn.experts.w3_sdnq.weight",
+      "blocks.17.ffn.experts.w1_sdnq.weight",
+      "blocks.17.ffn.experts.w2_sdnq.weight",
+      "blocks.17.ffn.experts.w3_sdnq.weight",
+      "blocks.18.ffn.experts.w1_sdnq.weight",
+      "blocks.18.ffn.experts.w2_sdnq.weight",
+      "blocks.18.ffn.experts.w3_sdnq.weight",
+      "blocks.19.ffn.experts.w1_sdnq.weight",
+      "blocks.19.ffn.experts.w2_sdnq.weight",
+      "blocks.19.ffn.experts.w3_sdnq.weight",
+      "blocks.20.ffn.experts.w1_sdnq.weight",
+      "blocks.20.ffn.experts.w2_sdnq.weight",
+      "blocks.20.ffn.experts.w3_sdnq.weight",
+      "blocks.21.ffn.experts.w1_sdnq.weight",
+      "blocks.21.ffn.experts.w2_sdnq.weight",
+      "blocks.21.ffn.experts.w3_sdnq.weight",
+      "blocks.22.ffn.experts.w1_sdnq.weight",
+      "blocks.22.ffn.experts.w2_sdnq.weight",
+      "blocks.22.ffn.experts.w3_sdnq.weight",
+      "blocks.23.ffn.experts.w1_sdnq.weight",
+      "blocks.23.ffn.experts.w2_sdnq.weight",
+      "blocks.23.ffn.experts.w3_sdnq.weight",
+      "blocks.24.ffn.experts.w1_sdnq.weight",
+      "blocks.24.ffn.experts.w2_sdnq.weight",
+      "blocks.24.ffn.experts.w3_sdnq.weight",
+      "blocks.25.ffn.experts.w1_sdnq.weight",
+      "blocks.25.ffn.experts.w2_sdnq.weight",
+      "blocks.25.ffn.experts.w3_sdnq.weight",
+      "blocks.26.ffn.experts.w1_sdnq.weight",
+      "blocks.26.ffn.experts.w2_sdnq.weight",
+      "blocks.26.ffn.experts.w3_sdnq.weight",
+      "blocks.27.ffn.experts.w1_sdnq.weight",
+      "blocks.27.ffn.experts.w2_sdnq.weight",
+      "blocks.27.ffn.experts.w3_sdnq.weight",
+      "blocks.28.ffn.experts.w1_sdnq.weight",
+      "blocks.28.ffn.experts.w2_sdnq.weight",
+      "blocks.28.ffn.experts.w3_sdnq.weight",
+      "blocks.29.ffn.experts.w1_sdnq.weight",
+      "blocks.29.ffn.experts.w2_sdnq.weight",
+      "blocks.29.ffn.experts.w3_sdnq.weight",
+      "blocks.30.ffn.experts.w1_sdnq.weight",
+      "blocks.30.ffn.experts.w2_sdnq.weight",
+      "blocks.30.ffn.experts.w3_sdnq.weight",
+      "blocks.31.ffn.experts.w1_sdnq.weight",
+      "blocks.31.ffn.experts.w2_sdnq.weight",
+      "blocks.31.ffn.experts.w3_sdnq.weight",
+      "blocks.32.ffn.experts.w1_sdnq.weight",
+      "blocks.32.ffn.experts.w2_sdnq.weight",
+      "blocks.32.ffn.experts.w3_sdnq.weight",
+      "blocks.33.ffn.experts.w1_sdnq.weight",
+      "blocks.33.ffn.experts.w2_sdnq.weight",
+      "blocks.33.ffn.experts.w3_sdnq.weight",
+      "blocks.34.ffn.experts.w1_sdnq.weight",
+      "blocks.34.ffn.experts.w2_sdnq.weight",
+      "blocks.34.ffn.experts.w3_sdnq.weight",
+      "blocks.35.ffn.experts.w1_sdnq.weight",
+      "blocks.35.ffn.experts.w2_sdnq.weight",
+      "blocks.35.ffn.experts.w3_sdnq.weight",
+      "blocks.36.ffn.experts.w1_sdnq.weight",
+      "blocks.36.ffn.experts.w2_sdnq.weight",
+      "blocks.36.ffn.experts.w3_sdnq.weight",
+      "blocks.37.ffn.experts.w1_sdnq.weight",
+      "blocks.37.ffn.experts.w2_sdnq.weight",
+      "blocks.37.ffn.experts.w3_sdnq.weight",
+      "blocks.38.ffn.experts.w1_sdnq.weight",
+      "blocks.38.ffn.experts.w2_sdnq.weight",
+      "blocks.38.ffn.experts.w3_sdnq.weight",
+      "blocks.39.ffn.experts.w1_sdnq.weight",
+      "blocks.39.ffn.experts.w2_sdnq.weight",
+      "blocks.39.ffn.experts.w3_sdnq.weight",
+      "blocks.40.ffn.experts.w1_sdnq.weight",
+      "blocks.40.ffn.experts.w2_sdnq.weight",
+      "blocks.40.ffn.experts.w3_sdnq.weight",
+      "blocks.41.ffn.experts.w1_sdnq.weight",
+      "blocks.41.ffn.experts.w2_sdnq.weight",
+      "blocks.41.ffn.experts.w3_sdnq.weight",
+      "blocks.42.ffn.experts.w1_sdnq.weight",
+      "blocks.42.ffn.experts.w2_sdnq.weight",
+      "blocks.42.ffn.experts.w3_sdnq.weight",
+      "blocks.43.ffn.experts.w1_sdnq.weight",
+      "blocks.43.ffn.experts.w2_sdnq.weight",
+      "blocks.43.ffn.experts.w3_sdnq.weight",
+      "blocks.44.ffn.experts.w1_sdnq.weight",
+      "blocks.44.ffn.experts.w2_sdnq.weight",
+      "blocks.44.ffn.experts.w3_sdnq.weight",
+      "blocks.45.ffn.experts.w1_sdnq.weight",
+      "blocks.45.ffn.experts.w2_sdnq.weight",
+      "blocks.45.ffn.experts.w3_sdnq.weight",
+      "blocks.46.ffn.experts.w1_sdnq.weight",
+      "blocks.46.ffn.experts.w2_sdnq.weight",
+      "blocks.46.ffn.experts.w3_sdnq.weight",
+      "blocks.47.ffn.experts.w1_sdnq.weight",
+      "blocks.47.ffn.experts.w2_sdnq.weight",
+      "blocks.47.ffn.experts.w3_sdnq.weight"
+    ],
+    "modules_to_not_use_matmul": [],
+    "non_blocking": false,
+    "quant_conv": false,
+    "quant_embedding": false,
+    "quant_method": "sdnq",
+    "quantization_device": "cuda",
+    "quantized_matmul_dtype": null,
+    "return_device": "cpu",
+    "sdnq_version": "0.2.2",
+    "svd_rank": 32,
+    "svd_steps": 8,
+    "use_dynamic_quantization": false,
+    "use_grad_ckpt": true,
+    "use_hadamard": false,
+    "use_quantized_matmul": false,
+    "use_quantized_matmul_conv": false,
+    "use_static_quantization": true,
+    "use_stochastic_rounding": false,
+    "use_svd": false,
+    "weights_dtype": "uint4"
+  },
+  "rope_theta": 256.0,
+  "routed_scaling_factor": 2.5,
+  "score_func": "sigmoid",
+  "text_dim": 2560,
+  "timestep_mlp_bias": true,
+  "topk_group": 2
+}

refiner/diffusion_pytorch_model-00001-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4fcfa25c57404a23e8da451e107b5533af5b579d907aa38b89f0d6a1a2828e7
+size 3903708600

refiner/diffusion_pytorch_model-00002-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2489df8e6a45a7883fcff10b34faa0209f07d7b97e6c5681439fafe9d4041303
+size 3907192200

refiner/diffusion_pytorch_model-00003-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0cf002ed56e1e04140afb7137365ea5d8a3497ef7a4c55911b3db2fdf6545f5
+size 3907192216

refiner/diffusion_pytorch_model-00004-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd87a652b9d279dbb5290a02c14ae6ce5987fdb0eb3caa30cea2980b2c63131
+size 3907192216

refiner/diffusion_pytorch_model-00005-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e5e9a038b797c68ea42398bbe425707c7b09414c352c6bfbd2923e7750f25ee
+size 3288080256

refiner/quantization_config.json

@@ -0,0 +1,215 @@
+{
+  "add_skip_keys": true,
+  "dequantize_fp32": false,
+  "dynamic_loss_threshold": null,
+  "group_size": 0,
+  "hadamard_group_size": 256,
+  "is_integer": true,
+  "is_training": false,
+  "minimum_allowed_numel": 16384,
+  "modules_dtype_dict": {},
+  "modules_quant_config": {},
+  "modules_to_not_convert": [
+    "time_embedder",
+    ".proj_out",
+    "time_modulation",
+    ".context_embedder",
+    ".t_embedder",
+    ".vid_out",
+    "norm_post_attn",
+    "time_text_embed",
+    "norm_out_modulation",
+    ".txt_in",
+    "patch_emb",
+    "patch_embedding",
+    ".condition_embedder",
+    "wte",
+    ".time_embed",
+    "scale_shift_table",
+    ".txt_out",
+    "norm1",
+    ".vid_in",
+    ".final_layer",
+    ".img_in",
+    "norm",
+    "patch_embed",
+    ".x_embedder",
+    "norm2",
+    ".emb_in",
+    "norm_k",
+    "multi_modal_projector",
+    ".img_out",
+    ".emb_out",
+    "norm_out",
+    "router",
+    "lm_head",
+    ".norm_out",
+    "norm_q",
+    ".y_embedder",
+    "norm_post_ffn",
+    "blocks.0.ffn.experts.w1_sdnq.weight",
+    "blocks.0.ffn.experts.w2_sdnq.weight",
+    "blocks.0.ffn.experts.w3_sdnq.weight",
+    "blocks.1.ffn.experts.w1_sdnq.weight",
+    "blocks.1.ffn.experts.w2_sdnq.weight",
+    "blocks.1.ffn.experts.w3_sdnq.weight",
+    "blocks.2.ffn.experts.w1_sdnq.weight",
+    "blocks.2.ffn.experts.w2_sdnq.weight",
+    "blocks.2.ffn.experts.w3_sdnq.weight",
+    "blocks.3.ffn.experts.w1_sdnq.weight",
+    "blocks.3.ffn.experts.w2_sdnq.weight",
+    "blocks.3.ffn.experts.w3_sdnq.weight",
+    "blocks.4.ffn.experts.w1_sdnq.weight",
+    "blocks.4.ffn.experts.w2_sdnq.weight",
+    "blocks.4.ffn.experts.w3_sdnq.weight",
+    "blocks.5.ffn.experts.w1_sdnq.weight",
+    "blocks.5.ffn.experts.w2_sdnq.weight",
+    "blocks.5.ffn.experts.w3_sdnq.weight",
+    "blocks.6.ffn.experts.w1_sdnq.weight",
+    "blocks.6.ffn.experts.w2_sdnq.weight",
+    "blocks.6.ffn.experts.w3_sdnq.weight",
+    "blocks.7.ffn.experts.w1_sdnq.weight",
+    "blocks.7.ffn.experts.w2_sdnq.weight",
+    "blocks.7.ffn.experts.w3_sdnq.weight",
+    "blocks.8.ffn.experts.w1_sdnq.weight",
+    "blocks.8.ffn.experts.w2_sdnq.weight",
+    "blocks.8.ffn.experts.w3_sdnq.weight",
+    "blocks.9.ffn.experts.w1_sdnq.weight",
+    "blocks.9.ffn.experts.w2_sdnq.weight",
+    "blocks.9.ffn.experts.w3_sdnq.weight",
+    "blocks.10.ffn.experts.w1_sdnq.weight",
+    "blocks.10.ffn.experts.w2_sdnq.weight",
+    "blocks.10.ffn.experts.w3_sdnq.weight",
+    "blocks.11.ffn.experts.w1_sdnq.weight",
+    "blocks.11.ffn.experts.w2_sdnq.weight",
+    "blocks.11.ffn.experts.w3_sdnq.weight",
+    "blocks.12.ffn.experts.w1_sdnq.weight",
+    "blocks.12.ffn.experts.w2_sdnq.weight",
+    "blocks.12.ffn.experts.w3_sdnq.weight",
+    "blocks.13.ffn.experts.w1_sdnq.weight",
+    "blocks.13.ffn.experts.w2_sdnq.weight",
+    "blocks.13.ffn.experts.w3_sdnq.weight",
+    "blocks.14.ffn.experts.w1_sdnq.weight",
+    "blocks.14.ffn.experts.w2_sdnq.weight",
+    "blocks.14.ffn.experts.w3_sdnq.weight",
+    "blocks.15.ffn.experts.w1_sdnq.weight",
+    "blocks.15.ffn.experts.w2_sdnq.weight",
+    "blocks.15.ffn.experts.w3_sdnq.weight",
+    "blocks.16.ffn.experts.w1_sdnq.weight",
+    "blocks.16.ffn.experts.w2_sdnq.weight",
+    "blocks.16.ffn.experts.w3_sdnq.weight",
+    "blocks.17.ffn.experts.w1_sdnq.weight",
+    "blocks.17.ffn.experts.w2_sdnq.weight",
+    "blocks.17.ffn.experts.w3_sdnq.weight",
+    "blocks.18.ffn.experts.w1_sdnq.weight",
+    "blocks.18.ffn.experts.w2_sdnq.weight",
+    "blocks.18.ffn.experts.w3_sdnq.weight",
+    "blocks.19.ffn.experts.w1_sdnq.weight",
+    "blocks.19.ffn.experts.w2_sdnq.weight",
+    "blocks.19.ffn.experts.w3_sdnq.weight",
+    "blocks.20.ffn.experts.w1_sdnq.weight",
+    "blocks.20.ffn.experts.w2_sdnq.weight",
+    "blocks.20.ffn.experts.w3_sdnq.weight",
+    "blocks.21.ffn.experts.w1_sdnq.weight",
+    "blocks.21.ffn.experts.w2_sdnq.weight",
+    "blocks.21.ffn.experts.w3_sdnq.weight",
+    "blocks.22.ffn.experts.w1_sdnq.weight",
+    "blocks.22.ffn.experts.w2_sdnq.weight",
+    "blocks.22.ffn.experts.w3_sdnq.weight",
+    "blocks.23.ffn.experts.w1_sdnq.weight",
+    "blocks.23.ffn.experts.w2_sdnq.weight",
+    "blocks.23.ffn.experts.w3_sdnq.weight",
+    "blocks.24.ffn.experts.w1_sdnq.weight",
+    "blocks.24.ffn.experts.w2_sdnq.weight",
+    "blocks.24.ffn.experts.w3_sdnq.weight",
+    "blocks.25.ffn.experts.w1_sdnq.weight",
+    "blocks.25.ffn.experts.w2_sdnq.weight",
+    "blocks.25.ffn.experts.w3_sdnq.weight",
+    "blocks.26.ffn.experts.w1_sdnq.weight",
+    "blocks.26.ffn.experts.w2_sdnq.weight",
+    "blocks.26.ffn.experts.w3_sdnq.weight",
+    "blocks.27.ffn.experts.w1_sdnq.weight",
+    "blocks.27.ffn.experts.w2_sdnq.weight",
+    "blocks.27.ffn.experts.w3_sdnq.weight",
+    "blocks.28.ffn.experts.w1_sdnq.weight",
+    "blocks.28.ffn.experts.w2_sdnq.weight",
+    "blocks.28.ffn.experts.w3_sdnq.weight",
+    "blocks.29.ffn.experts.w1_sdnq.weight",
+    "blocks.29.ffn.experts.w2_sdnq.weight",
+    "blocks.29.ffn.experts.w3_sdnq.weight",
+    "blocks.30.ffn.experts.w1_sdnq.weight",
+    "blocks.30.ffn.experts.w2_sdnq.weight",
+    "blocks.30.ffn.experts.w3_sdnq.weight",
+    "blocks.31.ffn.experts.w1_sdnq.weight",
+    "blocks.31.ffn.experts.w2_sdnq.weight",
+    "blocks.31.ffn.experts.w3_sdnq.weight",
+    "blocks.32.ffn.experts.w1_sdnq.weight",
+    "blocks.32.ffn.experts.w2_sdnq.weight",
+    "blocks.32.ffn.experts.w3_sdnq.weight",
+    "blocks.33.ffn.experts.w1_sdnq.weight",
+    "blocks.33.ffn.experts.w2_sdnq.weight",
+    "blocks.33.ffn.experts.w3_sdnq.weight",
+    "blocks.34.ffn.experts.w1_sdnq.weight",
+    "blocks.34.ffn.experts.w2_sdnq.weight",
+    "blocks.34.ffn.experts.w3_sdnq.weight",
+    "blocks.35.ffn.experts.w1_sdnq.weight",
+    "blocks.35.ffn.experts.w2_sdnq.weight",
+    "blocks.35.ffn.experts.w3_sdnq.weight",
+    "blocks.36.ffn.experts.w1_sdnq.weight",
+    "blocks.36.ffn.experts.w2_sdnq.weight",
+    "blocks.36.ffn.experts.w3_sdnq.weight",
+    "blocks.37.ffn.experts.w1_sdnq.weight",
+    "blocks.37.ffn.experts.w2_sdnq.weight",
+    "blocks.37.ffn.experts.w3_sdnq.weight",
+    "blocks.38.ffn.experts.w1_sdnq.weight",
+    "blocks.38.ffn.experts.w2_sdnq.weight",
+    "blocks.38.ffn.experts.w3_sdnq.weight",
+    "blocks.39.ffn.experts.w1_sdnq.weight",
+    "blocks.39.ffn.experts.w2_sdnq.weight",
+    "blocks.39.ffn.experts.w3_sdnq.weight",
+    "blocks.40.ffn.experts.w1_sdnq.weight",
+    "blocks.40.ffn.experts.w2_sdnq.weight",
+    "blocks.40.ffn.experts.w3_sdnq.weight",
+    "blocks.41.ffn.experts.w1_sdnq.weight",
+    "blocks.41.ffn.experts.w2_sdnq.weight",
+    "blocks.41.ffn.experts.w3_sdnq.weight",
+    "blocks.42.ffn.experts.w1_sdnq.weight",
+    "blocks.42.ffn.experts.w2_sdnq.weight",
+    "blocks.42.ffn.experts.w3_sdnq.weight",
+    "blocks.43.ffn.experts.w1_sdnq.weight",
+    "blocks.43.ffn.experts.w2_sdnq.weight",
+    "blocks.43.ffn.experts.w3_sdnq.weight",
+    "blocks.44.ffn.experts.w1_sdnq.weight",
+    "blocks.44.ffn.experts.w2_sdnq.weight",
+    "blocks.44.ffn.experts.w3_sdnq.weight",
+    "blocks.45.ffn.experts.w1_sdnq.weight",
+    "blocks.45.ffn.experts.w2_sdnq.weight",
+    "blocks.45.ffn.experts.w3_sdnq.weight",
+    "blocks.46.ffn.experts.w1_sdnq.weight",
+    "blocks.46.ffn.experts.w2_sdnq.weight",
+    "blocks.46.ffn.experts.w3_sdnq.weight",
+    "blocks.47.ffn.experts.w1_sdnq.weight",
+    "blocks.47.ffn.experts.w2_sdnq.weight",
+    "blocks.47.ffn.experts.w3_sdnq.weight"
+  ],
+  "modules_to_not_use_matmul": [],
+  "non_blocking": false,
+  "quant_conv": false,
+  "quant_embedding": false,
+  "quant_method": "sdnq",
+  "quantization_device": "cuda",
+  "quantized_matmul_dtype": null,
+  "return_device": "cpu",
+  "sdnq_version": "0.2.2",
+  "svd_rank": 32,
+  "svd_steps": 8,
+  "use_dynamic_quantization": false,
+  "use_grad_ckpt": true,
+  "use_hadamard": false,
+  "use_quantized_matmul": false,
+  "use_quantized_matmul_conv": false,
+  "use_static_quantization": true,
+  "use_stochastic_rounding": false,
+  "use_svd": false,
+  "weights_dtype": "uint4"
+}