hf_quant_config.json

{
    "producer": {
        "name": "modelopt",
        "version": "0.45.0"
    },
    "quantization": {
        "quant_algo": "NVFP4",
        "kv_cache_quant_algo": null,
        "group_size": 16,
        "exclude_modules": [
            "embed_tokens",
            "layers.0.self_attn*",
            "layers.1.self_attn*",
            "layers.10.self_attn*",
            "layers.11.self_attn*",
            "layers.12.self_attn*",
            "layers.13.self_attn*",
            "layers.14.self_attn*",
            "layers.15.self_attn*",
            "layers.16.self_attn*",
            "layers.17.self_attn*",
            "layers.18.self_attn*",
            "layers.19.self_attn*",
            "layers.2.self_attn*",
            "layers.20.self_attn*",
            "layers.21.self_attn*",
            "layers.22.self_attn*",
            "layers.23.self_attn*",
            "layers.24.self_attn*",
            "layers.25.self_attn*",
            "layers.26.self_attn*",
            "layers.27.self_attn*",
            "layers.28.self_attn*",
            "layers.29.self_attn*",
            "layers.3.self_attn*",
            "layers.30.self_attn*",
            "layers.31.self_attn*",
            "layers.32.self_attn*",
            "layers.33.self_attn*",
            "layers.4.self_attn*",
            "layers.5.self_attn*",
            "layers.6.self_attn*",
            "layers.7.self_attn*",
            "layers.8.self_attn*",
            "layers.9.self_attn*"
        ]
    }
}