{ "architectures": [ "Qwen3_5MoeQuantizedForConditionalGeneration" ], "dtype": "bfloat16", "image_token_id": 248056, "model_type": "qwen3_5_moe", "quip_params": { "K": 4, "L": 16, "V": 2, "codebook": "bitshift", "codebook_version": 0, "dead_experts": [ [ 0, 207, "gate_up_proj" ], [ 0, 207, "down_proj" ], [ 3, 72, "gate_up_proj" ], [ 3, 72, "down_proj" ], [ 3, 73, "gate_up_proj" ], [ 3, 73, "down_proj" ], [ 3, 99, "down_proj" ], [ 3, 166, "gate_up_proj" ], [ 3, 166, "down_proj" ], [ 3, 173, "gate_up_proj" ], [ 3, 173, "down_proj" ], [ 3, 180, "gate_up_proj" ], [ 3, 180, "down_proj" ], [ 3, 207, "down_proj" ], [ 3, 246, "gate_up_proj" ], [ 3, 246, "down_proj" ], [ 3, 251, "gate_up_proj" ], [ 3, 251, "down_proj" ], [ 3, 252, "gate_up_proj" ], [ 3, 252, "down_proj" ], [ 4, 62, "gate_up_proj" ], [ 4, 62, "down_proj" ], [ 4, 178, "gate_up_proj" ], [ 4, 181, "down_proj" ], [ 4, 235, "gate_up_proj" ], [ 4, 235, "down_proj" ], [ 4, 239, "gate_up_proj" ], [ 4, 239, "down_proj" ], [ 5, 65, "down_proj" ], [ 5, 97, "gate_up_proj" ], [ 5, 97, "down_proj" ], [ 5, 210, "gate_up_proj" ], [ 5, 210, "down_proj" ], [ 5, 231, "gate_up_proj" ], [ 5, 231, "down_proj" ], [ 6, 112, "down_proj" ], [ 6, 188, "gate_up_proj" ], [ 6, 188, "down_proj" ], [ 7, 164, "down_proj" ], [ 7, 241, "gate_up_proj" ], [ 7, 241, "down_proj" ], [ 8, 82, "gate_up_proj" ], [ 8, 82, "down_proj" ], [ 9, 101, "down_proj" ], [ 9, 140, "down_proj" ], [ 9, 162, "down_proj" ], [ 9, 221, "down_proj" ], [ 10, 6, "gate_up_proj" ], [ 10, 6, "down_proj" ], [ 10, 174, "down_proj" ], [ 10, 222, "down_proj" ], [ 13, 40, "gate_up_proj" ], [ 13, 40, "down_proj" ], [ 21, 253, "gate_up_proj" ], [ 21, 254, "gate_up_proj" ], [ 21, 255, "gate_up_proj" ], [ 22, 224, "gate_up_proj" ], [ 22, 225, "down_proj" ], [ 22, 226, "down_proj" ], [ 22, 227, "down_proj" ], [ 39, 146, "gate_up_proj" ], [ 39, 146, "down_proj" ] ], "decode_mode": "quantlut_sym", "expert_bits": 2, "num_experts": 256, "skip_list": [ "0_k", "0_o", "0_q", "0_v", "10_k", "10_o", "10_q", "10_v", "11_in_proj_qkv", "11_in_proj_z", "11_out_proj", "12_k", "12_o", "12_q", "12_v", "13_k", "13_o", "13_q", "13_v", "14_k", "14_o", "14_q", "14_v", "15_in_proj_qkv", "15_in_proj_z", "15_out_proj", "16_k", "16_o", "16_q", "16_v", "17_k", "17_o", "17_q", "17_v", "18_k", "18_o", "18_q", "18_v", "19_in_proj_qkv", "19_in_proj_z", "19_out_proj", "1_k", "1_o", "1_q", "1_v", "20_k", "20_o", "20_q", "20_v", "21_k", "21_o", "21_q", "21_v", "22_k", "22_o", "22_q", "22_v", "23_in_proj_qkv", "23_in_proj_z", "23_out_proj", "24_k", "24_o", "24_q", "24_v", "25_k", "25_o", "25_q", "25_v", "26_k", "26_o", "26_q", "26_v", "27_in_proj_qkv", "27_in_proj_z", "27_out_proj", "28_k", "28_o", "28_q", "28_v", "29_k", "29_o", "29_q", "29_v", "2_k", "2_o", "2_q", "2_v", "30_k", "30_o", "30_q", "30_v", "31_in_proj_qkv", "31_in_proj_z", "31_out_proj", "32_k", "32_o", "32_q", "32_v", "33_k", "33_o", "33_q", "33_v", "34_k", "34_o", "34_q", "34_v", "35_in_proj_qkv", "35_in_proj_z", "35_out_proj", "36_k", "36_o", "36_q", "36_v", "37_k", "37_o", "37_q", "37_v", "38_k", "38_o", "38_q", "38_v", "39_in_proj_qkv", "39_in_proj_z", "39_out_proj", "3_in_proj_qkv", "3_in_proj_z", "3_out_proj", "40_k", "40_o", "40_q", "40_v", "41_k", "41_o", "41_q", "41_v", "42_k", "42_o", "42_q", "42_v", "43_in_proj_qkv", "43_in_proj_z", "43_out_proj", "44_k", "44_o", "44_q", "44_v", "45_k", "45_o", "45_q", "45_v", "46_k", "46_o", "46_q", "46_v", "47_in_proj_qkv", "47_in_proj_z", "47_out_proj", "4_k", "4_o", "4_q", "4_v", "5_k", "5_o", "5_q", "5_v", "6_k", "6_o", "6_q", "6_v", "7_in_proj_qkv", "7_in_proj_z", "7_out_proj", "8_k", "8_o", "8_q", "8_v", "9_k", "9_o", "9_q", "9_v" ], "split_for_tp": false, "td_x": 16, "td_y": 16, "tlut_bits": 9 }, "text_config": { "attention_bias": false, "attention_dropout": 0.0, "attn_output_gate": true, "bos_token_id": null, "dtype": "bfloat16", "eos_token_id": 248044, "full_attention_interval": 4, "head_dim": 256, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "layer_types": [ "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention" ], "linear_conv_kernel_dim": 4, "linear_key_head_dim": 128, "linear_num_key_heads": 16, "linear_num_value_heads": 64, "linear_value_head_dim": 128, "mamba_ssm_dtype": "float32", "max_position_embeddings": 262144, "mlp_only_layers": [], "model_type": "qwen3_5_moe_text", "moe_intermediate_size": 1024, "mtp_num_hidden_layers": 1, "mtp_use_dedicated_embeddings": false, "num_attention_heads": 32, "num_experts": 256, "num_experts_per_tok": 8, "num_hidden_layers": 48, "num_key_value_heads": 2, "output_router_logits": false, "pad_token_id": null, "partial_rotary_factor": 0.25, "rms_norm_eps": 1e-06, "rope_parameters": { "mrope_interleaved": true, "mrope_section": [ 11, 11, 10 ], "partial_rotary_factor": 0.25, "rope_theta": 10000000, "rope_type": "default" }, "router_aux_loss_coef": 0.001, "shared_expert_intermediate_size": 1024, "tie_word_embeddings": false, "use_cache": true, "vocab_size": 248320 }, "tie_word_embeddings": false, "transformers_version": "5.3.0", "video_token_id": 248057, "vision_config": { "deepstack_visual_indexes": [], "depth": 27, "dtype": "bfloat16", "hidden_act": "gelu_pytorch_tanh", "hidden_size": 1152, "in_channels": 3, "initializer_range": 0.02, "intermediate_size": 4304, "model_type": "qwen3_5_moe", "num_heads": 16, "num_position_embeddings": 2304, "out_hidden_size": 3072, "patch_size": 16, "spatial_merge_size": 2, "temporal_patch_size": 2 }, "vision_end_token_id": 248054, "vision_start_token_id": 248053, "auto_map": { "AutoModel": "modeling_qwen3_5_moe_quantized.Qwen3_5MoeQuantizedForConditionalGeneration", "AutoModelForCausalLM": "modeling_qwen3_5_moe_quantized.Qwen3_5MoeQuantizedForConditionalGeneration" } }