| { | |
| "arch": "qwen3_5_moe", | |
| "T": 1024, | |
| "sizes": [ | |
| 1, | |
| 16 | |
| ], | |
| "config": { | |
| "attention_bias": false, | |
| "attention_dropout": 0.0, | |
| "attn_output_gate": true, | |
| "bos_token_id": 248044, | |
| "dtype": "bfloat16", | |
| "eos_token_id": 248044, | |
| "full_attention_interval": 4, | |
| "head_dim": 256, | |
| "hidden_act": "silu", | |
| "hidden_size": 2048, | |
| "initializer_range": 0.02, | |
| "layer_types": [ | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention" | |
| ], | |
| "linear_conv_kernel_dim": 4, | |
| "linear_key_head_dim": 128, | |
| "linear_num_key_heads": 16, | |
| "linear_num_value_heads": 32, | |
| "linear_value_head_dim": 128, | |
| "mamba_ssm_dtype": "float32", | |
| "max_position_embeddings": 262144, | |
| "model_type": "qwen3_5_moe_text", | |
| "moe_intermediate_size": 512, | |
| "mtp_num_hidden_layers": 1, | |
| "mtp_use_dedicated_embeddings": false, | |
| "num_attention_heads": 16, | |
| "num_experts": 256, | |
| "num_experts_per_tok": 8, | |
| "num_hidden_layers": 40, | |
| "num_key_value_heads": 2, | |
| "output_router_logits": false, | |
| "pad_token_id": null, | |
| "partial_rotary_factor": 0.25, | |
| "rms_norm_eps": 1e-06, | |
| "rope_parameters": { | |
| "mrope_interleaved": true, | |
| "mrope_section": [ | |
| 11, | |
| 11, | |
| 10 | |
| ], | |
| "partial_rotary_factor": 0.25, | |
| "rope_theta": 10000000, | |
| "rope_type": "default" | |
| }, | |
| "router_aux_loss_coef": 0.001, | |
| "shared_expert_intermediate_size": 512, | |
| "tie_word_embeddings": false, | |
| "use_cache": true, | |
| "vocab_size": 248320 | |
| }, | |
| "segments": { | |
| "seg00_S1.xml": { | |
| "index": 0, | |
| "consume_moe": null, | |
| "router_layer": 0, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg00.bin" | |
| }, | |
| "seg01_S1.xml": { | |
| "index": 1, | |
| "consume_moe": 0, | |
| "router_layer": 1, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg01.bin" | |
| }, | |
| "seg02_S1.xml": { | |
| "index": 2, | |
| "consume_moe": 1, | |
| "router_layer": 2, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg02.bin" | |
| }, | |
| "seg03_S1.xml": { | |
| "index": 3, | |
| "consume_moe": 2, | |
| "router_layer": 3, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg03.bin" | |
| }, | |
| "seg04_S1.xml": { | |
| "index": 4, | |
| "consume_moe": 3, | |
| "router_layer": 4, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg04.bin" | |
| }, | |
| "seg05_S1.xml": { | |
| "index": 5, | |
| "consume_moe": 4, | |
| "router_layer": 5, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg05.bin" | |
| }, | |
| "seg06_S1.xml": { | |
| "index": 6, | |
| "consume_moe": 5, | |
| "router_layer": 6, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg06.bin" | |
| }, | |
| "seg07_S1.xml": { | |
| "index": 7, | |
| "consume_moe": 6, | |
| "router_layer": 7, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg07.bin" | |
| }, | |
| "seg08_S1.xml": { | |
| "index": 8, | |
| "consume_moe": 7, | |
| "router_layer": 8, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg08.bin" | |
| }, | |
| "seg09_S1.xml": { | |
| "index": 9, | |
| "consume_moe": 8, | |
| "router_layer": 9, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg09.bin" | |
| }, | |
| "seg10_S1.xml": { | |
| "index": 10, | |
| "consume_moe": 9, | |
| "router_layer": 10, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg10.bin" | |
| }, | |
| "seg11_S1.xml": { | |
| "index": 11, | |
| "consume_moe": 10, | |
| "router_layer": 11, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg11.bin" | |
| }, | |
| "seg12_S1.xml": { | |
| "index": 12, | |
| "consume_moe": 11, | |
| "router_layer": 12, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg12.bin" | |
| }, | |
| "seg13_S1.xml": { | |
| "index": 13, | |
| "consume_moe": 12, | |
| "router_layer": 13, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg13.bin" | |
| }, | |
| "seg14_S1.xml": { | |
| "index": 14, | |
| "consume_moe": 13, | |
| "router_layer": 14, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg14.bin" | |
| }, | |
| "seg15_S1.xml": { | |
| "index": 15, | |
| "consume_moe": 14, | |
| "router_layer": 15, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg15.bin" | |
| }, | |
| "seg16_S1.xml": { | |
| "index": 16, | |
| "consume_moe": 15, | |
| "router_layer": 16, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg16.bin" | |
| }, | |
| "seg17_S1.xml": { | |
| "index": 17, | |
| "consume_moe": 16, | |
| "router_layer": 17, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg17.bin" | |
| }, | |
| "seg18_S1.xml": { | |
| "index": 18, | |
| "consume_moe": 17, | |
| "router_layer": 18, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg18.bin" | |
| }, | |
| "seg19_S1.xml": { | |
| "index": 19, | |
| "consume_moe": 18, | |
| "router_layer": 19, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg19.bin" | |
| }, | |
| "seg20_S1.xml": { | |
| "index": 20, | |
| "consume_moe": 19, | |
| "router_layer": 20, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg20.bin" | |
| }, | |
| "seg21_S1.xml": { | |
| "index": 21, | |
| "consume_moe": 20, | |
| "router_layer": 21, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg21.bin" | |
| }, | |
| "seg22_S1.xml": { | |
| "index": 22, | |
| "consume_moe": 21, | |
| "router_layer": 22, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg22.bin" | |
| }, | |
| "seg23_S1.xml": { | |
| "index": 23, | |
| "consume_moe": 22, | |
| "router_layer": 23, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg23.bin" | |
| }, | |
| "seg24_S1.xml": { | |
| "index": 24, | |
| "consume_moe": 23, | |
| "router_layer": 24, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg24.bin" | |
| }, | |
| "seg25_S1.xml": { | |
| "index": 25, | |
| "consume_moe": 24, | |
| "router_layer": 25, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg25.bin" | |
| }, | |
| "seg26_S1.xml": { | |
| "index": 26, | |
| "consume_moe": 25, | |
| "router_layer": 26, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg26.bin" | |
| }, | |
| "seg27_S1.xml": { | |
| "index": 27, | |
| "consume_moe": 26, | |
| "router_layer": 27, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg27.bin" | |
| }, | |
| "seg28_S1.xml": { | |
| "index": 28, | |
| "consume_moe": 27, | |
| "router_layer": 28, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg28.bin" | |
| }, | |
| "seg29_S1.xml": { | |
| "index": 29, | |
| "consume_moe": 28, | |
| "router_layer": 29, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg29.bin" | |
| }, | |
| "seg30_S1.xml": { | |
| "index": 30, | |
| "consume_moe": 29, | |
| "router_layer": 30, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg30.bin" | |
| }, | |
| "seg31_S1.xml": { | |
| "index": 31, | |
| "consume_moe": 30, | |
| "router_layer": 31, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg31.bin" | |
| }, | |
| "seg32_S1.xml": { | |
| "index": 32, | |
| "consume_moe": 31, | |
| "router_layer": 32, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg32.bin" | |
| }, | |
| "seg33_S1.xml": { | |
| "index": 33, | |
| "consume_moe": 32, | |
| "router_layer": 33, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg33.bin" | |
| }, | |
| "seg34_S1.xml": { | |
| "index": 34, | |
| "consume_moe": 33, | |
| "router_layer": 34, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg34.bin" | |
| }, | |
| "seg35_S1.xml": { | |
| "index": 35, | |
| "consume_moe": 34, | |
| "router_layer": 35, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg35.bin" | |
| }, | |
| "seg36_S1.xml": { | |
| "index": 36, | |
| "consume_moe": 35, | |
| "router_layer": 36, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg36.bin" | |
| }, | |
| "seg37_S1.xml": { | |
| "index": 37, | |
| "consume_moe": 36, | |
| "router_layer": 37, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg37.bin" | |
| }, | |
| "seg38_S1.xml": { | |
| "index": 38, | |
| "consume_moe": 37, | |
| "router_layer": 38, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg38.bin" | |
| }, | |
| "seg39_S1.xml": { | |
| "index": 39, | |
| "consume_moe": 38, | |
| "router_layer": 39, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg39.bin" | |
| }, | |
| "seg40_S1.xml": { | |
| "index": 40, | |
| "consume_moe": 39, | |
| "router_layer": null, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg40.bin" | |
| }, | |
| "seg41_S1.xml": { | |
| "index": 41, | |
| "consume_moe": null, | |
| "router_layer": null, | |
| "S": 1, | |
| "slots": 0, | |
| "head": true, | |
| "bin": "seg41.bin" | |
| }, | |
| "seg00_S16.xml": { | |
| "index": 0, | |
| "consume_moe": null, | |
| "router_layer": 0, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg00.bin" | |
| }, | |
| "seg01_S16.xml": { | |
| "index": 1, | |
| "consume_moe": 0, | |
| "router_layer": 1, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg01.bin" | |
| }, | |
| "seg02_S16.xml": { | |
| "index": 2, | |
| "consume_moe": 1, | |
| "router_layer": 2, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg02.bin" | |
| }, | |
| "seg03_S16.xml": { | |
| "index": 3, | |
| "consume_moe": 2, | |
| "router_layer": 3, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg03.bin" | |
| }, | |
| "seg04_S16.xml": { | |
| "index": 4, | |
| "consume_moe": 3, | |
| "router_layer": 4, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg04.bin" | |
| }, | |
| "seg05_S16.xml": { | |
| "index": 5, | |
| "consume_moe": 4, | |
| "router_layer": 5, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg05.bin" | |
| }, | |
| "seg06_S16.xml": { | |
| "index": 6, | |
| "consume_moe": 5, | |
| "router_layer": 6, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg06.bin" | |
| }, | |
| "seg07_S16.xml": { | |
| "index": 7, | |
| "consume_moe": 6, | |
| "router_layer": 7, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg07.bin" | |
| }, | |
| "seg08_S16.xml": { | |
| "index": 8, | |
| "consume_moe": 7, | |
| "router_layer": 8, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg08.bin" | |
| }, | |
| "seg09_S16.xml": { | |
| "index": 9, | |
| "consume_moe": 8, | |
| "router_layer": 9, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg09.bin" | |
| }, | |
| "seg10_S16.xml": { | |
| "index": 10, | |
| "consume_moe": 9, | |
| "router_layer": 10, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg10.bin" | |
| }, | |
| "seg11_S16.xml": { | |
| "index": 11, | |
| "consume_moe": 10, | |
| "router_layer": 11, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg11.bin" | |
| }, | |
| "seg12_S16.xml": { | |
| "index": 12, | |
| "consume_moe": 11, | |
| "router_layer": 12, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg12.bin" | |
| }, | |
| "seg13_S16.xml": { | |
| "index": 13, | |
| "consume_moe": 12, | |
| "router_layer": 13, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg13.bin" | |
| }, | |
| "seg14_S16.xml": { | |
| "index": 14, | |
| "consume_moe": 13, | |
| "router_layer": 14, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg14.bin" | |
| }, | |
| "seg15_S16.xml": { | |
| "index": 15, | |
| "consume_moe": 14, | |
| "router_layer": 15, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg15.bin" | |
| }, | |
| "seg16_S16.xml": { | |
| "index": 16, | |
| "consume_moe": 15, | |
| "router_layer": 16, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg16.bin" | |
| }, | |
| "seg17_S16.xml": { | |
| "index": 17, | |
| "consume_moe": 16, | |
| "router_layer": 17, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg17.bin" | |
| }, | |
| "seg18_S16.xml": { | |
| "index": 18, | |
| "consume_moe": 17, | |
| "router_layer": 18, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg18.bin" | |
| }, | |
| "seg19_S16.xml": { | |
| "index": 19, | |
| "consume_moe": 18, | |
| "router_layer": 19, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg19.bin" | |
| }, | |
| "seg20_S16.xml": { | |
| "index": 20, | |
| "consume_moe": 19, | |
| "router_layer": 20, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg20.bin" | |
| }, | |
| "seg21_S16.xml": { | |
| "index": 21, | |
| "consume_moe": 20, | |
| "router_layer": 21, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg21.bin" | |
| }, | |
| "seg22_S16.xml": { | |
| "index": 22, | |
| "consume_moe": 21, | |
| "router_layer": 22, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg22.bin" | |
| }, | |
| "seg23_S16.xml": { | |
| "index": 23, | |
| "consume_moe": 22, | |
| "router_layer": 23, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg23.bin" | |
| }, | |
| "seg24_S16.xml": { | |
| "index": 24, | |
| "consume_moe": 23, | |
| "router_layer": 24, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg24.bin" | |
| }, | |
| "seg25_S16.xml": { | |
| "index": 25, | |
| "consume_moe": 24, | |
| "router_layer": 25, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg25.bin" | |
| }, | |
| "seg26_S16.xml": { | |
| "index": 26, | |
| "consume_moe": 25, | |
| "router_layer": 26, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg26.bin" | |
| }, | |
| "seg27_S16.xml": { | |
| "index": 27, | |
| "consume_moe": 26, | |
| "router_layer": 27, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg27.bin" | |
| }, | |
| "seg28_S16.xml": { | |
| "index": 28, | |
| "consume_moe": 27, | |
| "router_layer": 28, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg28.bin" | |
| }, | |
| "seg29_S16.xml": { | |
| "index": 29, | |
| "consume_moe": 28, | |
| "router_layer": 29, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg29.bin" | |
| }, | |
| "seg30_S16.xml": { | |
| "index": 30, | |
| "consume_moe": 29, | |
| "router_layer": 30, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg30.bin" | |
| }, | |
| "seg31_S16.xml": { | |
| "index": 31, | |
| "consume_moe": 30, | |
| "router_layer": 31, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg31.bin" | |
| }, | |
| "seg32_S16.xml": { | |
| "index": 32, | |
| "consume_moe": 31, | |
| "router_layer": 32, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg32.bin" | |
| }, | |
| "seg33_S16.xml": { | |
| "index": 33, | |
| "consume_moe": 32, | |
| "router_layer": 33, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg33.bin" | |
| }, | |
| "seg34_S16.xml": { | |
| "index": 34, | |
| "consume_moe": 33, | |
| "router_layer": 34, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg34.bin" | |
| }, | |
| "seg35_S16.xml": { | |
| "index": 35, | |
| "consume_moe": 34, | |
| "router_layer": 35, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg35.bin" | |
| }, | |
| "seg36_S16.xml": { | |
| "index": 36, | |
| "consume_moe": 35, | |
| "router_layer": 36, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg36.bin" | |
| }, | |
| "seg37_S16.xml": { | |
| "index": 37, | |
| "consume_moe": 36, | |
| "router_layer": 37, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg37.bin" | |
| }, | |
| "seg38_S16.xml": { | |
| "index": 38, | |
| "consume_moe": 37, | |
| "router_layer": 38, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg38.bin" | |
| }, | |
| "seg39_S16.xml": { | |
| "index": 39, | |
| "consume_moe": 38, | |
| "router_layer": 39, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg39.bin" | |
| }, | |
| "seg40_S16.xml": { | |
| "index": 40, | |
| "consume_moe": 39, | |
| "router_layer": null, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg40.bin" | |
| }, | |
| "seg41_S16.xml": { | |
| "index": 41, | |
| "consume_moe": null, | |
| "router_layer": null, | |
| "S": 16, | |
| "slots": 0, | |
| "head": true, | |
| "bin": "seg41.bin" | |
| } | |
| }, | |
| "rope": { | |
| "dim": 64, | |
| "theta": 10000000, | |
| "mrope_section": [ | |
| 11, | |
| 11, | |
| 10 | |
| ], | |
| "mrope_interleaved": true | |
| }, | |
| "vision": { | |
| "type": "qwen", | |
| "grid": [ | |
| 32, | |
| 32 | |
| ], | |
| "tokens": 256, | |
| "merge": 2, | |
| "patch": 16, | |
| "temporal": 2, | |
| "mean": [ | |
| 0.5, | |
| 0.5, | |
| 0.5 | |
| ], | |
| "std": [ | |
| 0.5, | |
| 0.5, | |
| 0.5 | |
| ], | |
| "image_token_id": 248056, | |
| "image_px": [ | |
| 512, | |
| 512 | |
| ] | |
| }, | |
| "tables": { | |
| "embed": { | |
| "bits": 4, | |
| "group": 32 | |
| } | |
| } | |
| } |