| { | |
| "arch": "qwen3_moe", | |
| "T": 1024, | |
| "sizes": [ | |
| 1, | |
| 16 | |
| ], | |
| "config": { | |
| "architectures": [ | |
| "Qwen3MoeForCausalLM" | |
| ], | |
| "attention_bias": false, | |
| "attention_dropout": 0.0, | |
| "bos_token_id": 1, | |
| "decoder_sparse_step": 1, | |
| "dtype": "bfloat16", | |
| "eos_token_id": [ | |
| 2, | |
| 13 | |
| ], | |
| "head_dim": 128, | |
| "hidden_act": "silu", | |
| "hidden_size": 2560, | |
| "initializer_range": 0.006, | |
| "intermediate_size": 7680, | |
| "max_position_embeddings": 65536, | |
| "max_window_layers": 32, | |
| "mlp_only_layers": [], | |
| "model_type": "qwen3_moe", | |
| "moe_intermediate_size": 960, | |
| "norm_topk_prob": true, | |
| "num_attention_heads": 40, | |
| "num_experts": 128, | |
| "num_experts_per_tok": 8, | |
| "num_hidden_layers": 32, | |
| "num_key_value_heads": 4, | |
| "output_router_logits": false, | |
| "pad_token_id": 4, | |
| "rms_norm_eps": 1e-06, | |
| "rope_scaling": null, | |
| "rope_theta": 500000, | |
| "router_aux_loss_coef": 0.01, | |
| "sliding_window": null, | |
| "tie_word_embeddings": false, | |
| "torch_dtype": "bfloat16", | |
| "transformers_version": "4.51.0", | |
| "use_cache": false, | |
| "use_sliding_window": false, | |
| "vocab_size": 196608 | |
| }, | |
| "segments": { | |
| "seg00_S1.xml": { | |
| "index": 0, | |
| "consume_moe": null, | |
| "router_layer": 0, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg00.bin" | |
| }, | |
| "seg01_S1.xml": { | |
| "index": 1, | |
| "consume_moe": 0, | |
| "router_layer": 1, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg01.bin" | |
| }, | |
| "seg02_S1.xml": { | |
| "index": 2, | |
| "consume_moe": 1, | |
| "router_layer": 2, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg02.bin" | |
| }, | |
| "seg03_S1.xml": { | |
| "index": 3, | |
| "consume_moe": 2, | |
| "router_layer": 3, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg03.bin" | |
| }, | |
| "seg04_S1.xml": { | |
| "index": 4, | |
| "consume_moe": 3, | |
| "router_layer": 4, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg04.bin" | |
| }, | |
| "seg05_S1.xml": { | |
| "index": 5, | |
| "consume_moe": 4, | |
| "router_layer": 5, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg05.bin" | |
| }, | |
| "seg06_S1.xml": { | |
| "index": 6, | |
| "consume_moe": 5, | |
| "router_layer": 6, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg06.bin" | |
| }, | |
| "seg07_S1.xml": { | |
| "index": 7, | |
| "consume_moe": 6, | |
| "router_layer": 7, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg07.bin" | |
| }, | |
| "seg08_S1.xml": { | |
| "index": 8, | |
| "consume_moe": 7, | |
| "router_layer": 8, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg08.bin" | |
| }, | |
| "seg09_S1.xml": { | |
| "index": 9, | |
| "consume_moe": 8, | |
| "router_layer": 9, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg09.bin" | |
| }, | |
| "seg10_S1.xml": { | |
| "index": 10, | |
| "consume_moe": 9, | |
| "router_layer": 10, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg10.bin" | |
| }, | |
| "seg11_S1.xml": { | |
| "index": 11, | |
| "consume_moe": 10, | |
| "router_layer": 11, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg11.bin" | |
| }, | |
| "seg12_S1.xml": { | |
| "index": 12, | |
| "consume_moe": 11, | |
| "router_layer": 12, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg12.bin" | |
| }, | |
| "seg13_S1.xml": { | |
| "index": 13, | |
| "consume_moe": 12, | |
| "router_layer": 13, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg13.bin" | |
| }, | |
| "seg14_S1.xml": { | |
| "index": 14, | |
| "consume_moe": 13, | |
| "router_layer": 14, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg14.bin" | |
| }, | |
| "seg15_S1.xml": { | |
| "index": 15, | |
| "consume_moe": 14, | |
| "router_layer": 15, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg15.bin" | |
| }, | |
| "seg16_S1.xml": { | |
| "index": 16, | |
| "consume_moe": 15, | |
| "router_layer": 16, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg16.bin" | |
| }, | |
| "seg17_S1.xml": { | |
| "index": 17, | |
| "consume_moe": 16, | |
| "router_layer": 17, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg17.bin" | |
| }, | |
| "seg18_S1.xml": { | |
| "index": 18, | |
| "consume_moe": 17, | |
| "router_layer": 18, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg18.bin" | |
| }, | |
| "seg19_S1.xml": { | |
| "index": 19, | |
| "consume_moe": 18, | |
| "router_layer": 19, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg19.bin" | |
| }, | |
| "seg20_S1.xml": { | |
| "index": 20, | |
| "consume_moe": 19, | |
| "router_layer": 20, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg20.bin" | |
| }, | |
| "seg21_S1.xml": { | |
| "index": 21, | |
| "consume_moe": 20, | |
| "router_layer": 21, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg21.bin" | |
| }, | |
| "seg22_S1.xml": { | |
| "index": 22, | |
| "consume_moe": 21, | |
| "router_layer": 22, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg22.bin" | |
| }, | |
| "seg23_S1.xml": { | |
| "index": 23, | |
| "consume_moe": 22, | |
| "router_layer": 23, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg23.bin" | |
| }, | |
| "seg24_S1.xml": { | |
| "index": 24, | |
| "consume_moe": 23, | |
| "router_layer": 24, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg24.bin" | |
| }, | |
| "seg25_S1.xml": { | |
| "index": 25, | |
| "consume_moe": 24, | |
| "router_layer": 25, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg25.bin" | |
| }, | |
| "seg26_S1.xml": { | |
| "index": 26, | |
| "consume_moe": 25, | |
| "router_layer": 26, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg26.bin" | |
| }, | |
| "seg27_S1.xml": { | |
| "index": 27, | |
| "consume_moe": 26, | |
| "router_layer": 27, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg27.bin" | |
| }, | |
| "seg28_S1.xml": { | |
| "index": 28, | |
| "consume_moe": 27, | |
| "router_layer": 28, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg28.bin" | |
| }, | |
| "seg29_S1.xml": { | |
| "index": 29, | |
| "consume_moe": 28, | |
| "router_layer": 29, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg29.bin" | |
| }, | |
| "seg30_S1.xml": { | |
| "index": 30, | |
| "consume_moe": 29, | |
| "router_layer": 30, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg30.bin" | |
| }, | |
| "seg31_S1.xml": { | |
| "index": 31, | |
| "consume_moe": 30, | |
| "router_layer": 31, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg31.bin" | |
| }, | |
| "seg32_S1.xml": { | |
| "index": 32, | |
| "consume_moe": 31, | |
| "router_layer": null, | |
| "S": 1, | |
| "slots": 8, | |
| "bin": "seg32.bin" | |
| }, | |
| "seg33_S1.xml": { | |
| "index": 33, | |
| "consume_moe": null, | |
| "router_layer": null, | |
| "S": 1, | |
| "slots": 0, | |
| "head": true, | |
| "bin": "seg33.bin" | |
| }, | |
| "seg00_S16.xml": { | |
| "index": 0, | |
| "consume_moe": null, | |
| "router_layer": 0, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg00.bin" | |
| }, | |
| "seg01_S16.xml": { | |
| "index": 1, | |
| "consume_moe": 0, | |
| "router_layer": 1, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg01.bin" | |
| }, | |
| "seg02_S16.xml": { | |
| "index": 2, | |
| "consume_moe": 1, | |
| "router_layer": 2, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg02.bin" | |
| }, | |
| "seg03_S16.xml": { | |
| "index": 3, | |
| "consume_moe": 2, | |
| "router_layer": 3, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg03.bin" | |
| }, | |
| "seg04_S16.xml": { | |
| "index": 4, | |
| "consume_moe": 3, | |
| "router_layer": 4, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg04.bin" | |
| }, | |
| "seg05_S16.xml": { | |
| "index": 5, | |
| "consume_moe": 4, | |
| "router_layer": 5, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg05.bin" | |
| }, | |
| "seg06_S16.xml": { | |
| "index": 6, | |
| "consume_moe": 5, | |
| "router_layer": 6, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg06.bin" | |
| }, | |
| "seg07_S16.xml": { | |
| "index": 7, | |
| "consume_moe": 6, | |
| "router_layer": 7, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg07.bin" | |
| }, | |
| "seg08_S16.xml": { | |
| "index": 8, | |
| "consume_moe": 7, | |
| "router_layer": 8, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg08.bin" | |
| }, | |
| "seg09_S16.xml": { | |
| "index": 9, | |
| "consume_moe": 8, | |
| "router_layer": 9, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg09.bin" | |
| }, | |
| "seg10_S16.xml": { | |
| "index": 10, | |
| "consume_moe": 9, | |
| "router_layer": 10, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg10.bin" | |
| }, | |
| "seg11_S16.xml": { | |
| "index": 11, | |
| "consume_moe": 10, | |
| "router_layer": 11, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg11.bin" | |
| }, | |
| "seg12_S16.xml": { | |
| "index": 12, | |
| "consume_moe": 11, | |
| "router_layer": 12, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg12.bin" | |
| }, | |
| "seg13_S16.xml": { | |
| "index": 13, | |
| "consume_moe": 12, | |
| "router_layer": 13, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg13.bin" | |
| }, | |
| "seg14_S16.xml": { | |
| "index": 14, | |
| "consume_moe": 13, | |
| "router_layer": 14, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg14.bin" | |
| }, | |
| "seg15_S16.xml": { | |
| "index": 15, | |
| "consume_moe": 14, | |
| "router_layer": 15, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg15.bin" | |
| }, | |
| "seg16_S16.xml": { | |
| "index": 16, | |
| "consume_moe": 15, | |
| "router_layer": 16, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg16.bin" | |
| }, | |
| "seg17_S16.xml": { | |
| "index": 17, | |
| "consume_moe": 16, | |
| "router_layer": 17, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg17.bin" | |
| }, | |
| "seg18_S16.xml": { | |
| "index": 18, | |
| "consume_moe": 17, | |
| "router_layer": 18, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg18.bin" | |
| }, | |
| "seg19_S16.xml": { | |
| "index": 19, | |
| "consume_moe": 18, | |
| "router_layer": 19, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg19.bin" | |
| }, | |
| "seg20_S16.xml": { | |
| "index": 20, | |
| "consume_moe": 19, | |
| "router_layer": 20, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg20.bin" | |
| }, | |
| "seg21_S16.xml": { | |
| "index": 21, | |
| "consume_moe": 20, | |
| "router_layer": 21, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg21.bin" | |
| }, | |
| "seg22_S16.xml": { | |
| "index": 22, | |
| "consume_moe": 21, | |
| "router_layer": 22, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg22.bin" | |
| }, | |
| "seg23_S16.xml": { | |
| "index": 23, | |
| "consume_moe": 22, | |
| "router_layer": 23, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg23.bin" | |
| }, | |
| "seg24_S16.xml": { | |
| "index": 24, | |
| "consume_moe": 23, | |
| "router_layer": 24, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg24.bin" | |
| }, | |
| "seg25_S16.xml": { | |
| "index": 25, | |
| "consume_moe": 24, | |
| "router_layer": 25, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg25.bin" | |
| }, | |
| "seg26_S16.xml": { | |
| "index": 26, | |
| "consume_moe": 25, | |
| "router_layer": 26, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg26.bin" | |
| }, | |
| "seg27_S16.xml": { | |
| "index": 27, | |
| "consume_moe": 26, | |
| "router_layer": 27, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg27.bin" | |
| }, | |
| "seg28_S16.xml": { | |
| "index": 28, | |
| "consume_moe": 27, | |
| "router_layer": 28, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg28.bin" | |
| }, | |
| "seg29_S16.xml": { | |
| "index": 29, | |
| "consume_moe": 28, | |
| "router_layer": 29, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg29.bin" | |
| }, | |
| "seg30_S16.xml": { | |
| "index": 30, | |
| "consume_moe": 29, | |
| "router_layer": 30, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg30.bin" | |
| }, | |
| "seg31_S16.xml": { | |
| "index": 31, | |
| "consume_moe": 30, | |
| "router_layer": 31, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg31.bin" | |
| }, | |
| "seg32_S16.xml": { | |
| "index": 32, | |
| "consume_moe": 31, | |
| "router_layer": null, | |
| "S": 16, | |
| "slots": 128, | |
| "bin": "seg32.bin" | |
| }, | |
| "seg33_S16.xml": { | |
| "index": 33, | |
| "consume_moe": null, | |
| "router_layer": null, | |
| "S": 16, | |
| "slots": 0, | |
| "head": true, | |
| "bin": "seg33.bin" | |
| } | |
| }, | |
| "rope": { | |
| "dim": 128, | |
| "theta": 500000 | |
| }, | |
| "tables": { | |
| "embed": { | |
| "bits": 4, | |
| "group": 32 | |
| } | |
| } | |
| } |