{
 "arch": "qwen3_moe",
 "T": 1024,
 "sizes": [
  1,
  16
 ],
 "config": {
  "architectures": [
   "Qwen3MoeForCausalLM"
  ],
  "attention_bias": false,
  "attention_dropout": 0.0,
  "bos_token_id": 1,
  "decoder_sparse_step": 1,
  "dtype": "bfloat16",
  "eos_token_id": [
   2,
   13
  ],
  "head_dim": 128,
  "hidden_act": "silu",
  "hidden_size": 2560,
  "initializer_range": 0.006,
  "intermediate_size": 7680,
  "max_position_embeddings": 65536,
  "max_window_layers": 32,
  "mlp_only_layers": [],
  "model_type": "qwen3_moe",
  "moe_intermediate_size": 960,
  "norm_topk_prob": true,
  "num_attention_heads": 40,
  "num_experts": 128,
  "num_experts_per_tok": 8,
  "num_hidden_layers": 32,
  "num_key_value_heads": 4,
  "output_router_logits": false,
  "pad_token_id": 4,
  "rms_norm_eps": 1e-06,
  "rope_scaling": null,
  "rope_theta": 500000,
  "router_aux_loss_coef": 0.01,
  "sliding_window": null,
  "tie_word_embeddings": false,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.51.0",
  "use_cache": false,
  "use_sliding_window": false,
  "vocab_size": 196608
 },
 "segments": {
  "seg00_S1.xml": {
   "index": 0,
   "consume_moe": null,
   "router_layer": 0,
   "S": 1,
   "slots": 8,
   "bin": "seg00.bin"
  },
  "seg01_S1.xml": {
   "index": 1,
   "consume_moe": 0,
   "router_layer": 1,
   "S": 1,
   "slots": 8,
   "bin": "seg01.bin"
  },
  "seg02_S1.xml": {
   "index": 2,
   "consume_moe": 1,
   "router_layer": 2,
   "S": 1,
   "slots": 8,
   "bin": "seg02.bin"
  },
  "seg03_S1.xml": {
   "index": 3,
   "consume_moe": 2,
   "router_layer": 3,
   "S": 1,
   "slots": 8,
   "bin": "seg03.bin"
  },
  "seg04_S1.xml": {
   "index": 4,
   "consume_moe": 3,
   "router_layer": 4,
   "S": 1,
   "slots": 8,
   "bin": "seg04.bin"
  },
  "seg05_S1.xml": {
   "index": 5,
   "consume_moe": 4,
   "router_layer": 5,
   "S": 1,
   "slots": 8,
   "bin": "seg05.bin"
  },
  "seg06_S1.xml": {
   "index": 6,
   "consume_moe": 5,
   "router_layer": 6,
   "S": 1,
   "slots": 8,
   "bin": "seg06.bin"
  },
  "seg07_S1.xml": {
   "index": 7,
   "consume_moe": 6,
   "router_layer": 7,
   "S": 1,
   "slots": 8,
   "bin": "seg07.bin"
  },
  "seg08_S1.xml": {
   "index": 8,
   "consume_moe": 7,
   "router_layer": 8,
   "S": 1,
   "slots": 8,
   "bin": "seg08.bin"
  },
  "seg09_S1.xml": {
   "index": 9,
   "consume_moe": 8,
   "router_layer": 9,
   "S": 1,
   "slots": 8,
   "bin": "seg09.bin"
  },
  "seg10_S1.xml": {
   "index": 10,
   "consume_moe": 9,
   "router_layer": 10,
   "S": 1,
   "slots": 8,
   "bin": "seg10.bin"
  },
  "seg11_S1.xml": {
   "index": 11,
   "consume_moe": 10,
   "router_layer": 11,
   "S": 1,
   "slots": 8,
   "bin": "seg11.bin"
  },
  "seg12_S1.xml": {
   "index": 12,
   "consume_moe": 11,
   "router_layer": 12,
   "S": 1,
   "slots": 8,
   "bin": "seg12.bin"
  },
  "seg13_S1.xml": {
   "index": 13,
   "consume_moe": 12,
   "router_layer": 13,
   "S": 1,
   "slots": 8,
   "bin": "seg13.bin"
  },
  "seg14_S1.xml": {
   "index": 14,
   "consume_moe": 13,
   "router_layer": 14,
   "S": 1,
   "slots": 8,
   "bin": "seg14.bin"
  },
  "seg15_S1.xml": {
   "index": 15,
   "consume_moe": 14,
   "router_layer": 15,
   "S": 1,
   "slots": 8,
   "bin": "seg15.bin"
  },
  "seg16_S1.xml": {
   "index": 16,
   "consume_moe": 15,
   "router_layer": 16,
   "S": 1,
   "slots": 8,
   "bin": "seg16.bin"
  },
  "seg17_S1.xml": {
   "index": 17,
   "consume_moe": 16,
   "router_layer": 17,
   "S": 1,
   "slots": 8,
   "bin": "seg17.bin"
  },
  "seg18_S1.xml": {
   "index": 18,
   "consume_moe": 17,
   "router_layer": 18,
   "S": 1,
   "slots": 8,
   "bin": "seg18.bin"
  },
  "seg19_S1.xml": {
   "index": 19,
   "consume_moe": 18,
   "router_layer": 19,
   "S": 1,
   "slots": 8,
   "bin": "seg19.bin"
  },
  "seg20_S1.xml": {
   "index": 20,
   "consume_moe": 19,
   "router_layer": 20,
   "S": 1,
   "slots": 8,
   "bin": "seg20.bin"
  },
  "seg21_S1.xml": {
   "index": 21,
   "consume_moe": 20,
   "router_layer": 21,
   "S": 1,
   "slots": 8,
   "bin": "seg21.bin"
  },
  "seg22_S1.xml": {
   "index": 22,
   "consume_moe": 21,
   "router_layer": 22,
   "S": 1,
   "slots": 8,
   "bin": "seg22.bin"
  },
  "seg23_S1.xml": {
   "index": 23,
   "consume_moe": 22,
   "router_layer": 23,
   "S": 1,
   "slots": 8,
   "bin": "seg23.bin"
  },
  "seg24_S1.xml": {
   "index": 24,
   "consume_moe": 23,
   "router_layer": 24,
   "S": 1,
   "slots": 8,
   "bin": "seg24.bin"
  },
  "seg25_S1.xml": {
   "index": 25,
   "consume_moe": 24,
   "router_layer": 25,
   "S": 1,
   "slots": 8,
   "bin": "seg25.bin"
  },
  "seg26_S1.xml": {
   "index": 26,
   "consume_moe": 25,
   "router_layer": 26,
   "S": 1,
   "slots": 8,
   "bin": "seg26.bin"
  },
  "seg27_S1.xml": {
   "index": 27,
   "consume_moe": 26,
   "router_layer": 27,
   "S": 1,
   "slots": 8,
   "bin": "seg27.bin"
  },
  "seg28_S1.xml": {
   "index": 28,
   "consume_moe": 27,
   "router_layer": 28,
   "S": 1,
   "slots": 8,
   "bin": "seg28.bin"
  },
  "seg29_S1.xml": {
   "index": 29,
   "consume_moe": 28,
   "router_layer": 29,
   "S": 1,
   "slots": 8,
   "bin": "seg29.bin"
  },
  "seg30_S1.xml": {
   "index": 30,
   "consume_moe": 29,
   "router_layer": 30,
   "S": 1,
   "slots": 8,
   "bin": "seg30.bin"
  },
  "seg31_S1.xml": {
   "index": 31,
   "consume_moe": 30,
   "router_layer": 31,
   "S": 1,
   "slots": 8,
   "bin": "seg31.bin"
  },
  "seg32_S1.xml": {
   "index": 32,
   "consume_moe": 31,
   "router_layer": null,
   "S": 1,
   "slots": 8,
   "bin": "seg32.bin"
  },
  "seg33_S1.xml": {
   "index": 33,
   "consume_moe": null,
   "router_layer": null,
   "S": 1,
   "slots": 0,
   "head": true,
   "bin": "seg33.bin"
  },
  "seg00_S16.xml": {
   "index": 0,
   "consume_moe": null,
   "router_layer": 0,
   "S": 16,
   "slots": 128,
   "bin": "seg00.bin"
  },
  "seg01_S16.xml": {
   "index": 1,
   "consume_moe": 0,
   "router_layer": 1,
   "S": 16,
   "slots": 128,
   "bin": "seg01.bin"
  },
  "seg02_S16.xml": {
   "index": 2,
   "consume_moe": 1,
   "router_layer": 2,
   "S": 16,
   "slots": 128,
   "bin": "seg02.bin"
  },
  "seg03_S16.xml": {
   "index": 3,
   "consume_moe": 2,
   "router_layer": 3,
   "S": 16,
   "slots": 128,
   "bin": "seg03.bin"
  },
  "seg04_S16.xml": {
   "index": 4,
   "consume_moe": 3,
   "router_layer": 4,
   "S": 16,
   "slots": 128,
   "bin": "seg04.bin"
  },
  "seg05_S16.xml": {
   "index": 5,
   "consume_moe": 4,
   "router_layer": 5,
   "S": 16,
   "slots": 128,
   "bin": "seg05.bin"
  },
  "seg06_S16.xml": {
   "index": 6,
   "consume_moe": 5,
   "router_layer": 6,
   "S": 16,
   "slots": 128,
   "bin": "seg06.bin"
  },
  "seg07_S16.xml": {
   "index": 7,
   "consume_moe": 6,
   "router_layer": 7,
   "S": 16,
   "slots": 128,
   "bin": "seg07.bin"
  },
  "seg08_S16.xml": {
   "index": 8,
   "consume_moe": 7,
   "router_layer": 8,
   "S": 16,
   "slots": 128,
   "bin": "seg08.bin"
  },
  "seg09_S16.xml": {
   "index": 9,
   "consume_moe": 8,
   "router_layer": 9,
   "S": 16,
   "slots": 128,
   "bin": "seg09.bin"
  },
  "seg10_S16.xml": {
   "index": 10,
   "consume_moe": 9,
   "router_layer": 10,
   "S": 16,
   "slots": 128,
   "bin": "seg10.bin"
  },
  "seg11_S16.xml": {
   "index": 11,
   "consume_moe": 10,
   "router_layer": 11,
   "S": 16,
   "slots": 128,
   "bin": "seg11.bin"
  },
  "seg12_S16.xml": {
   "index": 12,
   "consume_moe": 11,
   "router_layer": 12,
   "S": 16,
   "slots": 128,
   "bin": "seg12.bin"
  },
  "seg13_S16.xml": {
   "index": 13,
   "consume_moe": 12,
   "router_layer": 13,
   "S": 16,
   "slots": 128,
   "bin": "seg13.bin"
  },
  "seg14_S16.xml": {
   "index": 14,
   "consume_moe": 13,
   "router_layer": 14,
   "S": 16,
   "slots": 128,
   "bin": "seg14.bin"
  },
  "seg15_S16.xml": {
   "index": 15,
   "consume_moe": 14,
   "router_layer": 15,
   "S": 16,
   "slots": 128,
   "bin": "seg15.bin"
  },
  "seg16_S16.xml": {
   "index": 16,
   "consume_moe": 15,
   "router_layer": 16,
   "S": 16,
   "slots": 128,
   "bin": "seg16.bin"
  },
  "seg17_S16.xml": {
   "index": 17,
   "consume_moe": 16,
   "router_layer": 17,
   "S": 16,
   "slots": 128,
   "bin": "seg17.bin"
  },
  "seg18_S16.xml": {
   "index": 18,
   "consume_moe": 17,
   "router_layer": 18,
   "S": 16,
   "slots": 128,
   "bin": "seg18.bin"
  },
  "seg19_S16.xml": {
   "index": 19,
   "consume_moe": 18,
   "router_layer": 19,
   "S": 16,
   "slots": 128,
   "bin": "seg19.bin"
  },
  "seg20_S16.xml": {
   "index": 20,
   "consume_moe": 19,
   "router_layer": 20,
   "S": 16,
   "slots": 128,
   "bin": "seg20.bin"
  },
  "seg21_S16.xml": {
   "index": 21,
   "consume_moe": 20,
   "router_layer": 21,
   "S": 16,
   "slots": 128,
   "bin": "seg21.bin"
  },
  "seg22_S16.xml": {
   "index": 22,
   "consume_moe": 21,
   "router_layer": 22,
   "S": 16,
   "slots": 128,
   "bin": "seg22.bin"
  },
  "seg23_S16.xml": {
   "index": 23,
   "consume_moe": 22,
   "router_layer": 23,
   "S": 16,
   "slots": 128,
   "bin": "seg23.bin"
  },
  "seg24_S16.xml": {
   "index": 24,
   "consume_moe": 23,
   "router_layer": 24,
   "S": 16,
   "slots": 128,
   "bin": "seg24.bin"
  },
  "seg25_S16.xml": {
   "index": 25,
   "consume_moe": 24,
   "router_layer": 25,
   "S": 16,
   "slots": 128,
   "bin": "seg25.bin"
  },
  "seg26_S16.xml": {
   "index": 26,
   "consume_moe": 25,
   "router_layer": 26,
   "S": 16,
   "slots": 128,
   "bin": "seg26.bin"
  },
  "seg27_S16.xml": {
   "index": 27,
   "consume_moe": 26,
   "router_layer": 27,
   "S": 16,
   "slots": 128,
   "bin": "seg27.bin"
  },
  "seg28_S16.xml": {
   "index": 28,
   "consume_moe": 27,
   "router_layer": 28,
   "S": 16,
   "slots": 128,
   "bin": "seg28.bin"
  },
  "seg29_S16.xml": {
   "index": 29,
   "consume_moe": 28,
   "router_layer": 29,
   "S": 16,
   "slots": 128,
   "bin": "seg29.bin"
  },
  "seg30_S16.xml": {
   "index": 30,
   "consume_moe": 29,
   "router_layer": 30,
   "S": 16,
   "slots": 128,
   "bin": "seg30.bin"
  },
  "seg31_S16.xml": {
   "index": 31,
   "consume_moe": 30,
   "router_layer": 31,
   "S": 16,
   "slots": 128,
   "bin": "seg31.bin"
  },
  "seg32_S16.xml": {
   "index": 32,
   "consume_moe": 31,
   "router_layer": null,
   "S": 16,
   "slots": 128,
   "bin": "seg32.bin"
  },
  "seg33_S16.xml": {
   "index": 33,
   "consume_moe": null,
   "router_layer": null,
   "S": 16,
   "slots": 0,
   "head": true,
   "bin": "seg33.bin"
  }
 },
 "rope": {
  "dim": 128,
  "theta": 500000
 },
 "tables": {
  "embed": {
   "bits": 4,
   "group": 32
  }
 }
}