Add files using upload-large-folder tool

config.json

@@ -0,0 +1,640 @@
+{
+  "architectures": [
+    "LongcatCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attention_method": "MLA",
+  "bos_token_id": 1,
+  "cli_factor": 2,
+  "disable_quant_module": [],
+  "dsa_mtp_cli": true,
+  "eos_token_id": 2,
+  "expert_ffn_hidden_size": 2048,
+  "ffn_hidden_size": 12288,
+  "hidden_size": 8192,
+  "index_head_dim": 128,
+  "index_init_tokens": 16,
+  "index_k_norm_type": "rms",
+  "index_local_tokens": 1024,
+  "index_n_heads": 32,
+  "index_topk": 2048,
+  "kv_lora_rank": 512,
+  "max_position_embeddings": 262144,
+  "mla_scale_kv_lora": true,
+  "mla_scale_q_lora": true,
+  "moe_impl": "mix",
+  "moe_switch_token_num": 1024,
+  "moe_topk": 12,
+  "mtp_disable_over_tokenizer": true,
+  "mtp_num_layers": 3,
+  "mtp_replicate_modules": true,
+  "n_routed_experts": 768,
+  "num_attention_heads": 64,
+  "num_layers": 38,
+  "oe_neighbor_num": 5,
+  "oe_split_num": 4,
+  "oe_vocab_size_ratio": 100.567,
+  "q_lora_rank": 1536,
+  "qk_nope_head_dim": 128,
+  "qk_rope_head_dim": 64,
+  "quantization_config": {
+    "activation_scheme": "dynamic",
+    "fmt": "e4m3",
+    "ignored_layers": [
+      "lm_head",
+      "model.embed_tokens",
+      "model.layers.0.input_layernorm.0",
+      "model.layers.0.input_layernorm.1",
+      "model.layers.0.mlp.router.classifier",
+      "model.layers.0.mlp.router.e_score_correction_bias",
+      "model.layers.0.post_attention_layernorm.0",
+      "model.layers.0.post_attention_layernorm.1",
+      "model.layers.0.self_attn.0.indexer.k_norm",
+      "model.layers.0.self_attn.0.indexer.weights_proj",
+      "model.layers.0.self_attn.0.indexer.wq_b",
+      "model.layers.0.self_attn.0.kv_a_layernorm",
+      "model.layers.0.self_attn.0.q_a_layernorm",
+      "model.layers.0.self_attn.1.kv_a_layernorm",
+      "model.layers.0.self_attn.1.q_a_layernorm",
+      "model.layers.1.input_layernorm.0",
+      "model.layers.1.input_layernorm.1",
+      "model.layers.1.mlp.router.classifier",
+      "model.layers.1.mlp.router.e_score_correction_bias",
+      "model.layers.1.post_attention_layernorm.0",
+      "model.layers.1.post_attention_layernorm.1",
+      "model.layers.1.self_attn.0.indexer.k_norm",
+      "model.layers.1.self_attn.0.indexer.weights_proj",
+      "model.layers.1.self_attn.0.indexer.wq_b",
+      "model.layers.1.self_attn.0.kv_a_layernorm",
+      "model.layers.1.self_attn.0.q_a_layernorm",
+      "model.layers.1.self_attn.1.kv_a_layernorm",
+      "model.layers.1.self_attn.1.q_a_layernorm",
+      "model.layers.10.input_layernorm.0",
+      "model.layers.10.input_layernorm.1",
+      "model.layers.10.mlp.router.classifier",
+      "model.layers.10.mlp.router.e_score_correction_bias",
+      "model.layers.10.post_attention_layernorm.0",
+      "model.layers.10.post_attention_layernorm.1",
+      "model.layers.10.self_attn.0.indexer.k_norm",
+      "model.layers.10.self_attn.0.indexer.weights_proj",
+      "model.layers.10.self_attn.0.indexer.wq_b",
+      "model.layers.10.self_attn.0.kv_a_layernorm",
+      "model.layers.10.self_attn.0.q_a_layernorm",
+      "model.layers.10.self_attn.1.kv_a_layernorm",
+      "model.layers.10.self_attn.1.q_a_layernorm",
+      "model.layers.11.input_layernorm.0",
+      "model.layers.11.input_layernorm.1",
+      "model.layers.11.mlp.router.classifier",
+      "model.layers.11.mlp.router.e_score_correction_bias",
+      "model.layers.11.post_attention_layernorm.0",
+      "model.layers.11.post_attention_layernorm.1",
+      "model.layers.11.self_attn.0.indexer.k_norm",
+      "model.layers.11.self_attn.0.indexer.weights_proj",
+      "model.layers.11.self_attn.0.indexer.wq_b",
+      "model.layers.11.self_attn.0.kv_a_layernorm",
+      "model.layers.11.self_attn.0.q_a_layernorm",
+      "model.layers.11.self_attn.1.kv_a_layernorm",
+      "model.layers.11.self_attn.1.q_a_layernorm",
+      "model.layers.12.input_layernorm.0",
+      "model.layers.12.input_layernorm.1",
+      "model.layers.12.mlp.router.classifier",
+      "model.layers.12.mlp.router.e_score_correction_bias",
+      "model.layers.12.post_attention_layernorm.0",
+      "model.layers.12.post_attention_layernorm.1",
+      "model.layers.12.self_attn.0.indexer.k_norm",
+      "model.layers.12.self_attn.0.indexer.weights_proj",
+      "model.layers.12.self_attn.0.indexer.wq_b",
+      "model.layers.12.self_attn.0.kv_a_layernorm",
+      "model.layers.12.self_attn.0.q_a_layernorm",
+      "model.layers.12.self_attn.1.kv_a_layernorm",
+      "model.layers.12.self_attn.1.q_a_layernorm",
+      "model.layers.13.input_layernorm.0",
+      "model.layers.13.input_layernorm.1",
+      "model.layers.13.mlp.router.classifier",
+      "model.layers.13.mlp.router.e_score_correction_bias",
+      "model.layers.13.post_attention_layernorm.0",
+      "model.layers.13.post_attention_layernorm.1",
+      "model.layers.13.self_attn.0.indexer.k_norm",
+      "model.layers.13.self_attn.0.indexer.weights_proj",
+      "model.layers.13.self_attn.0.indexer.wq_b",
+      "model.layers.13.self_attn.0.kv_a_layernorm",
+      "model.layers.13.self_attn.0.q_a_layernorm",
+      "model.layers.13.self_attn.1.kv_a_layernorm",
+      "model.layers.13.self_attn.1.q_a_layernorm",
+      "model.layers.14.input_layernorm.0",
+      "model.layers.14.input_layernorm.1",
+      "model.layers.14.mlp.router.classifier",
+      "model.layers.14.mlp.router.e_score_correction_bias",
+      "model.layers.14.post_attention_layernorm.0",
+      "model.layers.14.post_attention_layernorm.1",
+      "model.layers.14.self_attn.0.indexer.k_norm",
+      "model.layers.14.self_attn.0.indexer.weights_proj",
+      "model.layers.14.self_attn.0.indexer.wq_b",
+      "model.layers.14.self_attn.0.kv_a_layernorm",
+      "model.layers.14.self_attn.0.q_a_layernorm",
+      "model.layers.14.self_attn.1.kv_a_layernorm",
+      "model.layers.14.self_attn.1.q_a_layernorm",
+      "model.layers.15.input_layernorm.0",
+      "model.layers.15.input_layernorm.1",
+      "model.layers.15.mlp.router.classifier",
+      "model.layers.15.mlp.router.e_score_correction_bias",
+      "model.layers.15.post_attention_layernorm.0",
+      "model.layers.15.post_attention_layernorm.1",
+      "model.layers.15.self_attn.0.indexer.k_norm",
+      "model.layers.15.self_attn.0.indexer.weights_proj",
+      "model.layers.15.self_attn.0.indexer.wq_b",
+      "model.layers.15.self_attn.0.kv_a_layernorm",
+      "model.layers.15.self_attn.0.q_a_layernorm",
+      "model.layers.15.self_attn.1.kv_a_layernorm",
+      "model.layers.15.self_attn.1.q_a_layernorm",
+      "model.layers.16.input_layernorm.0",
+      "model.layers.16.input_layernorm.1",
+      "model.layers.16.mlp.router.classifier",
+      "model.layers.16.mlp.router.e_score_correction_bias",
+      "model.layers.16.post_attention_layernorm.0",
+      "model.layers.16.post_attention_layernorm.1",
+      "model.layers.16.self_attn.0.indexer.k_norm",
+      "model.layers.16.self_attn.0.indexer.weights_proj",
+      "model.layers.16.self_attn.0.indexer.wq_b",
+      "model.layers.16.self_attn.0.kv_a_layernorm",
+      "model.layers.16.self_attn.0.q_a_layernorm",
+      "model.layers.16.self_attn.1.kv_a_layernorm",
+      "model.layers.16.self_attn.1.q_a_layernorm",
+      "model.layers.17.input_layernorm.0",
+      "model.layers.17.input_layernorm.1",
+      "model.layers.17.mlp.router.classifier",
+      "model.layers.17.mlp.router.e_score_correction_bias",
+      "model.layers.17.post_attention_layernorm.0",
+      "model.layers.17.post_attention_layernorm.1",
+      "model.layers.17.self_attn.0.indexer.k_norm",
+      "model.layers.17.self_attn.0.indexer.weights_proj",
+      "model.layers.17.self_attn.0.indexer.wq_b",
+      "model.layers.17.self_attn.0.kv_a_layernorm",
+      "model.layers.17.self_attn.0.q_a_layernorm",
+      "model.layers.17.self_attn.1.kv_a_layernorm",
+      "model.layers.17.self_attn.1.q_a_layernorm",
+      "model.layers.18.input_layernorm.0",
+      "model.layers.18.input_layernorm.1",
+      "model.layers.18.mlp.router.classifier",
+      "model.layers.18.mlp.router.e_score_correction_bias",
+      "model.layers.18.post_attention_layernorm.0",
+      "model.layers.18.post_attention_layernorm.1",
+      "model.layers.18.self_attn.0.indexer.k_norm",
+      "model.layers.18.self_attn.0.indexer.weights_proj",
+      "model.layers.18.self_attn.0.indexer.wq_b",
+      "model.layers.18.self_attn.0.kv_a_layernorm",
+      "model.layers.18.self_attn.0.q_a_layernorm",
+      "model.layers.18.self_attn.1.kv_a_layernorm",
+      "model.layers.18.self_attn.1.q_a_layernorm",
+      "model.layers.19.input_layernorm.0",
+      "model.layers.19.input_layernorm.1",
+      "model.layers.19.mlp.router.classifier",
+      "model.layers.19.mlp.router.e_score_correction_bias",
+      "model.layers.19.post_attention_layernorm.0",
+      "model.layers.19.post_attention_layernorm.1",
+      "model.layers.19.self_attn.0.indexer.k_norm",
+      "model.layers.19.self_attn.0.indexer.weights_proj",
+      "model.layers.19.self_attn.0.indexer.wq_b",
+      "model.layers.19.self_attn.0.kv_a_layernorm",
+      "model.layers.19.self_attn.0.q_a_layernorm",
+      "model.layers.19.self_attn.1.kv_a_layernorm",
+      "model.layers.19.self_attn.1.q_a_layernorm",
+      "model.layers.2.input_layernorm.0",
+      "model.layers.2.input_layernorm.1",
+      "model.layers.2.mlp.router.classifier",
+      "model.layers.2.mlp.router.e_score_correction_bias",
+      "model.layers.2.post_attention_layernorm.0",
+      "model.layers.2.post_attention_layernorm.1",
+      "model.layers.2.self_attn.0.indexer.k_norm",
+      "model.layers.2.self_attn.0.indexer.weights_proj",
+      "model.layers.2.self_attn.0.indexer.wq_b",
+      "model.layers.2.self_attn.0.kv_a_layernorm",
+      "model.layers.2.self_attn.0.q_a_layernorm",
+      "model.layers.2.self_attn.1.kv_a_layernorm",
+      "model.layers.2.self_attn.1.q_a_layernorm",
+      "model.layers.20.input_layernorm.0",
+      "model.layers.20.input_layernorm.1",
+      "model.layers.20.mlp.router.classifier",
+      "model.layers.20.mlp.router.e_score_correction_bias",
+      "model.layers.20.post_attention_layernorm.0",
+      "model.layers.20.post_attention_layernorm.1",
+      "model.layers.20.self_attn.0.indexer.k_norm",
+      "model.layers.20.self_attn.0.indexer.weights_proj",
+      "model.layers.20.self_attn.0.indexer.wq_b",
+      "model.layers.20.self_attn.0.kv_a_layernorm",
+      "model.layers.20.self_attn.0.q_a_layernorm",
+      "model.layers.20.self_attn.1.kv_a_layernorm",
+      "model.layers.20.self_attn.1.q_a_layernorm",
+      "model.layers.21.input_layernorm.0",
+      "model.layers.21.input_layernorm.1",
+      "model.layers.21.mlp.router.classifier",
+      "model.layers.21.mlp.router.e_score_correction_bias",
+      "model.layers.21.post_attention_layernorm.0",
+      "model.layers.21.post_attention_layernorm.1",
+      "model.layers.21.self_attn.0.indexer.k_norm",
+      "model.layers.21.self_attn.0.indexer.weights_proj",
+      "model.layers.21.self_attn.0.indexer.wq_b",
+      "model.layers.21.self_attn.0.kv_a_layernorm",
+      "model.layers.21.self_attn.0.q_a_layernorm",
+      "model.layers.21.self_attn.1.kv_a_layernorm",
+      "model.layers.21.self_attn.1.q_a_layernorm",
+      "model.layers.22.input_layernorm.0",
+      "model.layers.22.input_layernorm.1",
+      "model.layers.22.mlp.router.classifier",
+      "model.layers.22.mlp.router.e_score_correction_bias",
+      "model.layers.22.post_attention_layernorm.0",
+      "model.layers.22.post_attention_layernorm.1",
+      "model.layers.22.self_attn.0.indexer.k_norm",
+      "model.layers.22.self_attn.0.indexer.weights_proj",
+      "model.layers.22.self_attn.0.indexer.wq_b",
+      "model.layers.22.self_attn.0.kv_a_layernorm",
+      "model.layers.22.self_attn.0.q_a_layernorm",
+      "model.layers.22.self_attn.1.kv_a_layernorm",
+      "model.layers.22.self_attn.1.q_a_layernorm",
+      "model.layers.23.input_layernorm.0",
+      "model.layers.23.input_layernorm.1",
+      "model.layers.23.mlp.router.classifier",
+      "model.layers.23.mlp.router.e_score_correction_bias",
+      "model.layers.23.post_attention_layernorm.0",
+      "model.layers.23.post_attention_layernorm.1",
+      "model.layers.23.self_attn.0.indexer.k_norm",
+      "model.layers.23.self_attn.0.indexer.weights_proj",
+      "model.layers.23.self_attn.0.indexer.wq_b",
+      "model.layers.23.self_attn.0.kv_a_layernorm",
+      "model.layers.23.self_attn.0.q_a_layernorm",
+      "model.layers.23.self_attn.1.kv_a_layernorm",
+      "model.layers.23.self_attn.1.q_a_layernorm",
+      "model.layers.24.input_layernorm.0",
+      "model.layers.24.input_layernorm.1",
+      "model.layers.24.mlp.router.classifier",
+      "model.layers.24.mlp.router.e_score_correction_bias",
+      "model.layers.24.post_attention_layernorm.0",
+      "model.layers.24.post_attention_layernorm.1",
+      "model.layers.24.self_attn.0.indexer.k_norm",
+      "model.layers.24.self_attn.0.indexer.weights_proj",
+      "model.layers.24.self_attn.0.indexer.wq_b",
+      "model.layers.24.self_attn.0.kv_a_layernorm",
+      "model.layers.24.self_attn.0.q_a_layernorm",
+      "model.layers.24.self_attn.1.kv_a_layernorm",
+      "model.layers.24.self_attn.1.q_a_layernorm",
+      "model.layers.25.input_layernorm.0",
+      "model.layers.25.input_layernorm.1",
+      "model.layers.25.mlp.router.classifier",
+      "model.layers.25.mlp.router.e_score_correction_bias",
+      "model.layers.25.post_attention_layernorm.0",
+      "model.layers.25.post_attention_layernorm.1",
+      "model.layers.25.self_attn.0.indexer.k_norm",
+      "model.layers.25.self_attn.0.indexer.weights_proj",
+      "model.layers.25.self_attn.0.indexer.wq_b",
+      "model.layers.25.self_attn.0.kv_a_layernorm",
+      "model.layers.25.self_attn.0.q_a_layernorm",
+      "model.layers.25.self_attn.1.kv_a_layernorm",
+      "model.layers.25.self_attn.1.q_a_layernorm",
+      "model.layers.26.input_layernorm.0",
+      "model.layers.26.input_layernorm.1",
+      "model.layers.26.mlp.router.classifier",
+      "model.layers.26.mlp.router.e_score_correction_bias",
+      "model.layers.26.post_attention_layernorm.0",
+      "model.layers.26.post_attention_layernorm.1",
+      "model.layers.26.self_attn.0.indexer.k_norm",
+      "model.layers.26.self_attn.0.indexer.weights_proj",
+      "model.layers.26.self_attn.0.indexer.wq_b",
+      "model.layers.26.self_attn.0.kv_a_layernorm",
+      "model.layers.26.self_attn.0.q_a_layernorm",
+      "model.layers.26.self_attn.1.kv_a_layernorm",
+      "model.layers.26.self_attn.1.q_a_layernorm",
+      "model.layers.27.input_layernorm.0",
+      "model.layers.27.input_layernorm.1",
+      "model.layers.27.mlp.router.classifier",
+      "model.layers.27.mlp.router.e_score_correction_bias",
+      "model.layers.27.post_attention_layernorm.0",
+      "model.layers.27.post_attention_layernorm.1",
+      "model.layers.27.self_attn.0.indexer.k_norm",
+      "model.layers.27.self_attn.0.indexer.weights_proj",
+      "model.layers.27.self_attn.0.indexer.wq_b",
+      "model.layers.27.self_attn.0.kv_a_layernorm",
+      "model.layers.27.self_attn.0.q_a_layernorm",
+      "model.layers.27.self_attn.1.kv_a_layernorm",
+      "model.layers.27.self_attn.1.q_a_layernorm",
+      "model.layers.28.input_layernorm.0",
+      "model.layers.28.input_layernorm.1",
+      "model.layers.28.mlp.router.classifier",
+      "model.layers.28.mlp.router.e_score_correction_bias",
+      "model.layers.28.post_attention_layernorm.0",
+      "model.layers.28.post_attention_layernorm.1",
+      "model.layers.28.self_attn.0.indexer.k_norm",
+      "model.layers.28.self_attn.0.indexer.weights_proj",
+      "model.layers.28.self_attn.0.indexer.wq_b",
+      "model.layers.28.self_attn.0.kv_a_layernorm",
+      "model.layers.28.self_attn.0.q_a_layernorm",
+      "model.layers.28.self_attn.1.kv_a_layernorm",
+      "model.layers.28.self_attn.1.q_a_layernorm",
+      "model.layers.29.input_layernorm.0",
+      "model.layers.29.input_layernorm.1",
+      "model.layers.29.mlp.router.classifier",
+      "model.layers.29.mlp.router.e_score_correction_bias",
+      "model.layers.29.post_attention_layernorm.0",
+      "model.layers.29.post_attention_layernorm.1",
+      "model.layers.29.self_attn.0.indexer.k_norm",
+      "model.layers.29.self_attn.0.indexer.weights_proj",
+      "model.layers.29.self_attn.0.indexer.wq_b",
+      "model.layers.29.self_attn.0.kv_a_layernorm",
+      "model.layers.29.self_attn.0.q_a_layernorm",
+      "model.layers.29.self_attn.1.kv_a_layernorm",
+      "model.layers.29.self_attn.1.q_a_layernorm",
+      "model.layers.3.input_layernorm.0",
+      "model.layers.3.input_layernorm.1",
+      "model.layers.3.mlp.router.classifier",
+      "model.layers.3.mlp.router.e_score_correction_bias",
+      "model.layers.3.post_attention_layernorm.0",
+      "model.layers.3.post_attention_layernorm.1",
+      "model.layers.3.self_attn.0.indexer.k_norm",
+      "model.layers.3.self_attn.0.indexer.weights_proj",
+      "model.layers.3.self_attn.0.indexer.wq_b",
+      "model.layers.3.self_attn.0.kv_a_layernorm",
+      "model.layers.3.self_attn.0.q_a_layernorm",
+      "model.layers.3.self_attn.1.kv_a_layernorm",
+      "model.layers.3.self_attn.1.q_a_layernorm",
+      "model.layers.30.input_layernorm.0",
+      "model.layers.30.input_layernorm.1",
+      "model.layers.30.mlp.router.classifier",
+      "model.layers.30.mlp.router.e_score_correction_bias",
+      "model.layers.30.post_attention_layernorm.0",
+      "model.layers.30.post_attention_layernorm.1",
+      "model.layers.30.self_attn.0.indexer.k_norm",
+      "model.layers.30.self_attn.0.indexer.weights_proj",
+      "model.layers.30.self_attn.0.indexer.wq_b",
+      "model.layers.30.self_attn.0.kv_a_layernorm",
+      "model.layers.30.self_attn.0.q_a_layernorm",
+      "model.layers.30.self_attn.1.kv_a_layernorm",
+      "model.layers.30.self_attn.1.q_a_layernorm",
+      "model.layers.31.input_layernorm.0",
+      "model.layers.31.input_layernorm.1",
+      "model.layers.31.mlp.router.classifier",
+      "model.layers.31.mlp.router.e_score_correction_bias",
+      "model.layers.31.post_attention_layernorm.0",
+      "model.layers.31.post_attention_layernorm.1",
+      "model.layers.31.self_attn.0.indexer.k_norm",
+      "model.layers.31.self_attn.0.indexer.weights_proj",
+      "model.layers.31.self_attn.0.indexer.wq_b",
+      "model.layers.31.self_attn.0.kv_a_layernorm",
+      "model.layers.31.self_attn.0.q_a_layernorm",
+      "model.layers.31.self_attn.1.kv_a_layernorm",
+      "model.layers.31.self_attn.1.q_a_layernorm",
+      "model.layers.32.input_layernorm.0",
+      "model.layers.32.input_layernorm.1",
+      "model.layers.32.mlp.router.classifier",
+      "model.layers.32.mlp.router.e_score_correction_bias",
+      "model.layers.32.post_attention_layernorm.0",
+      "model.layers.32.post_attention_layernorm.1",
+      "model.layers.32.self_attn.0.indexer.k_norm",
+      "model.layers.32.self_attn.0.indexer.weights_proj",
+      "model.layers.32.self_attn.0.indexer.wq_b",
+      "model.layers.32.self_attn.0.kv_a_layernorm",
+      "model.layers.32.self_attn.0.q_a_layernorm",
+      "model.layers.32.self_attn.1.kv_a_layernorm",
+      "model.layers.32.self_attn.1.q_a_layernorm",
+      "model.layers.33.input_layernorm.0",
+      "model.layers.33.input_layernorm.1",
+      "model.layers.33.mlp.router.classifier",
+      "model.layers.33.mlp.router.e_score_correction_bias",
+      "model.layers.33.post_attention_layernorm.0",
+      "model.layers.33.post_attention_layernorm.1",
+      "model.layers.33.self_attn.0.indexer.k_norm",
+      "model.layers.33.self_attn.0.indexer.weights_proj",
+      "model.layers.33.self_attn.0.indexer.wq_b",
+      "model.layers.33.self_attn.0.kv_a_layernorm",
+      "model.layers.33.self_attn.0.q_a_layernorm",
+      "model.layers.33.self_attn.1.kv_a_layernorm",
+      "model.layers.33.self_attn.1.q_a_layernorm",
+      "model.layers.34.input_layernorm.0",
+      "model.layers.34.input_layernorm.1",
+      "model.layers.34.mlp.router.classifier",
+      "model.layers.34.mlp.router.e_score_correction_bias",
+      "model.layers.34.post_attention_layernorm.0",
+      "model.layers.34.post_attention_layernorm.1",
+      "model.layers.34.self_attn.0.indexer.k_norm",
+      "model.layers.34.self_attn.0.indexer.weights_proj",
+      "model.layers.34.self_attn.0.indexer.wq_b",
+      "model.layers.34.self_attn.0.kv_a_layernorm",
+      "model.layers.34.self_attn.0.q_a_layernorm",
+      "model.layers.34.self_attn.1.kv_a_layernorm",
+      "model.layers.34.self_attn.1.q_a_layernorm",
+      "model.layers.35.input_layernorm.0",
+      "model.layers.35.input_layernorm.1",
+      "model.layers.35.mlp.router.classifier",
+      "model.layers.35.mlp.router.e_score_correction_bias",
+      "model.layers.35.post_attention_layernorm.0",
+      "model.layers.35.post_attention_layernorm.1",
+      "model.layers.35.self_attn.0.indexer.k_norm",
+      "model.layers.35.self_attn.0.indexer.weights_proj",
+      "model.layers.35.self_attn.0.indexer.wq_b",
+      "model.layers.35.self_attn.0.kv_a_layernorm",
+      "model.layers.35.self_attn.0.q_a_layernorm",
+      "model.layers.35.self_attn.1.kv_a_layernorm",
+      "model.layers.35.self_attn.1.q_a_layernorm",
+      "model.layers.36.input_layernorm.0",
+      "model.layers.36.input_layernorm.1",
+      "model.layers.36.mlp.router.classifier",
+      "model.layers.36.mlp.router.e_score_correction_bias",
+      "model.layers.36.post_attention_layernorm.0",
+      "model.layers.36.post_attention_layernorm.1",
+      "model.layers.36.self_attn.0.indexer.k_norm",
+      "model.layers.36.self_attn.0.indexer.weights_proj",
+      "model.layers.36.self_attn.0.indexer.wq_b",
+      "model.layers.36.self_attn.0.kv_a_layernorm",
+      "model.layers.36.self_attn.0.q_a_layernorm",
+      "model.layers.36.self_attn.1.kv_a_layernorm",
+      "model.layers.36.self_attn.1.q_a_layernorm",
+      "model.layers.37.input_layernorm.0",
+      "model.layers.37.input_layernorm.1",
+      "model.layers.37.mlp.router.classifier",
+      "model.layers.37.mlp.router.e_score_correction_bias",
+      "model.layers.37.post_attention_layernorm.0",
+      "model.layers.37.post_attention_layernorm.1",
+      "model.layers.37.self_attn.0.indexer.k_norm",
+      "model.layers.37.self_attn.0.indexer.weights_proj",
+      "model.layers.37.self_attn.0.indexer.wq_b",
+      "model.layers.37.self_attn.0.kv_a_layernorm",
+      "model.layers.37.self_attn.0.q_a_layernorm",
+      "model.layers.37.self_attn.1.kv_a_layernorm",
+      "model.layers.37.self_attn.1.q_a_layernorm",
+      "model.layers.4.input_layernorm.0",
+      "model.layers.4.input_layernorm.1",
+      "model.layers.4.mlp.router.classifier",
+      "model.layers.4.mlp.router.e_score_correction_bias",
+      "model.layers.4.post_attention_layernorm.0",
+      "model.layers.4.post_attention_layernorm.1",
+      "model.layers.4.self_attn.0.indexer.k_norm",
+      "model.layers.4.self_attn.0.indexer.weights_proj",
+      "model.layers.4.self_attn.0.indexer.wq_b",
+      "model.layers.4.self_attn.0.kv_a_layernorm",
+      "model.layers.4.self_attn.0.q_a_layernorm",
+      "model.layers.4.self_attn.1.kv_a_layernorm",
+      "model.layers.4.self_attn.1.q_a_layernorm",
+      "model.layers.5.input_layernorm.0",
+      "model.layers.5.input_layernorm.1",
+      "model.layers.5.mlp.router.classifier",
+      "model.layers.5.mlp.router.e_score_correction_bias",
+      "model.layers.5.post_attention_layernorm.0",
+      "model.layers.5.post_attention_layernorm.1",
+      "model.layers.5.self_attn.0.indexer.k_norm",
+      "model.layers.5.self_attn.0.indexer.weights_proj",
+      "model.layers.5.self_attn.0.indexer.wq_b",
+      "model.layers.5.self_attn.0.kv_a_layernorm",
+      "model.layers.5.self_attn.0.q_a_layernorm",
+      "model.layers.5.self_attn.1.kv_a_layernorm",
+      "model.layers.5.self_attn.1.q_a_layernorm",
+      "model.layers.6.input_layernorm.0",
+      "model.layers.6.input_layernorm.1",
+      "model.layers.6.mlp.router.classifier",
+      "model.layers.6.mlp.router.e_score_correction_bias",
+      "model.layers.6.post_attention_layernorm.0",
+      "model.layers.6.post_attention_layernorm.1",
+      "model.layers.6.self_attn.0.indexer.k_norm",
+      "model.layers.6.self_attn.0.indexer.weights_proj",
+      "model.layers.6.self_attn.0.indexer.wq_b",
+      "model.layers.6.self_attn.0.kv_a_layernorm",
+      "model.layers.6.self_attn.0.q_a_layernorm",
+      "model.layers.6.self_attn.1.kv_a_layernorm",
+      "model.layers.6.self_attn.1.q_a_layernorm",
+      "model.layers.7.input_layernorm.0",
+      "model.layers.7.input_layernorm.1",
+      "model.layers.7.mlp.router.classifier",
+      "model.layers.7.mlp.router.e_score_correction_bias",
+      "model.layers.7.post_attention_layernorm.0",
+      "model.layers.7.post_attention_layernorm.1",
+      "model.layers.7.self_attn.0.indexer.k_norm",
+      "model.layers.7.self_attn.0.indexer.weights_proj",
+      "model.layers.7.self_attn.0.indexer.wq_b",
+      "model.layers.7.self_attn.0.kv_a_layernorm",
+      "model.layers.7.self_attn.0.q_a_layernorm",
+      "model.layers.7.self_attn.1.kv_a_layernorm",
+      "model.layers.7.self_attn.1.q_a_layernorm",
+      "model.layers.8.input_layernorm.0",
+      "model.layers.8.input_layernorm.1",
+      "model.layers.8.mlp.router.classifier",
+      "model.layers.8.mlp.router.e_score_correction_bias",
+      "model.layers.8.post_attention_layernorm.0",
+      "model.layers.8.post_attention_layernorm.1",
+      "model.layers.8.self_attn.0.indexer.k_norm",
+      "model.layers.8.self_attn.0.indexer.weights_proj",
+      "model.layers.8.self_attn.0.indexer.wq_b",
+      "model.layers.8.self_attn.0.kv_a_layernorm",
+      "model.layers.8.self_attn.0.q_a_layernorm",
+      "model.layers.8.self_attn.1.kv_a_layernorm",
+      "model.layers.8.self_attn.1.q_a_layernorm",
+      "model.layers.9.input_layernorm.0",
+      "model.layers.9.input_layernorm.1",
+      "model.layers.9.mlp.router.classifier",
+      "model.layers.9.mlp.router.e_score_correction_bias",
+      "model.layers.9.post_attention_layernorm.0",
+      "model.layers.9.post_attention_layernorm.1",
+      "model.layers.9.self_attn.0.indexer.k_norm",
+      "model.layers.9.self_attn.0.indexer.weights_proj",
+      "model.layers.9.self_attn.0.indexer.wq_b",
+      "model.layers.9.self_attn.0.kv_a_layernorm",
+      "model.layers.9.self_attn.0.q_a_layernorm",
+      "model.layers.9.self_attn.1.kv_a_layernorm",
+      "model.layers.9.self_attn.1.q_a_layernorm",
+      "model.mtp.embed_tokens",
+      "model.mtp.layers.0.enorm.m",
+      "model.mtp.layers.0.hnorm.m",
+      "model.mtp.layers.0.input_layernorm",
+      "model.mtp.layers.0.post_attention_layernorm",
+      "model.mtp.layers.0.self_attn.indexer.k_norm",
+      "model.mtp.layers.0.self_attn.kv_a_layernorm",
+      "model.mtp.layers.0.self_attn.q_a_layernorm",
+      "model.mtp.ngram_embeddings.embedders.0",
+      "model.mtp.ngram_embeddings.embedders.1",
+      "model.mtp.ngram_embeddings.embedders.10",
+      "model.mtp.ngram_embeddings.embedders.11",
+      "model.mtp.ngram_embeddings.embedders.12",
+      "model.mtp.ngram_embeddings.embedders.13",
+      "model.mtp.ngram_embeddings.embedders.14",
+      "model.mtp.ngram_embeddings.embedders.15",
+      "model.mtp.ngram_embeddings.embedders.2",
+      "model.mtp.ngram_embeddings.embedders.3",
+      "model.mtp.ngram_embeddings.embedders.4",
+      "model.mtp.ngram_embeddings.embedders.5",
+      "model.mtp.ngram_embeddings.embedders.6",
+      "model.mtp.ngram_embeddings.embedders.7",
+      "model.mtp.ngram_embeddings.embedders.8",
+      "model.mtp.ngram_embeddings.embedders.9",
+      "model.mtp.ngram_embeddings.post_projs.0",
+      "model.mtp.ngram_embeddings.post_projs.1",
+      "model.mtp.ngram_embeddings.post_projs.10",
+      "model.mtp.ngram_embeddings.post_projs.11",
+      "model.mtp.ngram_embeddings.post_projs.12",
+      "model.mtp.ngram_embeddings.post_projs.13",
+      "model.mtp.ngram_embeddings.post_projs.14",
+      "model.mtp.ngram_embeddings.post_projs.15",
+      "model.mtp.ngram_embeddings.post_projs.2",
+      "model.mtp.ngram_embeddings.post_projs.3",
+      "model.mtp.ngram_embeddings.post_projs.4",
+      "model.mtp.ngram_embeddings.post_projs.5",
+      "model.mtp.ngram_embeddings.post_projs.6",
+      "model.mtp.ngram_embeddings.post_projs.7",
+      "model.mtp.ngram_embeddings.post_projs.8",
+      "model.mtp.ngram_embeddings.post_projs.9",
+      "model.mtp.norm",
+      "model.ngram_embeddings.embedders.0",
+      "model.ngram_embeddings.embedders.1",
+      "model.ngram_embeddings.embedders.10",
+      "model.ngram_embeddings.embedders.11",
+      "model.ngram_embeddings.embedders.12",
+      "model.ngram_embeddings.embedders.13",
+      "model.ngram_embeddings.embedders.14",
+      "model.ngram_embeddings.embedders.15",
+      "model.ngram_embeddings.embedders.2",
+      "model.ngram_embeddings.embedders.3",
+      "model.ngram_embeddings.embedders.4",
+      "model.ngram_embeddings.embedders.5",
+      "model.ngram_embeddings.embedders.6",
+      "model.ngram_embeddings.embedders.7",
+      "model.ngram_embeddings.embedders.8",
+      "model.ngram_embeddings.embedders.9",
+      "model.ngram_embeddings.post_projs.0",
+      "model.ngram_embeddings.post_projs.1",
+      "model.ngram_embeddings.post_projs.10",
+      "model.ngram_embeddings.post_projs.11",
+      "model.ngram_embeddings.post_projs.12",
+      "model.ngram_embeddings.post_projs.13",
+      "model.ngram_embeddings.post_projs.14",
+      "model.ngram_embeddings.post_projs.15",
+      "model.ngram_embeddings.post_projs.2",
+      "model.ngram_embeddings.post_projs.3",
+      "model.ngram_embeddings.post_projs.4",
+      "model.ngram_embeddings.post_projs.5",
+      "model.ngram_embeddings.post_projs.6",
+      "model.ngram_embeddings.post_projs.7",
+      "model.ngram_embeddings.post_projs.8",
+      "model.ngram_embeddings.post_projs.9",
+      "model.norm"
+    ],
+    "quant_method": "fp8",
+    "weight_block_size": [
+      128,
+      128
+    ]
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "factor": 120,
+    "mscale": 1,
+    "mscale_all_dim": 1,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "deepseek_yarn"
+  },
+  "rope_theta": 1000000.0,
+  "routed_scaling_factor": 9,
+  "use_cache": true,
+  "use_mla": 1,
+  "v_head_dim": 128,
+  "vocab_size": 163840,
+  "zero_expert_num": 128,
+  "zero_expert_type": "identity"
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 3,
+  "transformers_version": "4.55.0"
+}

megatron_config.json

@@ -0,0 +1 @@
+"{\"num_layers\": 38, \"encoder_num_layers\": 38, \"decoder_num_layers\": null, \"hidden_size\": 8192, \"kv_lora_rank\": 512, \"q_lora_rank\": 1536, \"qk_rope_head_dim\": 64, \"qk_nope_head_dim\": 128, \"v_head_dim\": 128, \"mla\": true, \"mla_recompute\": false, \"mla_recompute_layer_ids\": null, \"mla_scale_kv_lora\": 4.0, \"mla_scale_q_lora\": 2.309401076758503, \"mla_scale_v\": 1, \"base_size\": null, \"ffn_hidden_size\": 12288, \"expert_ffn_hidden_size\": 2048, \"num_attention_heads\": 64, \"kv_channels\": null, \"group_query_attention\": false, \"num_query_groups\": null, \"max_position_embeddings\": 131072, \"position_embedding_type\": \"rotary\", \"use_rotary_position_embeddings\": false, \"rotary_percent\": 1.0, \"rotary_seq_len_interpolation_factor\": null, \"use_rope_scaling\": false, \"add_position_embedding\": true, \"make_vocab_size_divisible_by\": 1, \"normalization\": \"RMSNorm\", \"norm_epsilon\": 1e-05, \"layernorm_epsilon\": 1e-05, \"apply_layernorm_1p\": false, \"apply_residual_connection_post_layernorm\": false, \"openai_gelu\": false, \"squared_relu\": false, \"swiglu\": true, \"squared_reglu\": false, \"fuse_swiglu_kernel\": false, \"onnx_safe\": null, \"bert_binary_head\": true, \"num_experts\": [768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, null, null, null], \"expert_interval\": 1, \"expert_model_parallel_size\": 256, \"use_ag_ep_comm_dispatcher\": false, \"moe_dispatch_type\": \"alltoall_abs\", \"zero_expert_num\": 128, \"zero_expert_type\": \"identity\", \"target_topk\": 8.0, \"only_adapt_ffn_bias\": true, \"skip_padding_tokens\": true, \"mock_data_for_skip_padding_val\": false, \"moe_expert_capacity_factor\": null, \"moe_expert_capacity_factor_in_fusion_operator\": null, \"moe_no_drop_ratio\": 0.0, \"simulated_expert_parallel\": 1, \"moe_pad_expert_input_to_capacity\": false, \"moe_token_drop_policy\": \"prob\", \"priority_type\": \"sum\", \"token_drop_during_eval\": false, \"dynamic_ep_drop_tokens\": false, \"dynamic_ep_drop_tokens_threshold\": null, \"router_combine_comm_opt\": false, \"enable_router_report\": true, \"router_report_interval\": 100, \"router_report_cost_log_interval\": 100, \"router_report_ep_group_percent\": 0.5, \"untie_embeddings_and_output_weights\": true, \"deepseek\": true, \"n_shared_experts\": null, \"norm_topk_prob\": false, \"routed_scaling_factor\": 9.0, \"sigmoid_router_score\": false, \"topk_method\": \"greedy\", \"loss_free_balance_rate\": 0.0, \"loss_free_balance_rate_log_interval\": 100, \"loss_free_decay_rule\": null, \"dynamic_update_loss_free_bias\": true, \"log_loss_free_bias_interval\": 50, \"use_optimized_loss_free_balance_impl\": true, \"use_padded_tokens_per_expert\": false, \"n_group\": 1, \"topk_group\": 1, \"device_load_balance_loss_coeff\": [0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001], \"communication_load_balance_loss_coeff\": null, \"use_modified_balance_loss\": false, \"output_z_loss\": false, \"output_z_loss_coeff\": 1e-05, \"hidden_z_loss_coeff\": 1e-07, \"shortcut_permute_alltoall_overlap\": false, \"shortcut_unpermute_alltoall_overlap\": false, \"use_alltoall_vc\": false, \"shortcut_issue_unpermute_alltoall_spec_point\": false, \"shortcut_tp_rs_overlap\": null, \"use_cp_token_balance\": false, \"att_mask_type\": \"full\", \"z_loss_type\": \"default\", \"enable_abstract_module\": true, \"enable_moe_chunk\": false, \"mtp_num_layers\": 3, \"mtp_loss_scaling_factor\": 0.1, \"mtp_replicate_modules\": true, \"mtp_layer_spec\": \"TransformerLayer\", \"mtp_transformer_impl\": \"local\", \"mtp_loss_scaling_factor_change_step\": -1, \"mtp_loss_scaling_factor_change_value\": 0.0, \"mtp_detach_gradient\": false, \"fix_mtp_loss_double_scale\": false, \"mtp_disable_over_tokenizer\": true, \"mtp_distill_from_main_model\": true, \"mtp_distill_top_n\": \"1\", \"mtp_loss_backward_at_once\": false, \"mtp_dedup_emb\": true, \"mtp_recompute_mlp\": false, \"mtp_acc_log_interval\": 5, \"moe_unpermute_impl\": \"scatter_add\", \"moe_unpermute_index_add_cast\": false, \"moe_ag_coalesced\": false, \"enable_gmm_unaligned_cutlass\": false, \"attention_dropout\": 0.0, \"hidden_dropout\": 0.0, \"weight_decay\": 0.01, \"start_weight_decay\": 0.01, \"end_weight_decay\": 0.01, \"weight_decay_incr_style\": \"constant\", \"clip_grad\": 1.0, \"clip_grad_non_matrix_only\": false, \"adam_beta1\": 0.9, \"adam_beta2\": 0.95, \"adam_eps\": 1e-16, \"sgd_momentum\": 0.9, \"muon_momentum\": 0.95, \"muon_nesterov\": false, \"muon_zeropower_backend\": \"optimal8\", \"muon_type\": \"bf16\", \"muon_comm_type\": \"fp32\", \"muon_ns_steps\": 5, \"muon_merge_moe_params\": false, \"muon_use_spectral_norm\": false, \"muon_adamw_lr\": 1.67e-06, \"attn_mult\": null, \"emb_mult\": 1.0, \"muon_qk_clip\": false, \"muon_qk_clip_factor\": 100.0, \"muon_dp_comm_overlap\": true, \"muon_overlap_with_dp_param_allgather\": true, \"muon_matched_adamw_rms\": 0.2, \"muon_matched_adamw_rms_source\": \"schedule\", \"muon_tp_parallel\": true, \"muon_error_skip\": false, \"muon_error_dump_path\": \"hdfs:///zw04mlnn01/checkpoint/EXE/wanghaibin09/sft_merge_for_0615/moe-pro-sft-128k-0620-train-data-ext-1ep-ep4-sampling-v1_pro_sft/muon_error_dumps\", \"report_muon_skip_params_rate\": false, \"report_muon_skip_params_rate_interval\": 20, \"use_adam_eps_to_control_muon_update\": true, \"use_adam_eps_to_control_muon_update_rate\": 1, \"muon_ns_pre_scale\": false, \"muon_disable_npu_optimize\": false, \"muon_scale_before_downcast\": true, \"muon_use_clamp_norm\": false, \"muon_update_ratio_report_interval\": 100, \"muon_split\": false, \"native_mm\": false, \"micro_batch_size\": 1, \"global_batch_size\": 256, \"rampup_batch_size\": null, \"rampup_stage_config\": null, \"recompute_granularity\": null, \"recompute_pp_forward\": false, \"recompute_pp_forward_num\": -1, \"check_for_nan_in_loss_and_grad\": true, \"distribute_saved_activations\": false, \"recompute_method\": null, \"recompute_num_layers\": null, \"load_iterations\": null, \"moe_infer\": false, \"expert_choice_infer_topk\": 1, \"profile\": false, \"profile_step_start\": 10, \"profile_step_end\": 12, \"profile_ranks\": [0], \"enable_mstx\": true, \"enable_mstx_profile_memory\": true, \"meta_grad_comm\": false, \"fused_moe_probs_with_swiglu\": true, \"use_fuse_probs_swiglu_op\": true, \"train_iters\": 1378, \"exit_iters\": null, \"train_samples\": null, \"train_tokens\": null, \"train_epochs\": 1, \"log_interval\": 1, \"exit_interval\": null, \"exit_duration_in_mins\": null, \"exit_signal_handler\": false, \"tensorboard_dir\": \"/tmp/moe-pro-sft-128k-0620-train-data-ext-1ep-ep4-sampling-v1\", \"masked_softmax_fusion\": false, \"swiglu_fusion\": false, \"bias_gelu_fusion\": false, \"bias_dropout_fusion\": true, \"apply_rope_fusion\": true, \"use_flash_attn\": true, \"sliding_window_attention\": \"0000000000000000000000000000000000000000000000000000000000000000000000000000000000\", \"sliding_window_left\": -1, \"sliding_window_right\": -1, \"duo_attention\": \"0000000000000000000000000000000000000000000000000000000000000000000000000000000000\", \"sink_num_blocks\": -1, \"recent_num_blocks\": -1, \"duo_sparsity\": 0.0, \"duo_head_score\": \"null\", \"cp_batch_p2p_comm\": false, \"flash_attn_deterministic\": true, \"qk_norm_type\": \"apex\", \"add_bias_linear\": false, \"optimizer\": \"muon\", \"dataloader_type\": \"cyclic\", \"eval_dataloader_type\": \"single\", \"skip_train_iteration_range\": null, \"skip_samples\": 0, \"skip_steps_by_config\": null, \"async_tensor_model_parallel_allreduce\": false, \"no_persist_layer_norm\": false, \"sequence_parallel\": true, \"gradient_accumulation_fusion\": true, \"save_step0_ckpt\": false, \"make_trainable\": true, \"medusa_num_heads\": null, \"medusa_num_layers\": null, \"use_distill_loss\": false, \"router_jitter_noise\": null, \"router_dtype\": \"float32\", \"router_score_full_precision\": true, \"router_bias\": false, \"router\": \"fast_token_choose\", \"router_splitk_threshold\": 8192, \"freeze_router_weight\": false, \"fake_uniform_router\": false, \"capacity_factor\": null, \"use_load_balance_loss\": true, \"use_load_balance_loss_in_sequence\": false, \"use_ep_drop\": false, \"use_bf16_buffer_allgather_parameters\": false, \"load_balance_loss_type\": \"z_loss\", \"moe_loss_coeff\": [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], \"only_z_loss_coeff\": 0.0, \"moe_topk\": 12, \"router_output_norm\": false, \"add_noise_for_expert_upcycling\": false, \"use_gigaflops_pre_token\": true, \"use_mcore_models\": true, \"expert_parallel\": false, \"ep_alltoall_opt\": true, \"manual_gc\": true, \"manual_gc_interval\": 2, \"manual_gc_eval\": true, \"grouped_gemm_gradient_accumulation_fusion\": true, \"use_grouped_gemm\": true, \"use_fuse_permute\": true, \"log_gmm_shape\": false, \"grouped_gemm_init_same_with_separate_expserts\": true, \"memory_optimize_in_sp\": false, \"memory_optimize_in_1f2b\": false, \"memory_optimize_in_1f2b_dense\": true, \"memory_optimize_in_sp_dense\": true, \"deterministic_mode\": true, \"use_hw_fa\": false, \"swiglu_recompute\": true, \"dynamic_moe_recompute\": false, \"moe_recompute_capacity_factor\": 2.0, \"rmsnorm_recompute\": false, \"fuse_router_topk_allgather\": false, \"disable_efficient_gqa\": false, \"exec_bit_flip_check_interval\": 500, \"fuse_cp_gqa_permute_ops\": true, \"defer_bit_flip_result_check\": true, \"exec_router_fwd_bit_flip_check_interval\": -1, \"exec_router_bwd_bit_flip_check_interval\": -1, \"bit_flip_check_first_iters\": 10, \"fuse_router_ops\": false, \"fuse_sparse_ops\": false, \"fuse_split_cat\": true, \"set_grad_none_in_gmm\": true, \"use_fuse_moe_loss\": false, \"set_stride_within_parallel_embedding_and_fa\": false, \"report_moe_loss_interval\": 0, \"enable_ema\": false, \"ema_factor\": 0.999, \"ema_interval\": 1, \"no_load_ema\": false, \"no_load_ema_change_dp\": false, \"use_chunked_output_layer\": true, \"output_layer_chunk_size\": 4096, \"chunked_output_layer_promotion\": true, \"chunked_output_layer_dw_add_fusion\": true, \"compute_z_loss_fast_mode\": false, \"oom_offload\": true, \"oom_offload_try_times\": 5, \"oom_offload_min_swap_tensor_size\": 1024, \"change_dp_use_gloo\": false, \"change_dp_opt\": false, \"change_dp_no_chunk_scatter\": true, \"scatter_chunk_size\": 0, \"available_free_memory_factor\": 0.5, \"seed\": 482, \"data_parallel_random_init\": false, \"init_method_std\": 0.00212, \"embedding_init_method\": 0.006, \"router_init_method_std\": null, \"init_method_xavier_uniform\": false, \"o_zero_init\": false, \"q_zero_init\": false, \"global_init_with_std_variance\": null, \"hccl_comm_cfgs\": \"{\\\"pp\\\":20,\\\"pp_list_0\\\":20,\\\"pp_list_1\\\":20,\\\"dp\\\":20,\\\"tp_cp\\\":20,\\\"tp\\\":100,\\\"ep\\\":200,\\\"dp_cp\\\":400,\\\"dp_modulo_exp\\\":200}\", \"lr\": 1.67e-06, \"enable_layer_lr\": false, \"layer_lr_switch_steps\": -1.0, \"lr_factor_per_layer\": null, \"report_layer_lr\": false, \"enable_block_lr\": false, \"filtered_keyword\": null, \"block_lr_switch_steps\": -1.0, \"block_lr_regex_config_file\": null, \"block_lr_embedding_layer\": null, \"block_lr_output_layer\": null, \"block_lr_norm_layer\": null, \"block_lr_dense_mlp_fc1\": null, \"block_lr_dense_mlp_fc2\": null, \"block_lr_moe_mlp_fc1\": null, \"block_lr_moe_mlp_fc2\": null, \"block_lr_router\": null, \"block_lr_attn_qkv\": null, \"block_lr_attn_proj\": null, \"report_block_lr\": false, \"lr_decay_style\": \"cosine\", \"lr_decay_iters\": 1378, \"lr_decay_by_update_step\": null, \"lr_decay_samples\": null, \"lr_decay_tokens\": null, \"lr_warmup_fraction\": 0.0, \"lr_warmup_iters\": 0, \"lr_warmup_samples\": 0, \"lr_warmup_tokens\": 0, \"lr_warmup_init\": 0.0, \"min_lr\": 0.0, \"override_opt_param_scheduler\": true, \"use_checkpoint_opt_param_scheduler\": false, \"mup_lr_scale_cond\": false, \"decay_frac_for_wsd\": null, \"emb_lr_scale_cond\": true, \"emb_lr_scale_base\": 1024, \"emb_no_wd_cond\": false, \"oe_proj_no_scale_lr\": false, \"ln_scale_lr_cond\": true, \"muon_reparam_scale_cond\": false, \"save\": \"hdfs:///zw04mlnn01/checkpoint/EXE/wanghaibin09/sft_merge_for_0615/moe-pro-sft-128k-0620-train-data-ext-1ep-ep4-sampling-v1_pro_sft\", \"save_interval\": 500, \"save_inner_interval\": 50, \"no_save_optim\": null, \"no_save_rng\": null, \"load\": \"hdfs:///zw04mlnn01/checkpoint/EXE/wanghaibin09/sft_merge_for_0615/moe-pro-sft-128k-0620-train-data-ext-1ep-ep4-sampling-v1_pro_sft\", \"load_unify\": true, \"save_on_load\": false, \"load_model_only\": null, \"no_load_optim\": null, \"no_load_optim_iteration\": null, \"no_load_rng\": null, \"finetune\": false, \"perform_initialization\": true, \"use_checkpoint_args\": false, \"exit_on_missing_checkpoint\": false, \"load_dense_model\": null, \"legacy_checkpoint\": false, \"load_no_bucket_checkpoint\": false, \"load_inequal_bucket_checkpoint\": true, \"distributed_checkpointing\": true, \"async_checkpointing\": false, \"async_checkpoint_barrier_timeout\": 600, \"max_num_disaster_checkpoints\": 2, \"load_model_optimize\": true, \"convert_dis_to_undis\": false, \"enable_checkpoint_pruning\": true, \"fp16\": false, \"bf16\": true, \"loss_scale\": null, \"initial_loss_scale\": 4294967296, \"min_loss_scale\": 1.0, \"loss_scale_window\": 1000, \"avg_loss_comm_optimize\": true, \"hysteresis\": 2, \"fp32_residual_connection\": false, \"apply_query_key_layer_scaling\": true, \"attention_softmax_in_fp32\": true, \"accumulate_allreduce_grads_in_fp32\": true, \"fp16_lm_cross_entropy\": false, \"tensor_model_parallel_size\": 4, \"tensor_model_parallel_size_moe\": 1, \"enable_grouped_moe\": false, \"moe_group_size\": null, \"grouped_moe_parallel_type\": \"ep\", \"grouped_moe_norm_type\": null, \"grouped_moe_norm_epsilon\": 1e-05, \"grouped_moe_proj_init_std\": null, \"grouped_moe_norm_scale\": 1.0, \"grouped_moe_router_splitk_threshold\": 8192, \"enable_router_sp2hp_overlap\": false, \"tensor_model_parallel_size_emb\": 256, \"use_a2a_in_embp\": true, \"tensor_model_parallel_size_emb_infer\": null, \"pipeline_model_parallel_size\": 7, \"vision_encoder_pipeline_model_parallel_size\": null, \"num_layers_per_pipeline_stage\": null, \"vision_encoder_num_layers_per_pipeline_stage\": null, \"pipeline_model_parallel_split_rank\": null, \"f1b2\": false, \"delay_p2p_comm_num\": 0, \"num_layers_per_virtual_pipeline_stage\": 3, \"use_two_p2p_comm_group\": true, \"overlap_p2p_comm\": false, \"distributed_backend\": \"nccl\", \"distributed_timeout_minutes\": 30, \"overlap_grad_reduce\": true, \"overlap_grad_reduce_in_chunk\": true, \"delay_grad_reduce\": true, \"bucket_size\": 500000000, \"disable_bucketing\": true, \"overlap_weight_allgather\": false, \"prefetch_bucket_cnt\": 2, \"max_running_allgather_kernel\": 2, \"grad_reduce_issue_speed\": 5, \"scatter_gather_tensors_in_pipeline\": true, \"use_ring_exchange_p2p\": false, \"local_rank\": 0, \"lazy_mpu_init\": null, \"use_cpu_initialization\": null, \"empty_unused_memory_level\": 0, \"standalone_embedding_stage\": false, \"balance_embedding_stage\": false, \"regard_embedding_as_transformer\": false, \"regard_unembedding_as_transformer\": false, \"padding_nooptransformer_num_sequential\": [1], \"padding_nooptransformer_num_reverse\": [3], \"use_distributed_optimizer\": true, \"context_parallel_size\": 64, \"cp_solution\": \"alltoall\", \"ag_kv_head_stride\": 2, \"cp_alltoall_solution\": true, \"cp_ag_reduce_in_fp32\": false, \"ignore_varlen_mfu\": false, \"variable_seq_lengths\": false, \"zero_bubble_pipeline_timers_start_iter\": 0, \"zero_bubble_pipeline_timers_end_iter\": 0, \"zero_bubble_max_pending_backward\": \"auto\", \"zero_bubble_adaptive_memory_limit_percentile\": 85, \"post_validation_alway_no_clip\": false, \"enable_optimizer_post_validation\": false, \"post_validation_async_recv_norm\": false, \"post_validation_after_step_num\": 1, \"post_validation_debug_log\": false, \"exactly_numeric_rollback\": false, \"enable_exactly_numeric_match\": false, \"enable_zero_bubble\": true, \"zero_bubble_v_schedule_dw_overlap\": false, \"zero_bubble_v_schedule_dw_overlap_opt\": false, \"zero_bubble_v_schedule\": true, \"zero_bubble_v_schedule_mem_setup\": \"zb\", \"allow_padding_num_layers\": false, \"zero_bubble_v_cooldown_opt\": false, \"zero_bubble_v_cooldown_opt_debug\": false, \"zero_bubble_v_dw_cp_overlap\": true, \"once_dw_extracted_num\": 4, \"inner_dw_queue_length\": 12, \"eval_iters\": 100, \"eval_interval\": 1000, \"eval_first_iter\": false, \"skip_train\": false, \"only_eval\": false, \"eval_ckpts\": null, \"use_eval_forward\": false, \"eval_padding_opt\": false, \"eval_global_batch_size\": 256, \"skip_seqlen_check\": false, \"data_path\": [\"/mnt/dolphinfs/ssd_pool/docker/user/hadoop-nlp-hl02/hadoop-aipnlp/RPG/wanghaibin09/tokenize/pro_sft_128k_0620_ep4_ext_sf_v2/LLM_SFT_20260620_ep4/pro_sft_128k_0620_ep4_ext_sf_v2/data/processed\"], \"split\": \"100\", \"train_data_path\": null, \"valid_data_path\": null, \"test_data_path\": null, \"data_cache_path\": null, \"vocab_size\": null, \"vocab_file\": \"/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/3A/code-and-math/fuhaomin/tokenizer/release/tokenizer/history/v3/260325\", \"merge_file\": null, \"vocab_extra_ids\": 0, \"seq_length\": 131072, \"encoder_seq_length\": 131072, \"decoder_seq_length\": null, \"retriever_seq_length\": 256, \"sample_rate\": 1.0, \"mask_prob\": 0.15, \"short_seq_prob\": 0.1, \"mmap_warmup\": false, \"num_workers\": 1, \"prefetch_factor\": 16, \"batch_data_scatter\": false, \"tokenizer_type\": \"BloomTokenizer\", \"tokenizer_model\": null, \"data_impl\": \"mmap\", \"eval_data_impl\": \"infer\", \"reset_position_ids\": false, \"reset_attention_mask\": false, \"eod_mask_loss\": true, \"bos_mask_loss\": false, \"data_configs\": \"\", \"num_data_splits\": 1, \"base_img_size\": 448, \"image_dir\": null, \"no_splicing_dataset\": false, \"proportion\": false, \"pre_concat_info\": \"\", \"data_ratio\": \"0\", \"vision_seq_length\": null, \"text_seq_length\": null, \"seq_length_per_image\": null, \"vision_tril_attention\": false, \"vision_class_token_length\": 1, \"vision_class_token\": true, \"image_processor\": \"blip2\", \"image_processor_path\": null, \"adlr_autoresume\": false, \"adlr_autoresume_interval\": 1000, \"ict_head_size\": null, \"biencoder_projection_dim\": 0, \"biencoder_shared_query_context_model\": false, \"ict_load\": null, \"bert_load\": null, \"titles_data_path\": null, \"query_in_block_prob\": 0.1, \"use_one_sent_docs\": false, \"evidence_data_path\": null, \"retriever_report_topk_accuracies\": [], \"retriever_score_scaling\": false, \"block_data_path\": null, \"embedding_path\": null, \"indexer_batch_size\": 128, \"indexer_log_interval\": 1000, \"num_classes\": 1000, \"img_h\": 224, \"img_w\": 224, \"num_channels\": 3, \"patch_dim\": 14, \"image_size\": 224, \"patch_size\": 14, \"max_image_num_per_micro_batch\": 1, \"classes_fraction\": 1.0, \"data_per_class_fraction\": 1.0, \"data_sharding\": true, \"head_lr_mult\": 1.0, \"vision_use_recompute\": false, \"vision_recompute_num_layers\": 1, \"vision_adapter_width\": 7168, \"layernorm_fp32\": false, \"quick_gelu\": false, \"vision_width\": 1664, \"vision_heads\": 16, \"vision_layers\": 48, \"vision_mlp_ratio\": 4, \"vision_hidden_act\": \"gelu\", \"vision_qkv_bias\": true, \"vision_pretraining\": false, \"vision_pretraining_type\": \"classify\", \"vision_backbone_type\": \"vit\", \"swin_backbone_type\": \"tiny\", \"mask_type\": \"random\", \"mask_factor\": 1.0, \"iter_per_epoch\": 1250, \"dino_local_img_size\": 96, \"dino_local_crops_number\": 10, \"dino_head_hidden_size\": 2048, \"dino_bottleneck_size\": 256, \"dino_freeze_last_layer\": 1, \"dino_norm_last_layer\": false, \"dino_warmup_teacher_temp\": 0.04, \"dino_teacher_temp\": 0.07, \"dino_warmup_teacher_temp_epochs\": 30, \"vision_tower\": null, \"vision_encoder_type\": \"openclip_vit\", \"unfreeze_vision\": false, \"mm_vision_select_layer\": -1, \"mm_adapter_type\": \"mlp\", \"resampler_output_dim\": null, \"resampler_n_queries\": 256, \"proj_output_dim\": null, \"pretrain_mm_mlp_adapter\": null, \"mm_projector_type\": \"linear\", \"pretrain_generation_adapter\": null, \"mm_patch_merge_type\": \"flat\", \"mm_vision_select_feature\": \"patch\", \"image_aspect_ratio\": \"pad\", \"mm_use_im_start_end\": false, \"mm_use_im_patch_token\": false, \"version\": \"v1\", \"vlm_sft\": false, \"vlm_task_type\": \"understanding\", \"loss_avg_by_token\": false, \"image_split\": -1, \"add_img_separator_token\": false, \"tune_llm_all\": false, \"tune_vit_all\": false, \"image_token_num\": -1, \"max_frame_num\": 100, \"read_fps\": false, \"keep_short_vid\": false, \"keep_long_vid\": false, \"keep_short_thred\": 64, \"keep_short_multi\": 4, \"sample_mid\": true, \"frame_num\": 8, \"vision_tower_lr_scale\": 0.1, \"vision_tower_lr_decay\": 1.0, \"vision_projector_lr_scale\": 1.0, \"gen_projector_lr_scale\": 10, \"no_shuffle_mllm_data\": false, \"data_index_dir\": null, \"gen_vit_data_parallel\": false, \"audio_vocab_size\": 8224, \"text_vocab_size\": 131072, \"audio_head_num\": 2, \"padding_idx\": 3, \"token_type_pause\": 1, \"token_type_end\": 2, \"token_type_audio\": 3, \"token_type_text\": 4, \"token_type_audio_text\": 5, \"token_type_asr\": 6, \"token_asr\": 102, \"seq_type_text\": 1, \"seq_type_audio\": 2, \"seq_type_asr\": 3, \"audio_loss_ratio\": 1, \"text_loss_ratio\": 1, \"pure_text_loss_ratio\": 1, \"audio_encoder_path\": null, \"audio_projector_path\": null, \"audio_output_layers_path\": null, \"audio_embeddings_path\": null, \"enable_audio_adaptor\": false, \"connector_type\": \"mlp\", \"adaptor_config_path\": null, \"tokenizer_name_or_path\": null, \"processor_config_file\": null, \"tokenizer_version\": null, \"data_file_or_path\": null, \"len_calc_downsample_rate\": 1, \"len_calc_processor_type\": \"dfsmn\", \"audio_sample_rate\": 16000, \"collate_sft_compute_assistant_only\": false, \"is_training\": false, \"special_tokens_learnable\": false, \"conversation_bos_token\": null, \"conversation_eos_token\": null, \"use_text_instruction\": false, \"add_round_idx\": false, \"freeze_audio_encoder\": false, \"freeze_connector\": false, \"freeze_language_model_without_embedding\": false, \"freeze_text_embedding\": false, \"freeze_audio_cb_0_embedding\": false, \"freeze_audio_cb_1_embedding\": false, \"freeze_whole_embedding\": false, \"log_params_norm\": false, \"log_num_zeros_in_grad\": false, \"timing_log_level\": 1, \"barrier_with_L1_time\": true, \"timing_log_option\": \"all\", \"tensorboard_log_interval\": 1, \"tensorboard_log_time_interval\": 50, \"log_time_interval\": 50, \"tensorboard_queue_size\": 1000, \"log_timers_to_tensorboard\": true, \"log_batch_size_to_tensorboard\": true, \"log_learning_rate_to_tensorboard\": true, \"log_learnig_rate_all_pp_rank\": false, \"log_loss_scale_to_tensorboard\": true, \"log_validation_ppl_to_tensorboard\": false, \"log_memory_to_tensorboard\": true, \"log_world_size_to_tensorboard\": false, \"tensorboard_log_level\": \"minimal\", \"params_init_log\": true, \"zb_debug_log\": false, \"zb_bc_repalce_ag\": true, \"log_experts_passed_token_to_tensorboard\": true, \"log_experts_passed_token_interval\": 100, \"report_drop_frac_capacity\": null, \"expert_level_log_interval_factor\": 10, \"wandb_project\": \"\", \"wandb_exp_name\": \"\", \"wandb_save_dir\": \"\", \"log_aggregated_val_loss\": true, \"router_info_log_interval\": null, \"collect_router_info_dp_ranks\": [0], \"collect_router_info_micro_batches\": [1], \"split_report_router_scores_grad\": true, \"inference_batch_times_seqlen_threshold\": 512, \"max_tokens_to_oom\": 12000, \"output_bert_embeddings\": false, \"bert_embedder_type\": \"megatron\", \"fp8\": null, \"fp8_margin\": 0, \"fp8_interval\": 1, \"transformer_impl\": \"shortcut\", \"fp8_amax_history_len\": 1, \"fp8_amax_compute_algo\": \"most_recent\", \"fp8_wgrad\": true, \"retro_workdir\": null, \"retro_add_retriever\": false, \"retro_cyclic_train_iters\": null, \"retro_encoder_layers\": 2, \"retro_encoder_hidden_dropout\": 0.1, \"retro_encoder_attention_dropout\": 0.1, \"retro_num_neighbors\": 2, \"retro_num_retrieved_chunks\": 2, \"retro_return_doc_ids\": false, \"model_spec\": null, \"cuda_event_trace\": true, \"dynamic_cuda_event_trace\": true, \"nccl_trace\": false, \"reduce_pipeline_data_io\": true, \"enable_step_sample_cache\": true, \"step_local_sample_sort_algo\": null, \"p2p_precision_split_factor\": 3, \"get_batch_in_cpu\": true, \"get_batch_deferred_on_this_cp_rank\": false, \"get_batch_cp_dedup\": false, \"keys_compute_w_at_once\": \"none\", \"hf_type\": \"llama\", \"trained_model\": null, \"ignore_mismatch_to_hf\": false, \"debug_log_path\": null, \"intm_dir\": null, \"multi_query_attention\": false, \"multi_query_group_num\": null, \"rmsnorm\": true, \"qk_norm\": false, \"add_qkv_bias\": false, \"add_layernorm_after_embedding\": false, \"intermediate_size\": null, \"rope_scaling_type\": \"yarn\", \"rope_scaling_factor\": 120.0, \"rope_scaling_low_freq_factor\": null, \"rope_scaling_high_freq_factor\": null, \"normalize_for_lm_head\": false, \"rope_theta\": 1000000.0, \"rope_scaling_beta_fast\": 32, \"rope_scaling_beta_slow\": 1, \"rope_scaling_mscale\": 1.0, \"rope_scaling_mscale_all_dim\": 1.0, \"original_max_position_embeddings\": 8192, \"varlen_attention\": true, \"varlen_end_to_end\": false, \"splicing_weight\": false, \"loss_weight_mode\": \"default\", \"uniform_loss_mask_for_test\": false, \"legacy_hf\": false, \"npu_mlp_embedding\": true, \"gpu_mlp_embedding\": false, \"output_logits_chunk_sum_cnt\": 1, \"attn_qk_norm\": false, \"profiler_enabled\": false, \"profiler_schedule_skip_first\": 0, \"profiler_schedule_wait_steps\": 1, \"profiler_schedule_warmup_steps\": 1, \"profiler_schedule_active_steps\": 3, \"profiler_schedule_repeat_times\": 1, \"profiler_schedule_rank_list\": \"*\", \"profiler_chrome_trace_dir\": \"\", \"memory_profiler_enabled\": false, \"memory_profiler_save_path\": null, \"memory_profiler_save_ranks\": null, \"simultaneous_writing_native_tensorboard\": false, \"hope_tracking.board\": true, \"hope_tracking.artifact\": false, \"multi_rank_tracking_enable\": true, \"multi_rank_full_tracking\": false, \"consume_same_data\": null, \"dataset_type\": \"chat_template\", \"eval_dataset_type\": \"chat\", \"eval_all_data\": false, \"topk_logits\": null, \"save_topk_router\": false, \"save_topk_dsa\": false, \"save_topk_logits\": true, \"topk_save_dir\": null, \"topk_check_data_path\": null, \"topk_check_data_path_interval\": 60, \"param_report\": \"200\", \"report_spectral_norm\": false, \"hope_tensorboard\": \"native\", \"legacy_version\": \"0.30\", \"splicing\": true, \"report_activation_norm\": true, \"report_mla_norm\": false, \"report_mla_chunk_size\": 2, \"report_activation_norm_interval\": 100, \"report_ep_unbalance_rate\": true, \"report_ep_unbalance_rate_interval\": 20, \"print_max_moe_recv_tokens\": true, \"print_max_moe_recv_tokens_interval\": 10, \"report_token_drop_rate\": false, \"report_token_drop_rate_interval\": 10, \"parallel_loader_num\": null, \"continue_training_iters\": null, \"post_training\": false, \"post_training_sft\": true, \"enable_unify_dp_broadcast\": false, \"unify_dp_broadcast_type\": \"parameter\", \"save_ckpt_to_memory\": false, \"load_iteration\": 500, \"offload_optimizer_by_step\": false, \"use_ares_writer\": false, \"exec_router_check_interval\": 25, \"dsa\": true, \"dsa_dense_warmup\": false, \"dsa_dense_warmup_with_fa\": false, \"dsa_dense_warmup_method\": \"forward-only\", \"index_n_heads\": 32, \"index_head_dim\": 128, \"index_topk\": 2048, \"dsa_distil_loss_weight\": 1.0, \"dsa_log_topk_acc\": false, \"dsa_log_topk_acc_interval\": 1, \"dsa_recompute_q_absorb\": true, \"dsa_recompute_topk\": false, \"cli_factor\": 2, \"cli_topk_offset\": 0, \"dsa_kv_block_size\": 1, \"dsa_q_block_size\": 1, \"dsa_num_init_token\": 16, \"dsa_num_local_token\": 1024, \"dsa_log_select_p_interval\": -1, \"dsa_log_select_p_per_head_interval\": -1, \"dsa_clip_grad_per_layer\": false, \"dsa_absorb_compute_select_p\": true, \"dsa_mtp_cli\": true, \"sft_token_affirmative\": false, \"dp_same_data\": false, \"warmup_model_before_train\": false, \"use_packed_indexed_ds\": false, \"use_packed_chat_ds\": false, \"packed_indexed_ds_method\": \"default\", \"dsa_bitflip_check_warmup_steps\": 5, \"dsa_bitflip_check_stable_steps\": 50, \"dsa_kl_loss_use_loss_mask\": false, \"record_spike_loss\": false, \"load_spike_loss_pre_steps\": false, \"spike_loss_data_dir\": null, \"spike_loss_method\": \"z_scores\", \"spike_loss_threshold\": 3, \"spike_loss_pre_steps\": 100, \"spike_loss_skip_samples\": 0, \"spike_loss_skip_update\": false, \"spike_loss_allowed_consecutive_steps\": 0, \"loss_window_size\": 0, \"spike_loss_allowed_total_steps\": 0, \"auto_rollback\": false, \"rollback_step_per_time\": 15, \"steps_left_to_compare\": 10, \"rollback_threshold\": 50, \"spike_step\": [-1], \"muon_error_step\": [-1], \"grad_norm_overflow_step\": [-1], \"qk_logits_overflow_step\": [-1], \"record_spike_grad_norm\": false, \"load_spike_grad_norm_pre_steps\": false, \"spike_grad_norm_data_dir\": null, \"spike_grad_norm_method\": \"z_scores\", \"spike_grad_norm_threshold\": 10, \"spike_grad_norm_pre_steps\": 100, \"spike_grad_norm_skip_samples\": 0, \"spike_grad_norm_skip_update\": false, \"spike_grad_norm_allowed_consecutive_steps\": 0, \"grad_norm_window_size\": 0, \"spike_grad_norm_allowed_total_steps\": 0, \"ref_ckpt_args_path_for_unify_model\": null, \"ref_ckpt_lr_warm_up_samples\": null, \"allow_unify_ignore_mismatched_keys\": true, \"model_register_model_family\": null, \"model_register_model_size\": null, \"model_register_model_stage\": null, \"model_register_model_name\": null, \"model_register_model_tag_base_sft\": null, \"model_register_model_vocab_dir_path\": null, \"model_register_model_series\": \"\", \"enable_custom_ops\": [], \"use_over_tokenizer\": true, \"oe_vocab_size_ratio\": 100.567, \"oe_split_num\": 4, \"oe_neighbor_num\": 5, \"report_oe_info_interval\": null, \"ignore_vocab_coprime\": false, \"output_conflict_info\": false, \"remove_oe_sync\": false, \"oe_merge_type\": \"2d_fat\", \"oe_recompute\": false, \"train_stop_iteration\": null, \"rank\": 0, \"world_size\": 14336, \"transformer_pipeline_model_parallel_size\": 7, \"data_parallel_size\": 8, \"embedding_num_layer\": 0, \"total_num_layers\": 42, \"padding_num_layer_sequential\": 1, \"padding_num_layer_reverse\": 3, \"virtual_pipeline_model_parallel_size\": 2, \"virtual_pipeline_total_num_layers\": 42, \"params_dtype\": {\"torch.dtype\": \"torch.bfloat16\"}, \"consumed_train_samples\": 352768, \"consumed_valid_samples\": 0, \"consumed_train_tokens\": 46238007296, \"gigaflos_no_embeds\": 23334606128688.055, \"spike_skip_samples\": 0, \"cp_ring_p2p_solution\": false, \"cp_allgather_solution_unbalance\": false, \"cp_allgather_solution_balance\": false, \"evaluating\": false, \"exec_bit_flip_check\": true, \"exec_router_fwd_bit_flip_check\": false, \"exec_router_bwd_bit_flip_check\": false, \"pre_step_losses\": [], \"pre_step_grad_norms\": [], \"padded_vocab_size\": 163840, \"device_arch\": \"910\", \"model_type\": {\"MegatronModelType\": \"ModelType.encoder_or_decoder\"}, \"checkpoint_data_parallel_size\": 8, \"iteration\": 1378, \"do_train\": 1, \"do_valid\": 0, \"do_test\": 0, \"curr_iteration\": 1377, \"skip_step_list\": []}"

model-00001-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:862131abdf7bc5a535981292ca0b83c205855f4dc4f08b85eb31f3be2ea53302
+size 16872370336

model-00002-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d77c5da7bd95d4fe14f630be3306508b693dbdea6e76e9e75491b0c032bf5af3
+size 16872349856

model-00003-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2013fff5230f33fe83a8f62e5bd50bb6f9b8ea8af243644d5766767084af6b8
+size 16872360096

model-00004-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63d2c7a1569d33500653c6a132e8d3035d6c804c52f108301dab25bbc234dd8c
+size 16872345760

model-00005-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fdd28890c7e5d93d82eb302187258bcc6c2291bdf8552533cdd72e34c16f8c4
+size 16872366240

model-00006-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:972a5672d8a17cf00b4ce39387051c159f0c1f1727c7576c403ddf95628e9b9b
+size 16872356000

model-00007-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a328a92b52ff9da55e9bb4627e8e2729decde4e9799341a7de09c570cd220e20
+size 13992122368

model-00008-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a04e9bf05ae693ee2726a53aeffa43141ded530e78f5b7363f94c399177692e
+size 16872343712

model-00009-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6a21ccd58ec2d9b6f2b78d85d596a01e0d062c9b3a157b5e2143d586daed707
+size 16872364192

model-00010-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f812af9dd855ee83d210b265d8487fb8a7aa78712284c8cb625063b95e26110
+size 16872374432

model-00011-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2b7829f335c08d38c54dbf70961d7ac47cc1ecde55fd6b0e1852110b1178d42
+size 16872353952

model-00012-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:721f4df587f77e225c53af4c9b4e7a7a5a9360b8f45780dbc94324a9e53217d8
+size 16872370336

model-00013-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93ddb7520c1893127c292c10d8f5349a64659ab571ce5139ed679af51bea3060
+size 16872349856

model-00014-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b50401233034f32b13fc637e18f438fc564ab2349878dc3dfab48d2c05b8f652
+size 16872360096

model-00015-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e4854fc72dd0d918c97429f560cdb323f188568c07ccdcf3c4b995adb22f18d
+size 13994195472

model-00016-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5497d49326bc5eabf522e6934dcf9651d6977850d6b17bb8aa2294fbc57ca055
+size 16872368288

model-00017-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:716a7e4e4ce81be3de9d1b9c1b61d15b24a8cf4c77f31aa04a399bf7f2725a08
+size 16872347808

model-00018-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1445caf42c4548a6c826a8b94caa993cfa399f160c2cf351d9088c9efba8ae91
+size 16872358048

model-00019-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:951c0f53fd2977ca556b3d6e5684e1b5474448babfd37205d105d77992f95876
+size 16872343712

model-00020-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff953654fe91987587fafaf5305f349ab2eb5d55e9f3c05ee2b27724687861a1
+size 16872364192

model-00021-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f30ff41862c1fa75f938489bc5901491780cb34acff5051d0c618b8ba56905cc
+size 16872374432

model-00022-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27c0d615fbb2d312666799fdba12e7a63c59d053259396ab08c20ed6147ec145
+size 16872353952

model-00023-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eaca710846e1a97b3ae143330566d3143f6813285180f3a3d741f4a635f9f65a
+size 13989035424

model-00024-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f227f3ae8a264ba4a0fc444231442c71d1ecffcdee1d2c18c7e33f350224cef8
+size 16872372384

model-00025-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2a012383acf60170ebe112c71cf3b9d069070ee3611da6fca43100d21645930
+size 16872351904

model-00026-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:674785a7112a530aa2b78feb8a78076105ee278a0a3ffbb793a47ef881727eb0
+size 16872362144

model-00027-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec76ca7c3b1741b55b37a43c0049a2288c3e707cb091201d106d81561daa9f1c
+size 16872368288

model-00028-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fd6161fc14a2f26fe4f44f0cf424648d491397ea5047d1c3bf2636cefdd10bc
+size 16872347808

model-00029-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:766516ddc3e9dbe1397c8a733f39e57ec2b87b3c260dacc3f3b42bfc9c0082a9
+size 16872358048

model-00030-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26a4591103f328458e3cb595b1e535b039a15a63c5ff2d8422ddeface20d4bf7
+size 13998690888

model-00031-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:599a29c7a09ff9ee2b40bf921036dcaed58c18d36cecdb81db33310454a92d0f
+size 13988110096

model-00032-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e631a5f02ff02e5fdc4d0ae5796ab1abadb529025cce7d2a66910f9b8084ed9
+size 16872345760

model-00033-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad093128de5d8f5e4ea29b268d8c0fe1c905799390003c4d470208221b4b3f4
+size 16872366240

model-00034-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86efc661b3673cb1b105b6562264bd03d1742e9a1557a267408ed9cc9d1a75ad
+size 16872356000

model-00035-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c69ab09dddb32e6fb1afbe66126939aac8ae82e36770a1e488769d5c356b02df
+size 16872372384

model-00036-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6185e64b0e019ddb8acb08b488ddcbf86da2e9803a3fb000b23179466076b5ce
+size 16872351904

model-00037-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba34e539afdab6572e1dd59b3f3fc2d37e26c77fda7c1259f70a0d233b7150a5
+size 16872362144

model-00038-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81755b11d253153b64c1ba601dc674ad495d2d85848084999fffc66b0e042061
+size 13986959720

model-00039-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd628fe871308f38acaaef4384f5c974d3e16ab1d41ba88cb324094a1edf2c1b
+size 13996483272

model-00072-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d54b73679d2caf2650ebd6bd8c50a0f5846a2345cb1f233bc544123e4343150
+size 13989157296

model-00077-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e3b4c48f7c395070163238ae19c10706b3f866910797f2322add7c8cca05e48
+size 13996499384

model-00081-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:becb669ae845f589a5a8b938de822dfe219f6387edf3b45861cd7bdd82cb9107
+size 13993353920

model-00088-of-00141.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8280dd3224cacbf91446e5c1b109d2e815f403097672c4ef33cc98544fb5553f
+size 13972380296

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<longcat_s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</longcat_s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<longcat_pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<longcat_unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenization_llama.py

@@ -0,0 +1,231 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tokenization classes for LLaMA."""
+import os
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import sentencepiece as spm
+
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
+
+PRETRAINED_VOCAB_FILES_MAP = {}
+
+
+class LlamaTokenizer(PreTrainedTokenizer):
+    """
+    Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file,
+        unk_token="<unk>",
+        bos_token="<s>",
+        eos_token="</s>",
+        sp_model_kwargs: Optional[Dict[str, Any]] = None,
+        add_bos_token=True,
+        add_eos_token=False,
+        decode_with_prefix_space=False,
+        **kwargs,
+    ):
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+        super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.decode_with_prefix_space = decode_with_prefix_space
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        self._no_prefix_space_tokens = None
+
+        """ Initialisation"""
+
+    @property
+    def no_prefix_space_tokens(self):
+        if self._no_prefix_space_tokens is None:
+            vocab = self.convert_ids_to_tokens(list(range(self.vocab_size)))
+            self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith("▁")}
+        return self._no_prefix_space_tokens
+
+    @property
+    def vocab_size(self):
+        """Returns vocab size"""
+        return self.sp_model.get_piece_size()
+
+    @property
+    def bos_token_id(self) -> Optional[int]:
+        return self.sp_model.bos_id()
+
+    @property
+    def eos_token_id(self) -> Optional[int]:
+        return self.sp_model.eos_id()
+
+    def get_vocab(self):
+        """Returns vocab as a dict"""
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text):
+        """Returns a tokenized string."""
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        token = self.sp_model.IdToPiece(index)
+        return token
+
+    def _maybe_add_prefix_space(self, tokens, decoded):
+        if tokens and tokens[0] not in self.no_prefix_space_tokens:
+            return " " + decoded
+        else:
+            return decoded
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        current_sub_tokens = []
+        out_string = ""
+        prev_is_special = False
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                if not prev_is_special:
+                    out_string += " "
+                out_string += self.sp_model.decode(current_sub_tokens) + token
+                prev_is_special = True
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+                prev_is_special = False
+        out_string += self.sp_model.decode(current_sub_tokens)
+        out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string)
+        return out_string
+
+    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        out_vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+        elif not os.path.isfile(self.vocab_file):
+            with open(out_vocab_file, "wb") as fi:
+                content_spiece_model = self.sp_model.serialized_model_proto()
+                fi.write(content_spiece_model)
+
+        return (out_vocab_file,)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        if self.add_bos_token:
+            bos_token_ids = [self.bos_token_id]
+        else:
+            bos_token_ids = []
+
+        output = bos_token_ids + token_ids_0
+
+        if token_ids_1 is not None:
+            output = output + token_ids_1
+
+        if self.add_eos_token:
+            output = output + [self.eos_token_id]
+
+        return output
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """
+        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` method.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        if token_ids_1 is None:
+            return [1] + ([0] * len(token_ids_0)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
+
+    def create_token_type_ids_from_sequences(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
+        use of token type ids, therefore a list of zeros is returned.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of zeros.
+        """
+        eos = [self.eos_token_id]
+
+        if token_ids_1 is None:
+            return len(token_ids_0 + eos) * [0]
+        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": true,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<longcat_s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</longcat_s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 131072,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<longcat_pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sp_model_kwargs": {},
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<longcat_unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "chat_template": "{%- set tool_choice = tool_choice | default('auto') -%}\n{%- set enable_thinking = enable_thinking | default(true) -%}\n{%- set save_history_reasoning_content = save_history_reasoning_content | default(true) -%}\n{%- set ns = namespace(final_messages = [], tool_types = [], last_query_index = -1) -%}\n{%- for message in messages -%}\n    {%- set role = message.get('role') -%}\n    {%- set c = message.get('content') -%}\n    {%- if c is string -%}\n        {%- set ns.final_messages = ns.final_messages + [message] -%}\n    {%- elif c is iterable and c is not mapping -%}\n        {%- set parts = namespace(text=[], thinking=[], tool_calls=[], tool_results=[]) -%}\n        {%- for item in c -%}\n            {%- if item['type'] == 'text' -%}\n                {%- set parts.text = parts.text + [item['text']] -%}\n            {%- elif item['type'] == 'thinking' -%}\n                {%- if item['thinking'] -%}\n                    {%- set parts.thinking = parts.thinking + [item['thinking']] -%}\n                {%- endif -%}\n            {%- elif item['type'] == 'tool_use' -%}\n                {%- set parts.tool_calls = parts.tool_calls + [{\n                    'id': item['id'],\n                    'type': 'function',\n                    'function': {\n                        'name': item['name'],\n                        'arguments': item['input']\n                    }\n                }] -%}\n            {%- elif item['type'] == 'tool_result' -%}\n                {%- set parts.tool_results = parts.tool_results + [{\n                    'role': 'tool',\n                    'name': item.get('name', ''),\n                    'tool_call_id': item['tool_use_id'],\n                    'content': item['content']\n                }] -%}\n            {%- endif -%}\n        {%- endfor -%}\n        {%- set ns.final_messages = ns.final_messages + parts.tool_results -%}\n        {%- if not (parts.tool_results | length > 0 and parts.text | length == 0 and role == 'user') -%}\n            {%- set ns.final_messages = ns.final_messages + [{\n                'role': role,\n                'content': parts.text | join(''),\n                'reasoning_content': parts.thinking | join(''),\n                'tool_calls': parts.tool_calls\n            }] -%}\n        {%- endif -%}\n    {%- else -%}\n        {%- set ns.final_messages = ns.final_messages + [message] -%}\n    {%- endif -%}\n{%- endfor -%}\n{%- set messages = ns.final_messages -%}\n\n{%- for idx in range(messages|length) -%}\n    {%- set msg = messages[idx] -%}\n    {%- if msg.role == 'user' -%}\n        {%- set ns.last_query_index = idx -%}\n    {%- endif -%}\n{%- endfor -%}\n\n{%- if tools and tool_choice != 'none' -%}\n    {{- \"<longcat_tool_declare>\\n\" -}}\n    {{- \"# Tools\\n\" -}}\n    {{- \"You have access to the following tools:\\n\\n\" -}}\n    {%- for tool in tools -%}\n        {%- if tool.type not in ns.tool_types -%}\n            {%- set ns.tool_types = ns.tool_types + [tool.type] -%}\n            {{- \"## Tool namespace: \" ~ tool.type ~ \"\\n\\n\" -}}\n        {%- endif -%}\n        {%- if tool.type == 'code_interpreter' -%}\n            {%- set tool = {\"type\":\"code_interpreter\",\"function\":{\"name\":\"code_interpreter_preview\",\"description\":\"The code will be executed in a stateful Jupyter notebook sandbox environment, only supports local computation, data processing, and file operations.\\nCode sandbox environment (network isolated) Any external network requests or online API calls are prohibited.\\nIf online functionality is needed, please use other permitted tools.\\nCode will respond with the output of the execution or time out after 60.0 seconds. \",\"parameters\":{\"type\":\"object\",\"properties\":{\"language\":{\"type\":\"string\",\"description\":\"The programming language of the code to be executed. Available values: python (Default), java, go, js, ts, c, c++.\"},\"code\":{\"type\":\"string\",\"description\":\"Python code to be executed must not include the following:\\n- Importing network libraries such as requests, httplib, etc.\\n- Any form of HTTP requests.\\n- External API calls.\\n- Network port operations. Example: ```python\\nimport pandas as pd\\npd.DataFrame({'A':[1,2]})\\n```\"},\"timeout\":{\"type\":\"number\",\"description\":\"The maximum execution time of the code, in seconds. Default is 60.0.\"}}},\"required\":[\"code\"]}} -%}\n        {%- endif -%}\n        {{- \"### Tool name: \" + tool.function.name + \"\\n\" -}}\n        {{- \"Description: \" + tool.function.description + \"\\n\\n\" -}}\n        {{- \"InputSchema: \" + tool.function.parameters | tojson(ensure_ascii=False) + \"\\n\\n\" -}}\n    {%- endfor -%}\n    {{- '**Note**: For each function call, output the function name and arguments within the following XML format:\\n<longcat_tool_call>{function-name}\\n<longcat_arg_key>{arg-key-1}</longcat_arg_key>\\n<longcat_arg_value>{arg-value-1}</longcat_arg_value>\\n<longcat_arg_key>{arg-key-2}</longcat_arg_key>\\n<longcat_arg_value>{arg-value-2}</longcat_arg_value>\\n...\\n</longcat_tool_call>\\n' -}}\n    {{- \"</longcat_tool_declare>\"-}}\n{%- endif -%}\n\n{%- for msg in messages -%}\n    {%- if msg.role == \"system\" -%}\n        {{- \"<longcat_system>\" + msg.content -}}\n    {%- elif msg.role == \"user\" -%}\n        {{- \"<longcat_user>\" -}}\n        {%- if msg[\"files\"] -%}\n            {{- '<longcat_files>\\n' ~ msg.files | tojson(indent=2) ~ '\\n</longcat_files>' -}}\n        {%- endif -%}\n        {{- msg.content -}}\n        {%- if save_history_reasoning_content and enable_thinking is not none -%}\n            {%- if loop.last or messages[loop.index0 + 1].role != 'assistant' -%}\n                {# pass #}\n            {%- else -%}\n                {%- if enable_thinking == true -%}\n                    {{- \" /think_on\" -}}\n                    {{- \" \" -}}\n                {%- elif enable_thinking == false -%}\n                    {{- \" /think_off \" -}}\n                {%- endif -%}\n            {%- endif -%}\n        {%- elif loop.index0 >= ns.last_query_index and enable_thinking is not none -%}\n            {%- if loop.last or messages[loop.index0 + 1].role != 'assistant' -%}\n                {# pass #}\n            {%- else -%}\n                {%- if enable_thinking == true -%}\n                    {{- \" /think_on\" -}}\n                    {{- \" \" -}}\n                {%- elif enable_thinking == false -%}\n                    {{- \" /think_off \" -}}\n                {%- endif -%}\n            {%- endif -%}\n        {%- endif -%}\n    {%- elif msg.role == \"assistant\" -%}\n        {{- \"<longcat_assistant>\" -}}\n        {%- if save_history_reasoning_content == true or loop.index0 > ns.last_query_index -%}\n            {%- if enable_thinking == true -%}\n                {%- if msg.reasoning_content -%}\n                    {%- set reasoning_content = msg.reasoning_content | trim -%}\n                    {{- \"<longcat_think>\\n\" ~ reasoning_content ~ \"\\n</longcat_think>\\n\" -}}\n                {%- else -%}\n                    {{- \"<longcat_think>\\n\\n</longcat_think>\\n\" -}}\n                {%- endif -%}\n            {%- elif enable_thinking == false -%}\n                {{- \"<longcat_think>\\n\\n</longcat_think>\\n\" -}}\n            {%- endif -%}\n        {%- endif -%}\n        {%- if msg.content -%}\n            {{- msg.content -}}\n        {%- endif -%}\n        {%- if msg.tool_calls -%}\n            {%- for tool_call in msg.tool_calls -%}\n                {{- \"<longcat_tool_call>\" ~ tool_call.function.name ~ \"\\n\" -}}\n                {%- set _args = tool_call.function.arguments -%}\n                {%- for k, v in _args.items() -%}\n                    {{- \"<longcat_arg_key>\" ~ k ~ \"</longcat_arg_key>\\n\" -}}\n                    {{- \"<longcat_arg_value>\" ~ (v if v is string else v | tojson(ensure_ascii=False)) ~ \"</longcat_arg_value>\\n\" -}}\n                {% endfor -%}\n                {{- \"</longcat_tool_call>\\n\" -}}\n            {%- endfor -%}\n        {%- endif -%}\n        {{- \"</longcat_s>\" -}}\n    {%- elif msg.role == \"tool\" -%}\n        {%- if messages[loop.index0 - 1].role != \"tool\" -%}\n            {{- \"<longcat_observation>\" -}}\n        {%- endif -%}\n        {{- \"<longcat_tool_response>\" ~ msg.name ~ \"\\n\" ~ msg.content ~ \"</longcat_tool_response>\" -}}\n        {%- if loop.index0 == messages|length - 1 -%}\n            {# pass #}\n        {%- elif (save_history_reasoning_content or loop.index0 > ns.last_query_index) and messages[loop.index0 + 1].role != \"tool\" -%}\n            {%- if enable_thinking == true -%}\n                {{- \" /think_on\" -}}\n                {{- \" \" -}}\n            {%- elif enable_thinking == false -%}\n                {{- \" /think_off \" -}}\n            {%- endif -%}\n        {%- endif -%}\n    {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {%- if enable_thinking == true -%}\n        {{- \" /think_on\" -}}\n        {{- \" <longcat_assistant><longcat_think>\" -}}\n    {%- elif enable_thinking == false -%}\n        {{- \" /think_off <longcat_assistant><longcat_think>\\n\\n</longcat_think>\\n\" -}}\n    {%- else -%}\n        {{- \" /think_off <longcat_assistant><longcat_think>\\n\\n</longcat_think>\\n\" -}}\n    {%- endif -%}\n{%- endif -%}"
+}