diff --git "a/freetoken_weight.json" "b/freetoken_weight.json"
new file mode 100644--- /dev/null
+++ "b/freetoken_weight.json"
@@ -0,0 +1 @@
+{"format": "freetoken_weight", "version": 1, "align": 4096, "shard_limit": 8589934592, "total_bytes": 78032617472, "tensors": [{"name": "model.layers.16.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 0, "nbytes": 81920}, {"name": "model.layers.16.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 81920, "nbytes": 20480}, {"name": "model.layers.16.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 102400, "nbytes": 6553600}, {"name": "model.layers.16.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 6656000, "nbytes": 6553600}, {"name": "model.layers.16.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 13209600, "nbytes": 96}, {"name": "model.layers.16.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 13213696, "nbytes": 81920}, {"name": "model.layers.16.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 13295616, "nbytes": 96}, {"name": "model.layers.16.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 13299712, "nbytes": 84377600}, {"name": "model.layers.16.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 97677312, "nbytes": 256}, {"name": "model.layers.16.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 97681408, "nbytes": 31457280}, {"name": "model.layers.16.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 129138688, "nbytes": 2621440}, {"name": "model.layers.16.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 131760128, "nbytes": 3276800}, {"name": "model.layers.16.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 135036928, "nbytes": 6553600}, {"name": "model.layers.16.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 141590528, "nbytes": 5120}, {"name": "model.layers.16.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 141598720, "nbytes": 81920}, {"name": "model.layers.16.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 141680640, "nbytes": 20480}, {"name": "model.layers.16.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 141701120, "nbytes": 6553600}, {"name": "model.layers.16.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 148254720, "nbytes": 6553600}, {"name": "model.layers.17.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 154808320, "nbytes": 81920}, {"name": "model.layers.17.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 154890240, "nbytes": 20480}, {"name": "model.layers.17.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 154910720, "nbytes": 6553600}, {"name": "model.layers.17.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 161464320, "nbytes": 6553600}, {"name": "model.layers.17.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 168017920, "nbytes": 96}, {"name": "model.layers.17.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 168022016, "nbytes": 81920}, {"name": "model.layers.17.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 168103936, "nbytes": 96}, {"name": "model.layers.17.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 168108032, "nbytes": 84377600}, {"name": "model.layers.17.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 252485632, "nbytes": 256}, {"name": "model.layers.17.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 252489728, "nbytes": 31457280}, {"name": "model.layers.17.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 283947008, "nbytes": 2621440}, {"name": "model.layers.17.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 286568448, "nbytes": 3276800}, {"name": "model.layers.17.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 289845248, "nbytes": 6553600}, {"name": "model.layers.17.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 296398848, "nbytes": 5120}, {"name": "model.layers.17.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 296407040, "nbytes": 81920}, {"name": "model.layers.17.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 296488960, "nbytes": 20480}, {"name": "model.layers.17.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 296509440, "nbytes": 6553600}, {"name": "model.layers.17.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 303063040, "nbytes": 6553600}, {"name": "model.layers.18.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 309616640, "nbytes": 81920}, {"name": "model.layers.18.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 309698560, "nbytes": 20480}, {"name": "model.layers.18.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 309719040, "nbytes": 6553600}, {"name": "model.layers.18.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 316272640, "nbytes": 6553600}, {"name": "model.layers.18.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 322826240, "nbytes": 96}, {"name": "model.layers.18.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 322830336, "nbytes": 81920}, {"name": "model.layers.18.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 322912256, "nbytes": 96}, {"name": "model.layers.18.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 322916352, "nbytes": 84377600}, {"name": "model.layers.18.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 407293952, "nbytes": 256}, {"name": "model.layers.18.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 407298048, "nbytes": 31457280}, {"name": "model.layers.18.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 438755328, "nbytes": 2621440}, {"name": "model.layers.18.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 441376768, "nbytes": 3276800}, {"name": "model.layers.18.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 444653568, "nbytes": 6553600}, {"name": "model.layers.18.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 451207168, "nbytes": 5120}, {"name": "model.layers.18.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 451215360, "nbytes": 81920}, {"name": "model.layers.18.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 451297280, "nbytes": 20480}, {"name": "model.layers.18.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 451317760, "nbytes": 6553600}, {"name": "model.layers.18.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 457871360, "nbytes": 6553600}, {"name": "model.layers.19.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 464424960, "nbytes": 81920}, {"name": "model.layers.19.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 464506880, "nbytes": 20480}, {"name": "model.layers.19.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 464527360, "nbytes": 6553600}, {"name": "model.layers.19.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 471080960, "nbytes": 6553600}, {"name": "model.layers.19.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 477634560, "nbytes": 2621440}, {"name": "model.layers.19.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 480256000, "nbytes": 3276800}, {"name": "model.layers.19.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 483532800, "nbytes": 6553600}, {"name": "model.layers.19.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 490086400, "nbytes": 5120}, {"name": "model.layers.19.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 490094592, "nbytes": 81920}, {"name": "model.layers.19.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 490176512, "nbytes": 20480}, {"name": "model.layers.19.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 490196992, "nbytes": 6553600}, {"name": "model.layers.19.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 496750592, "nbytes": 6553600}, {"name": "model.layers.19.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 503304192, "nbytes": 3276800}, {"name": "model.layers.19.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 506580992, "nbytes": 256}, {"name": "model.layers.19.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 506585088, "nbytes": 256}, {"name": "model.layers.19.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 506589184, "nbytes": 512}, {"name": "model.layers.19.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 506593280, "nbytes": 31457280}, {"name": "model.layers.19.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 538050560, "nbytes": 512}, {"name": "model.layers.19.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 538054656, "nbytes": 68157440}, {"name": "model.layers.20.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 606212096, "nbytes": 81920}, {"name": "model.layers.20.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 606294016, "nbytes": 20480}, {"name": "model.layers.20.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 606314496, "nbytes": 6553600}, {"name": "model.layers.20.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 612868096, "nbytes": 6553600}, {"name": "model.layers.20.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 619421696, "nbytes": 96}, {"name": "model.layers.20.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 619425792, "nbytes": 81920}, {"name": "model.layers.20.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 619507712, "nbytes": 96}, {"name": "model.layers.20.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 619511808, "nbytes": 84377600}, {"name": "model.layers.20.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 703889408, "nbytes": 256}, {"name": "model.layers.20.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 703893504, "nbytes": 31457280}, {"name": "model.layers.20.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 735350784, "nbytes": 2621440}, {"name": "model.layers.20.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 737972224, "nbytes": 3276800}, {"name": "model.layers.20.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 741249024, "nbytes": 6553600}, {"name": "model.layers.20.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 747802624, "nbytes": 5120}, {"name": "model.layers.20.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 747810816, "nbytes": 81920}, {"name": "model.layers.20.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 747892736, "nbytes": 20480}, {"name": "model.layers.20.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 747913216, "nbytes": 6553600}, {"name": "model.layers.20.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 754466816, "nbytes": 6553600}, {"name": "model.layers.21.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 761020416, "nbytes": 81920}, {"name": "model.layers.21.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 761102336, "nbytes": 20480}, {"name": "model.layers.21.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 761122816, "nbytes": 6553600}, {"name": "model.layers.21.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 767676416, "nbytes": 6553600}, {"name": "model.layers.21.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 774230016, "nbytes": 96}, {"name": "model.layers.21.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 774234112, "nbytes": 81920}, {"name": "model.layers.21.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 774316032, "nbytes": 96}, {"name": "model.layers.21.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 774320128, "nbytes": 84377600}, {"name": "model.layers.21.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 858697728, "nbytes": 256}, {"name": "model.layers.21.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 858701824, "nbytes": 31457280}, {"name": "model.layers.21.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 890159104, "nbytes": 2621440}, {"name": "model.layers.21.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 892780544, "nbytes": 3276800}, {"name": "model.layers.21.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 896057344, "nbytes": 6553600}, {"name": "model.layers.21.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 902610944, "nbytes": 5120}, {"name": "model.layers.21.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 902619136, "nbytes": 81920}, {"name": "model.layers.21.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 902701056, "nbytes": 20480}, {"name": "model.layers.21.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 902721536, "nbytes": 6553600}, {"name": "model.layers.21.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 909275136, "nbytes": 6553600}, {"name": "model.layers.22.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 915828736, "nbytes": 81920}, {"name": "model.layers.22.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 915910656, "nbytes": 20480}, {"name": "model.layers.22.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 915931136, "nbytes": 6553600}, {"name": "model.layers.22.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 922484736, "nbytes": 6553600}, {"name": "model.layers.22.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 929038336, "nbytes": 96}, {"name": "model.layers.22.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 929042432, "nbytes": 81920}, {"name": "model.layers.22.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 929124352, "nbytes": 96}, {"name": "model.layers.22.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 929128448, "nbytes": 84377600}, {"name": "model.layers.22.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 1013506048, "nbytes": 256}, {"name": "model.layers.22.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 1013510144, "nbytes": 31457280}, {"name": "model.layers.22.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 1044967424, "nbytes": 2621440}, {"name": "model.layers.22.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 1047588864, "nbytes": 3276800}, {"name": "model.layers.22.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 1050865664, "nbytes": 6553600}, {"name": "model.layers.22.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 1057419264, "nbytes": 5120}, {"name": "model.layers.22.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1057427456, "nbytes": 81920}, {"name": "model.layers.22.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1057509376, "nbytes": 20480}, {"name": "model.layers.22.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1057529856, "nbytes": 6553600}, {"name": "model.layers.22.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1064083456, "nbytes": 6553600}, {"name": "model.layers.23.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1070637056, "nbytes": 81920}, {"name": "model.layers.23.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1070718976, "nbytes": 20480}, {"name": "model.layers.23.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1070739456, "nbytes": 6553600}, {"name": "model.layers.23.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1077293056, "nbytes": 6553600}, {"name": "model.layers.23.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 1083846656, "nbytes": 2621440}, {"name": "model.layers.23.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 1086468096, "nbytes": 3276800}, {"name": "model.layers.23.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 1089744896, "nbytes": 6553600}, {"name": "model.layers.23.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 1096298496, "nbytes": 5120}, {"name": "model.layers.23.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1096306688, "nbytes": 81920}, {"name": "model.layers.23.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1096388608, "nbytes": 20480}, {"name": "model.layers.23.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1096409088, "nbytes": 6553600}, {"name": "model.layers.23.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1102962688, "nbytes": 6553600}, {"name": "model.layers.23.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 1109516288, "nbytes": 3276800}, {"name": "model.layers.23.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 1112793088, "nbytes": 256}, {"name": "model.layers.23.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 1112797184, "nbytes": 256}, {"name": "model.layers.23.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 1112801280, "nbytes": 512}, {"name": "model.layers.23.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 1112805376, "nbytes": 31457280}, {"name": "model.layers.23.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 1144262656, "nbytes": 512}, {"name": "model.layers.23.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 1144266752, "nbytes": 68157440}, {"name": "model.layers.24.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1212424192, "nbytes": 81920}, {"name": "model.layers.24.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1212506112, "nbytes": 20480}, {"name": "model.layers.24.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1212526592, "nbytes": 6553600}, {"name": "model.layers.24.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1219080192, "nbytes": 6553600}, {"name": "model.layers.24.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1225633792, "nbytes": 96}, {"name": "model.layers.24.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 1225637888, "nbytes": 81920}, {"name": "model.layers.24.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1225719808, "nbytes": 96}, {"name": "model.layers.24.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 1225723904, "nbytes": 84377600}, {"name": "model.layers.24.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 1310101504, "nbytes": 256}, {"name": "model.layers.24.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 1310105600, "nbytes": 31457280}, {"name": "model.layers.24.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 1341562880, "nbytes": 2621440}, {"name": "model.layers.24.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 1344184320, "nbytes": 3276800}, {"name": "model.layers.24.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 1347461120, "nbytes": 6553600}, {"name": "model.layers.24.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 1354014720, "nbytes": 5120}, {"name": "model.layers.24.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1354022912, "nbytes": 81920}, {"name": "model.layers.24.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1354104832, "nbytes": 20480}, {"name": "model.layers.24.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1354125312, "nbytes": 6553600}, {"name": "model.layers.24.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1360678912, "nbytes": 6553600}, {"name": "model.layers.25.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1367232512, "nbytes": 81920}, {"name": "model.layers.25.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1367314432, "nbytes": 20480}, {"name": "model.layers.25.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1367334912, "nbytes": 6553600}, {"name": "model.layers.25.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1373888512, "nbytes": 6553600}, {"name": "model.layers.25.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1380442112, "nbytes": 96}, {"name": "model.layers.25.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 1380446208, "nbytes": 81920}, {"name": "model.layers.25.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1380528128, "nbytes": 96}, {"name": "model.layers.25.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 1380532224, "nbytes": 84377600}, {"name": "model.layers.25.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 1464909824, "nbytes": 256}, {"name": "model.layers.25.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 1464913920, "nbytes": 31457280}, {"name": "model.layers.25.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 1496371200, "nbytes": 2621440}, {"name": "model.layers.25.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 1498992640, "nbytes": 3276800}, {"name": "model.layers.25.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 1502269440, "nbytes": 6553600}, {"name": "model.layers.25.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 1508823040, "nbytes": 5120}, {"name": "model.layers.25.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1508831232, "nbytes": 81920}, {"name": "model.layers.25.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1508913152, "nbytes": 20480}, {"name": "model.layers.25.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1508933632, "nbytes": 6553600}, {"name": "model.layers.25.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1515487232, "nbytes": 6553600}, {"name": "model.layers.26.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1522040832, "nbytes": 81920}, {"name": "model.layers.26.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1522122752, "nbytes": 20480}, {"name": "model.layers.26.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1522143232, "nbytes": 6553600}, {"name": "model.layers.26.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1528696832, "nbytes": 6553600}, {"name": "model.layers.26.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1535250432, "nbytes": 96}, {"name": "model.layers.26.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 1535254528, "nbytes": 81920}, {"name": "model.layers.26.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1535336448, "nbytes": 96}, {"name": "model.layers.26.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 1535340544, "nbytes": 84377600}, {"name": "model.layers.26.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 1619718144, "nbytes": 256}, {"name": "model.layers.26.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 1619722240, "nbytes": 31457280}, {"name": "model.layers.26.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 1651179520, "nbytes": 2621440}, {"name": "model.layers.26.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 1653800960, "nbytes": 3276800}, {"name": "model.layers.26.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 1657077760, "nbytes": 6553600}, {"name": "model.layers.26.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 1663631360, "nbytes": 5120}, {"name": "model.layers.26.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1663639552, "nbytes": 81920}, {"name": "model.layers.26.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1663721472, "nbytes": 20480}, {"name": "model.layers.26.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1663741952, "nbytes": 6553600}, {"name": "model.layers.26.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1670295552, "nbytes": 6553600}, {"name": "model.layers.27.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1676849152, "nbytes": 81920}, {"name": "model.layers.27.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1676931072, "nbytes": 20480}, {"name": "model.layers.27.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1676951552, "nbytes": 6553600}, {"name": "model.layers.27.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1683505152, "nbytes": 6553600}, {"name": "model.layers.27.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 1690058752, "nbytes": 2621440}, {"name": "model.layers.27.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 1692680192, "nbytes": 3276800}, {"name": "model.layers.27.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 1695956992, "nbytes": 6553600}, {"name": "model.layers.27.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 1702510592, "nbytes": 5120}, {"name": "model.layers.27.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1702518784, "nbytes": 81920}, {"name": "model.layers.27.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1702600704, "nbytes": 20480}, {"name": "model.layers.27.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1702621184, "nbytes": 6553600}, {"name": "model.layers.27.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1709174784, "nbytes": 6553600}, {"name": "model.layers.27.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 1715728384, "nbytes": 3276800}, {"name": "model.layers.27.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 1719005184, "nbytes": 256}, {"name": "model.layers.27.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 1719009280, "nbytes": 256}, {"name": "model.layers.27.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 1719013376, "nbytes": 512}, {"name": "model.layers.27.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 1719017472, "nbytes": 31457280}, {"name": "model.layers.27.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 1750474752, "nbytes": 512}, {"name": "model.layers.27.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 1750478848, "nbytes": 68157440}, {"name": "model.layers.28.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1818636288, "nbytes": 81920}, {"name": "model.layers.28.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1818718208, "nbytes": 20480}, {"name": "model.layers.28.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1818738688, "nbytes": 6553600}, {"name": "model.layers.28.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1825292288, "nbytes": 6553600}, {"name": "model.layers.28.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1831845888, "nbytes": 96}, {"name": "model.layers.28.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 1831849984, "nbytes": 81920}, {"name": "model.layers.28.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1831931904, "nbytes": 96}, {"name": "model.layers.28.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 1831936000, "nbytes": 84377600}, {"name": "model.layers.28.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 1916313600, "nbytes": 256}, {"name": "model.layers.28.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 1916317696, "nbytes": 31457280}, {"name": "model.layers.28.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 1947774976, "nbytes": 2621440}, {"name": "model.layers.28.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 1950396416, "nbytes": 3276800}, {"name": "model.layers.28.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 1953673216, "nbytes": 6553600}, {"name": "model.layers.28.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 1960226816, "nbytes": 5120}, {"name": "model.layers.28.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1960235008, "nbytes": 81920}, {"name": "model.layers.28.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1960316928, "nbytes": 20480}, {"name": "model.layers.28.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1960337408, "nbytes": 6553600}, {"name": "model.layers.28.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1966891008, "nbytes": 6553600}, {"name": "model.layers.29.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 1973444608, "nbytes": 81920}, {"name": "model.layers.29.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 1973526528, "nbytes": 20480}, {"name": "model.layers.29.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 1973547008, "nbytes": 6553600}, {"name": "model.layers.29.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 1980100608, "nbytes": 6553600}, {"name": "model.layers.29.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1986654208, "nbytes": 96}, {"name": "model.layers.29.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 1986658304, "nbytes": 81920}, {"name": "model.layers.29.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 1986740224, "nbytes": 96}, {"name": "model.layers.29.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 1986744320, "nbytes": 84377600}, {"name": "model.layers.29.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2071121920, "nbytes": 256}, {"name": "model.layers.29.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 2071126016, "nbytes": 31457280}, {"name": "model.layers.29.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 2102583296, "nbytes": 2621440}, {"name": "model.layers.29.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 2105204736, "nbytes": 3276800}, {"name": "model.layers.29.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 2108481536, "nbytes": 6553600}, {"name": "model.layers.29.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 2115035136, "nbytes": 5120}, {"name": "model.layers.29.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2115043328, "nbytes": 81920}, {"name": "model.layers.29.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2115125248, "nbytes": 20480}, {"name": "model.layers.29.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2115145728, "nbytes": 6553600}, {"name": "model.layers.29.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2121699328, "nbytes": 6553600}, {"name": "model.layers.30.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2128252928, "nbytes": 81920}, {"name": "model.layers.30.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2128334848, "nbytes": 20480}, {"name": "model.layers.30.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2128355328, "nbytes": 6553600}, {"name": "model.layers.30.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2134908928, "nbytes": 6553600}, {"name": "model.layers.30.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 2141462528, "nbytes": 96}, {"name": "model.layers.30.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 2141466624, "nbytes": 81920}, {"name": "model.layers.30.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 2141548544, "nbytes": 96}, {"name": "model.layers.30.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 2141552640, "nbytes": 84377600}, {"name": "model.layers.30.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2225930240, "nbytes": 256}, {"name": "model.layers.30.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 2225934336, "nbytes": 31457280}, {"name": "model.layers.30.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 2257391616, "nbytes": 2621440}, {"name": "model.layers.30.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 2260013056, "nbytes": 3276800}, {"name": "model.layers.30.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 2263289856, "nbytes": 6553600}, {"name": "model.layers.30.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 2269843456, "nbytes": 5120}, {"name": "model.layers.30.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2269851648, "nbytes": 81920}, {"name": "model.layers.30.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2269933568, "nbytes": 20480}, {"name": "model.layers.30.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2269954048, "nbytes": 6553600}, {"name": "model.layers.30.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2276507648, "nbytes": 6553600}, {"name": "model.layers.31.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2283061248, "nbytes": 81920}, {"name": "model.layers.31.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2283143168, "nbytes": 20480}, {"name": "model.layers.31.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2283163648, "nbytes": 6553600}, {"name": "model.layers.31.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2289717248, "nbytes": 6553600}, {"name": "model.layers.31.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 2296270848, "nbytes": 2621440}, {"name": "model.layers.31.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 2298892288, "nbytes": 3276800}, {"name": "model.layers.31.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 2302169088, "nbytes": 6553600}, {"name": "model.layers.31.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 2308722688, "nbytes": 5120}, {"name": "model.layers.31.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2308730880, "nbytes": 81920}, {"name": "model.layers.31.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2308812800, "nbytes": 20480}, {"name": "model.layers.31.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2308833280, "nbytes": 6553600}, {"name": "model.layers.31.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2315386880, "nbytes": 6553600}, {"name": "model.layers.31.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 2321940480, "nbytes": 3276800}, {"name": "model.layers.31.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2325217280, "nbytes": 256}, {"name": "model.layers.31.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2325221376, "nbytes": 256}, {"name": "model.layers.31.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 2325225472, "nbytes": 512}, {"name": "model.layers.31.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 2325229568, "nbytes": 31457280}, {"name": "model.layers.31.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 2356686848, "nbytes": 512}, {"name": "model.layers.31.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 2356690944, "nbytes": 68157440}, {"name": "model.layers.32.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2424848384, "nbytes": 81920}, {"name": "model.layers.32.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2424930304, "nbytes": 20480}, {"name": "model.layers.32.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2424950784, "nbytes": 6553600}, {"name": "model.layers.32.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2431504384, "nbytes": 6553600}, {"name": "model.layers.32.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 2438057984, "nbytes": 96}, {"name": "model.layers.32.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 2438062080, "nbytes": 81920}, {"name": "model.layers.32.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 2438144000, "nbytes": 96}, {"name": "model.layers.32.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 2438148096, "nbytes": 84377600}, {"name": "model.layers.32.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2522525696, "nbytes": 256}, {"name": "model.layers.32.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 2522529792, "nbytes": 31457280}, {"name": "model.layers.32.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 2553987072, "nbytes": 2621440}, {"name": "model.layers.32.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 2556608512, "nbytes": 3276800}, {"name": "model.layers.32.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 2559885312, "nbytes": 6553600}, {"name": "model.layers.32.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 2566438912, "nbytes": 5120}, {"name": "model.layers.32.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2566447104, "nbytes": 81920}, {"name": "model.layers.32.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2566529024, "nbytes": 20480}, {"name": "model.layers.32.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2566549504, "nbytes": 6553600}, {"name": "model.layers.32.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2573103104, "nbytes": 6553600}, {"name": "model.layers.33.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2579656704, "nbytes": 81920}, {"name": "model.layers.33.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2579738624, "nbytes": 20480}, {"name": "model.layers.33.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2579759104, "nbytes": 6553600}, {"name": "model.layers.33.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2586312704, "nbytes": 6553600}, {"name": "model.layers.33.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 2592866304, "nbytes": 96}, {"name": "model.layers.33.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 2592870400, "nbytes": 81920}, {"name": "model.layers.33.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 2592952320, "nbytes": 96}, {"name": "model.layers.33.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 2592956416, "nbytes": 84377600}, {"name": "model.layers.33.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2677334016, "nbytes": 256}, {"name": "model.layers.33.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 2677338112, "nbytes": 31457280}, {"name": "model.layers.33.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 2708795392, "nbytes": 2621440}, {"name": "model.layers.33.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 2711416832, "nbytes": 3276800}, {"name": "model.layers.33.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 2714693632, "nbytes": 6553600}, {"name": "model.layers.33.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 2721247232, "nbytes": 5120}, {"name": "model.layers.33.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2721255424, "nbytes": 81920}, {"name": "model.layers.33.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2721337344, "nbytes": 20480}, {"name": "model.layers.33.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2721357824, "nbytes": 6553600}, {"name": "model.layers.33.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2727911424, "nbytes": 6553600}, {"name": "model.layers.34.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2734465024, "nbytes": 81920}, {"name": "model.layers.34.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2734546944, "nbytes": 20480}, {"name": "model.layers.34.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2734567424, "nbytes": 6553600}, {"name": "model.layers.34.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2741121024, "nbytes": 6553600}, {"name": "model.layers.34.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 2747674624, "nbytes": 96}, {"name": "model.layers.34.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 2747678720, "nbytes": 81920}, {"name": "model.layers.34.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 2747760640, "nbytes": 96}, {"name": "model.layers.34.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 2747764736, "nbytes": 84377600}, {"name": "model.layers.34.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2832142336, "nbytes": 256}, {"name": "model.layers.34.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 2832146432, "nbytes": 31457280}, {"name": "model.layers.34.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 2863603712, "nbytes": 2621440}, {"name": "model.layers.34.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 2866225152, "nbytes": 3276800}, {"name": "model.layers.34.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 2869501952, "nbytes": 6553600}, {"name": "model.layers.34.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 2876055552, "nbytes": 5120}, {"name": "model.layers.34.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2876063744, "nbytes": 81920}, {"name": "model.layers.34.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2876145664, "nbytes": 20480}, {"name": "model.layers.34.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2876166144, "nbytes": 6553600}, {"name": "model.layers.34.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2882719744, "nbytes": 6553600}, {"name": "model.layers.35.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2889273344, "nbytes": 81920}, {"name": "model.layers.35.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2889355264, "nbytes": 20480}, {"name": "model.layers.35.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2889375744, "nbytes": 6553600}, {"name": "model.layers.35.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2895929344, "nbytes": 6553600}, {"name": "model.layers.35.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 2902482944, "nbytes": 2621440}, {"name": "model.layers.35.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 2905104384, "nbytes": 3276800}, {"name": "model.layers.35.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 2908381184, "nbytes": 6553600}, {"name": "model.layers.35.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 2914934784, "nbytes": 5120}, {"name": "model.layers.35.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 2914942976, "nbytes": 81920}, {"name": "model.layers.35.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 2915024896, "nbytes": 20480}, {"name": "model.layers.35.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 2915045376, "nbytes": 6553600}, {"name": "model.layers.35.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 2921598976, "nbytes": 6553600}, {"name": "model.layers.35.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 2928152576, "nbytes": 3276800}, {"name": "model.layers.35.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2931429376, "nbytes": 256}, {"name": "model.layers.35.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2931433472, "nbytes": 256}, {"name": "model.layers.35.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 2931437568, "nbytes": 512}, {"name": "model.layers.35.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 2931441664, "nbytes": 31457280}, {"name": "model.layers.35.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 2962898944, "nbytes": 512}, {"name": "model.layers.35.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 2962903040, "nbytes": 68157440}, {"name": "model.layers.36.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3031060480, "nbytes": 81920}, {"name": "model.layers.36.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3031142400, "nbytes": 20480}, {"name": "model.layers.36.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3031162880, "nbytes": 6553600}, {"name": "model.layers.36.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3037716480, "nbytes": 6553600}, {"name": "model.layers.36.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3044270080, "nbytes": 96}, {"name": "model.layers.36.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 3044274176, "nbytes": 81920}, {"name": "model.layers.36.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3044356096, "nbytes": 96}, {"name": "model.layers.36.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 3044360192, "nbytes": 84377600}, {"name": "model.layers.36.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3128737792, "nbytes": 256}, {"name": "model.layers.36.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 3128741888, "nbytes": 31457280}, {"name": "model.layers.36.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 3160199168, "nbytes": 2621440}, {"name": "model.layers.36.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 3162820608, "nbytes": 3276800}, {"name": "model.layers.36.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 3166097408, "nbytes": 6553600}, {"name": "model.layers.36.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 3172651008, "nbytes": 5120}, {"name": "model.layers.36.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3172659200, "nbytes": 81920}, {"name": "model.layers.36.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3172741120, "nbytes": 20480}, {"name": "model.layers.36.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3172761600, "nbytes": 6553600}, {"name": "model.layers.36.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3179315200, "nbytes": 6553600}, {"name": "model.layers.37.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3185868800, "nbytes": 81920}, {"name": "model.layers.37.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3185950720, "nbytes": 20480}, {"name": "model.layers.37.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3185971200, "nbytes": 6553600}, {"name": "model.layers.37.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3192524800, "nbytes": 6553600}, {"name": "model.layers.37.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3199078400, "nbytes": 96}, {"name": "model.layers.37.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 3199082496, "nbytes": 81920}, {"name": "model.layers.37.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3199164416, "nbytes": 96}, {"name": "model.layers.37.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 3199168512, "nbytes": 84377600}, {"name": "model.layers.37.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3283546112, "nbytes": 256}, {"name": "model.layers.37.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 3283550208, "nbytes": 31457280}, {"name": "model.layers.37.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 3315007488, "nbytes": 2621440}, {"name": "model.layers.37.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 3317628928, "nbytes": 3276800}, {"name": "model.layers.37.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 3320905728, "nbytes": 6553600}, {"name": "model.layers.37.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 3327459328, "nbytes": 5120}, {"name": "model.layers.37.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3327467520, "nbytes": 81920}, {"name": "model.layers.37.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3327549440, "nbytes": 20480}, {"name": "model.layers.37.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3327569920, "nbytes": 6553600}, {"name": "model.layers.37.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3334123520, "nbytes": 6553600}, {"name": "model.layers.38.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3340677120, "nbytes": 81920}, {"name": "model.layers.38.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3340759040, "nbytes": 20480}, {"name": "model.layers.38.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3340779520, "nbytes": 6553600}, {"name": "model.layers.38.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3347333120, "nbytes": 6553600}, {"name": "model.layers.38.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3353886720, "nbytes": 96}, {"name": "model.layers.38.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 3353890816, "nbytes": 81920}, {"name": "model.layers.38.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3353972736, "nbytes": 96}, {"name": "model.layers.38.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 3353976832, "nbytes": 84377600}, {"name": "model.layers.38.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3438354432, "nbytes": 256}, {"name": "model.layers.38.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 3438358528, "nbytes": 31457280}, {"name": "model.layers.38.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 3469815808, "nbytes": 2621440}, {"name": "model.layers.38.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 3472437248, "nbytes": 3276800}, {"name": "model.layers.38.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 3475714048, "nbytes": 6553600}, {"name": "model.layers.38.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 3482267648, "nbytes": 5120}, {"name": "model.layers.38.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3482275840, "nbytes": 81920}, {"name": "model.layers.38.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3482357760, "nbytes": 20480}, {"name": "model.layers.38.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3482378240, "nbytes": 6553600}, {"name": "model.layers.38.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3488931840, "nbytes": 6553600}, {"name": "model.layers.39.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3495485440, "nbytes": 81920}, {"name": "model.layers.39.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3495567360, "nbytes": 20480}, {"name": "model.layers.39.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3495587840, "nbytes": 6553600}, {"name": "model.layers.39.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3502141440, "nbytes": 6553600}, {"name": "model.layers.39.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 3508695040, "nbytes": 2621440}, {"name": "model.layers.39.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 3511316480, "nbytes": 3276800}, {"name": "model.layers.39.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 3514593280, "nbytes": 6553600}, {"name": "model.layers.39.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 3521146880, "nbytes": 5120}, {"name": "model.layers.39.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3521155072, "nbytes": 81920}, {"name": "model.layers.39.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3521236992, "nbytes": 20480}, {"name": "model.layers.39.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3521257472, "nbytes": 6553600}, {"name": "model.layers.39.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3527811072, "nbytes": 6553600}, {"name": "model.layers.39.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 3534364672, "nbytes": 3276800}, {"name": "model.layers.39.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3537641472, "nbytes": 256}, {"name": "model.layers.39.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3537645568, "nbytes": 256}, {"name": "model.layers.39.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 3537649664, "nbytes": 512}, {"name": "model.layers.39.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 3537653760, "nbytes": 31457280}, {"name": "model.layers.39.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 3569111040, "nbytes": 512}, {"name": "model.layers.39.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 3569115136, "nbytes": 68157440}, {"name": "model.layers.40.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3637272576, "nbytes": 81920}, {"name": "model.layers.40.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3637354496, "nbytes": 20480}, {"name": "model.layers.40.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3637374976, "nbytes": 6553600}, {"name": "model.layers.40.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3643928576, "nbytes": 6553600}, {"name": "model.layers.40.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3650482176, "nbytes": 96}, {"name": "model.layers.40.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 3650486272, "nbytes": 81920}, {"name": "model.layers.40.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3650568192, "nbytes": 96}, {"name": "model.layers.40.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 3650572288, "nbytes": 84377600}, {"name": "model.layers.40.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3734949888, "nbytes": 256}, {"name": "model.layers.40.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 3734953984, "nbytes": 31457280}, {"name": "model.layers.40.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 3766411264, "nbytes": 2621440}, {"name": "model.layers.40.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 3769032704, "nbytes": 3276800}, {"name": "model.layers.40.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 3772309504, "nbytes": 6553600}, {"name": "model.layers.40.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 3778863104, "nbytes": 5120}, {"name": "model.layers.40.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3778871296, "nbytes": 81920}, {"name": "model.layers.40.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3778953216, "nbytes": 20480}, {"name": "model.layers.40.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3778973696, "nbytes": 6553600}, {"name": "model.layers.40.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3785527296, "nbytes": 6553600}, {"name": "model.layers.41.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3792080896, "nbytes": 81920}, {"name": "model.layers.41.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3792162816, "nbytes": 20480}, {"name": "model.layers.41.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3792183296, "nbytes": 6553600}, {"name": "model.layers.41.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3798736896, "nbytes": 6553600}, {"name": "model.layers.41.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3805290496, "nbytes": 96}, {"name": "model.layers.41.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 3805294592, "nbytes": 81920}, {"name": "model.layers.41.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3805376512, "nbytes": 96}, {"name": "model.layers.41.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 3805380608, "nbytes": 84377600}, {"name": "model.layers.41.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3889758208, "nbytes": 256}, {"name": "model.layers.41.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 3889762304, "nbytes": 31457280}, {"name": "model.layers.41.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 3921219584, "nbytes": 2621440}, {"name": "model.layers.41.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 3923841024, "nbytes": 3276800}, {"name": "model.layers.41.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 3927117824, "nbytes": 6553600}, {"name": "model.layers.41.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 3933671424, "nbytes": 5120}, {"name": "model.layers.41.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3933679616, "nbytes": 81920}, {"name": "model.layers.41.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3933761536, "nbytes": 20480}, {"name": "model.layers.41.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3933782016, "nbytes": 6553600}, {"name": "model.layers.41.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3940335616, "nbytes": 6553600}, {"name": "model.layers.42.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 3946889216, "nbytes": 81920}, {"name": "model.layers.42.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 3946971136, "nbytes": 20480}, {"name": "model.layers.42.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 3946991616, "nbytes": 6553600}, {"name": "model.layers.42.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 3953545216, "nbytes": 6553600}, {"name": "model.layers.42.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3960098816, "nbytes": 96}, {"name": "model.layers.42.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 3960102912, "nbytes": 81920}, {"name": "model.layers.42.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 3960184832, "nbytes": 96}, {"name": "model.layers.42.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 3960188928, "nbytes": 84377600}, {"name": "model.layers.42.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4044566528, "nbytes": 256}, {"name": "model.layers.42.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 4044570624, "nbytes": 31457280}, {"name": "model.layers.42.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 4076027904, "nbytes": 2621440}, {"name": "model.layers.42.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 4078649344, "nbytes": 3276800}, {"name": "model.layers.42.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 4081926144, "nbytes": 6553600}, {"name": "model.layers.42.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 4088479744, "nbytes": 5120}, {"name": "model.layers.42.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4088487936, "nbytes": 81920}, {"name": "model.layers.42.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4088569856, "nbytes": 20480}, {"name": "model.layers.42.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4088590336, "nbytes": 6553600}, {"name": "model.layers.42.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4095143936, "nbytes": 6553600}, {"name": "model.layers.43.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4101697536, "nbytes": 81920}, {"name": "model.layers.43.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4101779456, "nbytes": 20480}, {"name": "model.layers.43.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4101799936, "nbytes": 6553600}, {"name": "model.layers.43.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4108353536, "nbytes": 6553600}, {"name": "model.layers.43.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 4114907136, "nbytes": 2621440}, {"name": "model.layers.43.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 4117528576, "nbytes": 3276800}, {"name": "model.layers.43.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 4120805376, "nbytes": 6553600}, {"name": "model.layers.43.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 4127358976, "nbytes": 5120}, {"name": "model.layers.43.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4127367168, "nbytes": 81920}, {"name": "model.layers.43.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4127449088, "nbytes": 20480}, {"name": "model.layers.43.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4127469568, "nbytes": 6553600}, {"name": "model.layers.43.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4134023168, "nbytes": 6553600}, {"name": "model.layers.43.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 4140576768, "nbytes": 3276800}, {"name": "model.layers.43.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4143853568, "nbytes": 256}, {"name": "model.layers.43.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4143857664, "nbytes": 256}, {"name": "model.layers.43.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 4143861760, "nbytes": 512}, {"name": "model.layers.43.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 4143865856, "nbytes": 31457280}, {"name": "model.layers.43.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 4175323136, "nbytes": 512}, {"name": "model.layers.43.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 4175327232, "nbytes": 68157440}, {"name": "model.layers.44.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4243484672, "nbytes": 81920}, {"name": "model.layers.44.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4243566592, "nbytes": 20480}, {"name": "model.layers.44.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4243587072, "nbytes": 6553600}, {"name": "model.layers.44.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4250140672, "nbytes": 6553600}, {"name": "model.layers.44.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 4256694272, "nbytes": 96}, {"name": "model.layers.44.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 4256698368, "nbytes": 81920}, {"name": "model.layers.44.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 4256780288, "nbytes": 96}, {"name": "model.layers.44.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 4256784384, "nbytes": 84377600}, {"name": "model.layers.44.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4341161984, "nbytes": 256}, {"name": "model.layers.44.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 4341166080, "nbytes": 31457280}, {"name": "model.layers.44.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 4372623360, "nbytes": 2621440}, {"name": "model.layers.44.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 4375244800, "nbytes": 3276800}, {"name": "model.layers.44.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 4378521600, "nbytes": 6553600}, {"name": "model.layers.44.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 4385075200, "nbytes": 5120}, {"name": "model.layers.44.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4385083392, "nbytes": 81920}, {"name": "model.layers.44.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4385165312, "nbytes": 20480}, {"name": "model.layers.44.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4385185792, "nbytes": 6553600}, {"name": "model.layers.44.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4391739392, "nbytes": 6553600}, {"name": "model.layers.45.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4398292992, "nbytes": 81920}, {"name": "model.layers.45.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4398374912, "nbytes": 20480}, {"name": "model.layers.45.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4398395392, "nbytes": 6553600}, {"name": "model.layers.45.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4404948992, "nbytes": 6553600}, {"name": "model.layers.45.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 4411502592, "nbytes": 96}, {"name": "model.layers.45.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 4411506688, "nbytes": 81920}, {"name": "model.layers.45.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 4411588608, "nbytes": 96}, {"name": "model.layers.45.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 4411592704, "nbytes": 84377600}, {"name": "model.layers.45.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4495970304, "nbytes": 256}, {"name": "model.layers.45.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 4495974400, "nbytes": 31457280}, {"name": "model.layers.45.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 4527431680, "nbytes": 2621440}, {"name": "model.layers.45.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 4530053120, "nbytes": 3276800}, {"name": "model.layers.45.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 4533329920, "nbytes": 6553600}, {"name": "model.layers.45.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 4539883520, "nbytes": 5120}, {"name": "model.layers.45.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4539891712, "nbytes": 81920}, {"name": "model.layers.45.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4539973632, "nbytes": 20480}, {"name": "model.layers.45.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4539994112, "nbytes": 6553600}, {"name": "model.layers.45.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4546547712, "nbytes": 6553600}, {"name": "model.layers.46.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4553101312, "nbytes": 81920}, {"name": "model.layers.46.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4553183232, "nbytes": 20480}, {"name": "model.layers.46.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4553203712, "nbytes": 6553600}, {"name": "model.layers.46.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4559757312, "nbytes": 6553600}, {"name": "model.layers.46.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 4566310912, "nbytes": 96}, {"name": "model.layers.46.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 4566315008, "nbytes": 81920}, {"name": "model.layers.46.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 4566396928, "nbytes": 96}, {"name": "model.layers.46.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 4566401024, "nbytes": 84377600}, {"name": "model.layers.46.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4650778624, "nbytes": 256}, {"name": "model.layers.46.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 4650782720, "nbytes": 31457280}, {"name": "model.layers.46.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 4682240000, "nbytes": 2621440}, {"name": "model.layers.46.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 4684861440, "nbytes": 3276800}, {"name": "model.layers.46.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 4688138240, "nbytes": 6553600}, {"name": "model.layers.46.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 4694691840, "nbytes": 5120}, {"name": "model.layers.46.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4694700032, "nbytes": 81920}, {"name": "model.layers.46.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4694781952, "nbytes": 20480}, {"name": "model.layers.46.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4694802432, "nbytes": 6553600}, {"name": "model.layers.46.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4701356032, "nbytes": 6553600}, {"name": "model.layers.47.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4707909632, "nbytes": 81920}, {"name": "model.layers.47.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4707991552, "nbytes": 20480}, {"name": "model.layers.47.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4708012032, "nbytes": 6553600}, {"name": "model.layers.47.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4714565632, "nbytes": 6553600}, {"name": "model.layers.47.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 4721119232, "nbytes": 2621440}, {"name": "model.layers.47.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 4723740672, "nbytes": 3276800}, {"name": "model.layers.47.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 4727017472, "nbytes": 6553600}, {"name": "model.layers.47.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 4733571072, "nbytes": 5120}, {"name": "model.layers.47.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 4733579264, "nbytes": 81920}, {"name": "model.layers.47.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 4733661184, "nbytes": 20480}, {"name": "model.layers.47.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 4733681664, "nbytes": 6553600}, {"name": "model.layers.47.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 4740235264, "nbytes": 6553600}, {"name": "model.layers.47.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 4746788864, "nbytes": 3276800}, {"name": "model.layers.47.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4750065664, "nbytes": 256}, {"name": "model.layers.47.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4750069760, "nbytes": 256}, {"name": "model.layers.47.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 4750073856, "nbytes": 512}, {"name": "model.layers.47.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 4750077952, "nbytes": 31457280}, {"name": "model.layers.47.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 4781535232, "nbytes": 512}, {"name": "model.layers.47.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 4781539328, "nbytes": 68157440}, {"name": "lm_head.weight", "kind": "weight", "dtype": "bfloat16", "shape": [248320, 2560], "global_off": 4849696768, "nbytes": 1271398400}, {"name": "model.embed_tokens.weight", "kind": "weight", "dtype": "bfloat16", "shape": [248320, 2560], "global_off": 6121095168, "nbytes": 1271398400}, {"name": "model.hyper_connection_mixer.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7392493568, "nbytes": 20480}, {"name": "model.hyper_connection_mixer.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 7392514048, "nbytes": 6553600}, {"name": "model.hyper_connection_mixer.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 7399067648, "nbytes": 6553600}, {"name": "model.layers.0.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 7405621248, "nbytes": 81920}, {"name": "model.layers.0.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7405703168, "nbytes": 20480}, {"name": "model.layers.0.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 7405723648, "nbytes": 6553600}, {"name": "model.layers.0.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 7412277248, "nbytes": 6553600}, {"name": "model.layers.0.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 7418830848, "nbytes": 96}, {"name": "model.layers.0.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 7418834944, "nbytes": 81920}, {"name": "model.layers.0.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 7418916864, "nbytes": 96}, {"name": "model.layers.0.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 7418920960, "nbytes": 84377600}, {"name": "model.layers.0.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 7503298560, "nbytes": 256}, {"name": "model.layers.0.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 7503302656, "nbytes": 31457280}, {"name": "model.layers.0.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 7534759936, "nbytes": 2621440}, {"name": "model.layers.0.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 7537381376, "nbytes": 3276800}, {"name": "model.layers.0.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 7540658176, "nbytes": 6553600}, {"name": "model.layers.0.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 7547211776, "nbytes": 5120}, {"name": "model.layers.0.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 7547219968, "nbytes": 81920}, {"name": "model.layers.0.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7547301888, "nbytes": 20480}, {"name": "model.layers.0.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 7547322368, "nbytes": 6553600}, {"name": "model.layers.0.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 7553875968, "nbytes": 6553600}, {"name": "model.layers.1.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 7560429568, "nbytes": 81920}, {"name": "model.layers.1.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7560511488, "nbytes": 20480}, {"name": "model.layers.1.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 7560531968, "nbytes": 6553600}, {"name": "model.layers.1.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 7567085568, "nbytes": 6553600}, {"name": "model.layers.1.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 7573639168, "nbytes": 96}, {"name": "model.layers.1.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 7573643264, "nbytes": 81920}, {"name": "model.layers.1.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 7573725184, "nbytes": 96}, {"name": "model.layers.1.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 7573729280, "nbytes": 84377600}, {"name": "model.layers.1.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 7658106880, "nbytes": 256}, {"name": "model.layers.1.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 7658110976, "nbytes": 31457280}, {"name": "model.layers.1.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 7689568256, "nbytes": 2621440}, {"name": "model.layers.1.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 7692189696, "nbytes": 3276800}, {"name": "model.layers.1.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 7695466496, "nbytes": 6553600}, {"name": "model.layers.1.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 7702020096, "nbytes": 5120}, {"name": "model.layers.1.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 7702028288, "nbytes": 81920}, {"name": "model.layers.1.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7702110208, "nbytes": 20480}, {"name": "model.layers.1.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 7702130688, "nbytes": 6553600}, {"name": "model.layers.1.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 7708684288, "nbytes": 6553600}, {"name": "model.layers.1.ple.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 7715237888, "nbytes": 81920}, {"name": "model.layers.1.ple.key_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 2560], "global_off": 7715319808, "nbytes": 52428800}, {"name": "model.layers.1.ple.norm_conv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7767748608, "nbytes": 20480}, {"name": "model.layers.1.ple.norm_key.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7767769088, "nbytes": 20480}, {"name": "model.layers.1.ple.norm_query.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7767789568, "nbytes": 20480}, {"name": "model.layers.1.ple.value_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 2560], "global_off": 7767810048, "nbytes": 13107200}, {"name": "model.layers.10.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 7780917248, "nbytes": 81920}, {"name": "model.layers.10.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7780999168, "nbytes": 20480}, {"name": "model.layers.10.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 7781019648, "nbytes": 6553600}, {"name": "model.layers.10.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 7787573248, "nbytes": 6553600}, {"name": "model.layers.10.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 7794126848, "nbytes": 96}, {"name": "model.layers.10.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 7794130944, "nbytes": 81920}, {"name": "model.layers.10.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 7794212864, "nbytes": 96}, {"name": "model.layers.10.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 7794216960, "nbytes": 84377600}, {"name": "model.layers.10.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 7878594560, "nbytes": 256}, {"name": "model.layers.10.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 7878598656, "nbytes": 31457280}, {"name": "model.layers.10.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 7910055936, "nbytes": 2621440}, {"name": "model.layers.10.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 7912677376, "nbytes": 3276800}, {"name": "model.layers.10.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 7915954176, "nbytes": 6553600}, {"name": "model.layers.10.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 7922507776, "nbytes": 5120}, {"name": "model.layers.10.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 7922515968, "nbytes": 81920}, {"name": "model.layers.10.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7922597888, "nbytes": 20480}, {"name": "model.layers.10.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 7922618368, "nbytes": 6553600}, {"name": "model.layers.10.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 7929171968, "nbytes": 6553600}, {"name": "model.layers.11.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 7935725568, "nbytes": 81920}, {"name": "model.layers.11.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7935807488, "nbytes": 20480}, {"name": "model.layers.11.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 7935827968, "nbytes": 6553600}, {"name": "model.layers.11.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 7942381568, "nbytes": 6553600}, {"name": "model.layers.11.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 7948935168, "nbytes": 2621440}, {"name": "model.layers.11.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 7951556608, "nbytes": 3276800}, {"name": "model.layers.11.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 7954833408, "nbytes": 6553600}, {"name": "model.layers.11.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 7961387008, "nbytes": 5120}, {"name": "model.layers.11.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 7961395200, "nbytes": 81920}, {"name": "model.layers.11.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 7961477120, "nbytes": 20480}, {"name": "model.layers.11.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 7961497600, "nbytes": 6553600}, {"name": "model.layers.11.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 7968051200, "nbytes": 6553600}, {"name": "model.layers.11.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 7974604800, "nbytes": 3276800}, {"name": "model.layers.11.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 7977881600, "nbytes": 256}, {"name": "model.layers.11.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 7977885696, "nbytes": 256}, {"name": "model.layers.11.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 7977889792, "nbytes": 512}, {"name": "model.layers.11.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 7977893888, "nbytes": 31457280}, {"name": "model.layers.11.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 8009351168, "nbytes": 512}, {"name": "model.layers.11.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 8009355264, "nbytes": 68157440}, {"name": "model.layers.12.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8077512704, "nbytes": 81920}, {"name": "model.layers.12.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8077594624, "nbytes": 20480}, {"name": "model.layers.12.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8077615104, "nbytes": 6553600}, {"name": "model.layers.12.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8084168704, "nbytes": 6553600}, {"name": "model.layers.12.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8090722304, "nbytes": 96}, {"name": "model.layers.12.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 8090726400, "nbytes": 81920}, {"name": "model.layers.12.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8090808320, "nbytes": 96}, {"name": "model.layers.12.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 8090812416, "nbytes": 84377600}, {"name": "model.layers.12.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8175190016, "nbytes": 256}, {"name": "model.layers.12.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 8175194112, "nbytes": 31457280}, {"name": "model.layers.12.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 8206651392, "nbytes": 2621440}, {"name": "model.layers.12.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 8209272832, "nbytes": 3276800}, {"name": "model.layers.12.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 8212549632, "nbytes": 6553600}, {"name": "model.layers.12.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 8219103232, "nbytes": 5120}, {"name": "model.layers.12.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8219111424, "nbytes": 81920}, {"name": "model.layers.12.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8219193344, "nbytes": 20480}, {"name": "model.layers.12.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8219213824, "nbytes": 6553600}, {"name": "model.layers.12.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8225767424, "nbytes": 6553600}, {"name": "model.layers.13.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8232321024, "nbytes": 81920}, {"name": "model.layers.13.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8232402944, "nbytes": 20480}, {"name": "model.layers.13.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8232423424, "nbytes": 6553600}, {"name": "model.layers.13.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8238977024, "nbytes": 6553600}, {"name": "model.layers.13.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8245530624, "nbytes": 96}, {"name": "model.layers.13.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 8245534720, "nbytes": 81920}, {"name": "model.layers.13.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8245616640, "nbytes": 96}, {"name": "model.layers.13.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 8245620736, "nbytes": 84377600}, {"name": "model.layers.13.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8329998336, "nbytes": 256}, {"name": "model.layers.13.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 8330002432, "nbytes": 31457280}, {"name": "model.layers.13.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 8361459712, "nbytes": 2621440}, {"name": "model.layers.13.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 8364081152, "nbytes": 3276800}, {"name": "model.layers.13.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 8367357952, "nbytes": 6553600}, {"name": "model.layers.13.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 8373911552, "nbytes": 5120}, {"name": "model.layers.13.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8373919744, "nbytes": 81920}, {"name": "model.layers.13.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8374001664, "nbytes": 20480}, {"name": "model.layers.13.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8374022144, "nbytes": 6553600}, {"name": "model.layers.13.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8380575744, "nbytes": 6553600}, {"name": "model.layers.14.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8387129344, "nbytes": 81920}, {"name": "model.layers.14.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8387211264, "nbytes": 20480}, {"name": "model.layers.14.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8387231744, "nbytes": 6553600}, {"name": "model.layers.14.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8393785344, "nbytes": 6553600}, {"name": "model.layers.14.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8400338944, "nbytes": 96}, {"name": "model.layers.14.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 8400343040, "nbytes": 81920}, {"name": "model.layers.14.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8400424960, "nbytes": 96}, {"name": "model.layers.14.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 8400429056, "nbytes": 84377600}, {"name": "model.layers.14.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8484806656, "nbytes": 256}, {"name": "model.layers.14.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 8484810752, "nbytes": 31457280}, {"name": "model.layers.14.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 8516268032, "nbytes": 2621440}, {"name": "model.layers.14.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 8518889472, "nbytes": 3276800}, {"name": "model.layers.14.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 8522166272, "nbytes": 6553600}, {"name": "model.layers.14.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 8528719872, "nbytes": 5120}, {"name": "model.layers.14.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8528728064, "nbytes": 81920}, {"name": "model.layers.14.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8528809984, "nbytes": 20480}, {"name": "model.layers.14.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8528830464, "nbytes": 6553600}, {"name": "model.layers.14.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8535384064, "nbytes": 6553600}, {"name": "model.layers.15.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8541937664, "nbytes": 81920}, {"name": "model.layers.15.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8542019584, "nbytes": 20480}, {"name": "model.layers.15.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8542040064, "nbytes": 6553600}, {"name": "model.layers.15.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8548593664, "nbytes": 6553600}, {"name": "model.layers.15.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 8555147264, "nbytes": 2621440}, {"name": "model.layers.15.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 8557768704, "nbytes": 3276800}, {"name": "model.layers.15.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 8561045504, "nbytes": 6553600}, {"name": "model.layers.15.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 8567599104, "nbytes": 5120}, {"name": "model.layers.15.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8567607296, "nbytes": 81920}, {"name": "model.layers.15.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8567689216, "nbytes": 20480}, {"name": "model.layers.15.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8567709696, "nbytes": 6553600}, {"name": "model.layers.15.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8574263296, "nbytes": 6553600}, {"name": "model.layers.15.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 8580816896, "nbytes": 3276800}, {"name": "model.layers.15.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8584093696, "nbytes": 256}, {"name": "model.layers.15.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8584097792, "nbytes": 256}, {"name": "model.layers.15.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 8584101888, "nbytes": 512}, {"name": "model.layers.15.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 8584105984, "nbytes": 68157440}, {"name": "model.layers.15.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 8652263424, "nbytes": 31457280}, {"name": "model.layers.15.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 8683720704, "nbytes": 512}, {"name": "model.layers.2.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8683724800, "nbytes": 81920}, {"name": "model.layers.2.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8683806720, "nbytes": 20480}, {"name": "model.layers.2.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8683827200, "nbytes": 6553600}, {"name": "model.layers.2.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8690380800, "nbytes": 6553600}, {"name": "model.layers.2.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8696934400, "nbytes": 96}, {"name": "model.layers.2.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 8696938496, "nbytes": 81920}, {"name": "model.layers.2.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8697020416, "nbytes": 96}, {"name": "model.layers.2.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 8697024512, "nbytes": 84377600}, {"name": "model.layers.2.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8781402112, "nbytes": 256}, {"name": "model.layers.2.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 8781406208, "nbytes": 31457280}, {"name": "model.layers.2.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 8812863488, "nbytes": 2621440}, {"name": "model.layers.2.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 8815484928, "nbytes": 3276800}, {"name": "model.layers.2.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 8818761728, "nbytes": 6553600}, {"name": "model.layers.2.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 8825315328, "nbytes": 5120}, {"name": "model.layers.2.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8825323520, "nbytes": 81920}, {"name": "model.layers.2.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8825405440, "nbytes": 20480}, {"name": "model.layers.2.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8825425920, "nbytes": 6553600}, {"name": "model.layers.2.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8831979520, "nbytes": 6553600}, {"name": "model.layers.3.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8838533120, "nbytes": 81920}, {"name": "model.layers.3.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8838615040, "nbytes": 20480}, {"name": "model.layers.3.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8838635520, "nbytes": 6553600}, {"name": "model.layers.3.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8845189120, "nbytes": 6553600}, {"name": "model.layers.3.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 8851742720, "nbytes": 2621440}, {"name": "model.layers.3.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 8854364160, "nbytes": 3276800}, {"name": "model.layers.3.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 8857640960, "nbytes": 6553600}, {"name": "model.layers.3.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 8864194560, "nbytes": 5120}, {"name": "model.layers.3.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8864202752, "nbytes": 81920}, {"name": "model.layers.3.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8864284672, "nbytes": 20480}, {"name": "model.layers.3.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8864305152, "nbytes": 6553600}, {"name": "model.layers.3.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8870858752, "nbytes": 6553600}, {"name": "model.layers.3.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 8877412352, "nbytes": 3276800}, {"name": "model.layers.3.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8880689152, "nbytes": 256}, {"name": "model.layers.3.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8880693248, "nbytes": 256}, {"name": "model.layers.3.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 8880697344, "nbytes": 512}, {"name": "model.layers.3.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 8880701440, "nbytes": 31457280}, {"name": "model.layers.3.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 8912158720, "nbytes": 512}, {"name": "model.layers.3.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 8912162816, "nbytes": 68157440}, {"name": "model.layers.4.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 8980320256, "nbytes": 81920}, {"name": "model.layers.4.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 8980402176, "nbytes": 20480}, {"name": "model.layers.4.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 8980422656, "nbytes": 6553600}, {"name": "model.layers.4.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 8986976256, "nbytes": 6553600}, {"name": "model.layers.4.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8993529856, "nbytes": 96}, {"name": "model.layers.4.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 8993533952, "nbytes": 81920}, {"name": "model.layers.4.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 8993615872, "nbytes": 96}, {"name": "model.layers.4.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 8993619968, "nbytes": 84377600}, {"name": "model.layers.4.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9077997568, "nbytes": 256}, {"name": "model.layers.4.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 9078001664, "nbytes": 31457280}, {"name": "model.layers.4.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 9109458944, "nbytes": 2621440}, {"name": "model.layers.4.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 9112080384, "nbytes": 3276800}, {"name": "model.layers.4.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 9115357184, "nbytes": 6553600}, {"name": "model.layers.4.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 9121910784, "nbytes": 5120}, {"name": "model.layers.4.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9121918976, "nbytes": 81920}, {"name": "model.layers.4.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9122000896, "nbytes": 20480}, {"name": "model.layers.4.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9122021376, "nbytes": 6553600}, {"name": "model.layers.4.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9128574976, "nbytes": 6553600}, {"name": "model.layers.5.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9135128576, "nbytes": 81920}, {"name": "model.layers.5.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9135210496, "nbytes": 20480}, {"name": "model.layers.5.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9135230976, "nbytes": 6553600}, {"name": "model.layers.5.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9141784576, "nbytes": 6553600}, {"name": "model.layers.5.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 9148338176, "nbytes": 96}, {"name": "model.layers.5.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 9148342272, "nbytes": 81920}, {"name": "model.layers.5.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 9148424192, "nbytes": 96}, {"name": "model.layers.5.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 9148428288, "nbytes": 84377600}, {"name": "model.layers.5.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9232805888, "nbytes": 256}, {"name": "model.layers.5.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 9232809984, "nbytes": 31457280}, {"name": "model.layers.5.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 9264267264, "nbytes": 2621440}, {"name": "model.layers.5.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 9266888704, "nbytes": 3276800}, {"name": "model.layers.5.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 9270165504, "nbytes": 6553600}, {"name": "model.layers.5.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 9276719104, "nbytes": 5120}, {"name": "model.layers.5.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9276727296, "nbytes": 81920}, {"name": "model.layers.5.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9276809216, "nbytes": 20480}, {"name": "model.layers.5.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9276829696, "nbytes": 6553600}, {"name": "model.layers.5.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9283383296, "nbytes": 6553600}, {"name": "model.layers.6.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9289936896, "nbytes": 81920}, {"name": "model.layers.6.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9290018816, "nbytes": 20480}, {"name": "model.layers.6.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9290039296, "nbytes": 6553600}, {"name": "model.layers.6.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9296592896, "nbytes": 6553600}, {"name": "model.layers.6.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 9303146496, "nbytes": 96}, {"name": "model.layers.6.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 9303150592, "nbytes": 81920}, {"name": "model.layers.6.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 9303232512, "nbytes": 96}, {"name": "model.layers.6.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 9303236608, "nbytes": 84377600}, {"name": "model.layers.6.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9387614208, "nbytes": 256}, {"name": "model.layers.6.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 9387618304, "nbytes": 31457280}, {"name": "model.layers.6.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 9419075584, "nbytes": 2621440}, {"name": "model.layers.6.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 9421697024, "nbytes": 3276800}, {"name": "model.layers.6.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 9424973824, "nbytes": 6553600}, {"name": "model.layers.6.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 9431527424, "nbytes": 5120}, {"name": "model.layers.6.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9431535616, "nbytes": 81920}, {"name": "model.layers.6.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9431617536, "nbytes": 20480}, {"name": "model.layers.6.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9431638016, "nbytes": 6553600}, {"name": "model.layers.6.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9438191616, "nbytes": 6553600}, {"name": "model.layers.7.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9444745216, "nbytes": 81920}, {"name": "model.layers.7.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9444827136, "nbytes": 20480}, {"name": "model.layers.7.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9444847616, "nbytes": 6553600}, {"name": "model.layers.7.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9451401216, "nbytes": 6553600}, {"name": "model.layers.7.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 9457954816, "nbytes": 2621440}, {"name": "model.layers.7.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 9460576256, "nbytes": 3276800}, {"name": "model.layers.7.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 9463853056, "nbytes": 6553600}, {"name": "model.layers.7.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 9470406656, "nbytes": 5120}, {"name": "model.layers.7.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9470414848, "nbytes": 81920}, {"name": "model.layers.7.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9470496768, "nbytes": 20480}, {"name": "model.layers.7.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9470517248, "nbytes": 6553600}, {"name": "model.layers.7.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9477070848, "nbytes": 6553600}, {"name": "model.layers.7.self_attn.index_qk_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [640, 2560], "global_off": 9483624448, "nbytes": 3276800}, {"name": "model.layers.7.self_attn.index_k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9486901248, "nbytes": 256}, {"name": "model.layers.7.self_attn.index_q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9486905344, "nbytes": 256}, {"name": "model.layers.7.self_attn.k_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 9486909440, "nbytes": 512}, {"name": "model.layers.7.self_attn.o_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 9486913536, "nbytes": 31457280}, {"name": "model.layers.7.self_attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256], "global_off": 9518370816, "nbytes": 512}, {"name": "model.layers.7.self_attn.qkv_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [13312, 2560], "global_off": 9518374912, "nbytes": 68157440}, {"name": "model.layers.8.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9586532352, "nbytes": 81920}, {"name": "model.layers.8.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9586614272, "nbytes": 20480}, {"name": "model.layers.8.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9586634752, "nbytes": 6553600}, {"name": "model.layers.8.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9593188352, "nbytes": 6553600}, {"name": "model.layers.8.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 9599741952, "nbytes": 96}, {"name": "model.layers.8.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 9599746048, "nbytes": 81920}, {"name": "model.layers.8.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 9599827968, "nbytes": 96}, {"name": "model.layers.8.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 9599832064, "nbytes": 84377600}, {"name": "model.layers.8.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9684209664, "nbytes": 256}, {"name": "model.layers.8.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 9684213760, "nbytes": 31457280}, {"name": "model.layers.8.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 9715671040, "nbytes": 2621440}, {"name": "model.layers.8.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 9718292480, "nbytes": 3276800}, {"name": "model.layers.8.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 9721569280, "nbytes": 6553600}, {"name": "model.layers.8.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 9728122880, "nbytes": 5120}, {"name": "model.layers.8.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9728131072, "nbytes": 81920}, {"name": "model.layers.8.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9728212992, "nbytes": 20480}, {"name": "model.layers.8.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9728233472, "nbytes": 6553600}, {"name": "model.layers.8.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9734787072, "nbytes": 6553600}, {"name": "model.layers.9.attn_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9741340672, "nbytes": 81920}, {"name": "model.layers.9.attn_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9741422592, "nbytes": 20480}, {"name": "model.layers.9.attn_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9741443072, "nbytes": 6553600}, {"name": "model.layers.9.attn_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9747996672, "nbytes": 6553600}, {"name": "model.layers.9.linear_attn.A_log", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 9754550272, "nbytes": 96}, {"name": "model.layers.9.linear_attn.conv1d.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 1, 4], "global_off": 9754554368, "nbytes": 81920}, {"name": "model.layers.9.linear_attn.dt_bias", "kind": "weight", "dtype": "bfloat16", "shape": [48], "global_off": 9754636288, "nbytes": 96}, {"name": "model.layers.9.linear_attn.in_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [16480, 2560], "global_off": 9754640384, "nbytes": 84377600}, {"name": "model.layers.9.linear_attn.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9839017984, "nbytes": 256}, {"name": "model.layers.9.linear_attn.out_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 6144], "global_off": 9839022080, "nbytes": 31457280}, {"name": "model.layers.9.mlp.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 2560], "global_off": 9870479360, "nbytes": 2621440}, {"name": "model.layers.9.mlp.shared_expert.down_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [2560, 640], "global_off": 9873100800, "nbytes": 3276800}, {"name": "model.layers.9.mlp.shared_expert.gate_up_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1280, 2560], "global_off": 9876377600, "nbytes": 6553600}, {"name": "model.layers.9.mlp.shared_expert_gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1, 2560], "global_off": 9882931200, "nbytes": 5120}, {"name": "model.layers.9.mlp_hyper_connection.block_inject_weight.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4, 10240], "global_off": 9882939392, "nbytes": 81920}, {"name": "model.layers.9.mlp_hyper_connection.hc_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240], "global_off": 9883021312, "nbytes": 20480}, {"name": "model.layers.9.mlp_hyper_connection.input_mix_weight_down.weight", "kind": "weight", "dtype": "bfloat16", "shape": [320, 10240], "global_off": 9883041792, "nbytes": 6553600}, {"name": "model.layers.9.mlp_hyper_connection.input_mix_weight_up.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10240, 320], "global_off": 9889595392, "nbytes": 6553600}, {"name": "gate_up_packed#L00000", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 9896148992, "nbytes": 838860800}, {"name": "gate_up_scale#L00000", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 10735009792, "nbytes": 104857600}, {"name": "gate_up_global#L00000", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 10839867392, "nbytes": 1310720}, {"name": "down_packed#L00000", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 10841178112, "nbytes": 419430400}, {"name": "down_scale#L00000", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 11260608512, "nbytes": 52428800}, {"name": "down_global#L00000", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 11313037312, "nbytes": 2621440}, {"name": "gate_up_packed#L00001", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 11315658752, "nbytes": 838860800}, {"name": "gate_up_scale#L00001", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 12154519552, "nbytes": 104857600}, {"name": "gate_up_global#L00001", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 12259377152, "nbytes": 1310720}, {"name": "down_packed#L00001", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 12260687872, "nbytes": 419430400}, {"name": "down_scale#L00001", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 12680118272, "nbytes": 52428800}, {"name": "down_global#L00001", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 12732547072, "nbytes": 2621440}, {"name": "gate_up_packed#L00002", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 12735168512, "nbytes": 838860800}, {"name": "gate_up_scale#L00002", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 13574029312, "nbytes": 104857600}, {"name": "gate_up_global#L00002", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 13678886912, "nbytes": 1310720}, {"name": "down_packed#L00002", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 13680197632, "nbytes": 419430400}, {"name": "down_scale#L00002", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 14099628032, "nbytes": 52428800}, {"name": "down_global#L00002", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 14152056832, "nbytes": 2621440}, {"name": "gate_up_packed#L00003", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 14154678272, "nbytes": 838860800}, {"name": "gate_up_scale#L00003", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 14993539072, "nbytes": 104857600}, {"name": "gate_up_global#L00003", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 15098396672, "nbytes": 1310720}, {"name": "down_packed#L00003", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 15099707392, "nbytes": 419430400}, {"name": "down_scale#L00003", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 15519137792, "nbytes": 52428800}, {"name": "down_global#L00003", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 15571566592, "nbytes": 2621440}, {"name": "gate_up_packed#L00004", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 15574188032, "nbytes": 838860800}, {"name": "gate_up_scale#L00004", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 16413048832, "nbytes": 104857600}, {"name": "gate_up_global#L00004", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 16517906432, "nbytes": 1310720}, {"name": "down_packed#L00004", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 16519217152, "nbytes": 419430400}, {"name": "down_scale#L00004", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 16938647552, "nbytes": 52428800}, {"name": "down_global#L00004", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 16991076352, "nbytes": 2621440}, {"name": "gate_up_packed#L00005", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 16993697792, "nbytes": 838860800}, {"name": "gate_up_scale#L00005", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 17832558592, "nbytes": 104857600}, {"name": "gate_up_global#L00005", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 17937416192, "nbytes": 1310720}, {"name": "down_packed#L00005", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 17938726912, "nbytes": 419430400}, {"name": "down_scale#L00005", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 18358157312, "nbytes": 52428800}, {"name": "down_global#L00005", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 18410586112, "nbytes": 2621440}, {"name": "gate_up_packed#L00006", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 18413207552, "nbytes": 838860800}, {"name": "gate_up_scale#L00006", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 19252068352, "nbytes": 104857600}, {"name": "gate_up_global#L00006", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 19356925952, "nbytes": 1310720}, {"name": "down_packed#L00006", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 19358236672, "nbytes": 419430400}, {"name": "down_scale#L00006", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 19777667072, "nbytes": 52428800}, {"name": "down_global#L00006", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 19830095872, "nbytes": 2621440}, {"name": "gate_up_packed#L00007", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 19832717312, "nbytes": 838860800}, {"name": "gate_up_scale#L00007", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 20671578112, "nbytes": 104857600}, {"name": "gate_up_global#L00007", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 20776435712, "nbytes": 1310720}, {"name": "down_packed#L00007", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 20777746432, "nbytes": 419430400}, {"name": "down_scale#L00007", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 21197176832, "nbytes": 52428800}, {"name": "down_global#L00007", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 21249605632, "nbytes": 2621440}, {"name": "gate_up_packed#L00008", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 21252227072, "nbytes": 838860800}, {"name": "gate_up_scale#L00008", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 22091087872, "nbytes": 104857600}, {"name": "gate_up_global#L00008", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 22195945472, "nbytes": 1310720}, {"name": "down_packed#L00008", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 22197256192, "nbytes": 419430400}, {"name": "down_scale#L00008", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 22616686592, "nbytes": 52428800}, {"name": "down_global#L00008", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 22669115392, "nbytes": 2621440}, {"name": "gate_up_packed#L00009", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 22671736832, "nbytes": 838860800}, {"name": "gate_up_scale#L00009", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 23510597632, "nbytes": 104857600}, {"name": "gate_up_global#L00009", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 23615455232, "nbytes": 1310720}, {"name": "down_packed#L00009", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 23616765952, "nbytes": 419430400}, {"name": "down_scale#L00009", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 24036196352, "nbytes": 52428800}, {"name": "down_global#L00009", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 24088625152, "nbytes": 2621440}, {"name": "gate_up_packed#L00010", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 24091246592, "nbytes": 838860800}, {"name": "gate_up_scale#L00010", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 24930107392, "nbytes": 104857600}, {"name": "gate_up_global#L00010", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 25034964992, "nbytes": 1310720}, {"name": "down_packed#L00010", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 25036275712, "nbytes": 419430400}, {"name": "down_scale#L00010", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 25455706112, "nbytes": 52428800}, {"name": "down_global#L00010", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 25508134912, "nbytes": 2621440}, {"name": "gate_up_packed#L00011", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 25510756352, "nbytes": 838860800}, {"name": "gate_up_scale#L00011", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 26349617152, "nbytes": 104857600}, {"name": "gate_up_global#L00011", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 26454474752, "nbytes": 1310720}, {"name": "down_packed#L00011", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 26455785472, "nbytes": 419430400}, {"name": "down_scale#L00011", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 26875215872, "nbytes": 52428800}, {"name": "down_global#L00011", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 26927644672, "nbytes": 2621440}, {"name": "gate_up_packed#L00012", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 26930266112, "nbytes": 838860800}, {"name": "gate_up_scale#L00012", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 27769126912, "nbytes": 104857600}, {"name": "gate_up_global#L00012", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 27873984512, "nbytes": 1310720}, {"name": "down_packed#L00012", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 27875295232, "nbytes": 419430400}, {"name": "down_scale#L00012", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 28294725632, "nbytes": 52428800}, {"name": "down_global#L00012", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 28347154432, "nbytes": 2621440}, {"name": "gate_up_packed#L00013", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 28349775872, "nbytes": 838860800}, {"name": "gate_up_scale#L00013", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 29188636672, "nbytes": 104857600}, {"name": "gate_up_global#L00013", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 29293494272, "nbytes": 1310720}, {"name": "down_packed#L00013", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 29294804992, "nbytes": 419430400}, {"name": "down_scale#L00013", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 29714235392, "nbytes": 52428800}, {"name": "down_global#L00013", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 29766664192, "nbytes": 2621440}, {"name": "gate_up_packed#L00014", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 29769285632, "nbytes": 838860800}, {"name": "gate_up_scale#L00014", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 30608146432, "nbytes": 104857600}, {"name": "gate_up_global#L00014", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 30713004032, "nbytes": 1310720}, {"name": "down_packed#L00014", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 30714314752, "nbytes": 419430400}, {"name": "down_scale#L00014", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 31133745152, "nbytes": 52428800}, {"name": "down_global#L00014", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 31186173952, "nbytes": 2621440}, {"name": "gate_up_packed#L00015", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 31188795392, "nbytes": 838860800}, {"name": "gate_up_scale#L00015", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 32027656192, "nbytes": 104857600}, {"name": "gate_up_global#L00015", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 32132513792, "nbytes": 1310720}, {"name": "down_packed#L00015", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 32133824512, "nbytes": 419430400}, {"name": "down_scale#L00015", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 32553254912, "nbytes": 52428800}, {"name": "down_global#L00015", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 32605683712, "nbytes": 2621440}, {"name": "gate_up_packed#L00016", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 32608305152, "nbytes": 838860800}, {"name": "gate_up_scale#L00016", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 33447165952, "nbytes": 104857600}, {"name": "gate_up_global#L00016", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 33552023552, "nbytes": 1310720}, {"name": "down_packed#L00016", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 33553334272, "nbytes": 419430400}, {"name": "down_scale#L00016", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 33972764672, "nbytes": 52428800}, {"name": "down_global#L00016", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 34025193472, "nbytes": 2621440}, {"name": "gate_up_packed#L00017", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 34027814912, "nbytes": 838860800}, {"name": "gate_up_scale#L00017", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 34866675712, "nbytes": 104857600}, {"name": "gate_up_global#L00017", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 34971533312, "nbytes": 1310720}, {"name": "down_packed#L00017", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 34972844032, "nbytes": 419430400}, {"name": "down_scale#L00017", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 35392274432, "nbytes": 52428800}, {"name": "down_global#L00017", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 35444703232, "nbytes": 2621440}, {"name": "gate_up_packed#L00018", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 35447324672, "nbytes": 838860800}, {"name": "gate_up_scale#L00018", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 36286185472, "nbytes": 104857600}, {"name": "gate_up_global#L00018", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 36391043072, "nbytes": 1310720}, {"name": "down_packed#L00018", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 36392353792, "nbytes": 419430400}, {"name": "down_scale#L00018", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 36811784192, "nbytes": 52428800}, {"name": "down_global#L00018", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 36864212992, "nbytes": 2621440}, {"name": "gate_up_packed#L00019", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 36866834432, "nbytes": 838860800}, {"name": "gate_up_scale#L00019", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 37705695232, "nbytes": 104857600}, {"name": "gate_up_global#L00019", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 37810552832, "nbytes": 1310720}, {"name": "down_packed#L00019", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 37811863552, "nbytes": 419430400}, {"name": "down_scale#L00019", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 38231293952, "nbytes": 52428800}, {"name": "down_global#L00019", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 38283722752, "nbytes": 2621440}, {"name": "gate_up_packed#L00020", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 38286344192, "nbytes": 838860800}, {"name": "gate_up_scale#L00020", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 39125204992, "nbytes": 104857600}, {"name": "gate_up_global#L00020", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 39230062592, "nbytes": 1310720}, {"name": "down_packed#L00020", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 39231373312, "nbytes": 419430400}, {"name": "down_scale#L00020", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 39650803712, "nbytes": 52428800}, {"name": "down_global#L00020", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 39703232512, "nbytes": 2621440}, {"name": "gate_up_packed#L00021", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 39705853952, "nbytes": 838860800}, {"name": "gate_up_scale#L00021", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 40544714752, "nbytes": 104857600}, {"name": "gate_up_global#L00021", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 40649572352, "nbytes": 1310720}, {"name": "down_packed#L00021", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 40650883072, "nbytes": 419430400}, {"name": "down_scale#L00021", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 41070313472, "nbytes": 52428800}, {"name": "down_global#L00021", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 41122742272, "nbytes": 2621440}, {"name": "gate_up_packed#L00022", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 41125363712, "nbytes": 838860800}, {"name": "gate_up_scale#L00022", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 41964224512, "nbytes": 104857600}, {"name": "gate_up_global#L00022", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 42069082112, "nbytes": 1310720}, {"name": "down_packed#L00022", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 42070392832, "nbytes": 419430400}, {"name": "down_scale#L00022", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 42489823232, "nbytes": 52428800}, {"name": "down_global#L00022", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 42542252032, "nbytes": 2621440}, {"name": "gate_up_packed#L00023", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 42544873472, "nbytes": 838860800}, {"name": "gate_up_scale#L00023", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 43383734272, "nbytes": 104857600}, {"name": "gate_up_global#L00023", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 43488591872, "nbytes": 1310720}, {"name": "down_packed#L00023", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 43489902592, "nbytes": 419430400}, {"name": "down_scale#L00023", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 43909332992, "nbytes": 52428800}, {"name": "down_global#L00023", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 43961761792, "nbytes": 2621440}, {"name": "gate_up_packed#L00024", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 43964383232, "nbytes": 838860800}, {"name": "gate_up_scale#L00024", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 44803244032, "nbytes": 104857600}, {"name": "gate_up_global#L00024", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 44908101632, "nbytes": 1310720}, {"name": "down_packed#L00024", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 44909412352, "nbytes": 419430400}, {"name": "down_scale#L00024", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 45328842752, "nbytes": 52428800}, {"name": "down_global#L00024", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 45381271552, "nbytes": 2621440}, {"name": "gate_up_packed#L00025", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 45383892992, "nbytes": 838860800}, {"name": "gate_up_scale#L00025", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 46222753792, "nbytes": 104857600}, {"name": "gate_up_global#L00025", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 46327611392, "nbytes": 1310720}, {"name": "down_packed#L00025", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 46328922112, "nbytes": 419430400}, {"name": "down_scale#L00025", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 46748352512, "nbytes": 52428800}, {"name": "down_global#L00025", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 46800781312, "nbytes": 2621440}, {"name": "gate_up_packed#L00026", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 46803402752, "nbytes": 838860800}, {"name": "gate_up_scale#L00026", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 47642263552, "nbytes": 104857600}, {"name": "gate_up_global#L00026", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 47747121152, "nbytes": 1310720}, {"name": "down_packed#L00026", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 47748431872, "nbytes": 419430400}, {"name": "down_scale#L00026", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 48167862272, "nbytes": 52428800}, {"name": "down_global#L00026", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 48220291072, "nbytes": 2621440}, {"name": "gate_up_packed#L00027", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 48222912512, "nbytes": 838860800}, {"name": "gate_up_scale#L00027", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 49061773312, "nbytes": 104857600}, {"name": "gate_up_global#L00027", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 49166630912, "nbytes": 1310720}, {"name": "down_packed#L00027", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 49167941632, "nbytes": 419430400}, {"name": "down_scale#L00027", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 49587372032, "nbytes": 52428800}, {"name": "down_global#L00027", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 49639800832, "nbytes": 2621440}, {"name": "gate_up_packed#L00028", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 49642422272, "nbytes": 838860800}, {"name": "gate_up_scale#L00028", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 50481283072, "nbytes": 104857600}, {"name": "gate_up_global#L00028", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 50586140672, "nbytes": 1310720}, {"name": "down_packed#L00028", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 50587451392, "nbytes": 419430400}, {"name": "down_scale#L00028", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 51006881792, "nbytes": 52428800}, {"name": "down_global#L00028", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 51059310592, "nbytes": 2621440}, {"name": "gate_up_packed#L00029", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 51061932032, "nbytes": 838860800}, {"name": "gate_up_scale#L00029", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 51900792832, "nbytes": 104857600}, {"name": "gate_up_global#L00029", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 52005650432, "nbytes": 1310720}, {"name": "down_packed#L00029", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 52006961152, "nbytes": 419430400}, {"name": "down_scale#L00029", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 52426391552, "nbytes": 52428800}, {"name": "down_global#L00029", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 52478820352, "nbytes": 2621440}, {"name": "gate_up_packed#L00030", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 52481441792, "nbytes": 838860800}, {"name": "gate_up_scale#L00030", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 53320302592, "nbytes": 104857600}, {"name": "gate_up_global#L00030", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 53425160192, "nbytes": 1310720}, {"name": "down_packed#L00030", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 53426470912, "nbytes": 419430400}, {"name": "down_scale#L00030", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 53845901312, "nbytes": 52428800}, {"name": "down_global#L00030", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 53898330112, "nbytes": 2621440}, {"name": "gate_up_packed#L00031", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 53900951552, "nbytes": 838860800}, {"name": "gate_up_scale#L00031", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 54739812352, "nbytes": 104857600}, {"name": "gate_up_global#L00031", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 54844669952, "nbytes": 1310720}, {"name": "down_packed#L00031", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 54845980672, "nbytes": 419430400}, {"name": "down_scale#L00031", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 55265411072, "nbytes": 52428800}, {"name": "down_global#L00031", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 55317839872, "nbytes": 2621440}, {"name": "gate_up_packed#L00032", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 55320461312, "nbytes": 838860800}, {"name": "gate_up_scale#L00032", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 56159322112, "nbytes": 104857600}, {"name": "gate_up_global#L00032", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 56264179712, "nbytes": 1310720}, {"name": "down_packed#L00032", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 56265490432, "nbytes": 419430400}, {"name": "down_scale#L00032", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 56684920832, "nbytes": 52428800}, {"name": "down_global#L00032", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 56737349632, "nbytes": 2621440}, {"name": "gate_up_packed#L00033", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 56739971072, "nbytes": 838860800}, {"name": "gate_up_scale#L00033", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 57578831872, "nbytes": 104857600}, {"name": "gate_up_global#L00033", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 57683689472, "nbytes": 1310720}, {"name": "down_packed#L00033", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 57685000192, "nbytes": 419430400}, {"name": "down_scale#L00033", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 58104430592, "nbytes": 52428800}, {"name": "down_global#L00033", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 58156859392, "nbytes": 2621440}, {"name": "gate_up_packed#L00034", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 58159480832, "nbytes": 838860800}, {"name": "gate_up_scale#L00034", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 58998341632, "nbytes": 104857600}, {"name": "gate_up_global#L00034", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 59103199232, "nbytes": 1310720}, {"name": "down_packed#L00034", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 59104509952, "nbytes": 419430400}, {"name": "down_scale#L00034", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 59523940352, "nbytes": 52428800}, {"name": "down_global#L00034", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 59576369152, "nbytes": 2621440}, {"name": "gate_up_packed#L00035", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 59578990592, "nbytes": 838860800}, {"name": "gate_up_scale#L00035", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 60417851392, "nbytes": 104857600}, {"name": "gate_up_global#L00035", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 60522708992, "nbytes": 1310720}, {"name": "down_packed#L00035", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 60524019712, "nbytes": 419430400}, {"name": "down_scale#L00035", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 60943450112, "nbytes": 52428800}, {"name": "down_global#L00035", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 60995878912, "nbytes": 2621440}, {"name": "gate_up_packed#L00036", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 60998500352, "nbytes": 838860800}, {"name": "gate_up_scale#L00036", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 61837361152, "nbytes": 104857600}, {"name": "gate_up_global#L00036", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 61942218752, "nbytes": 1310720}, {"name": "down_packed#L00036", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 61943529472, "nbytes": 419430400}, {"name": "down_scale#L00036", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 62362959872, "nbytes": 52428800}, {"name": "down_global#L00036", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 62415388672, "nbytes": 2621440}, {"name": "gate_up_packed#L00037", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 62418010112, "nbytes": 838860800}, {"name": "gate_up_scale#L00037", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 63256870912, "nbytes": 104857600}, {"name": "gate_up_global#L00037", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 63361728512, "nbytes": 1310720}, {"name": "down_packed#L00037", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 63363039232, "nbytes": 419430400}, {"name": "down_scale#L00037", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 63782469632, "nbytes": 52428800}, {"name": "down_global#L00037", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 63834898432, "nbytes": 2621440}, {"name": "gate_up_packed#L00038", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 63837519872, "nbytes": 838860800}, {"name": "gate_up_scale#L00038", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 64676380672, "nbytes": 104857600}, {"name": "gate_up_global#L00038", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 64781238272, "nbytes": 1310720}, {"name": "down_packed#L00038", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 64782548992, "nbytes": 419430400}, {"name": "down_scale#L00038", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 65201979392, "nbytes": 52428800}, {"name": "down_global#L00038", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 65254408192, "nbytes": 2621440}, {"name": "gate_up_packed#L00039", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 65257029632, "nbytes": 838860800}, {"name": "gate_up_scale#L00039", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 66095890432, "nbytes": 104857600}, {"name": "gate_up_global#L00039", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 66200748032, "nbytes": 1310720}, {"name": "down_packed#L00039", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 66202058752, "nbytes": 419430400}, {"name": "down_scale#L00039", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 66621489152, "nbytes": 52428800}, {"name": "down_global#L00039", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 66673917952, "nbytes": 2621440}, {"name": "gate_up_packed#L00040", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 66676539392, "nbytes": 838860800}, {"name": "gate_up_scale#L00040", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 67515400192, "nbytes": 104857600}, {"name": "gate_up_global#L00040", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 67620257792, "nbytes": 1310720}, {"name": "down_packed#L00040", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 67621568512, "nbytes": 419430400}, {"name": "down_scale#L00040", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 68040998912, "nbytes": 52428800}, {"name": "down_global#L00040", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 68093427712, "nbytes": 2621440}, {"name": "gate_up_packed#L00041", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 68096049152, "nbytes": 838860800}, {"name": "gate_up_scale#L00041", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 68934909952, "nbytes": 104857600}, {"name": "gate_up_global#L00041", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 69039767552, "nbytes": 1310720}, {"name": "down_packed#L00041", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 69041078272, "nbytes": 419430400}, {"name": "down_scale#L00041", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 69460508672, "nbytes": 52428800}, {"name": "down_global#L00041", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 69512937472, "nbytes": 2621440}, {"name": "gate_up_packed#L00042", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 69515558912, "nbytes": 838860800}, {"name": "gate_up_scale#L00042", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 70354419712, "nbytes": 104857600}, {"name": "gate_up_global#L00042", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 70459277312, "nbytes": 1310720}, {"name": "down_packed#L00042", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 70460588032, "nbytes": 419430400}, {"name": "down_scale#L00042", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 70880018432, "nbytes": 52428800}, {"name": "down_global#L00042", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 70932447232, "nbytes": 2621440}, {"name": "gate_up_packed#L00043", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 70935068672, "nbytes": 838860800}, {"name": "gate_up_scale#L00043", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 71773929472, "nbytes": 104857600}, {"name": "gate_up_global#L00043", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 71878787072, "nbytes": 1310720}, {"name": "down_packed#L00043", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 71880097792, "nbytes": 419430400}, {"name": "down_scale#L00043", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 72299528192, "nbytes": 52428800}, {"name": "down_global#L00043", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 72351956992, "nbytes": 2621440}, {"name": "gate_up_packed#L00044", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 72354578432, "nbytes": 838860800}, {"name": "gate_up_scale#L00044", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 73193439232, "nbytes": 104857600}, {"name": "gate_up_global#L00044", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 73298296832, "nbytes": 1310720}, {"name": "down_packed#L00044", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 73299607552, "nbytes": 419430400}, {"name": "down_scale#L00044", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 73719037952, "nbytes": 52428800}, {"name": "down_global#L00044", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 73771466752, "nbytes": 2621440}, {"name": "gate_up_packed#L00045", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 73774088192, "nbytes": 838860800}, {"name": "gate_up_scale#L00045", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 74612948992, "nbytes": 104857600}, {"name": "gate_up_global#L00045", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 74717806592, "nbytes": 1310720}, {"name": "down_packed#L00045", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 74719117312, "nbytes": 419430400}, {"name": "down_scale#L00045", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 75138547712, "nbytes": 52428800}, {"name": "down_global#L00045", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 75190976512, "nbytes": 2621440}, {"name": "gate_up_packed#L00046", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 75193597952, "nbytes": 838860800}, {"name": "gate_up_scale#L00046", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 76032458752, "nbytes": 104857600}, {"name": "gate_up_global#L00046", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 76137316352, "nbytes": 1310720}, {"name": "down_packed#L00046", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 76138627072, "nbytes": 419430400}, {"name": "down_scale#L00046", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 76558057472, "nbytes": 52428800}, {"name": "down_global#L00046", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 76610486272, "nbytes": 2621440}, {"name": "gate_up_packed#L00047", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 1280, 1280], "global_off": 76613107712, "nbytes": 838860800}, {"name": "gate_up_scale#L00047", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 1280, 160], "global_off": 77451968512, "nbytes": 104857600}, {"name": "gate_up_global#L00047", "kind": "experts_bank", "dtype": "float16", "shape": [512, 1280], "global_off": 77556826112, "nbytes": 1310720}, {"name": "down_packed#L00047", "kind": "experts_bank", "dtype": "uint8", "shape": [512, 2560, 320], "global_off": 77558136832, "nbytes": 419430400}, {"name": "down_scale#L00047", "kind": "experts_bank", "dtype": "float8_e4m3fn", "shape": [512, 2560, 40], "global_off": 77977567232, "nbytes": 52428800}, {"name": "down_global#L00047", "kind": "experts_bank", "dtype": "float16", "shape": [512, 2560], "global_off": 78029996032, "nbytes": 2621440}], "shards": [{"file": "freetoken-00000.ftw", "global_off": 0, "nbytes": 8584105984}, {"file": "freetoken-00001.ftw", "global_off": 8584105984, "nbytes": 8409591808}, {"file": "freetoken-00002.ftw", "global_off": 16993697792, "nbytes": 8517058560}, {"file": "freetoken-00003.ftw", "global_off": 25510756352, "nbytes": 8517058560}, {"file": "freetoken-00004.ftw", "global_off": 34027814912, "nbytes": 8517058560}, {"file": "freetoken-00005.ftw", "global_off": 42544873472, "nbytes": 8517058560}, {"file": "freetoken-00006.ftw", "global_off": 51061932032, "nbytes": 8517058560}, {"file": "freetoken-00007.ftw", "global_off": 59578990592, "nbytes": 8517058560}, {"file": "freetoken-00008.ftw", "global_off": 68096049152, "nbytes": 8517058560}, {"file": "freetoken-00009.ftw", "global_off": 76613107712, "nbytes": 1419509760}], "source_model_path": "/home/lidaiqing/.sparklab/models/qwen3.8-flash-next/source/103a76083161", "fingerprint": "47e11ddb878adf4c", "quant_format": "nvfp4", "expert_quantization": null, "expert_bank_num_layers": 48, "counts": {"weight": 887, "experts_bank": 288}, "copied_metadata": ["README.md", ".gitattributes", "tokenizer_config.json", "chat_template.jinja", "generation_config.json", "config.json", "vocab.json", "video_preprocessor_config.json", "tokenizer.json", "LICENSE", "preprocessor_config.json", "merges.txt"], "external_artifacts": [{"kind": "qwen4_ngram", "schema_version": "1.0", "file": "qwen4_ngram.bin", "dtype": "bfloat16", "rows": 320001536, "dim": 160, "nbytes": 102400491520, "parts": 128}]}
\ No newline at end of file