diff --git "a/compatibility/mac-check/checkpoint-headers.json" "b/compatibility/mac-check/checkpoint-headers.json" new file mode 100644--- /dev/null +++ "b/compatibility/mac-check/checkpoint-headers.json" @@ -0,0 +1,28068 @@ +{ + "language_model.model.embed_tokens.biases": { + "data_offsets": [ + 2991199424, + 3030930624 + ], + "dtype": "BF16", + "shape": [ + 248320, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.embed_tokens.scales": { + "data_offsets": [ + 3130496704, + 3170227904 + ], + "dtype": "BF16", + "shape": [ + 248320, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.embed_tokens.weight": { + "data_offsets": [ + 1072236352, + 1707935552 + ], + "dtype": "U32", + "shape": [ + 248320, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.input_layernorm.weight": { + "data_offsets": [ + 2359641056, + 2359651296 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.A_log": { + "data_offsets": [ + 5282602912, + 5282603008 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.conv1d.weight": { + "data_offsets": [ + 2591968928, + 2592050848 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.dt_bias": { + "data_offsets": [ + 3982025024, + 3982025120 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 870346272, + 870353952 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 5280825920, + 5280833600 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 2212639520, + 2212762400 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4978187232, + 4978194912 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 2406181856, + 2406189536 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 2084293920, + 2084416800 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 2088267040, + 2089905440 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4824483968, + 4826122368 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4238483904, + 4264698304 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 1900260544, + 1901243584 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2625626208, + 2626609248 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 1952392480, + 1968121120 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.norm.weight": { + "data_offsets": [ + 2213745440, + 2213745696 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.out_proj.biases": { + "data_offsets": [ + 2212762400, + 2213745440 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.out_proj.scales": { + "data_offsets": [ + 663200960, + 664184000 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.linear_attn.out_proj.weight": { + "data_offsets": [ + 5282610688, + 5298339328 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.mlp.down_proj.biases": { + "data_offsets": [ + 3521490880, + 3524276160 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.mlp.down_proj.scales": { + "data_offsets": [ + 2141023520, + 2143808800 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.mlp.down_proj.weight": { + "data_offsets": [ + 4846275040, + 4890839520 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.mlp.gate_proj.biases": { + "data_offsets": [ + 2163323680, + 2166108960 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.mlp.gate_proj.scales": { + "data_offsets": [ + 2085399840, + 2088185120 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.mlp.gate_proj.weight": { + "data_offsets": [ + 2456498752, + 2501063232 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.mlp.up_proj.biases": { + "data_offsets": [ + 1066658112, + 1069443392 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.mlp.up_proj.scales": { + "data_offsets": [ + 2143808800, + 2146594080 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.mlp.up_proj.weight": { + "data_offsets": [ + 3526047680, + 3570612160 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.0.post_attention_layernorm.weight": { + "data_offsets": [ + 1713740288, + 1713750528 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.input_layernorm.weight": { + "data_offsets": [ + 1897546944, + 1897557184 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.A_log": { + "data_offsets": [ + 1713719616, + 1713719712 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.conv1d.weight": { + "data_offsets": [ + 1897557184, + 1897639104 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.dt_bias": { + "data_offsets": [ + 1716781568, + 1716781664 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 1744634464, + 1744642144 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 5162033632, + 5162041312 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 2501063232, + 2501186112 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2921214144, + 2921221824 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 1069443392, + 1069451072 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 1845650624, + 1845773504 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 5280833600, + 5282472000 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2972839104, + 2974477504 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1786667968, + 1812882368 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 1766187968, + 1767171008 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2626609248, + 2627592288 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2835108544, + 2850837184 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.norm.weight": { + "data_offsets": [ + 1765941440, + 1765941696 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.out_proj.biases": { + "data_offsets": [ + 1712531008, + 1713514048 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.out_proj.scales": { + "data_offsets": [ + 1901243584, + 1902226624 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.linear_attn.out_proj.weight": { + "data_offsets": [ + 1770939328, + 1786667968 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.mlp.down_proj.biases": { + "data_offsets": [ + 413951168, + 416736448 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.mlp.down_proj.scales": { + "data_offsets": [ + 1063865152, + 1066650432 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.mlp.down_proj.weight": { + "data_offsets": [ + 2168075040, + 2212639520 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.mlp.gate_proj.biases": { + "data_offsets": [ + 908365344, + 911150624 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.mlp.gate_proj.scales": { + "data_offsets": [ + 2135452960, + 2138238240 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.mlp.gate_proj.weight": { + "data_offsets": [ + 2220399392, + 2264963872 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.mlp.up_proj.biases": { + "data_offsets": [ + 1989594400, + 1992379680 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.mlp.up_proj.scales": { + "data_offsets": [ + 911150624, + 913935904 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.mlp.up_proj.weight": { + "data_offsets": [ + 4058415520, + 4102980000 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.1.post_attention_layernorm.weight": { + "data_offsets": [ + 1713719712, + 1713729952 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.input_layernorm.weight": { + "data_offsets": [ + 3521480640, + 3521490880 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.A_log": { + "data_offsets": [ + 3659224192, + 3659224288 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.conv1d.weight": { + "data_offsets": [ + 382742208, + 382824128 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.dt_bias": { + "data_offsets": [ + 3659224096, + 3659224192 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 870338592, + 870346272 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 411158208, + 411165888 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1063742272, + 1063865152 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 3659216320, + 3659224000 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3643480000, + 3643487680 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3596826560, + 3596949440 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3524286400, + 3525924800 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4603161632, + 4604800032 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 3492480960, + 3518695360 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4398156480, + 4399139520 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 746611136, + 747594176 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2675925184, + 2691653824 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.norm.weight": { + "data_offsets": [ + 5327121600, + 5327121856 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.out_proj.biases": { + "data_offsets": [ + 3735246048, + 3736229088 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.out_proj.scales": { + "data_offsets": [ + 4823362592, + 4824345632 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.linear_attn.out_proj.weight": { + "data_offsets": [ + 1744642144, + 1760370784 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.mlp.down_proj.biases": { + "data_offsets": [ + 1946821920, + 1949607200 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.mlp.down_proj.scales": { + "data_offsets": [ + 3844872928, + 3847658208 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.mlp.down_proj.weight": { + "data_offsets": [ + 3800298208, + 3844862688 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.mlp.gate_proj.biases": { + "data_offsets": [ + 1891976384, + 1894761664 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.mlp.gate_proj.scales": { + "data_offsets": [ + 3797512928, + 3800298208 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.mlp.gate_proj.weight": { + "data_offsets": [ + 3736236768, + 3780801248 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.mlp.up_proj.biases": { + "data_offsets": [ + 2606989152, + 2609774432 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.mlp.up_proj.scales": { + "data_offsets": [ + 3847658208, + 3850443488 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.mlp.up_proj.weight": { + "data_offsets": [ + 2315076576, + 2359641056 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.10.post_attention_layernorm.weight": { + "data_offsets": [ + 2314942848, + 2314953088 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.input_layernorm.weight": { + "data_offsets": [ + 3844862688, + 3844872928 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.mlp.down_proj.biases": { + "data_offsets": [ + 3963500864, + 3966286144 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.mlp.down_proj.scales": { + "data_offsets": [ + 3960707904, + 3963493184 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.mlp.down_proj.weight": { + "data_offsets": [ + 3378100160, + 3422664640 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.mlp.gate_proj.biases": { + "data_offsets": [ + 3957922624, + 3960707904 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.mlp.gate_proj.scales": { + "data_offsets": [ + 2918428864, + 2921214144 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.mlp.gate_proj.weight": { + "data_offsets": [ + 3913358144, + 3957922624 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.mlp.up_proj.biases": { + "data_offsets": [ + 5170407392, + 5173192672 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.mlp.up_proj.scales": { + "data_offsets": [ + 4026589600, + 4029374880 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.mlp.up_proj.weight": { + "data_offsets": [ + 3982025120, + 4026589600 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.post_attention_layernorm.weight": { + "data_offsets": [ + 1902226624, + 1902236864 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.k_norm.weight": { + "data_offsets": [ + 1766105536, + 1766106048 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.k_proj.biases": { + "data_offsets": [ + 2605752160, + 2605916000 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.k_proj.scales": { + "data_offsets": [ + 3889273664, + 3889437504 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.k_proj.weight": { + "data_offsets": [ + 3886652128, + 3889273568 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.o_proj.biases": { + "data_offsets": [ + 2409968192, + 2410951232 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.o_proj.scales": { + "data_offsets": [ + 2406189536, + 2407172576 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.o_proj.weight": { + "data_offsets": [ + 3897629504, + 3913358144 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.q_norm.weight": { + "data_offsets": [ + 2605916000, + 2605916512 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.q_proj.biases": { + "data_offsets": [ + 3884686048, + 3886652128 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.q_proj.scales": { + "data_offsets": [ + 3641513920, + 3643480000 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.q_proj.weight": { + "data_offsets": [ + 3850443488, + 3881900768 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.v_proj.biases": { + "data_offsets": [ + 3422674880, + 3422838720 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.v_proj.scales": { + "data_offsets": [ + 1765941696, + 1766105536 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.11.self_attn.v_proj.weight": { + "data_offsets": [ + 1763320000, + 1765941440 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.input_layernorm.weight": { + "data_offsets": [ + 5170397152, + 5170407392 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.A_log": { + "data_offsets": [ + 4102987680, + 4102987776 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.conv1d.weight": { + "data_offsets": [ + 4029497760, + 4029579680 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.dt_bias": { + "data_offsets": [ + 2315076480, + 2315076576 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 4824476192, + 4824483872 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 2309528352, + 2309536032 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 3525924800, + 3526047680 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4102980000, + 4102987680 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 2755003584, + 2755011264 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 5298367584, + 5298490464 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4890839520, + 4892477920 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4055794080, + 4057432480 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4029579680, + 4055794080 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4057432480, + 4058415520 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 4775193632, + 4776176672 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 1970097440, + 1985826080 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.norm.weight": { + "data_offsets": [ + 4102987872, + 4102988128 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.out_proj.biases": { + "data_offsets": [ + 4144603488, + 4145586528 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.out_proj.scales": { + "data_offsets": [ + 4140835168, + 4141818208 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.linear_attn.out_proj.weight": { + "data_offsets": [ + 4122485088, + 4138213728 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.mlp.down_proj.biases": { + "data_offsets": [ + 1709745728, + 1712531008 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.mlp.down_proj.scales": { + "data_offsets": [ + 5304061024, + 5306846304 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.mlp.down_proj.weight": { + "data_offsets": [ + 4193919424, + 4238483904 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.mlp.gate_proj.biases": { + "data_offsets": [ + 4191134144, + 4193919424 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.mlp.gate_proj.scales": { + "data_offsets": [ + 743564736, + 746350016 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.mlp.gate_proj.weight": { + "data_offsets": [ + 4146569664, + 4191134144 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.mlp.up_proj.biases": { + "data_offsets": [ + 4377912000, + 4380697280 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.mlp.up_proj.scales": { + "data_offsets": [ + 4356448960, + 4359234240 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.mlp.up_proj.weight": { + "data_offsets": [ + 4264698560, + 4309263040 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.12.post_attention_layernorm.weight": { + "data_offsets": [ + 2214818592, + 2214828832 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.input_layernorm.weight": { + "data_offsets": [ + 1713730048, + 1713740288 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.A_log": { + "data_offsets": [ + 4400808640, + 4400808736 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.conv1d.weight": { + "data_offsets": [ + 4380697280, + 4380779200 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.dt_bias": { + "data_offsets": [ + 1713729952, + 1713730048 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 4400793280, + 4400800960 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 4400785600, + 4400793280 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1009209152, + 1009332032 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 746480576, + 746488256 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 4399139520, + 4399147200 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 2787615424, + 2787738304 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4380779200, + 4382417600 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4828907648, + 4830546048 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 752199616, + 778414016 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 748595136, + 749578176 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 3780801248, + 3781784288 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 4382417600, + 4398146240 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.norm.weight": { + "data_offsets": [ + 4416537376, + 4416537632 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.out_proj.biases": { + "data_offsets": [ + 1812882368, + 1813865408 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.out_proj.scales": { + "data_offsets": [ + 3084949184, + 3085932224 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.linear_attn.out_proj.weight": { + "data_offsets": [ + 4419322912, + 4435051552 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.mlp.down_proj.biases": { + "data_offsets": [ + 4598410272, + 4601195552 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.mlp.down_proj.scales": { + "data_offsets": [ + 4590054432, + 4592839712 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.mlp.down_proj.weight": { + "data_offsets": [ + 4545489952, + 4590054432 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.mlp.gate_proj.biases": { + "data_offsets": [ + 4539919392, + 4542704672 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.mlp.gate_proj.scales": { + "data_offsets": [ + 3881900768, + 3884686048 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.mlp.gate_proj.weight": { + "data_offsets": [ + 4479626272, + 4524190752 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.mlp.up_proj.biases": { + "data_offsets": [ + 4746193952, + 4748979232 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.mlp.up_proj.scales": { + "data_offsets": [ + 4679347232, + 4682132512 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.mlp.up_proj.weight": { + "data_offsets": [ + 4604800032, + 4649364512 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.13.post_attention_layernorm.weight": { + "data_offsets": [ + 4479616032, + 4479626272 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.input_layernorm.weight": { + "data_offsets": [ + 4892477920, + 4892488160 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.A_log": { + "data_offsets": [ + 4830546048, + 4830546144 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.conv1d.weight": { + "data_offsets": [ + 1013223232, + 1013305152 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.dt_bias": { + "data_offsets": [ + 4824483872, + 4824483968 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2213745696, + 2213753376 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 724495296, + 724502976 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 4824353312, + 4824476192 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4824345632, + 4824353312 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 2921221824, + 2921229504 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 1009332032, + 1009454912 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 1742996064, + 1744634464 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4820741152, + 4822379552 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4748979232, + 4775193632 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4822379552, + 4823362592 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2084416800, + 2085399840 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 4682132512, + 4697861152 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.norm.weight": { + "data_offsets": [ + 4830546144, + 4830546400 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.out_proj.biases": { + "data_offsets": [ + 2359651296, + 2360634336 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.out_proj.scales": { + "data_offsets": [ + 907218464, + 908201504 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.linear_attn.out_proj.weight": { + "data_offsets": [ + 4830546400, + 4846275040 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.mlp.down_proj.biases": { + "data_offsets": [ + 4892498400, + 4895283680 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.mlp.down_proj.scales": { + "data_offsets": [ + 820193312, + 822978592 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.mlp.down_proj.weight": { + "data_offsets": [ + 4435051552, + 4479616032 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.mlp.gate_proj.biases": { + "data_offsets": [ + 1842865344, + 1845650624 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.mlp.gate_proj.scales": { + "data_offsets": [ + 3892058944, + 3894844224 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.mlp.gate_proj.weight": { + "data_offsets": [ + 4776176672, + 4820741152 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.mlp.up_proj.biases": { + "data_offsets": [ + 4898068960, + 4900854240 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.mlp.up_proj.scales": { + "data_offsets": [ + 4895283680, + 4898068960 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.mlp.up_proj.weight": { + "data_offsets": [ + 3204716224, + 3249280704 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.14.post_attention_layernorm.weight": { + "data_offsets": [ + 4892488160, + 4892498400 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.input_layernorm.weight": { + "data_offsets": [ + 1970087200, + 1970097440 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.mlp.down_proj.biases": { + "data_offsets": [ + 5162041312, + 5164826592 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.mlp.down_proj.scales": { + "data_offsets": [ + 1069451072, + 1072236352 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.mlp.down_proj.weight": { + "data_offsets": [ + 5117469152, + 5162033632 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.mlp.gate_proj.biases": { + "data_offsets": [ + 4978194912, + 4980980192 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.mlp.gate_proj.scales": { + "data_offsets": [ + 5025554912, + 5028340192 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.mlp.gate_proj.weight": { + "data_offsets": [ + 4980990432, + 5025554912 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.mlp.up_proj.biases": { + "data_offsets": [ + 5167611872, + 5170397152 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.mlp.up_proj.scales": { + "data_offsets": [ + 5164826592, + 5167611872 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.mlp.up_proj.weight": { + "data_offsets": [ + 3596949440, + 3641513920 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.post_attention_layernorm.weight": { + "data_offsets": [ + 4980980192, + 4980990432 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.k_norm.weight": { + "data_offsets": [ + 873139232, + 873139744 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.k_proj.biases": { + "data_offsets": [ + 724666816, + 724830656 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.k_proj.scales": { + "data_offsets": [ + 908201504, + 908365344 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.k_proj.weight": { + "data_offsets": [ + 4542868512, + 4545489952 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.o_proj.biases": { + "data_offsets": [ + 4118716768, + 4119699808 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.o_proj.scales": { + "data_offsets": [ + 2410951232, + 2411934272 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.o_proj.weight": { + "data_offsets": [ + 3966286144, + 3982014784 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.q_norm.weight": { + "data_offsets": [ + 4978186720, + 4978187232 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.q_proj.biases": { + "data_offsets": [ + 3252557504, + 3254523584 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.q_proj.scales": { + "data_offsets": [ + 1760370784, + 1762336864 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.q_proj.weight": { + "data_offsets": [ + 4945418720, + 4976876000 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.v_proj.biases": { + "data_offsets": [ + 4978022880, + 4978186720 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.v_proj.scales": { + "data_offsets": [ + 4976876000, + 4977039840 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.15.self_attn.v_proj.weight": { + "data_offsets": [ + 2593771264, + 2596392704 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.input_layernorm.weight": { + "data_offsets": [ + 3468386240, + 3468396480 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.A_log": { + "data_offsets": [ + 2592050848, + 2592050944 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.conv1d.weight": { + "data_offsets": [ + 2593689344, + 2593771264 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.dt_bias": { + "data_offsets": [ + 2545750592, + 2545750688 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 5180409312, + 5180416992 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 5180401632, + 5180409312 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 2625503328, + 2625626208 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 507094208, + 507101888 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 5183202272, + 5183209952 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 980086496, + 980209376 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 5173192672, + 5174831072 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2313304448, + 2314942848 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4649364512, + 4675578912 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2146604320, + 2147587360 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 4138213728, + 4139196768 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3781784288, + 3797512928 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.norm.weight": { + "data_offsets": [ + 4264698304, + 4264698560 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.out_proj.biases": { + "data_offsets": [ + 2309536128, + 2310519168 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.out_proj.scales": { + "data_offsets": [ + 2974477504, + 2975460544 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.linear_attn.out_proj.weight": { + "data_offsets": [ + 583124160, + 598852800 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.mlp.down_proj.biases": { + "data_offsets": [ + 5275124800, + 5277910080 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.mlp.down_proj.scales": { + "data_offsets": [ + 4119699808, + 4122485088 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.mlp.down_proj.weight": { + "data_offsets": [ + 5230560224, + 5275124704 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.mlp.gate_proj.biases": { + "data_offsets": [ + 4826122368, + 4828907648 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.mlp.gate_proj.scales": { + "data_offsets": [ + 5227774432, + 5230559712 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.mlp.gate_proj.weight": { + "data_offsets": [ + 5183209952, + 5227774432 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.mlp.up_proj.biases": { + "data_offsets": [ + 5277910080, + 5280695360 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.mlp.up_proj.scales": { + "data_offsets": [ + 5174831072, + 5177616352 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.mlp.up_proj.weight": { + "data_offsets": [ + 4698844192, + 4743408672 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.16.post_attention_layernorm.weight": { + "data_offsets": [ + 2599177984, + 2599188224 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.input_layernorm.weight": { + "data_offsets": [ + 5298347104, + 5298357344 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.A_log": { + "data_offsets": [ + 5298347008, + 5298347104 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.conv1d.weight": { + "data_offsets": [ + 2088185120, + 2088267040 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.dt_bias": { + "data_offsets": [ + 783001536, + 783001632 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 5298339328, + 5298347008 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 5282603008, + 5282610688 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 5282480032, + 5282602912 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 5282472000, + 5282479680 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 5280818240, + 5280825920 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 5280695360, + 5280818240 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4399147200, + 4400785600 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2592050944, + 2593689344 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 3570612160, + 3596826560 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2166108960, + 2167092000 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 4145586624, + 4146569664 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2147587360, + 2163316000 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.norm.weight": { + "data_offsets": [ + 1707935552, + 1707935808 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.out_proj.biases": { + "data_offsets": [ + 4675578912, + 4676561952 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.out_proj.scales": { + "data_offsets": [ + 2605924192, + 2606907232 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.linear_attn.out_proj.weight": { + "data_offsets": [ + 367013568, + 382742208 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.mlp.down_proj.biases": { + "data_offsets": [ + 2214828832, + 2217614112 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.mlp.down_proj.scales": { + "data_offsets": [ + 5298490464, + 5301275744 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.mlp.down_proj.weight": { + "data_offsets": [ + 3332542400, + 3377106880 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.mlp.gate_proj.biases": { + "data_offsets": [ + 5180416992, + 5183202272 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.mlp.gate_proj.scales": { + "data_offsets": [ + 2755994304, + 2758779584 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.mlp.gate_proj.weight": { + "data_offsets": [ + 1847411904, + 1891976384 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.mlp.up_proj.biases": { + "data_offsets": [ + 5306846304, + 5309631584 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.mlp.up_proj.scales": { + "data_offsets": [ + 5301275744, + 5304061024 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.mlp.up_proj.weight": { + "data_offsets": [ + 3690681568, + 3735246048 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.17.post_attention_layernorm.weight": { + "data_offsets": [ + 5298357344, + 5298367584 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.input_layernorm.weight": { + "data_offsets": [ + 367003328, + 367013568 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.A_log": { + "data_offsets": [ + 935521920, + 935522016 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.conv1d.weight": { + "data_offsets": [ + 3173013184, + 3173095104 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.dt_bias": { + "data_offsets": [ + 5282479936, + 5282480032 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 747594176, + 747601856 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 5327252416, + 5327260096 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 618349760, + 618472640 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 5327244736, + 5327252416 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3033726144, + 3033733824 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 5327121856, + 5327244736 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 5309631680, + 5311270080 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 888868384, + 890506784 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 2760417984, + 2786632384 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 382839488, + 383822528 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 916024960, + 917008000 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 5311392960, + 5327121600 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.norm.weight": { + "data_offsets": [ + 1842865088, + 1842865344 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.out_proj.biases": { + "data_offsets": [ + 1012240192, + 1013223232 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.out_proj.scales": { + "data_offsets": [ + 5327260096, + 5328243136 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.linear_attn.out_proj.weight": { + "data_offsets": [ + 891489824, + 907218464 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.mlp.down_proj.biases": { + "data_offsets": [ + 314083008, + 316868288 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.mlp.down_proj.scales": { + "data_offsets": [ + 316868288, + 319653568 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.mlp.down_proj.weight": { + "data_offsets": [ + 319653568, + 364218048 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.mlp.gate_proj.biases": { + "data_offsets": [ + 3030930624, + 3033715904 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.mlp.gate_proj.scales": { + "data_offsets": [ + 364218048, + 367003328 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.mlp.gate_proj.weight": { + "data_offsets": [ + 3085932224, + 3130496704 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.mlp.up_proj.biases": { + "data_offsets": [ + 263825088, + 266610368 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.mlp.up_proj.scales": { + "data_offsets": [ + 266610368, + 269395648 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.mlp.up_proj.weight": { + "data_offsets": [ + 269395648, + 313960128 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.18.post_attention_layernorm.weight": { + "data_offsets": [ + 3033715904, + 3033726144 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.input_layernorm.weight": { + "data_offsets": [ + 206797504, + 206807744 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.mlp.down_proj.biases": { + "data_offsets": [ + 2599198560, + 2601983840 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.mlp.down_proj.scales": { + "data_offsets": [ + 156652224, + 159437504 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.mlp.down_proj.weight": { + "data_offsets": [ + 159437504, + 204001984 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.mlp.gate_proj.biases": { + "data_offsets": [ + 204001984, + 206787264 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.mlp.gate_proj.scales": { + "data_offsets": [ + 2407172576, + 2409957856 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.mlp.gate_proj.weight": { + "data_offsets": [ + 1019177792, + 1063742272 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.mlp.up_proj.biases": { + "data_offsets": [ + 109302464, + 112087744 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.mlp.up_proj.scales": { + "data_offsets": [ + 1013428032, + 1016213312 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.mlp.up_proj.weight": { + "data_offsets": [ + 112087744, + 156652224 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.post_attention_layernorm.weight": { + "data_offsets": [ + 206787264, + 206797504 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.k_norm.weight": { + "data_offsets": [ + 206807744, + 206808256 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.k_proj.biases": { + "data_offsets": [ + 228271808, + 228435648 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.k_proj.scales": { + "data_offsets": [ + 4542704672, + 4542868512 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.k_proj.weight": { + "data_offsets": [ + 783001632, + 785623072 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.o_proj.biases": { + "data_offsets": [ + 206808768, + 207791808 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.o_proj.scales": { + "data_offsets": [ + 207791808, + 208774848 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.o_proj.weight": { + "data_offsets": [ + 208774848, + 224503488 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.q_norm.weight": { + "data_offsets": [ + 206808256, + 206808768 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.q_proj.biases": { + "data_offsets": [ + 228435648, + 230401728 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.q_proj.scales": { + "data_offsets": [ + 230401728, + 232367808 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.q_proj.weight": { + "data_offsets": [ + 232367808, + 263825088 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.v_proj.biases": { + "data_offsets": [ + 782837696, + 783001536 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.v_proj.scales": { + "data_offsets": [ + 224503488, + 224667328 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.19.self_attn.v_proj.weight": { + "data_offsets": [ + 224667328, + 227288768 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.input_layernorm.weight": { + "data_offsets": [ + 3524276160, + 3524286400 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.A_log": { + "data_offsets": [ + 2409957856, + 2409957952 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.conv1d.weight": { + "data_offsets": [ + 2606907232, + 2606989152 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.dt_bias": { + "data_offsets": [ + 2628575328, + 2628575424 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3736229088, + 3736236768 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 2547396768, + 2547404448 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 2691794624, + 2691917504 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2691653824, + 2691661504 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3037591744, + 3037599424 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 5311270080, + 5311392960 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 412312768, + 413951168 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2758779584, + 2760417984 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1813865408, + 1840079808 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2627592288, + 2628575328 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 1988611360, + 1989594400 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2609774432, + 2625503072 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.norm.weight": { + "data_offsets": [ + 2214818336, + 2214818592 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.out_proj.biases": { + "data_offsets": [ + 3033815744, + 3034798784 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.out_proj.scales": { + "data_offsets": [ + 2786632384, + 2787615424 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.linear_attn.out_proj.weight": { + "data_offsets": [ + 727828416, + 743557056 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.mlp.down_proj.biases": { + "data_offsets": [ + 2907607744, + 2910393024 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.mlp.down_proj.scales": { + "data_offsets": [ + 4592839712, + 4595624992 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.mlp.down_proj.weight": { + "data_offsets": [ + 2501186112, + 2545750592 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.mlp.gate_proj.biases": { + "data_offsets": [ + 2602966880, + 2605752160 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.mlp.gate_proj.scales": { + "data_offsets": [ + 3894844224, + 3897629504 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.mlp.gate_proj.weight": { + "data_offsets": [ + 1992379680, + 2036944160 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.mlp.up_proj.biases": { + "data_offsets": [ + 2596392704, + 2599177984 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.mlp.up_proj.scales": { + "data_offsets": [ + 2310519168, + 2313304448 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.mlp.up_proj.weight": { + "data_offsets": [ + 2411934272, + 2456498752 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.2.post_attention_layernorm.weight": { + "data_offsets": [ + 2790523584, + 2790533824 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.input_layernorm.weight": { + "data_offsets": [ + 60009312, + 60019552 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.A_log": { + "data_offsets": [ + 3889273568, + 3889273664 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.conv1d.weight": { + "data_offsets": [ + 724838336, + 724920256 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.dt_bias": { + "data_offsets": [ + 77714528, + 77714624 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2710431424, + 2710439104 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3963493184, + 3963500864 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 313960128, + 314083008 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 78697664, + 78705344 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 2910393024, + 2910400704 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 78705344, + 78828224 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 79811264, + 81449664 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 81449664, + 83088064 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 83088064, + 109302464 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 78828224, + 79811264 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 227288768, + 228271808 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3643487680, + 3659216320 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.norm.weight": { + "data_offsets": [ + 77714272, + 77714528 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.out_proj.biases": { + "data_offsets": [ + 60019552, + 61002592 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.out_proj.scales": { + "data_offsets": [ + 61002592, + 61985632 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.linear_attn.out_proj.weight": { + "data_offsets": [ + 61985632, + 77714272 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.mlp.down_proj.biases": { + "data_offsets": [ + 7078752, + 9864032 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.mlp.down_proj.scales": { + "data_offsets": [ + 9864032, + 12649312 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.mlp.down_proj.weight": { + "data_offsets": [ + 12649312, + 57213792 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.mlp.gate_proj.biases": { + "data_offsets": [ + 57213792, + 59999072 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.mlp.gate_proj.scales": { + "data_offsets": [ + 4743408672, + 4746193952 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.mlp.gate_proj.weight": { + "data_offsets": [ + 5028340192, + 5072904672 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.mlp.up_proj.biases": { + "data_offsets": [ + 4293472, + 7078752 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.mlp.up_proj.scales": { + "data_offsets": [ + 3489695680, + 3492480960 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.mlp.up_proj.weight": { + "data_offsets": [ + 679930816, + 724495296 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.20.post_attention_layernorm.weight": { + "data_offsets": [ + 59999072, + 60009312 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.input_layernorm.weight": { + "data_offsets": [ + 3982014784, + 3982025024 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.A_log": { + "data_offsets": [ + 10496, + 10592 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.conv1d.weight": { + "data_offsets": [ + 1713514048, + 1713595968 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.dt_bias": { + "data_offsets": [ + 4145586528, + 4145586624 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 10592, + 18272 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 18272, + 25952 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1716658688, + 1716781568 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 411150528, + 411158208 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 25952, + 33632 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 4029374880, + 4029497760 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 1016672, + 2655072 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2655072, + 4293472 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 383945408, + 410159808 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4359234240, + 4360217280 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 33632, + 1016672 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 873139744, + 888868384 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.norm.weight": { + "data_offsets": [ + 10240, + 10496 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.out_proj.biases": { + "data_offsets": [ + 4977039840, + 4978022880 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.out_proj.scales": { + "data_offsets": [ + 77714624, + 78697664 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.linear_attn.out_proj.weight": { + "data_offsets": [ + 4400808736, + 4416537376 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.mlp.gate_proj.biases": { + "data_offsets": [ + 2860257984, + 2863043264 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.mlp.gate_proj.scales": { + "data_offsets": [ + 2850837184, + 2853622464 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.mlp.gate_proj.weight": { + "data_offsets": [ + 2360634336, + 2405198816 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.post_attention_layernorm.weight": { + "data_offsets": [ + 0, + 10240 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.input_layernorm.weight": { + "data_offsets": [ + 2146594080, + 2146604320 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.mlp.down_proj.biases": { + "data_offsets": [ + 4676561952, + 4679347232 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.mlp.down_proj.scales": { + "data_offsets": [ + 3082163904, + 3084949184 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.mlp.down_proj.weight": { + "data_offsets": [ + 3037599424, + 3082163904 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.mlp.gate_proj.biases": { + "data_offsets": [ + 817408032, + 820193312 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.mlp.gate_proj.scales": { + "data_offsets": [ + 3034798784, + 3037584064 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.mlp.gate_proj.weight": { + "data_offsets": [ + 2547404448, + 2591968928 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.mlp.up_proj.biases": { + "data_offsets": [ + 917008000, + 919793280 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.mlp.up_proj.scales": { + "data_offsets": [ + 2856407744, + 2859193024 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.mlp.up_proj.weight": { + "data_offsets": [ + 2264963872, + 2309528352 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.post_attention_layernorm.weight": { + "data_offsets": [ + 2991189184, + 2991199424 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.k_norm.weight": { + "data_offsets": [ + 1713719104, + 1713719616 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.k_proj.biases": { + "data_offsets": [ + 2921229504, + 2921393344 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.k_proj.scales": { + "data_offsets": [ + 1709581888, + 1709745728 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.k_proj.weight": { + "data_offsets": [ + 2915807424, + 2918428864 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.o_proj.biases": { + "data_offsets": [ + 2167092000, + 2168075040 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.o_proj.scales": { + "data_offsets": [ + 2601983840, + 2602966880 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.o_proj.weight": { + "data_offsets": [ + 2975460544, + 2991189184 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.q_norm.weight": { + "data_offsets": [ + 5230559712, + 5230560224 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.q_proj.biases": { + "data_offsets": [ + 913936000, + 915902080 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.q_proj.scales": { + "data_offsets": [ + 4601195552, + 4603161632 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.q_proj.weight": { + "data_offsets": [ + 551666368, + 583123648 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.v_proj.biases": { + "data_offsets": [ + 2924342464, + 2924506304 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.v_proj.scales": { + "data_offsets": [ + 2924178624, + 2924342464 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.3.self_attn.v_proj.weight": { + "data_offsets": [ + 4309263040, + 4311884480 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.input_layernorm.weight": { + "data_offsets": [ + 2691661504, + 2691671744 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.A_log": { + "data_offsets": [ + 1946811584, + 1946811680 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.conv1d.weight": { + "data_offsets": [ + 2859193024, + 2859274944 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.dt_bias": { + "data_offsets": [ + 2309536032, + 2309536128 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 1016213312, + 1016220992 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 1019170112, + 1019177792 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1713750528, + 1713873408 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 679923136, + 679930816 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 1709574208, + 1709581888 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 746357696, + 746480576 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3173095104, + 3174733504 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 750561216, + 752199616 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1716781664, + 1742996064 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2910400704, + 2911383744 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2969070784, + 2970053824 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 4524190752, + 4539919392 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.norm.weight": { + "data_offsets": [ + 5282479680, + 5282479936 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.out_proj.biases": { + "data_offsets": [ + 2213753376, + 2214736416 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.out_proj.scales": { + "data_offsets": [ + 1762336864, + 1763319904 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.linear_attn.out_proj.weight": { + "data_offsets": [ + 3468396480, + 3484125120 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.mlp.down_proj.biases": { + "data_offsets": [ + 1006423776, + 1009209056 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.mlp.down_proj.scales": { + "data_offsets": [ + 1894761664, + 1897546944 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.mlp.down_proj.weight": { + "data_offsets": [ + 4311884480, + 4356448960 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.mlp.gate_proj.biases": { + "data_offsets": [ + 2970053824, + 2972839104 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.mlp.gate_proj.scales": { + "data_offsets": [ + 1009454912, + 1012240192 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.mlp.gate_proj.weight": { + "data_offsets": [ + 2790544064, + 2835108544 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.mlp.up_proj.biases": { + "data_offsets": [ + 3518695360, + 3521480640 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.mlp.up_proj.scales": { + "data_offsets": [ + 1016384832, + 1019170112 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.mlp.up_proj.weight": { + "data_offsets": [ + 2039729440, + 2084293920 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.4.post_attention_layernorm.weight": { + "data_offsets": [ + 4398146240, + 4398156480 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.input_layernorm.weight": { + "data_offsets": [ + 1902236864, + 1902247104 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.A_log": { + "data_offsets": [ + 1763319904, + 1763320000 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.conv1d.weight": { + "data_offsets": [ + 3033733824, + 3033815744 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.dt_bias": { + "data_offsets": [ + 913935904, + 913936000 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3037584064, + 3037591744 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 1066650432, + 1066658112 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1013305152, + 1013428032 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2545750688, + 2545758368 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3270252224, + 3270259904 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 915902080, + 916024960 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 2911383744, + 2913022144 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4139196768, + 4140835168 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 980209376, + 1006423776 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 410159808, + 411142848 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 4697861152, + 4698844192 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 4362183360, + 4377912000 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.norm.weight": { + "data_offsets": [ + 1713595968, + 1713596224 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.out_proj.biases": { + "data_offsets": [ + 890506784, + 891489824 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.out_proj.scales": { + "data_offsets": [ + 1767171008, + 1768154048 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.linear_attn.out_proj.weight": { + "data_offsets": [ + 2691917504, + 2707646144 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.mlp.down_proj.biases": { + "data_offsets": [ + 2921393344, + 2924178624 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.mlp.down_proj.scales": { + "data_offsets": [ + 4595624992, + 4598410272 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.mlp.down_proj.weight": { + "data_offsets": [ + 2710439104, + 2755003584 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.mlp.gate_proj.biases": { + "data_offsets": [ + 3170227904, + 3173013184 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.mlp.gate_proj.scales": { + "data_offsets": [ + 1949607200, + 1952392480 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.mlp.gate_proj.weight": { + "data_offsets": [ + 825763872, + 870328352 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.mlp.up_proj.biases": { + "data_offsets": [ + 2138238240, + 2141023520 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.mlp.up_proj.scales": { + "data_offsets": [ + 811837472, + 814622752 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.mlp.up_proj.weight": { + "data_offsets": [ + 2863043264, + 2907607744 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.5.post_attention_layernorm.weight": { + "data_offsets": [ + 2409957952, + 2409968192 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.input_layernorm.weight": { + "data_offsets": [ + 664184000, + 664194240 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.A_log": { + "data_offsets": [ + 2599188224, + 2599188320 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.conv1d.weight": { + "data_offsets": [ + 507012288, + 507094208 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.dt_bias": { + "data_offsets": [ + 4102987776, + 4102987872 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2163316000, + 2163323680 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 724830656, + 724838336 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 724920256, + 725043136 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 743557056, + 743564736 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 746350016, + 746357696 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 746488256, + 746611136 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 1707935808, + 1709574208 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 778414016, + 780052416 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 785623072, + 811837472 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2405198816, + 2406181856 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 749578176, + 750561216 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 919793280, + 935521920 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.norm.weight": { + "data_offsets": [ + 679922880, + 679923136 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.out_proj.biases": { + "data_offsets": [ + 3467403200, + 3468386240 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.out_proj.scales": { + "data_offsets": [ + 2755011264, + 2755994304 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.linear_attn.out_proj.weight": { + "data_offsets": [ + 664194240, + 679922880 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.mlp.down_proj.biases": { + "data_offsets": [ + 822978592, + 825763872 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.mlp.down_proj.scales": { + "data_offsets": [ + 2036944160, + 2039729440 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.mlp.down_proj.weight": { + "data_offsets": [ + 2924506304, + 2969070784 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.mlp.gate_proj.biases": { + "data_offsets": [ + 780052416, + 782837696 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.mlp.gate_proj.scales": { + "data_offsets": [ + 1840079808, + 1842865088 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.mlp.gate_proj.weight": { + "data_offsets": [ + 618636480, + 663200960 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.mlp.up_proj.biases": { + "data_offsets": [ + 1985826080, + 1988611360 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.mlp.up_proj.scales": { + "data_offsets": [ + 4141818208, + 4144603488 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.mlp.up_proj.weight": { + "data_offsets": [ + 4900854240, + 4945418720 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.6.post_attention_layernorm.weight": { + "data_offsets": [ + 1946811680, + 1946821920 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.input_layernorm.weight": { + "data_offsets": [ + 2599188320, + 2599198560 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.mlp.down_proj.biases": { + "data_offsets": [ + 725043136, + 727828416 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.mlp.down_proj.scales": { + "data_offsets": [ + 2787738304, + 2790523584 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.mlp.down_proj.weight": { + "data_offsets": [ + 462447808, + 507012288 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.mlp.gate_proj.biases": { + "data_offsets": [ + 2853622464, + 2856407744 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.mlp.gate_proj.scales": { + "data_offsets": [ + 615564480, + 618349760 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.mlp.gate_proj.weight": { + "data_offsets": [ + 507101888, + 551666368 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.mlp.up_proj.biases": { + "data_offsets": [ + 459662528, + 462447808 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.mlp.up_proj.scales": { + "data_offsets": [ + 2217614112, + 2220399392 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.mlp.up_proj.weight": { + "data_offsets": [ + 2090888480, + 2135452960 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.post_attention_layernorm.weight": { + "data_offsets": [ + 748584896, + 748595136 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.k_norm.weight": { + "data_offsets": [ + 2314953088, + 2314953600 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.k_proj.biases": { + "data_offsets": [ + 412148928, + 412312768 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.k_proj.scales": { + "data_offsets": [ + 1016220992, + 1016384832 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.k_proj.weight": { + "data_offsets": [ + 3889437504, + 3892058944 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.o_proj.biases": { + "data_offsets": [ + 598852800, + 599835840 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.o_proj.scales": { + "data_offsets": [ + 3203733184, + 3204716224 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.o_proj.weight": { + "data_offsets": [ + 599835840, + 615564480 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.q_norm.weight": { + "data_offsets": [ + 583123648, + 583124160 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.q_proj.biases": { + "data_offsets": [ + 4360217280, + 4362183360 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.q_proj.scales": { + "data_offsets": [ + 1968121120, + 1970087200 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.q_proj.weight": { + "data_offsets": [ + 3659224288, + 3690681568 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.v_proj.biases": { + "data_offsets": [ + 724502976, + 724666816 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.v_proj.scales": { + "data_offsets": [ + 618472640, + 618636480 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.7.self_attn.v_proj.weight": { + "data_offsets": [ + 1897639104, + 1900260544 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.input_layernorm.weight": { + "data_offsets": [ + 870328352, + 870338592 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.A_log": { + "data_offsets": [ + 5275124704, + 5275124800 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.conv1d.weight": { + "data_offsets": [ + 1766106048, + 1766187968 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.dt_bias": { + "data_offsets": [ + 3659224000, + 3659224096 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 382824128, + 382831808 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 382831808, + 382839488 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 383822528, + 383945408 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4400800960, + 4400808640 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 411142848, + 411150528 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 2691671744, + 2691794624 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 1845773504, + 1847411904 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2545758368, + 2547396768 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 433448128, + 459662528 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2089905440, + 2090888480 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 747601856, + 748584896 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 416736448, + 432465088 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.norm.weight": { + "data_offsets": [ + 2625503072, + 2625503328 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.out_proj.biases": { + "data_offsets": [ + 411165888, + 412148928 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.out_proj.scales": { + "data_offsets": [ + 432465088, + 433448128 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.linear_attn.out_proj.weight": { + "data_offsets": [ + 4102988128, + 4118716768 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.mlp.down_proj.biases": { + "data_offsets": [ + 3174733504, + 3177518784 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.mlp.down_proj.scales": { + "data_offsets": [ + 814622752, + 817408032 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.mlp.down_proj.weight": { + "data_offsets": [ + 2628575424, + 2673139904 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.mlp.gate_proj.biases": { + "data_offsets": [ + 2673139904, + 2675925184 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.mlp.gate_proj.scales": { + "data_offsets": [ + 4416537632, + 4419322912 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.mlp.gate_proj.weight": { + "data_offsets": [ + 935522016, + 980086496 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.mlp.up_proj.biases": { + "data_offsets": [ + 5177616352, + 5180401632 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.mlp.up_proj.scales": { + "data_offsets": [ + 870353952, + 873139232 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.mlp.up_proj.weight": { + "data_offsets": [ + 1902247104, + 1946811584 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.8.post_attention_layernorm.weight": { + "data_offsets": [ + 2790533824, + 2790544064 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.input_layernorm.weight": { + "data_offsets": [ + 3378089920, + 3378100160 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.A_log": { + "data_offsets": [ + 1009209056, + 1009209152 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.conv1d.weight": { + "data_offsets": [ + 2214736416, + 2214818336 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.dt_bias": { + "data_offsets": [ + 5309631584, + 5309631680 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3272241344, + 3272249024 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3272233664, + 3272241344 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1713596224, + 1713719104 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2605916512, + 2605924192 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3272225984, + 3272233664 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 2314953600, + 2315076480 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3250919104, + 3252557504 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 3249280704, + 3250919104 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 3177518784, + 3203733184 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 3271242944, + 3272225984 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 3270259904, + 3271242944 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3254523584, + 3270252224 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.norm.weight": { + "data_offsets": [ + 3272249024, + 3272249280 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.out_proj.biases": { + "data_offsets": [ + 3377106880, + 3378089920 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.out_proj.scales": { + "data_offsets": [ + 2859274944, + 2860257984 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.linear_attn.out_proj.weight": { + "data_offsets": [ + 3272249280, + 3287977920 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.mlp.down_proj.biases": { + "data_offsets": [ + 1768154048, + 1770939328 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.mlp.down_proj.scales": { + "data_offsets": [ + 3486910400, + 3489695680 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.mlp.down_proj.weight": { + "data_offsets": [ + 3287977920, + 3332542400 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.mlp.gate_proj.biases": { + "data_offsets": [ + 1713873408, + 1716658688 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.mlp.gate_proj.scales": { + "data_offsets": [ + 3484125120, + 3486910400 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.mlp.gate_proj.weight": { + "data_offsets": [ + 3422838720, + 3467403200 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.mlp.up_proj.biases": { + "data_offsets": [ + 2707646144, + 2710431424 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.mlp.up_proj.scales": { + "data_offsets": [ + 2913022144, + 2915807424 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.mlp.up_proj.weight": { + "data_offsets": [ + 5072904672, + 5117469152 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.9.post_attention_layernorm.weight": { + "data_offsets": [ + 3422664640, + 3422674880 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00001-of-00003.safetensors" + }, + "language_model.model.layers.21.mlp.down_proj.biases": { + "data_offsets": [ + 2015722656, + 2018507936 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.21.mlp.down_proj.scales": { + "data_offsets": [ + 2067834016, + 2070619296 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.21.mlp.down_proj.weight": { + "data_offsets": [ + 3596768544, + 3641333024 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.21.mlp.up_proj.biases": { + "data_offsets": [ + 1247809312, + 1250594592 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.21.mlp.up_proj.scales": { + "data_offsets": [ + 1565419232, + 1568204512 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.21.mlp.up_proj.weight": { + "data_offsets": [ + 5234773472, + 5279337952 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.input_layernorm.weight": { + "data_offsets": [ + 2891374496, + 2891384736 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.A_log": { + "data_offsets": [ + 2293206656, + 2293206752 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.conv1d.weight": { + "data_offsets": [ + 844682496, + 844764416 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.dt_bias": { + "data_offsets": [ + 2403486432, + 2403486528 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 1966554784, + 1966562464 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 752724288, + 752731968 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 3159811168, + 3159934048 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2308935392, + 2308943072 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 2245749280, + 2245756960 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 2188737696, + 2188860576 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4699813888, + 4701452288 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2738427200, + 2740065600 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1536288992, + 1562503392 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2644211264, + 2645194304 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 1965564064, + 1966547104 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2226416160, + 2242144800 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.norm.weight": { + "data_offsets": [ + 2446110528, + 2446110784 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.out_proj.biases": { + "data_offsets": [ + 1864773344, + 1865756384 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.out_proj.scales": { + "data_offsets": [ + 2540769344, + 2541752384 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.linear_attn.out_proj.weight": { + "data_offsets": [ + 3350848864, + 3366577504 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.mlp.down_proj.biases": { + "data_offsets": [ + 1777937632, + 1780722912 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.mlp.down_proj.scales": { + "data_offsets": [ + 1861988064, + 1864773344 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.mlp.down_proj.weight": { + "data_offsets": [ + 3090983008, + 3135547488 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.mlp.gate_proj.biases": { + "data_offsets": [ + 1899190336, + 1901975616 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.mlp.gate_proj.scales": { + "data_offsets": [ + 2816284480, + 2819069760 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.mlp.gate_proj.weight": { + "data_offsets": [ + 5019141600, + 5063706080 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.mlp.up_proj.biases": { + "data_offsets": [ + 898685888, + 901471168 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.mlp.up_proj.scales": { + "data_offsets": [ + 2989706752, + 2992492032 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.mlp.up_proj.weight": { + "data_offsets": [ + 1686842592, + 1731407072 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.22.post_attention_layernorm.weight": { + "data_offsets": [ + 1107377184, + 1107387424 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.input_layernorm.weight": { + "data_offsets": [ + 3553996064, + 3554006304 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.mlp.down_proj.biases": { + "data_offsets": [ + 1901975616, + 1904760896 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.mlp.down_proj.scales": { + "data_offsets": [ + 1486153440, + 1488938720 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.mlp.down_proj.weight": { + "data_offsets": [ + 1107633184, + 1152197664 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.mlp.gate_proj.biases": { + "data_offsets": [ + 1959993504, + 1962778784 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.mlp.gate_proj.scales": { + "data_offsets": [ + 3916751744, + 3919537024 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.mlp.gate_proj.weight": { + "data_offsets": [ + 2493583424, + 2538147904 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.mlp.up_proj.biases": { + "data_offsets": [ + 2645194304, + 2647979584 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.mlp.up_proj.scales": { + "data_offsets": [ + 1488938720, + 1491724000 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.mlp.up_proj.weight": { + "data_offsets": [ + 1439950560, + 1484515040 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.post_attention_layernorm.weight": { + "data_offsets": [ + 2293196416, + 2293206656 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.k_norm.weight": { + "data_offsets": [ + 1536288480, + 1536288992 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.k_proj.biases": { + "data_offsets": [ + 1799565024, + 1799728864 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.k_proj.scales": { + "data_offsets": [ + 4526424736, + 4526588576 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.k_proj.weight": { + "data_offsets": [ + 3437126080, + 3439747520 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.o_proj.biases": { + "data_offsets": [ + 1966562976, + 1967546016 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.o_proj.scales": { + "data_offsets": [ + 5282123744, + 5283106784 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.o_proj.weight": { + "data_offsets": [ + 1881485376, + 1897214016 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.q_norm.weight": { + "data_offsets": [ + 1959992992, + 1959993504 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.q_proj.biases": { + "data_offsets": [ + 2065857696, + 2067823776 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.q_proj.scales": { + "data_offsets": [ + 1675691232, + 1677657312 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.q_proj.weight": { + "data_offsets": [ + 1568204512, + 1599661792 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.v_proj.biases": { + "data_offsets": [ + 2180210336, + 2180374176 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.v_proj.scales": { + "data_offsets": [ + 746989888, + 747153728 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.23.self_attn.v_proj.weight": { + "data_offsets": [ + 2897956352, + 2900577792 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.input_layernorm.weight": { + "data_offsets": [ + 2875348896, + 2875359136 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.A_log": { + "data_offsets": [ + 4795286624, + 4795286720 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.conv1d.weight": { + "data_offsets": [ + 2617904704, + 2617986624 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.dt_bias": { + "data_offsets": [ + 2188860576, + 2188860672 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2411852608, + 2411860288 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 901471168, + 901478848 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 2819069760, + 2819192640 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2770066240, + 2770073920 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3703859232, + 3703866912 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 2891087776, + 2891210656 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4221546048, + 4223184448 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 3470255040, + 3471893440 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 2650764864, + 2676979264 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 1970175136, + 1971158176 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2398079712, + 2399062752 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2722698560, + 2738427200 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.norm.weight": { + "data_offsets": [ + 3211052128, + 3211052384 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.out_proj.biases": { + "data_offsets": [ + 2538147904, + 2539130944 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.out_proj.scales": { + "data_offsets": [ + 4138676288, + 4139659328 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.linear_attn.out_proj.weight": { + "data_offsets": [ + 3472876480, + 3488605120 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.mlp.down_proj.biases": { + "data_offsets": [ + 2995277312, + 2998062592 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.mlp.down_proj.scales": { + "data_offsets": [ + 671352128, + 674137408 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.mlp.down_proj.weight": { + "data_offsets": [ + 2945142272, + 2989706752 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.mlp.gate_proj.biases": { + "data_offsets": [ + 2193647968, + 2196433248 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.mlp.gate_proj.scales": { + "data_offsets": [ + 2891402752, + 2894188032 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.mlp.gate_proj.weight": { + "data_offsets": [ + 1491724000, + 1536288480 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.mlp.up_proj.biases": { + "data_offsets": [ + 3434340800, + 3437126080 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.mlp.up_proj.scales": { + "data_offsets": [ + 3088189952, + 3090975232 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.mlp.up_proj.weight": { + "data_offsets": [ + 2449018944, + 2493583424 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.24.post_attention_layernorm.weight": { + "data_offsets": [ + 2891384736, + 2891394976 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.input_layernorm.weight": { + "data_offsets": [ + 1104581568, + 1104591808 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.A_log": { + "data_offsets": [ + 2222647648, + 2222647744 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.conv1d.weight": { + "data_offsets": [ + 1389480160, + 1389562080 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.dt_bias": { + "data_offsets": [ + 1904768576, + 1904768672 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3463675840, + 3463683520 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 1966547104, + 1966554784 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 3383960000, + 3384082880 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 1562626272, + 1562633952 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 2071602336, + 2071610016 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 1562503392, + 1562626272 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 1484515040, + 1486153440 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 3972432000, + 3974070400 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 2196433248, + 2222647648 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 1683227872, + 1684210912 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 3139315808, + 3140298848 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 1599661792, + 1615390432 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.norm.weight": { + "data_offsets": [ + 1865756384, + 1865756640 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.out_proj.biases": { + "data_offsets": [ + 3491400640, + 3492383680 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.out_proj.scales": { + "data_offsets": [ + 1340880672, + 1341863712 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.linear_attn.out_proj.weight": { + "data_offsets": [ + 3439747520, + 3455476160 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.mlp.down_proj.biases": { + "data_offsets": [ + 1389684960, + 1392470240 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.mlp.down_proj.scales": { + "data_offsets": [ + 2245846656, + 2248631936 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.mlp.down_proj.weight": { + "data_offsets": [ + 2827835296, + 2872399776 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.mlp.gate_proj.biases": { + "data_offsets": [ + 3459244480, + 3462029760 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.mlp.gate_proj.scales": { + "data_offsets": [ + 3495005120, + 3497790400 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.mlp.gate_proj.weight": { + "data_offsets": [ + 3856458624, + 3901023104 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.mlp.up_proj.biases": { + "data_offsets": [ + 3205481568, + 3208266848 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.mlp.up_proj.scales": { + "data_offsets": [ + 2222647744, + 2225433024 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.mlp.up_proj.weight": { + "data_offsets": [ + 3735651872, + 3780216352 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.25.post_attention_layernorm.weight": { + "data_offsets": [ + 2190834432, + 2190844672 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.input_layernorm.weight": { + "data_offsets": [ + 4226110528, + 4226120768 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.A_log": { + "data_offsets": [ + 1389479968, + 1389480064 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.conv1d.weight": { + "data_offsets": [ + 4791920224, + 4792002144 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.dt_bias": { + "data_offsets": [ + 1389480064, + 1389480160 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2177573536, + 2177581216 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 2193640288, + 2193647968 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1389562080, + 1389684960 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2586316864, + 2586324544 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 1392470240, + 1392477920 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 1392477920, + 1392600800 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3546623008, + 3548261408 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 3543993888, + 3545632288 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 3304236384, + 3330450784 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2177588896, + 2178571936 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 3545632288, + 3546615328 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 1865756640, + 1881485280 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.norm.weight": { + "data_offsets": [ + 1389479712, + 1389479968 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.out_proj.biases": { + "data_offsets": [ + 1776954592, + 1777937632 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.out_proj.scales": { + "data_offsets": [ + 3087206912, + 3088189952 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.linear_attn.out_proj.weight": { + "data_offsets": [ + 3140298848, + 3156027488 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.mlp.down_proj.biases": { + "data_offsets": [ + 851336032, + 854121312 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.mlp.down_proj.scales": { + "data_offsets": [ + 1232162080, + 1234947360 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.mlp.down_proj.weight": { + "data_offsets": [ + 797316928, + 841881408 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.mlp.gate_proj.biases": { + "data_offsets": [ + 1239371040, + 1242156320 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.mlp.gate_proj.scales": { + "data_offsets": [ + 1242156320, + 1244941600 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.mlp.gate_proj.weight": { + "data_offsets": [ + 3159934048, + 3204498528 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.mlp.up_proj.biases": { + "data_offsets": [ + 1223796000, + 1226581280 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.mlp.up_proj.scales": { + "data_offsets": [ + 1226581280, + 1229366560 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.mlp.up_proj.weight": { + "data_offsets": [ + 1250594592, + 1295159072 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.26.post_attention_layernorm.weight": { + "data_offsets": [ + 1341863712, + 1341873952 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.input_layernorm.weight": { + "data_offsets": [ + 845755232, + 845765472 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.mlp.down_proj.biases": { + "data_offsets": [ + 848550752, + 851336032 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.mlp.down_proj.scales": { + "data_offsets": [ + 5349953504, + 5352738784 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.mlp.down_proj.weight": { + "data_offsets": [ + 3233457504, + 3278021984 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.mlp.gate_proj.biases": { + "data_offsets": [ + 2406282048, + 2409067328 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.mlp.gate_proj.scales": { + "data_offsets": [ + 1101796288, + 1104581568 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.mlp.gate_proj.weight": { + "data_offsets": [ + 1816440544, + 1861005024 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.mlp.up_proj.biases": { + "data_offsets": [ + 1234947360, + 1237732640 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.mlp.up_proj.scales": { + "data_offsets": [ + 3156027488, + 3158812768 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.mlp.up_proj.weight": { + "data_offsets": [ + 3042642432, + 3087206912 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.post_attention_layernorm.weight": { + "data_offsets": [ + 2067823776, + 2067834016 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.k_norm.weight": { + "data_offsets": [ + 1798417632, + 1798418144 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.k_proj.biases": { + "data_offsets": [ + 1107387424, + 1107551264 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.k_proj.scales": { + "data_offsets": [ + 1170711584, + 1170875424 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.k_proj.weight": { + "data_offsets": [ + 1684221152, + 1686842592 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.o_proj.biases": { + "data_offsets": [ + 1861005024, + 1861988064 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.o_proj.scales": { + "data_offsets": [ + 3458261440, + 3459244480 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.o_proj.weight": { + "data_offsets": [ + 1325152032, + 1340880672 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.q_norm.weight": { + "data_offsets": [ + 3542354880, + 3542355392 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.q_proj.biases": { + "data_offsets": [ + 1170875424, + 1172841504 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.q_proj.scales": { + "data_offsets": [ + 4430813856, + 4432779936 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.q_proj.weight": { + "data_offsets": [ + 1192338720, + 1223796000 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.v_proj.biases": { + "data_offsets": [ + 2647979584, + 2648143424 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.v_proj.scales": { + "data_offsets": [ + 3703866912, + 3704030752 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.27.self_attn.v_proj.weight": { + "data_offsets": [ + 1967546016, + 1970167456 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.input_layernorm.weight": { + "data_offsets": [ + 1232151840, + 1232162080 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.A_log": { + "data_offsets": [ + 2821977920, + 2821978016 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.conv1d.weight": { + "data_offsets": [ + 1101714368, + 1101796288 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.dt_bias": { + "data_offsets": [ + 3383959904, + 3383960000 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 5066665440, + 5066673120 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 4817010880, + 4817018560 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 2446110784, + 2446233664 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2177581216, + 2177588896 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 1035179968, + 1035187648 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3431432640, + 3431555520 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 1050916288, + 1052554688 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 1052554688, + 1054193088 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1295159072, + 1321373472 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 3204498528, + 3205481568 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2769072960, + 2770056000 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 1035187648, + 1050916288 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.norm.weight": { + "data_offsets": [ + 3972431744, + 3972432000 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.out_proj.biases": { + "data_offsets": [ + 3135547488, + 3136530528 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.out_proj.scales": { + "data_offsets": [ + 1321383712, + 1322366752 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.linear_attn.out_proj.weight": { + "data_offsets": [ + 2414653248, + 2430381888 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.mlp.down_proj.biases": { + "data_offsets": [ + 1341873952, + 1344659232 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.mlp.down_proj.scales": { + "data_offsets": [ + 3208266848, + 3211052128 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.mlp.down_proj.weight": { + "data_offsets": [ + 4314520224, + 4359084704 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.mlp.gate_proj.biases": { + "data_offsets": [ + 3920520064, + 3923305344 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.mlp.gate_proj.scales": { + "data_offsets": [ + 1245024032, + 1247809312 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.mlp.gate_proj.weight": { + "data_offsets": [ + 2021293216, + 2065857696 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.mlp.up_proj.biases": { + "data_offsets": [ + 1080415168, + 1083200448 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.mlp.up_proj.scales": { + "data_offsets": [ + 1229366560, + 1232151840 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.mlp.up_proj.weight": { + "data_offsets": [ + 2353515232, + 2398079712 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.28.post_attention_layernorm.weight": { + "data_offsets": [ + 1389223712, + 1389233952 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.input_layernorm.weight": { + "data_offsets": [ + 752742208, + 752752448 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.A_log": { + "data_offsets": [ + 841881664, + 841881760 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.conv1d.weight": { + "data_offsets": [ + 1389397792, + 1389479712 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.dt_bias": { + "data_offsets": [ + 844674720, + 844674816 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2245756960, + 2245764640 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3158812768, + 3158820448 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 2177450656, + 2177573536 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 1675683552, + 1675691232 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 844674816, + 844682496 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3332089184, + 3332212064 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3926221184, + 3927859584 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 3542355488, + 3543993888 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 2104050336, + 2130264736 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 3232351584, + 3233334624 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2823944096, + 2824927136 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 4749977184, + 4765705824 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.norm.weight": { + "data_offsets": [ + 841881408, + 841881664 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.out_proj.biases": { + "data_offsets": [ + 2225433120, + 2226416160 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.out_proj.scales": { + "data_offsets": [ + 844772192, + 845755232 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.linear_attn.out_proj.weight": { + "data_offsets": [ + 1782688992, + 1798417632 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.mlp.down_proj.biases": { + "data_offsets": [ + 749939008, + 752724288 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.mlp.down_proj.scales": { + "data_offsets": [ + 1930993312, + 1933778592 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.mlp.down_proj.weight": { + "data_offsets": [ + 946043328, + 990607808 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.mlp.gate_proj.biases": { + "data_offsets": [ + 3428647360, + 3431432640 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.mlp.gate_proj.scales": { + "data_offsets": [ + 3229566304, + 3232351584 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.mlp.gate_proj.weight": { + "data_offsets": [ + 1971158176, + 2015722656 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.mlp.up_proj.biases": { + "data_offsets": [ + 2872399776, + 2875185056 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.mlp.up_proj.scales": { + "data_offsets": [ + 1104591808, + 1107377088 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.mlp.up_proj.weight": { + "data_offsets": [ + 1344659232, + 1389223712 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.29.post_attention_layernorm.weight": { + "data_offsets": [ + 752731968, + 752742208 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.input_layernorm.weight": { + "data_offsets": [ + 4603931296, + 4603941536 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.A_log": { + "data_offsets": [ + 2190844672, + 2190844768 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.conv1d.weight": { + "data_offsets": [ + 746907968, + 746989888 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.dt_bias": { + "data_offsets": [ + 2225433024, + 2225433120 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2998062592, + 2998070272 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 990607808, + 990615488 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 671229248, + 671352128 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 674137408, + 674145088 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 844764512, + 844772192 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3332212064, + 3332334944 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3462037440, + 3463675840 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 1237732640, + 1239371040 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 720693568, + 746907968 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 674145088, + 675128128 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 719702848, + 720685888 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3335120224, + 3350848864 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.norm.weight": { + "data_offsets": [ + 670245952, + 670246208 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.out_proj.biases": { + "data_offsets": [ + 2070619296, + 2071602336 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.out_proj.scales": { + "data_offsets": [ + 3641333024, + 3642316064 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.linear_attn.out_proj.weight": { + "data_offsets": [ + 654517312, + 670245952 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.mlp.down_proj.biases": { + "data_offsets": [ + 1962778784, + 1965564064 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.mlp.down_proj.scales": { + "data_offsets": [ + 3556955424, + 3559740704 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.mlp.down_proj.weight": { + "data_offsets": [ + 4956219104, + 5000783584 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.mlp.gate_proj.biases": { + "data_offsets": [ + 3554006304, + 3556791584 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.mlp.gate_proj.scales": { + "data_offsets": [ + 3551210528, + 3553995808 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.mlp.gate_proj.weight": { + "data_offsets": [ + 2676979264, + 2721543744 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.mlp.up_proj.biases": { + "data_offsets": [ + 2992492032, + 2995277312 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.mlp.up_proj.scales": { + "data_offsets": [ + 3559740704, + 3562525984 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.mlp.up_proj.weight": { + "data_offsets": [ + 4654266368, + 4698830848 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.30.post_attention_layernorm.weight": { + "data_offsets": [ + 651885632, + 651895872 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.input_layernorm.weight": { + "data_offsets": [ + 1898197056, + 1898207296 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.mlp.down_proj.biases": { + "data_offsets": [ + 3694028832, + 3696814112 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.mlp.down_proj.scales": { + "data_offsets": [ + 3691243296, + 3694028576 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.mlp.down_proj.weight": { + "data_offsets": [ + 675138368, + 719702848 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.mlp.gate_proj.biases": { + "data_offsets": [ + 3688458016, + 3691243296 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.mlp.gate_proj.scales": { + "data_offsets": [ + 1172841504, + 1175626784 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.mlp.gate_proj.weight": { + "data_offsets": [ + 3643893536, + 3688458016 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.mlp.up_proj.biases": { + "data_offsets": [ + 3704030752, + 3706816032 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.mlp.up_proj.scales": { + "data_offsets": [ + 3699435552, + 3702220832 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.mlp.up_proj.weight": { + "data_offsets": [ + 3384082880, + 3428647360 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.post_attention_layernorm.weight": { + "data_offsets": [ + 3643883296, + 3643893536 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.k_norm.weight": { + "data_offsets": [ + 3643882784, + 3643883296 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.k_proj.biases": { + "data_offsets": [ + 3642316064, + 3642479904 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.k_proj.scales": { + "data_offsets": [ + 3596604704, + 3596768544 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.k_proj.weight": { + "data_offsets": [ + 3593983264, + 3596604704 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.o_proj.biases": { + "data_offsets": [ + 3642899744, + 3643882784 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.o_proj.scales": { + "data_offsets": [ + 3158820448, + 3159803488 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.o_proj.weight": { + "data_offsets": [ + 3366585184, + 3382313824 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.q_norm.weight": { + "data_offsets": [ + 1966562464, + 1966562976 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.q_proj.biases": { + "data_offsets": [ + 4652300032, + 4654266112 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.q_proj.scales": { + "data_offsets": [ + 1780722912, + 1782688992 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.q_proj.weight": { + "data_offsets": [ + 3562525984, + 3593983264 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.v_proj.biases": { + "data_offsets": [ + 2875185056, + 2875348896 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.v_proj.scales": { + "data_offsets": [ + 3642572064, + 3642735904 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.31.self_attn.v_proj.weight": { + "data_offsets": [ + 3492383680, + 3495005120 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.input_layernorm.weight": { + "data_offsets": [ + 2403486528, + 2403496768 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.A_log": { + "data_offsets": [ + 2245846560, + 2245846656 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.conv1d.weight": { + "data_offsets": [ + 2245764640, + 2245846560 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.dt_bias": { + "data_offsets": [ + 3542355392, + 3542355488 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3090975328, + 3090983008 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 720685888, + 720693568 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 2617781824, + 2617904704 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4832747648, + 4832755328 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3469254080, + 3469261760 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 5352861664, + 5352984544 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3780216352, + 3781854752 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 3706816032, + 3708454432 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 2617996864, + 2644211264 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 3798566432, + 3799549472 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 3797583392, + 3798566432 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3781854752, + 3797583392 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.norm.weight": { + "data_offsets": [ + 2721543744, + 2721544000 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.out_proj.biases": { + "data_offsets": [ + 3799549568, + 3800532608 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.out_proj.scales": { + "data_offsets": [ + 4042911808, + 4043894848 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.linear_attn.out_proj.weight": { + "data_offsets": [ + 1615390432, + 1631119072 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.mlp.down_proj.biases": { + "data_offsets": [ + 3466468800, + 3469254080 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.mlp.down_proj.scales": { + "data_offsets": [ + 3847890048, + 3850675328 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.mlp.down_proj.weight": { + "data_offsets": [ + 3803317888, + 3847882368 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.mlp.gate_proj.biases": { + "data_offsets": [ + 2819192640, + 2821977920 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.mlp.gate_proj.scales": { + "data_offsets": [ + 3800532608, + 3803317888 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.mlp.gate_proj.weight": { + "data_offsets": [ + 990615488, + 1035179968 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.mlp.up_proj.biases": { + "data_offsets": [ + 3853468544, + 3856253824 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.mlp.up_proj.scales": { + "data_offsets": [ + 3850683264, + 3853468544 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.mlp.up_proj.weight": { + "data_offsets": [ + 2308943072, + 2353507552 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.32.post_attention_layernorm.weight": { + "data_offsets": [ + 3491390400, + 3491400640 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.input_layernorm.weight": { + "data_offsets": [ + 3995551808, + 3995562048 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.A_log": { + "data_offsets": [ + 3976863456, + 3976863552 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.conv1d.weight": { + "data_offsets": [ + 3856253824, + 3856335744 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.dt_bias": { + "data_offsets": [ + 3976863360, + 3976863456 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3976855680, + 3976863360 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3927859584, + 3927867264 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 3926098304, + 3926221184 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 3159803488, + 3159811168 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3923305344, + 3923313024 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 2827712416, + 2827835296 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 2178571936, + 2180210336 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 3702220832, + 3703859232 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 2740065600, + 2766280000 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 3919537024, + 3920520064 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2648143424, + 2649126464 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3901023104, + 3916751744 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.norm.weight": { + "data_offsets": [ + 3976863552, + 3976863808 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.out_proj.biases": { + "data_offsets": [ + 3993585728, + 3994568768 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.out_proj.scales": { + "data_offsets": [ + 3992602688, + 3993585728 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.linear_attn.out_proj.weight": { + "data_offsets": [ + 3976874048, + 3992602688 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.mlp.down_proj.biases": { + "data_offsets": [ + 4043894848, + 4046680128 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.mlp.down_proj.scales": { + "data_offsets": [ + 3995562048, + 3998347328 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.mlp.down_proj.weight": { + "data_offsets": [ + 752752448, + 797316928 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.mlp.gate_proj.biases": { + "data_offsets": [ + 2190844768, + 2193630048 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.mlp.gate_proj.scales": { + "data_offsets": [ + 3488605120, + 3491390400 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.mlp.gate_proj.weight": { + "data_offsets": [ + 2998070272, + 3042634752 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.mlp.up_proj.biases": { + "data_offsets": [ + 4135809088, + 4138594368 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.mlp.up_proj.scales": { + "data_offsets": [ + 2824927136, + 2827712416 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.mlp.up_proj.weight": { + "data_offsets": [ + 4091244608, + 4135809088 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.33.post_attention_layernorm.weight": { + "data_offsets": [ + 4649489056, + 4649499296 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.input_layernorm.weight": { + "data_offsets": [ + 4223184448, + 4223194688 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.A_log": { + "data_offsets": [ + 844764416, + 844764512 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.conv1d.weight": { + "data_offsets": [ + 4138594368, + 4138676288 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.dt_bias": { + "data_offsets": [ + 4226120768, + 4226120864 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2411860288, + 2411867968 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 2722527040, + 2722534720 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 4225987648, + 4226110528 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4225979968, + 4225987648 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 1080407488, + 1080415168 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 4218637888, + 4218760768 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4216999488, + 4218637888 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4169813568, + 4171451968 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4140642368, + 4166856768 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4811015360, + 4811998400 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 4704429664, + 4705412704 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 1154982944, + 1170711584 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.norm.weight": { + "data_offsets": [ + 1192338464, + 1192338720 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.out_proj.biases": { + "data_offsets": [ + 4241849504, + 4242832544 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.out_proj.scales": { + "data_offsets": [ + 1175626784, + 1176609824 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.linear_attn.out_proj.weight": { + "data_offsets": [ + 4226120864, + 4241849504 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.mlp.down_proj.biases": { + "data_offsets": [ + 4311734944, + 4314520224 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.mlp.down_proj.scales": { + "data_offsets": [ + 4308826784, + 4311612064 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.mlp.down_proj.weight": { + "data_offsets": [ + 4245625504, + 4290189984 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.mlp.gate_proj.biases": { + "data_offsets": [ + 4242832544, + 4245617824 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.mlp.gate_proj.scales": { + "data_offsets": [ + 2399062752, + 2401848032 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.mlp.gate_proj.weight": { + "data_offsets": [ + 901478848, + 946043328 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.mlp.up_proj.biases": { + "data_offsets": [ + 3974070400, + 3976855680 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.mlp.up_proj.scales": { + "data_offsets": [ + 3455476160, + 3458261440 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.mlp.up_proj.weight": { + "data_offsets": [ + 4171451968, + 4216016448 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.34.post_attention_layernorm.weight": { + "data_offsets": [ + 1684210912, + 1684221152 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.input_layernorm.weight": { + "data_offsets": [ + 1321373472, + 1321383712 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.mlp.down_proj.biases": { + "data_offsets": [ + 4649514656, + 4652299936 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.mlp.down_proj.scales": { + "data_offsets": [ + 4477344416, + 4480129696 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.mlp.down_proj.weight": { + "data_offsets": [ + 4432779936, + 4477344416 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.mlp.gate_proj.biases": { + "data_offsets": [ + 4428028576, + 4430813856 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.mlp.gate_proj.scales": { + "data_offsets": [ + 4425120416, + 4427905696 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.mlp.gate_proj.weight": { + "data_offsets": [ + 2248631936, + 2293196416 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.mlp.up_proj.biases": { + "data_offsets": [ + 3332334944, + 3335120224 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.mlp.up_proj.scales": { + "data_offsets": [ + 3431555520, + 3434340800 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.mlp.up_proj.weight": { + "data_offsets": [ + 5283106784, + 5327671264 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.post_attention_layernorm.weight": { + "data_offsets": [ + 2193630048, + 2193640288 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.k_norm.weight": { + "data_offsets": [ + 4425119904, + 4425120416 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.k_proj.biases": { + "data_offsets": [ + 3556791584, + 3556955424 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.k_proj.scales": { + "data_offsets": [ + 4361050784, + 4361214624 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.k_proj.weight": { + "data_offsets": [ + 2130264736, + 2132886176 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.o_proj.biases": { + "data_offsets": [ + 1731407072, + 1732390112 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.o_proj.scales": { + "data_offsets": [ + 4424129184, + 4425112224 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.o_proj.weight": { + "data_offsets": [ + 4363836064, + 4379564704 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.q_norm.weight": { + "data_offsets": [ + 1244941600, + 1244942112 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.q_proj.biases": { + "data_offsets": [ + 4359084704, + 4361050784 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.q_proj.scales": { + "data_offsets": [ + 3549244448, + 3551210528 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.q_proj.weight": { + "data_offsets": [ + 5111237600, + 5142694880 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.v_proj.biases": { + "data_offsets": [ + 3642735904, + 3642899744 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.v_proj.scales": { + "data_offsets": [ + 2891210656, + 2891374496 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.35.self_attn.v_proj.weight": { + "data_offsets": [ + 4361214624, + 4363836064 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.input_layernorm.weight": { + "data_offsets": [ + 4702443008, + 4702453248 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.A_log": { + "data_offsets": [ + 4652299936, + 4652300032 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.conv1d.weight": { + "data_offsets": [ + 4526342816, + 4526424736 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.dt_bias": { + "data_offsets": [ + 1107377088, + 1107377184 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 4649506976, + 4649514656 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 4649499296, + 4649506976 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 4427905696, + 4428028576 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4425112224, + 4425119904 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 4166856768, + 4166864448 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 4311612064, + 4311734944 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 2814638400, + 2816276800 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 3330450784, + 3332089184 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4529373856, + 4555588256 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4603941536, + 4604924576 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 4602938016, + 4603921056 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 1176609824, + 1192338464 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.norm.weight": { + "data_offsets": [ + 4654266112, + 4654266368 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.out_proj.biases": { + "data_offsets": [ + 4701452288, + 4702435328 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.out_proj.scales": { + "data_offsets": [ + 4698830848, + 4699813888 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.linear_attn.out_proj.weight": { + "data_offsets": [ + 4290189984, + 4305918624 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.mlp.down_proj.biases": { + "data_offsets": [ + 845765472, + 848550752 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.mlp.down_proj.scales": { + "data_offsets": [ + 2183167136, + 2185952416 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.mlp.down_proj.weight": { + "data_offsets": [ + 4558373536, + 4602938016 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.mlp.gate_proj.biases": { + "data_offsets": [ + 2403496768, + 2406282048 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.mlp.gate_proj.scales": { + "data_offsets": [ + 4305918624, + 4308703904 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.mlp.gate_proj.weight": { + "data_offsets": [ + 4046680128, + 4091244608 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.mlp.up_proj.biases": { + "data_offsets": [ + 841889440, + 844674720 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.mlp.up_proj.scales": { + "data_offsets": [ + 3923313024, + 3926098304 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.mlp.up_proj.weight": { + "data_offsets": [ + 4705412704, + 4749977184 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.36.post_attention_layernorm.weight": { + "data_offsets": [ + 4702453248, + 4702463488 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.input_layernorm.weight": { + "data_offsets": [ + 2770056000, + 2770066240 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.A_log": { + "data_offsets": [ + 854121312, + 854121408 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.conv1d.weight": { + "data_offsets": [ + 1244942112, + 1245024032 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.dt_bias": { + "data_offsets": [ + 4817018560, + 4817018656 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 4816020160, + 4816027840 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3042634752, + 3042642432 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 4815897280, + 4816020160 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 3382313824, + 3382321504 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 4815889600, + 4815897280 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 4815766720, + 4815889600 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4793640544, + 4795278944 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4792002144, + 4793640544 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4765705824, + 4791920224 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4811998400, + 4812981440 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2242144800, + 2243127840 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 4795286720, + 4811015360 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.norm.weight": { + "data_offsets": [ + 4817018752, + 4817019008 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.out_proj.biases": { + "data_offsets": [ + 1898207296, + 1899190336 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.out_proj.scales": { + "data_offsets": [ + 4832755328, + 4833738368 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.linear_attn.out_proj.weight": { + "data_offsets": [ + 4817019008, + 4832747648 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.mlp.down_proj.biases": { + "data_offsets": [ + 4886658688, + 4889443968 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.mlp.down_proj.scales": { + "data_offsets": [ + 1322366752, + 1325152032 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.mlp.down_proj.weight": { + "data_offsets": [ + 1631119072, + 1675683552 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.mlp.gate_proj.biases": { + "data_offsets": [ + 2180381856, + 2183167136 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.mlp.gate_proj.scales": { + "data_offsets": [ + 4839308928, + 4842094208 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.mlp.gate_proj.weight": { + "data_offsets": [ + 4842094208, + 4886658688 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.mlp.up_proj.biases": { + "data_offsets": [ + 2766280000, + 2769065280 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.mlp.up_proj.scales": { + "data_offsets": [ + 1680442592, + 1683227872 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.mlp.up_proj.weight": { + "data_offsets": [ + 3497790400, + 3542354880 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.37.post_attention_layernorm.weight": { + "data_offsets": [ + 4526332576, + 4526342816 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.input_layernorm.weight": { + "data_offsets": [ + 5064689120, + 5064699360 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.A_log": { + "data_offsets": [ + 2891394976, + 2891395072 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.conv1d.weight": { + "data_offsets": [ + 4907957888, + 4908039808 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.dt_bias": { + "data_offsets": [ + 3799549472, + 3799549568 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 4245617824, + 4245625504 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 2188860672, + 2188868352 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 3856335744, + 3856458624 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 5000783584, + 5000791264 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3462029760, + 3462037440 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 4308703904, + 4308826784 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4908050144, + 4909688544 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2649126464, + 2650764864 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1054193088, + 1080407488 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4910671584, + 4911654624 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 4909688544, + 4910671584 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 4892229248, + 4907957888 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.norm.weight": { + "data_offsets": [ + 5000791264, + 5000791520 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.out_proj.biases": { + "data_offsets": [ + 5063706080, + 5064689120 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.out_proj.scales": { + "data_offsets": [ + 3469272000, + 3470255040 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.linear_attn.out_proj.weight": { + "data_offsets": [ + 5001774560, + 5017503200 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.mlp.down_proj.biases": { + "data_offsets": [ + 3226781024, + 3229566304 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.mlp.down_proj.scales": { + "data_offsets": [ + 3136530528, + 3139315808 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.mlp.down_proj.weight": { + "data_offsets": [ + 5142694880, + 5187259360 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.mlp.gate_proj.biases": { + "data_offsets": [ + 747153728, + 749939008 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.mlp.gate_proj.scales": { + "data_offsets": [ + 4526588576, + 4529373856 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.mlp.gate_proj.weight": { + "data_offsets": [ + 5066673120, + 5111237600 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.mlp.up_proj.biases": { + "data_offsets": [ + 1562633952, + 1565419232 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.mlp.up_proj.scales": { + "data_offsets": [ + 2018507936, + 2021293216 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.mlp.up_proj.weight": { + "data_offsets": [ + 5187259360, + 5231823840 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.38.post_attention_layernorm.weight": { + "data_offsets": [ + 3976863808, + 3976874048 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.input_layernorm.weight": { + "data_offsets": [ + 3469261760, + 3469272000 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.mlp.down_proj.biases": { + "data_offsets": [ + 2411867968, + 2414653248 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.mlp.down_proj.scales": { + "data_offsets": [ + 2446233664, + 2449018944 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.mlp.down_proj.weight": { + "data_offsets": [ + 3998347328, + 4042911808 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.mlp.gate_proj.biases": { + "data_offsets": [ + 5327671264, + 5330456544 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.mlp.gate_proj.scales": { + "data_offsets": [ + 5279338464, + 5282123744 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.mlp.gate_proj.weight": { + "data_offsets": [ + 4604924576, + 4649489056 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.mlp.up_proj.biases": { + "data_offsets": [ + 1098929088, + 1101714368 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.mlp.up_proj.scales": { + "data_offsets": [ + 5330456544, + 5333241824 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.mlp.up_proj.weight": { + "data_offsets": [ + 2541752384, + 2586316864 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.post_attention_layernorm.weight": { + "data_offsets": [ + 2617986624, + 2617996864 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.k_norm.weight": { + "data_offsets": [ + 5279337952, + 5279338464 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.k_proj.biases": { + "data_offsets": [ + 5231823840, + 5231987680 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.k_proj.scales": { + "data_offsets": [ + 4166864448, + 4167028288 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.k_proj.weight": { + "data_offsets": [ + 2243127840, + 2245749280 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.o_proj.biases": { + "data_offsets": [ + 2721544000, + 2722527040 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.o_proj.scales": { + "data_offsets": [ + 1815457504, + 1816440544 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.o_proj.weight": { + "data_offsets": [ + 1799728864, + 1815457504 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.q_norm.weight": { + "data_offsets": [ + 5231987680, + 5231988192 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.q_proj.biases": { + "data_offsets": [ + 2188868352, + 2190834432 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.q_proj.scales": { + "data_offsets": [ + 4702463488, + 4704429568 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.q_proj.weight": { + "data_offsets": [ + 2586324544, + 2617781824 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.v_proj.biases": { + "data_offsets": [ + 1389233952, + 1389397792 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.v_proj.scales": { + "data_offsets": [ + 2722534720, + 2722698560 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.39.self_attn.v_proj.weight": { + "data_offsets": [ + 3696814112, + 3699435552 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.input_layernorm.weight": { + "data_offsets": [ + 1904768672, + 1904778912 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.A_log": { + "data_offsets": [ + 5353107424, + 5353107520 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.conv1d.weight": { + "data_offsets": [ + 3642479904, + 3642561824 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.dt_bias": { + "data_offsets": [ + 4908039808, + 4908039904 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3546615328, + 3546623008 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3366577504, + 3366585184 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 5352984544, + 5353107424 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2891395072, + 2891402752 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3847882368, + 3847890048 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 5352738784, + 5352861664 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3382321504, + 3383959904 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4480129696, + 4481768096 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1933778592, + 1959992992 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 5348970464, + 5349953504 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 4139659328, + 4140642368 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 5333241824, + 5348970464 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.norm.weight": { + "data_offsets": [ + 5353107520, + 5353107776 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.out_proj.biases": { + "data_offsets": [ + 5353107776, + 5354090816 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.out_proj.scales": { + "data_offsets": [ + 4816027840, + 4817010880 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.linear_attn.out_proj.weight": { + "data_offsets": [ + 2875359136, + 2891087776 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.mlp.down_proj.biases": { + "data_offsets": [ + 643519552, + 646304832 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.mlp.down_proj.scales": { + "data_offsets": [ + 646304832, + 649090112 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.mlp.down_proj.weight": { + "data_offsets": [ + 1732390112, + 1776954592 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.mlp.gate_proj.biases": { + "data_offsets": [ + 649090112, + 651875392 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.mlp.gate_proj.scales": { + "data_offsets": [ + 4223194688, + 4225979968 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.mlp.gate_proj.weight": { + "data_offsets": [ + 2900577792, + 2945142272 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.mlp.up_proj.biases": { + "data_offsets": [ + 1437165280, + 1439950560 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.mlp.up_proj.scales": { + "data_offsets": [ + 4836523648, + 4839308928 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.mlp.up_proj.weight": { + "data_offsets": [ + 854121408, + 898685888 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.40.post_attention_layernorm.weight": { + "data_offsets": [ + 651875392, + 651885632 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.input_layernorm.weight": { + "data_offsets": [ + 4908039904, + 4908050144 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.A_log": { + "data_offsets": [ + 1881485280, + 1881485376 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.conv1d.weight": { + "data_offsets": [ + 1107551264, + 1107633184 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.dt_bias": { + "data_offsets": [ + 620778976, + 620779072 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 620779072, + 620786752 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3850675584, + 3850683264 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 620786752, + 620909632 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 841881760, + 841889440 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 4702435328, + 4702443008 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3233334624, + 3233457504 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 640242752, + 641881152 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 641881152, + 643519552 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 3278021984, + 3304236384 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 622548032, + 623531072 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 623531072, + 624514112 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 624514112, + 640242752 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.norm.weight": { + "data_offsets": [ + 620778720, + 620778976 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.out_proj.biases": { + "data_offsets": [ + 5000791520, + 5001774560 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.out_proj.scales": { + "data_offsets": [ + 619795680, + 620778720 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.linear_attn.out_proj.weight": { + "data_offsets": [ + 1083200448, + 1098929088 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.mlp.down_proj.biases": { + "data_offsets": [ + 563949280, + 566734560 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.mlp.down_proj.scales": { + "data_offsets": [ + 566734560, + 569519840 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.mlp.down_proj.weight": { + "data_offsets": [ + 2770073920, + 2814638400 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.mlp.gate_proj.biases": { + "data_offsets": [ + 569519840, + 572305120 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.mlp.gate_proj.scales": { + "data_offsets": [ + 572312800, + 575098080 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.mlp.gate_proj.weight": { + "data_offsets": [ + 575098080, + 619662560 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.mlp.up_proj.biases": { + "data_offsets": [ + 513814240, + 516599520 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.mlp.up_proj.scales": { + "data_offsets": [ + 516599520, + 519384800 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.mlp.up_proj.weight": { + "data_offsets": [ + 519384800, + 563949280 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.41.post_attention_layernorm.weight": { + "data_offsets": [ + 619662560, + 619672800 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.input_layernorm.weight": { + "data_offsets": [ + 437956320, + 437966560 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.A_log": { + "data_offsets": [ + 4704429568, + 4704429664 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.conv1d.weight": { + "data_offsets": [ + 513732320, + 513814240 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.dt_bias": { + "data_offsets": [ + 4817018656, + 4817018752 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 439932640, + 439940320 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 2353507552, + 2353515232 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 439940320, + 440063200 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 440063200, + 440070880 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 484635360, + 484643040 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 484643040, + 484765920 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 485871840, + 487510240 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2401848032, + 2403486432 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 487510240, + 513724640 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 670246208, + 671229248 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 484765920, + 485748960 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3211052384, + 3226781024 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.norm.weight": { + "data_offsets": [ + 3694028576, + 3694028832 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.out_proj.biases": { + "data_offsets": [ + 437966560, + 438949600 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.out_proj.scales": { + "data_offsets": [ + 438949600, + 439932640 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.linear_attn.out_proj.weight": { + "data_offsets": [ + 2293206752, + 2308935392 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.mlp.down_proj.biases": { + "data_offsets": [ + 343246560, + 346031840 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.mlp.down_proj.scales": { + "data_offsets": [ + 346031840, + 348817120 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.mlp.down_proj.weight": { + "data_offsets": [ + 348817120, + 393381600 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.mlp.gate_proj.biases": { + "data_offsets": [ + 1677657312, + 1680442592 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.mlp.gate_proj.scales": { + "data_offsets": [ + 4218760768, + 4221546048 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.mlp.gate_proj.weight": { + "data_offsets": [ + 393381600, + 437946080 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.mlp.up_proj.biases": { + "data_offsets": [ + 2895171072, + 2897956352 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.mlp.up_proj.scales": { + "data_offsets": [ + 340461280, + 343246560 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.mlp.up_proj.weight": { + "data_offsets": [ + 1392600800, + 1437165280 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.42.post_attention_layernorm.weight": { + "data_offsets": [ + 437946080, + 437956320 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.input_layernorm.weight": { + "data_offsets": [ + 319962240, + 319972480 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.mlp.down_proj.biases": { + "data_offsets": [ + 272612480, + 275397760 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.mlp.down_proj.scales": { + "data_offsets": [ + 4833738368, + 4836523648 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.mlp.down_proj.weight": { + "data_offsets": [ + 275397760, + 319962240 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.mlp.gate_proj.biases": { + "data_offsets": [ + 4167028288, + 4169813568 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.mlp.gate_proj.scales": { + "data_offsets": [ + 2409067328, + 2411852608 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.mlp.gate_proj.weight": { + "data_offsets": [ + 4481768096, + 4526332576 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.mlp.up_proj.biases": { + "data_offsets": [ + 3463683520, + 3466468800 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.mlp.up_proj.scales": { + "data_offsets": [ + 225262720, + 228048000 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.mlp.up_proj.weight": { + "data_offsets": [ + 228048000, + 272612480 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.post_attention_layernorm.weight": { + "data_offsets": [ + 3642561824, + 3642572064 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.k_norm.weight": { + "data_offsets": [ + 319972480, + 319972992 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.k_proj.biases": { + "data_offsets": [ + 340133504, + 340297344 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.k_proj.scales": { + "data_offsets": [ + 340297344, + 340461184 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.k_proj.weight": { + "data_offsets": [ + 651895872, + 654517312 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.o_proj.biases": { + "data_offsets": [ + 4216016448, + 4216999488 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.o_proj.scales": { + "data_offsets": [ + 3548261408, + 3549244448 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.o_proj.weight": { + "data_offsets": [ + 321619584, + 337348224 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.q_norm.weight": { + "data_offsets": [ + 319980672, + 319981184 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.q_proj.biases": { + "data_offsets": [ + 2821978016, + 2823944096 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.q_proj.scales": { + "data_offsets": [ + 5064699360, + 5066665440 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.q_proj.weight": { + "data_offsets": [ + 2072593056, + 2104050336 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.v_proj.biases": { + "data_offsets": [ + 1799401184, + 1799565024 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.v_proj.scales": { + "data_offsets": [ + 337348224, + 337512064 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.43.self_attn.v_proj.weight": { + "data_offsets": [ + 337512064, + 340133504 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.input_layernorm.weight": { + "data_offsets": [ + 204731168, + 204741408 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.A_log": { + "data_offsets": [ + 340461184, + 340461280 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.conv1d.weight": { + "data_offsets": [ + 225180800, + 225262720 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.dt_bias": { + "data_offsets": [ + 221453344, + 221453440 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2180374176, + 2180381856 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 1970167456, + 1970175136 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 485748960, + 485871840 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2816276800, + 2816284480 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 1904760896, + 1904768576 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 221453440, + 221576320 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 319981184, + 321619584 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 223542400, + 225180800 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 3709437472, + 3735651872 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 221576320, + 222559360 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 222559360, + 223542400 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2430381888, + 2446110528 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.norm.weight": { + "data_offsets": [ + 221453088, + 221453344 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.out_proj.biases": { + "data_offsets": [ + 2071610016, + 2072593056 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.out_proj.scales": { + "data_offsets": [ + 204741408, + 205724448 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.linear_attn.out_proj.weight": { + "data_offsets": [ + 205724448, + 221453088 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.mlp.down_proj.biases": { + "data_offsets": [ + 154585888, + 157371168 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.mlp.down_proj.scales": { + "data_offsets": [ + 157371168, + 160156448 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.mlp.down_proj.weight": { + "data_offsets": [ + 160156448, + 204720928 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.mlp.gate_proj.biases": { + "data_offsets": [ + 5231988192, + 5234773472 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.mlp.gate_proj.scales": { + "data_offsets": [ + 2185952416, + 2188737696 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.mlp.gate_proj.weight": { + "data_offsets": [ + 4911654624, + 4956219104 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.mlp.up_proj.biases": { + "data_offsets": [ + 4812981440, + 4815766720 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.mlp.up_proj.scales": { + "data_offsets": [ + 4889443968, + 4892229248 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.mlp.up_proj.weight": { + "data_offsets": [ + 4379564704, + 4424129184 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.44.post_attention_layernorm.weight": { + "data_offsets": [ + 204720928, + 204731168 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.input_layernorm.weight": { + "data_offsets": [ + 4603921056, + 4603931296 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.A_log": { + "data_offsets": [ + 136023136, + 136023232 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.conv1d.weight": { + "data_offsets": [ + 154503968, + 154585888 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.dt_bias": { + "data_offsets": [ + 136023232, + 136023328 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 136023328, + 136031008 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 572305120, + 572312800 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 619672800, + 619795680 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4795278944, + 4795286624 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 513724640, + 513732320 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 136031008, + 136153888 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 152865568, + 154503968 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2539130944, + 2540769344 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1904778912, + 1930993312 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 3708454432, + 3709437472 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 136153888, + 137136928 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 137136928, + 152865568 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.norm.weight": { + "data_offsets": [ + 3553995808, + 3553996064 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.out_proj.biases": { + "data_offsets": [ + 3471893440, + 3472876480 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.out_proj.scales": { + "data_offsets": [ + 1897214016, + 1898197056 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.linear_attn.out_proj.weight": { + "data_offsets": [ + 120294496, + 136023136 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.mlp.down_proj.biases": { + "data_offsets": [ + 64578656, + 67363936 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.mlp.down_proj.scales": { + "data_offsets": [ + 67363936, + 70149216 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.mlp.down_proj.weight": { + "data_offsets": [ + 3927867264, + 3972431744 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.mlp.gate_proj.biases": { + "data_offsets": [ + 70149216, + 72934496 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.mlp.gate_proj.scales": { + "data_offsets": [ + 72934496, + 75719776 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.mlp.gate_proj.weight": { + "data_offsets": [ + 75719776, + 120284256 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.mlp.up_proj.biases": { + "data_offsets": [ + 1152197664, + 1154982944 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.mlp.up_proj.scales": { + "data_offsets": [ + 4555588256, + 4558373536 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.mlp.up_proj.weight": { + "data_offsets": [ + 2132886176, + 2177450656 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.45.post_attention_layernorm.weight": { + "data_offsets": [ + 120284256, + 120294496 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.input_layernorm.weight": { + "data_offsets": [ + 5570560, + 5580800 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.A_log": { + "data_offsets": [ + 3090975232, + 3090975328 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.conv1d.weight": { + "data_offsets": [ + 64496736, + 64578656 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.dt_bias": { + "data_offsets": [ + 21309440, + 21309536 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 319972992, + 319980672 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 2769065280, + 2769072960 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 21309536, + 21432416 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 21432416, + 21440096 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 21440096, + 21447776 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 21447776, + 21570656 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 5017503200, + 5019141600 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 620909632, + 622548032 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 38282336, + 64496736 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 3994568768, + 3995551808 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 21570656, + 22553696 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 22553696, + 38282336 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.norm.weight": { + "data_offsets": [ + 3850675328, + 3850675584 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.out_proj.biases": { + "data_offsets": [ + 1798418144, + 1799401184 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.out_proj.scales": { + "data_offsets": [ + 2894188032, + 2895171072 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.linear_attn.out_proj.weight": { + "data_offsets": [ + 5580800, + 21309440 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.mlp.gate_proj.biases": { + "data_offsets": [ + 0, + 2785280 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.mlp.gate_proj.scales": { + "data_offsets": [ + 2785280, + 5570560 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.mlp.gate_proj.weight": { + "data_offsets": [ + 440070880, + 484635360 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.model.layers.46.post_attention_layernorm.weight": { + "data_offsets": [ + 675128128, + 675138368 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00002-of-00003.safetensors" + }, + "language_model.lm_head.biases": { + "data_offsets": [ + 5061177600, + 5100908800 + ], + "dtype": "BF16", + "shape": [ + 248320, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.lm_head.scales": { + "data_offsets": [ + 5020451072, + 5060182272 + ], + "dtype": "BF16", + "shape": [ + 248320, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.lm_head.weight": { + "data_offsets": [ + 2619728192, + 3255427392 + ], + "dtype": "U32", + "shape": [ + 248320, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.46.mlp.down_proj.biases": { + "data_offsets": [ + 1824698656, + 1827483936 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.46.mlp.down_proj.scales": { + "data_offsets": [ + 2476975680, + 2479760960 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.46.mlp.down_proj.weight": { + "data_offsets": [ + 3372890272, + 3417454752 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.46.mlp.up_proj.biases": { + "data_offsets": [ + 1905940992, + 1908726272 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.46.mlp.up_proj.scales": { + "data_offsets": [ + 2067706624, + 2070491904 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.46.mlp.up_proj.weight": { + "data_offsets": [ + 3631507072, + 3676071552 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.input_layernorm.weight": { + "data_offsets": [ + 2219991776, + 2220002016 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.mlp.down_proj.biases": { + "data_offsets": [ + 3512160736, + 3514946016 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.mlp.down_proj.scales": { + "data_offsets": [ + 2213922016, + 2216707296 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.mlp.down_proj.weight": { + "data_offsets": [ + 1489974560, + 1534539040 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.mlp.gate_proj.biases": { + "data_offsets": [ + 2124431360, + 2127216640 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.mlp.gate_proj.scales": { + "data_offsets": [ + 2192620512, + 2195405792 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.mlp.gate_proj.weight": { + "data_offsets": [ + 1026955168, + 1071519648 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.mlp.up_proj.biases": { + "data_offsets": [ + 2064921344, + 2067706624 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.mlp.up_proj.scales": { + "data_offsets": [ + 1955943680, + 1958728960 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.mlp.up_proj.weight": { + "data_offsets": [ + 3531057472, + 3575621952 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.post_attention_layernorm.weight": { + "data_offsets": [ + 1127313536, + 1127323776 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.k_norm.weight": { + "data_offsets": [ + 1645495136, + 1645495648 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.k_proj.biases": { + "data_offsets": [ + 1824534816, + 1824698656 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.k_proj.scales": { + "data_offsets": [ + 1081868192, + 1082032032 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.k_proj.weight": { + "data_offsets": [ + 2474341696, + 2476963136 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.o_proj.biases": { + "data_offsets": [ + 2269343616, + 2270326656 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.o_proj.scales": { + "data_offsets": [ + 1485529376, + 1486512416 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.o_proj.weight": { + "data_offsets": [ + 3357161376, + 3372890016 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.q_norm.weight": { + "data_offsets": [ + 1882282240, + 1882282752 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.q_proj.biases": { + "data_offsets": [ + 4841336512, + 4843302592 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.q_proj.scales": { + "data_offsets": [ + 1643365216, + 1645331296 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.q_proj.weight": { + "data_offsets": [ + 1650766944, + 1682224224 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.v_proj.biases": { + "data_offsets": [ + 1828599072, + 1828762912 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.v_proj.scales": { + "data_offsets": [ + 1953293056, + 1953456896 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.47.self_attn.v_proj.weight": { + "data_offsets": [ + 2274250112, + 2276871552 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.input_layernorm.weight": { + "data_offsets": [ + 1639295840, + 1639306080 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.A_log": { + "data_offsets": [ + 2270481600, + 2270481696 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.conv1d.weight": { + "data_offsets": [ + 1573760288, + 1573842208 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.dt_bias": { + "data_offsets": [ + 1876235552, + 1876235648 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3723265664, + 3723273344 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 1479205664, + 1479213344 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1690317760, + 1690440640 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 1735258912, + 1735266592 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 1828466976, + 1828474656 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 602945600, + 603068480 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 2218353376, + 2219991776 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 1199127168, + 1200765568 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1534539040, + 1560753440 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 3862359744, + 3863342784 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 1564202272, + 1565185312 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2128199680, + 2143928320 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.norm.weight": { + "data_offsets": [ + 3282788672, + 3282788928 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.out_proj.biases": { + "data_offsets": [ + 2191637472, + 2192620512 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.out_proj.scales": { + "data_offsets": [ + 1463601184, + 1464584224 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.linear_attn.out_proj.weight": { + "data_offsets": [ + 2195405792, + 2211134432 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.mlp.down_proj.biases": { + "data_offsets": [ + 1248133248, + 1250918528 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.mlp.down_proj.scales": { + "data_offsets": [ + 4774467072, + 4777252352 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.mlp.down_proj.weight": { + "data_offsets": [ + 1370701600, + 1415266080 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.mlp.gate_proj.biases": { + "data_offsets": [ + 1473627424, + 1476412704 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.mlp.gate_proj.scales": { + "data_offsets": [ + 1687530176, + 1690315456 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.mlp.gate_proj.weight": { + "data_offsets": [ + 1418051360, + 1462615840 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.mlp.up_proj.biases": { + "data_offsets": [ + 1476412704, + 1479197984 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.mlp.up_proj.scales": { + "data_offsets": [ + 2280642432, + 2283427712 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.mlp.up_proj.weight": { + "data_offsets": [ + 1828762912, + 1873327392 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.48.post_attention_layernorm.weight": { + "data_offsets": [ + 2382290880, + 2382301120 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.input_layernorm.weight": { + "data_offsets": [ + 2458351040, + 2458361280 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.A_log": { + "data_offsets": [ + 3283897152, + 3283897248 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.conv1d.weight": { + "data_offsets": [ + 3782691456, + 3782773376 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.dt_bias": { + "data_offsets": [ + 4652357984, + 4652358080 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 959398912, + 959406592 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 2407521024, + 2407528704 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 591790400, + 591913280 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2381015424, + 2381023104 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 4149921568, + 4149929248 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3282791232, + 3282914112 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 2144294208, + 2145932608 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2072137984, + 2073776384 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 2353817984, + 2380032384 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 1486512416, + 1487495456 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2190654432, + 2191637472 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2383578880, + 2399307520 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.norm.weight": { + "data_offsets": [ + 3514958560, + 3514958816 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.out_proj.biases": { + "data_offsets": [ + 1642382176, + 1643365216 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.out_proj.scales": { + "data_offsets": [ + 2454888800, + 2455871840 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.linear_attn.out_proj.weight": { + "data_offsets": [ + 1623567200, + 1639295840 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.mlp.down_proj.biases": { + "data_offsets": [ + 2501435712, + 2504220992 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.mlp.down_proj.scales": { + "data_offsets": [ + 3528272096, + 3531057376 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.mlp.down_proj.weight": { + "data_offsets": [ + 425401248, + 469965728 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.mlp.gate_proj.biases": { + "data_offsets": [ + 2498650432, + 2501435712 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.mlp.gate_proj.scales": { + "data_offsets": [ + 2480136512, + 2482921792 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.mlp.gate_proj.weight": { + "data_offsets": [ + 1735266592, + 1779831072 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.mlp.up_proj.biases": { + "data_offsets": [ + 1620775008, + 1623560288 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.mlp.up_proj.scales": { + "data_offsets": [ + 1415266080, + 1418051360 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.mlp.up_proj.weight": { + "data_offsets": [ + 2530599232, + 2575163712 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.49.post_attention_layernorm.weight": { + "data_offsets": [ + 4567044000, + 4567054240 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.input_layernorm.weight": { + "data_offsets": [ + 3514946016, + 3514956256 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.A_log": { + "data_offsets": [ + 549856160, + 549856256 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.conv1d.weight": { + "data_offsets": [ + 2479929408, + 2480011328 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.dt_bias": { + "data_offsets": [ + 2274250016, + 2274250112 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3496269472, + 3496277152 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3357153696, + 3357161376 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 4510188704, + 4510311584 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 3356430752, + 3356438432 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 2382425536, + 2382433216 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 4648449632, + 4648572512 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 928408256, + 930046656 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 2070499584, + 2072137984 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 3255591232, + 3281805632 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2460173504, + 2461156544 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 3282914112, + 3283897152 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 4892807168, + 4908535808 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.norm.weight": { + "data_offsets": [ + 3372890016, + 3372890272 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.out_proj.biases": { + "data_offsets": [ + 2381152896, + 2382135936 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.out_proj.scales": { + "data_offsets": [ + 4616552096, + 4617535136 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.linear_attn.out_proj.weight": { + "data_offsets": [ + 3496432096, + 3512160736 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.mlp.down_proj.biases": { + "data_offsets": [ + 2461280960, + 2464066240 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.mlp.down_proj.scales": { + "data_offsets": [ + 3514958816, + 3517744096 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.mlp.down_proj.weight": { + "data_offsets": [ + 2309253504, + 2353817984 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.mlp.gate_proj.biases": { + "data_offsets": [ + 4555902880, + 4558688160 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.mlp.gate_proj.scales": { + "data_offsets": [ + 4355602624, + 4358387904 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.mlp.gate_proj.weight": { + "data_offsets": [ + 2575163712, + 2619728192 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.mlp.up_proj.biases": { + "data_offsets": [ + 1482662176, + 1485447456 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.mlp.up_proj.scales": { + "data_offsets": [ + 4558688160, + 4561473440 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.mlp.up_proj.weight": { + "data_offsets": [ + 1779968032, + 1824532512 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.50.post_attention_layernorm.weight": { + "data_offsets": [ + 1086199616, + 1086209856 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.input_layernorm.weight": { + "data_offsets": [ + 2476965440, + 2476975680 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.mlp.down_proj.biases": { + "data_offsets": [ + 2399307520, + 2402092800 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.mlp.down_proj.scales": { + "data_offsets": [ + 1682265792, + 1685051072 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.mlp.down_proj.weight": { + "data_offsets": [ + 1958728960, + 2003293440 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.mlp.gate_proj.biases": { + "data_offsets": [ + 1560753440, + 1563538720 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.mlp.gate_proj.scales": { + "data_offsets": [ + 1297124224, + 1299909504 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.mlp.gate_proj.weight": { + "data_offsets": [ + 1299918112, + 1344482592 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.mlp.up_proj.biases": { + "data_offsets": [ + 1245347968, + 1248133248 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.mlp.up_proj.scales": { + "data_offsets": [ + 4661083072, + 4663868352 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.mlp.up_proj.weight": { + "data_offsets": [ + 1252557440, + 1297121920 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.post_attention_layernorm.weight": { + "data_offsets": [ + 861628480, + 861638720 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.k_norm.weight": { + "data_offsets": [ + 1365294368, + 1365294880 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.k_proj.biases": { + "data_offsets": [ + 2530435392, + 2530599232 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.k_proj.scales": { + "data_offsets": [ + 3255427392, + 3255591232 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.k_proj.weight": { + "data_offsets": [ + 3575621952, + 3578243392 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.o_proj.biases": { + "data_offsets": [ + 1462615840, + 1463598880 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.o_proj.scales": { + "data_offsets": [ + 1347558944, + 1348541984 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.o_proj.weight": { + "data_offsets": [ + 1348678944, + 1364407584 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.q_norm.weight": { + "data_offsets": [ + 1347558432, + 1347558944 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.q_proj.biases": { + "data_offsets": [ + 3526298336, + 3528264416 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.q_proj.scales": { + "data_offsets": [ + 1566212128, + 1568178208 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.q_proj.weight": { + "data_offsets": [ + 3462026912, + 3493484192 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.v_proj.biases": { + "data_offsets": [ + 1364407584, + 1364571424 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.v_proj.scales": { + "data_offsets": [ + 3721463424, + 3721627264 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.51.self_attn.v_proj.weight": { + "data_offsets": [ + 1071644064, + 1074265504 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.input_layernorm.weight": { + "data_offsets": [ + 4358400448, + 4358410688 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.A_log": { + "data_offsets": [ + 2454888704, + 2454888800 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.conv1d.weight": { + "data_offsets": [ + 3356348832, + 3356430752 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.dt_bias": { + "data_offsets": [ + 1682224224, + 1682224320 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2464221184, + 2464228864 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 1123701376, + 1123709056 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1123709056, + 1123831936 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2458363584, + 2458371264 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3727477376, + 3727485056 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 1620652128, + 1620775008 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 2458371264, + 2460009664 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 1125675136, + 1127313536 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 1172871296, + 1199085696 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 1639306336, + 1640289376 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2276873856, + 2277856896 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3334227616, + 3349956256 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.norm.weight": { + "data_offsets": [ + 1690440640, + 1690440896 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.out_proj.biases": { + "data_offsets": [ + 4318031296, + 4319014336 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.out_proj.scales": { + "data_offsets": [ + 1087514080, + 1088497120 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.linear_attn.out_proj.weight": { + "data_offsets": [ + 1089487168, + 1105215808 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.mlp.down_proj.biases": { + "data_offsets": [ + 1024008032, + 1026793312 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.mlp.down_proj.scales": { + "data_offsets": [ + 1470842144, + 1473627424 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.mlp.down_proj.weight": { + "data_offsets": [ + 1690443296, + 1735007776 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.mlp.gate_proj.biases": { + "data_offsets": [ + 1079082912, + 1081868192 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.mlp.gate_proj.scales": { + "data_offsets": [ + 1344482592, + 1347267872 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.mlp.gate_proj.weight": { + "data_offsets": [ + 4511338400, + 4555902880 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.mlp.up_proj.biases": { + "data_offsets": [ + 869850752, + 872636032 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.mlp.up_proj.scales": { + "data_offsets": [ + 1010323200, + 1013108480 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.mlp.up_proj.weight": { + "data_offsets": [ + 1128306816, + 1172871296 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.52.post_attention_layernorm.weight": { + "data_offsets": [ + 1245330048, + 1245340288 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.input_layernorm.weight": { + "data_offsets": [ + 1348666400, + 1348676640 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.A_log": { + "data_offsets": [ + 1089487072, + 1089487168 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.conv1d.weight": { + "data_offsets": [ + 960402176, + 960484096 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.dt_bias": { + "data_offsets": [ + 1086530944, + 1086531040 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2070491904, + 2070499584 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 930046656, + 930054336 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1876112672, + 1876235552 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 931045056, + 931052736 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 932691136, + 932698816 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 2480011328, + 2480134208 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 1250919040, + 1252557440 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 931052736, + 932691136 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 932935680, + 959150080 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2380032384, + 2381015424 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 2382433216, + 2383416256 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 1887730944, + 1903459584 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.norm.weight": { + "data_offsets": [ + 918491584, + 918491840 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.out_proj.biases": { + "data_offsets": [ + 917506240, + 918489280 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.out_proj.scales": { + "data_offsets": [ + 2270481696, + 2271464736 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.linear_attn.out_proj.weight": { + "data_offsets": [ + 2049192704, + 2064921344 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.mlp.down_proj.biases": { + "data_offsets": [ + 867054048, + 869839328 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.mlp.down_proj.scales": { + "data_offsets": [ + 788062016, + 790847296 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.mlp.down_proj.weight": { + "data_offsets": [ + 2220985056, + 2265549536 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.mlp.gate_proj.biases": { + "data_offsets": [ + 864268768, + 867054048 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.mlp.gate_proj.scales": { + "data_offsets": [ + 2452103424, + 2454888704 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.mlp.gate_proj.weight": { + "data_offsets": [ + 872917696, + 917482176 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.mlp.up_proj.biases": { + "data_offsets": [ + 4507403424, + 4510188704 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.mlp.up_proj.scales": { + "data_offsets": [ + 3352580000, + 3355365280 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.mlp.up_proj.weight": { + "data_offsets": [ + 689014336, + 733578816 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.53.post_attention_layernorm.weight": { + "data_offsets": [ + 2265558144, + 2265568384 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.input_layernorm.weight": { + "data_offsets": [ + 1365284128, + 1365294368 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.A_log": { + "data_offsets": [ + 1024007936, + 1024008032 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.conv1d.weight": { + "data_offsets": [ + 1123954816, + 1124036736 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.dt_bias": { + "data_offsets": [ + 2190654336, + 2190654432 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2299322496, + 2299330176 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 930054336, + 930062016 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 2381030016, + 2381152896 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 1470834464, + 1470842144 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 4507385760, + 4507393440 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3724256384, + 3724379264 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 1124036736, + 1125675136 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 3866128064, + 3867766464 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 835414080, + 861628480 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 1086531040, + 1087514080 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 959416832, + 960399872 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 628721216, + 644449856 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.norm.weight": { + "data_offsets": [ + 1882281984, + 1882282240 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.out_proj.biases": { + "data_offsets": [ + 1127323776, + 1128306816 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.out_proj.scales": { + "data_offsets": [ + 601962560, + 602945600 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.linear_attn.out_proj.weight": { + "data_offsets": [ + 612984896, + 628713536 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.mlp.down_proj.biases": { + "data_offsets": [ + 1479876896, + 1482662176 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.mlp.down_proj.scales": { + "data_offsets": [ + 586064896, + 588850176 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.mlp.down_proj.weight": { + "data_offsets": [ + 1574121568, + 1618686048 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.mlp.gate_proj.biases": { + "data_offsets": [ + 4617535232, + 4620320512 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.mlp.gate_proj.scales": { + "data_offsets": [ + 589005120, + 591790400 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.mlp.gate_proj.weight": { + "data_offsets": [ + 2407538944, + 2452103424 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.mlp.up_proj.biases": { + "data_offsets": [ + 1468049184, + 1470834464 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.mlp.up_proj.scales": { + "data_offsets": [ + 583279616, + 586064896 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.mlp.up_proj.weight": { + "data_offsets": [ + 3286682528, + 3331247008 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.54.post_attention_layernorm.weight": { + "data_offsets": [ + 3782773376, + 3782783616 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.input_layernorm.weight": { + "data_offsets": [ + 425391008, + 425401248 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.mlp.down_proj.biases": { + "data_offsets": [ + 3582166656, + 3584951936 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.mlp.down_proj.scales": { + "data_offsets": [ + 1876669952, + 1879455232 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.mlp.down_proj.weight": { + "data_offsets": [ + 2079866880, + 2124431360 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.mlp.gate_proj.biases": { + "data_offsets": [ + 422605728, + 425391008 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.mlp.gate_proj.scales": { + "data_offsets": [ + 1568180512, + 1570965792 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.mlp.gate_proj.weight": { + "data_offsets": [ + 644449856, + 689014336 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.mlp.up_proj.biases": { + "data_offsets": [ + 5120278592, + 5123063872 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.mlp.up_proj.scales": { + "data_offsets": [ + 3691800192, + 3694585472 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.mlp.up_proj.weight": { + "data_offsets": [ + 3584951936, + 3629516416 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.post_attention_layernorm.weight": { + "data_offsets": [ + 1779957792, + 1779968032 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.k_norm.weight": { + "data_offsets": [ + 3954110816, + 3954111328 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.k_proj.biases": { + "data_offsets": [ + 549692320, + 549856160 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.k_proj.scales": { + "data_offsets": [ + 2283427712, + 2283591552 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.k_proj.weight": { + "data_offsets": [ + 4281012352, + 4283633792 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.o_proj.biases": { + "data_offsets": [ + 470711200, + 471694240 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.o_proj.scales": { + "data_offsets": [ + 516258720, + 517241760 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.o_proj.weight": { + "data_offsets": [ + 518235040, + 533963680 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.q_norm.weight": { + "data_offsets": [ + 1468048672, + 1468049184 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.q_proj.biases": { + "data_offsets": [ + 549856256, + 551822336 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.q_proj.scales": { + "data_offsets": [ + 1618686048, + 1620652128 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.q_proj.weight": { + "data_offsets": [ + 551822336, + 583279616 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.v_proj.biases": { + "data_offsets": [ + 1086209856, + 1086373696 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.v_proj.scales": { + "data_offsets": [ + 4655143360, + 4655307200 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.55.self_attn.v_proj.weight": { + "data_offsets": [ + 1365294880, + 1367916320 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.input_layernorm.weight": { + "data_offsets": [ + 3782783616, + 3782793856 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.A_log": { + "data_offsets": [ + 2458350944, + 2458351040 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.conv1d.weight": { + "data_offsets": [ + 470629280, + 470711200 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.dt_bias": { + "data_offsets": [ + 3531057376, + 3531057472 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3727425664, + 3727433344 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3724632704, + 3724640384 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 3724509824, + 3724632704 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4230391616, + 4230399296 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3724502144, + 3724509824 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3724379264, + 3724502144 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3721627264, + 3723265664 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 5016027392, + 5017665792 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 3694585472, + 3720799872 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 1013108480, + 1014091520 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 4506279840, + 4507262880 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 1107972736, + 1123701376 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.norm.weight": { + "data_offsets": [ + 3727433344, + 3727433600 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.out_proj.biases": { + "data_offsets": [ + 3281805632, + 3282788672 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.out_proj.scales": { + "data_offsets": [ + 2404571904, + 2405554944 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.linear_attn.out_proj.weight": { + "data_offsets": [ + 4692950464, + 4708679104 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.mlp.down_proj.biases": { + "data_offsets": [ + 2271464736, + 2274250016 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.mlp.down_proj.scales": { + "data_offsets": [ + 3830574784, + 3833360064 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.mlp.down_proj.weight": { + "data_offsets": [ + 3785875648, + 3830440128 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.mlp.gate_proj.biases": { + "data_offsets": [ + 4102481184, + 4105266464 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.mlp.gate_proj.scales": { + "data_offsets": [ + 3782811008, + 3785596288 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.mlp.gate_proj.weight": { + "data_offsets": [ + 960484096, + 1005048576 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.mlp.up_proj.biases": { + "data_offsets": [ + 3833360064, + 3836145344 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.mlp.up_proj.scales": { + "data_offsets": [ + 1873327392, + 1876112672 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.mlp.up_proj.weight": { + "data_offsets": [ + 471694240, + 516258720 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.56.post_attention_layernorm.weight": { + "data_offsets": [ + 3782800768, + 3782811008 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.input_layernorm.weight": { + "data_offsets": [ + 3357143456, + 3357153696 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.A_log": { + "data_offsets": [ + 3998931648, + 3998931744 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.conv1d.weight": { + "data_offsets": [ + 1485447456, + 1485529376 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.dt_bias": { + "data_offsets": [ + 3998931552, + 3998931648 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 3998800992, + 3998808672 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 601952576, + 601960256 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1107694912, + 1107817792 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 2216707296, + 2216714976 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3528264416, + 3528272096 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3954111328, + 3954234208 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 3930971584, + 3932609984 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 5116379552, + 5118017952 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 3836145344, + 3862359744 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 3950342496, + 3951325536 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 1467063328, + 1468046368 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3932612288, + 3948340928 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.norm.weight": { + "data_offsets": [ + 4001046816, + 4001047072 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.out_proj.biases": { + "data_offsets": [ + 2265568384, + 2266551424 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.out_proj.scales": { + "data_offsets": [ + 4793001408, + 4793984448 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.linear_attn.out_proj.weight": { + "data_offsets": [ + 4001047072, + 4016775712 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.mlp.down_proj.biases": { + "data_offsets": [ + 4099693600, + 4102478880 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.mlp.down_proj.scales": { + "data_offsets": [ + 4096908320, + 4099693600 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.mlp.down_proj.weight": { + "data_offsets": [ + 4049556256, + 4094120736 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.mlp.gate_proj.biases": { + "data_offsets": [ + 4020554272, + 4023339552 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.mlp.gate_proj.scales": { + "data_offsets": [ + 3522530016, + 3525315296 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.mlp.gate_proj.weight": { + "data_offsets": [ + 2003301120, + 2047865600 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.mlp.up_proj.biases": { + "data_offsets": [ + 4149929248, + 4152714528 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.mlp.up_proj.scales": { + "data_offsets": [ + 4201958432, + 4204743712 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.mlp.up_proj.weight": { + "data_offsets": [ + 4105266464, + 4149830944 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.57.post_attention_layernorm.weight": { + "data_offsets": [ + 4019560992, + 4019571232 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.input_layernorm.weight": { + "data_offsets": [ + 3830564544, + 3830574784 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.A_log": { + "data_offsets": [ + 4230388864, + 4230388960 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.conv1d.weight": { + "data_offsets": [ + 4153699872, + 4153781792 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.dt_bias": { + "data_offsets": [ + 4230388768, + 4230388864 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 1955936000, + 1955943680 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 1479197984, + 1479205664 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 4453110208, + 4453233088 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4156403232, + 4156410912 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 1364571424, + 1364579104 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 3998808672, + 3998931552 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4153781792, + 4155420192 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 1084561216, + 1086199616 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4911321088, + 4937535488 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4155420192, + 4156403232 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 517252000, + 518235040 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 2283593856, + 2299322496 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.norm.weight": { + "data_offsets": [ + 4230389056, + 4230389312 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.out_proj.biases": { + "data_offsets": [ + 2127216640, + 2128199680 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.out_proj.scales": { + "data_offsets": [ + 5124882112, + 5125865152 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.linear_attn.out_proj.weight": { + "data_offsets": [ + 3676071552, + 3691800192 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.mlp.down_proj.biases": { + "data_offsets": [ + 3867766464, + 3870551744 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.mlp.down_proj.scales": { + "data_offsets": [ + 4283788736, + 4286574016 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.mlp.down_proj.weight": { + "data_offsets": [ + 4232325760, + 4276890240 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.mlp.gate_proj.biases": { + "data_offsets": [ + 3863342784, + 3866128064 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.mlp.gate_proj.scales": { + "data_offsets": [ + 2211134432, + 2213919712 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.mlp.gate_proj.weight": { + "data_offsets": [ + 3728200320, + 3772764800 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.mlp.up_proj.biases": { + "data_offsets": [ + 3283897248, + 3286682528 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.mlp.up_proj.scales": { + "data_offsets": [ + 3724640384, + 3727425664 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.mlp.up_proj.weight": { + "data_offsets": [ + 790847296, + 835411776 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.58.post_attention_layernorm.weight": { + "data_offsets": [ + 4232315520, + 4232325760 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.input_layernorm.weight": { + "data_offsets": [ + 4358387904, + 4358398144 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.mlp.down_proj.biases": { + "data_offsets": [ + 4450324928, + 4453110208 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.mlp.down_proj.scales": { + "data_offsets": [ + 4094123040, + 4096908320 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.mlp.down_proj.weight": { + "data_offsets": [ + 4361195968, + 4405760448 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.mlp.gate_proj.biases": { + "data_offsets": [ + 1647974752, + 1650760032 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.mlp.gate_proj.scales": { + "data_offsets": [ + 4358410688, + 4361195968 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.mlp.gate_proj.weight": { + "data_offsets": [ + 4156410912, + 4200975392 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.mlp.up_proj.biases": { + "data_offsets": [ + 4500864512, + 4503649792 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.mlp.up_proj.scales": { + "data_offsets": [ + 4497924288, + 4500709568 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.mlp.up_proj.weight": { + "data_offsets": [ + 4453357504, + 4497921984 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.post_attention_layernorm.weight": { + "data_offsets": [ + 3772764800, + 3772775040 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.k_norm.weight": { + "data_offsets": [ + 4321351872, + 4321352384 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.k_proj.biases": { + "data_offsets": [ + 4321188032, + 4321351872 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.k_proj.scales": { + "data_offsets": [ + 4321024192, + 4321188032 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.k_proj.weight": { + "data_offsets": [ + 1882324224, + 1884945664 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.o_proj.biases": { + "data_offsets": [ + 1088504032, + 1089487072 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.o_proj.scales": { + "data_offsets": [ + 3331247008, + 3332230048 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.o_proj.weight": { + "data_offsets": [ + 4204743712, + 4220472352 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.q_norm.weight": { + "data_offsets": [ + 1250918528, + 1250919040 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.q_proj.biases": { + "data_offsets": [ + 4319058112, + 4321024192 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.q_proj.scales": { + "data_offsets": [ + 2405554944, + 2407521024 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.q_proj.weight": { + "data_offsets": [ + 4286574016, + 4318031296 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.v_proj.biases": { + "data_offsets": [ + 2479760960, + 2479924800 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.v_proj.scales": { + "data_offsets": [ + 1645331296, + 1645495136 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.59.self_attn.v_proj.weight": { + "data_offsets": [ + 4771720448, + 4774341888 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.input_layernorm.weight": { + "data_offsets": [ + 5120143936, + 5120154176 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.A_log": { + "data_offsets": [ + 4230388960, + 4230389056 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.conv1d.weight": { + "data_offsets": [ + 2143930624, + 2144012544 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.dt_bias": { + "data_offsets": [ + 1690440896, + 1690440992 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 2003293440, + 2003301120 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 4321352384, + 4321360064 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 591913280, + 592036160 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 1245340288, + 1245347968 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 4507393440, + 4507401120 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 4507262880, + 4507385760 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 2216714976, + 2218353376 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4503649792, + 4505288192 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4023341856, + 4049556256 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 4505296800, + 4506279840 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 3355365280, + 3356348320 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 533963680, + 549692320 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.norm.weight": { + "data_offsets": [ + 2280642176, + 2280642432 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.out_proj.biases": { + "data_offsets": [ + 1827483936, + 1828466976 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.out_proj.scales": { + "data_offsets": [ + 4510311584, + 4511294624 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.linear_attn.out_proj.weight": { + "data_offsets": [ + 5125865152, + 5141593792 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.mlp.down_proj.biases": { + "data_offsets": [ + 1007527680, + 1010312960 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.mlp.down_proj.scales": { + "data_offsets": [ + 3578243392, + 3581028672 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.mlp.down_proj.weight": { + "data_offsets": [ + 4567054240, + 4611618720 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.mlp.gate_proj.biases": { + "data_offsets": [ + 4564258720, + 4567044000 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.mlp.gate_proj.scales": { + "data_offsets": [ + 4561473440, + 4564258720 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.mlp.gate_proj.weight": { + "data_offsets": [ + 3886407104, + 3930971584 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.mlp.up_proj.biases": { + "data_offsets": [ + 3951325536, + 3954110816 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.mlp.up_proj.scales": { + "data_offsets": [ + 1367916320, + 1370701600 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.mlp.up_proj.weight": { + "data_offsets": [ + 3417454752, + 3462019232 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.60.post_attention_layernorm.weight": { + "data_offsets": [ + 959406592, + 959416832 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.input_layernorm.weight": { + "data_offsets": [ + 917496000, + 917506240 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.A_log": { + "data_offsets": [ + 4622799616, + 4622799712 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.conv1d.weight": { + "data_offsets": [ + 932853760, + 932935680 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.dt_bias": { + "data_offsets": [ + 4617535136, + 4617535232 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 4616544416, + 4616552096 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 3520531680, + 3520539360 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 1123831936, + 1123954816 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 4149830944, + 4149838624 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 3462019232, + 3462026912 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 4232037696, + 4232160576 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4611618720, + 4613257120 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4230399296, + 4232037696 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 2504220992, + 2530435392 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 2073776384, + 2074759424 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 3581028672, + 3582011712 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 3870676160, + 3886404800 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.norm.weight": { + "data_offsets": [ + 4632718432, + 4632718688 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.out_proj.biases": { + "data_offsets": [ + 2220002016, + 2220985056 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.out_proj.scales": { + "data_offsets": [ + 4648572512, + 4649555552 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.linear_attn.out_proj.weight": { + "data_offsets": [ + 4632720992, + 4648449632 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.mlp.down_proj.biases": { + "data_offsets": [ + 1076256160, + 1079041440 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.mlp.down_proj.scales": { + "data_offsets": [ + 4652358080, + 4655143360 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.mlp.down_proj.weight": { + "data_offsets": [ + 4405760448, + 4450324928 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.mlp.gate_proj.biases": { + "data_offsets": [ + 3517746400, + 3520531680 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.mlp.gate_proj.scales": { + "data_offsets": [ + 4649572704, + 4652357984 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.mlp.gate_proj.weight": { + "data_offsets": [ + 1908728576, + 1953293056 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.mlp.up_proj.biases": { + "data_offsets": [ + 4663868352, + 4666653632 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.mlp.up_proj.scales": { + "data_offsets": [ + 4655307200, + 4658092480 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.mlp.up_proj.weight": { + "data_offsets": [ + 1200765568, + 1245330048 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.61.post_attention_layernorm.weight": { + "data_offsets": [ + 4649555552, + 4649565792 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.input_layernorm.weight": { + "data_offsets": [ + 4792991168, + 4793001408 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.A_log": { + "data_offsets": [ + 4777260128, + 4777260224 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.conv1d.weight": { + "data_offsets": [ + 4666653632, + 4666735552 + ], + "dtype": "BF16", + "shape": [ + 10240, + 4, + 1 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.dt_bias": { + "data_offsets": [ + 4777260032, + 4777260128 + ], + "dtype": "BF16", + "shape": [ + 48 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_a.biases": { + "data_offsets": [ + 628713536, + 628721216 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_a.scales": { + "data_offsets": [ + 4777252352, + 4777260032 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_a.weight": { + "data_offsets": [ + 4774341888, + 4774464768 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_b.biases": { + "data_offsets": [ + 5061167616, + 5061175296 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_b.scales": { + "data_offsets": [ + 2383571200, + 2383578880 + ], + "dtype": "BF16", + "shape": [ + 48, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_b.weight": { + "data_offsets": [ + 4727030784, + 4727153664 + ], + "dtype": "U32", + "shape": [ + 48, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_qkv.biases": { + "data_offsets": [ + 4709661440, + 4711299840 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_qkv.scales": { + "data_offsets": [ + 4276892544, + 4278530944 + ], + "dtype": "BF16", + "shape": [ + 10240, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_qkv.weight": { + "data_offsets": [ + 4666735552, + 4692949952 + ], + "dtype": "U32", + "shape": [ + 10240, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_z.biases": { + "data_offsets": [ + 5060184576, + 5061167616 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_z.scales": { + "data_offsets": [ + 930062016, + 931045056 + ], + "dtype": "BF16", + "shape": [ + 6144, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.in_proj_z.weight": { + "data_offsets": [ + 4711302144, + 4727030784 + ], + "dtype": "U32", + "shape": [ + 6144, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.norm.weight": { + "data_offsets": [ + 1639306080, + 1639306336 + ], + "dtype": "BF16", + "shape": [ + 128 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.out_proj.biases": { + "data_offsets": [ + 4152714528, + 4153697568 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.out_proj.scales": { + "data_offsets": [ + 3723273344, + 3724256384 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.linear_attn.out_proj.weight": { + "data_offsets": [ + 4777260224, + 4792988864 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.mlp.down_proj.biases": { + "data_offsets": [ + 3493484192, + 3496269472 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.mlp.down_proj.scales": { + "data_offsets": [ + 2266558336, + 2269343616 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.mlp.down_proj.weight": { + "data_offsets": [ + 4846097088, + 4890661568 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.mlp.gate_proj.biases": { + "data_offsets": [ + 4843302592, + 4846087872 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.mlp.gate_proj.scales": { + "data_offsets": [ + 4838548928, + 4841334208 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.mlp.gate_proj.weight": { + "data_offsets": [ + 4793984448, + 4838548928 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.mlp.up_proj.biases": { + "data_offsets": [ + 4908535808, + 4911321088 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.mlp.up_proj.scales": { + "data_offsets": [ + 1884945664, + 1887730944 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.mlp.up_proj.weight": { + "data_offsets": [ + 3954236512, + 3998800992 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.62.post_attention_layernorm.weight": { + "data_offsets": [ + 1010312960, + 1010323200 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.input_layernorm.weight": { + "data_offsets": [ + 4965879808, + 4965890048 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.mlp.down_proj.biases": { + "data_offsets": [ + 2277856896, + 2280642176 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.mlp.down_proj.scales": { + "data_offsets": [ + 1879455232, + 1882240512 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.mlp.down_proj.weight": { + "data_offsets": [ + 4727153664, + 4771718144 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.mlp.gate_proj.biases": { + "data_offsets": [ + 5010454528, + 5013239808 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.mlp.gate_proj.scales": { + "data_offsets": [ + 4016775712, + 4019560992 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.mlp.gate_proj.weight": { + "data_offsets": [ + 4965890048, + 5010454528 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.mlp.up_proj.biases": { + "data_offsets": [ + 5017665792, + 5020451072 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.mlp.up_proj.scales": { + "data_offsets": [ + 5013242112, + 5016027392 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.mlp.up_proj.weight": { + "data_offsets": [ + 733578816, + 778143296 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.post_attention_layernorm.weight": { + "data_offsets": [ + 3727485056, + 3727495296 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.k_norm.weight": { + "data_offsets": [ + 869850240, + 869850752 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.k_proj.biases": { + "data_offsets": [ + 4658092480, + 4658256320 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.k_proj.scales": { + "data_offsets": [ + 1347270176, + 1347434016 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.k_proj.weight": { + "data_offsets": [ + 2077245440, + 2079866880 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.o_proj.biases": { + "data_offsets": [ + 1565185312, + 1566168352 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.o_proj.scales": { + "data_offsets": [ + 4200975392, + 4201958432 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.o_proj.weight": { + "data_offsets": [ + 2482921792, + 2498650432 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.q_norm.weight": { + "data_offsets": [ + 4692949952, + 4692950464 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.q_proj.biases": { + "data_offsets": [ + 1640416096, + 1642382176 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.q_proj.scales": { + "data_offsets": [ + 4937535488, + 4939501568 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.q_proj.weight": { + "data_offsets": [ + 4321360064, + 4352817344 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.v_proj.biases": { + "data_offsets": [ + 4708679104, + 4708842944 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.v_proj.scales": { + "data_offsets": [ + 4965715968, + 4965879808 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.layers.63.self_attn.v_proj.weight": { + "data_offsets": [ + 861638720, + 864260160 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "language_model.model.norm.weight": { + "data_offsets": [ + 2407528704, + 2407538944 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.fc.biases": { + "data_offsets": [ + 114698240, + 116336640 + ], + "dtype": "BF16", + "shape": [ + 5120, + 160 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.fc.scales": { + "data_offsets": [ + 116336640, + 117975040 + ], + "dtype": "BF16", + "shape": [ + 5120, + 160 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.fc.weight": { + "data_offsets": [ + 4939501568, + 4965715968 + ], + "dtype": "U32", + "shape": [ + 5120, + 1280 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.input_layernorm.weight": { + "data_offsets": [ + 395871456, + 395881696 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.mlp.down_proj.biases": { + "data_offsets": [ + 297543872, + 300329152 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.mlp.down_proj.scales": { + "data_offsets": [ + 2795520, + 5580800 + ], + "dtype": "BF16", + "shape": [ + 5120, + 272 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.mlp.down_proj.weight": { + "data_offsets": [ + 5580800, + 50145280 + ], + "dtype": "U32", + "shape": [ + 5120, + 2176 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.mlp.gate_proj.biases": { + "data_offsets": [ + 4658256320, + 4661041600 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.mlp.gate_proj.scales": { + "data_offsets": [ + 4352817344, + 4355602624 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.mlp.gate_proj.weight": { + "data_offsets": [ + 50145280, + 94709760 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.mlp.up_proj.biases": { + "data_offsets": [ + 10240, + 2795520 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.mlp.up_proj.scales": { + "data_offsets": [ + 1570965792, + 1573751072 + ], + "dtype": "BF16", + "shape": [ + 17408, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.mlp.up_proj.weight": { + "data_offsets": [ + 2145932608, + 2190497088 + ], + "dtype": "U32", + "shape": [ + 17408, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.post_attention_layernorm.weight": { + "data_offsets": [ + 517241760, + 517252000 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.k_norm.weight": { + "data_offsets": [ + 3356348320, + 3356348832 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.k_proj.biases": { + "data_offsets": [ + 342233632, + 342397472 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.k_proj.scales": { + "data_offsets": [ + 110602240, + 110766080 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.k_proj.weight": { + "data_offsets": [ + 3349956256, + 3352577696 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.o_proj.biases": { + "data_offsets": [ + 3525315296, + 3526298336 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.o_proj.scales": { + "data_offsets": [ + 4019571232, + 4020554272 + ], + "dtype": "BF16", + "shape": [ + 5120, + 96 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.o_proj.weight": { + "data_offsets": [ + 94709760, + 110438400 + ], + "dtype": "U32", + "shape": [ + 5120, + 768 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.q_norm.weight": { + "data_offsets": [ + 5124052160, + 5124052672 + ], + "dtype": "BF16", + "shape": [ + 256 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.q_proj.biases": { + "data_offsets": [ + 110766080, + 112732160 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.q_proj.scales": { + "data_offsets": [ + 112732160, + 114698240 + ], + "dtype": "BF16", + "shape": [ + 12288, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.q_proj.weight": { + "data_offsets": [ + 251668736, + 283126016 + ], + "dtype": "U32", + "shape": [ + 12288, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.v_proj.biases": { + "data_offsets": [ + 2460009664, + 2460173504 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.v_proj.scales": { + "data_offsets": [ + 110438400, + 110602240 + ], + "dtype": "BF16", + "shape": [ + 1024, + 80 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.layers.0.self_attn.v_proj.weight": { + "data_offsets": [ + 4613257120, + 4615878560 + ], + "dtype": "U32", + "shape": [ + 1024, + 640 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.norm.weight": { + "data_offsets": [ + 0, + 10240 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.pre_fc_norm_embedding.weight": { + "data_offsets": [ + 120618304, + 120628544 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "mtp.pre_fc_norm_hidden.weight": { + "data_offsets": [ + 120630848, + 120641088 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.attn.proj.bias": { + "data_offsets": [ + 5123882368, + 5123884672 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.attn.proj.biases": { + "data_offsets": [ + 4319016640, + 4319058112 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.attn.proj.scales": { + "data_offsets": [ + 2464228864, + 2464270336 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.attn.proj.weight": { + "data_offsets": [ + 5123063872, + 5123727424 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.attn.qkv.bias": { + "data_offsets": [ + 917482176, + 917489088 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.attn.qkv.biases": { + "data_offsets": [ + 3998931744, + 3999056160 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.attn.qkv.scales": { + "data_offsets": [ + 5120154176, + 5120278592 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.attn.qkv.weight": { + "data_offsets": [ + 5118144672, + 5120135328 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.mlp.linear_fc1.bias": { + "data_offsets": [ + 3948343232, + 3948351840 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.mlp.linear_fc1.biases": { + "data_offsets": [ + 5123894912, + 5124049856 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.mlp.linear_fc1.scales": { + "data_offsets": [ + 4232160576, + 4232315520 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.mlp.linear_fc1.weight": { + "data_offsets": [ + 4620320512, + 4622799616 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.mlp.linear_fc2.bias": { + "data_offsets": [ + 5124049856, + 5124052160 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.mlp.linear_fc2.weight": { + "data_offsets": [ + 1014091520, + 1024007936 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.norm1.bias": { + "data_offsets": [ + 2479924800, + 2479927104 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.norm1.weight": { + "data_offsets": [ + 5118017952, + 5118020256 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.norm2.bias": { + "data_offsets": [ + 4615878560, + 4615880864 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.0.norm2.weight": { + "data_offsets": [ + 5013239808, + 5013242112 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.attn.proj.bias": { + "data_offsets": [ + 4774464768, + 4774467072 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.attn.proj.biases": { + "data_offsets": [ + 5100908800, + 5100950272 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.attn.proj.scales": { + "data_offsets": [ + 3356438432, + 3356479904 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.attn.proj.weight": { + "data_offsets": [ + 5124052672, + 5124716224 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.attn.qkv.bias": { + "data_offsets": [ + 3782793856, + 3782800768 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.attn.qkv.biases": { + "data_offsets": [ + 4453233088, + 4453357504 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.attn.qkv.scales": { + "data_offsets": [ + 1779831072, + 1779955488 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.attn.qkv.weight": { + "data_offsets": [ + 3520539360, + 3522530016 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.mlp.linear_fc1.bias": { + "data_offsets": [ + 5141593792, + 5141602400 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.mlp.linear_fc1.biases": { + "data_offsets": [ + 1086373696, + 1086528640 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.mlp.linear_fc1.scales": { + "data_offsets": [ + 4283633792, + 4283788736 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.mlp.linear_fc1.weight": { + "data_offsets": [ + 1903461888, + 1905940992 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.mlp.linear_fc2.bias": { + "data_offsets": [ + 5141643872, + 5141646176 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.mlp.linear_fc2.weight": { + "data_offsets": [ + 3772775040, + 3782691456 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.norm1.bias": { + "data_offsets": [ + 918489280, + 918491584 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.norm1.weight": { + "data_offsets": [ + 5061175296, + 5061177600 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.norm2.bias": { + "data_offsets": [ + 1779955488, + 1779957792 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.1.norm2.weight": { + "data_offsets": [ + 4319014336, + 4319016640 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.attn.proj.bias": { + "data_offsets": [ + 251624960, + 251627264 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.attn.proj.biases": { + "data_offsets": [ + 251627264, + 251668736 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.attn.proj.scales": { + "data_offsets": [ + 283280960, + 283322432 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.attn.proj.weight": { + "data_offsets": [ + 1479213344, + 1479876896 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.attn.qkv.bias": { + "data_offsets": [ + 350474656, + 350481568 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.attn.qkv.biases": { + "data_offsets": [ + 959274496, + 959398912 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.attn.qkv.scales": { + "data_offsets": [ + 395884000, + 396008416 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.attn.qkv.weight": { + "data_offsets": [ + 283322432, + 285313088 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.mlp.linear_fc1.bias": { + "data_offsets": [ + 251616352, + 251624960 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.mlp.linear_fc1.biases": { + "data_offsets": [ + 2190497088, + 2190652032 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.mlp.linear_fc1.scales": { + "data_offsets": [ + 1107817792, + 1107972736 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.mlp.linear_fc1.weight": { + "data_offsets": [ + 380479616, + 382958720 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.mlp.linear_fc2.bias": { + "data_offsets": [ + 241697632, + 241699936 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.mlp.linear_fc2.weight": { + "data_offsets": [ + 603068480, + 612984896 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.norm1.bias": { + "data_offsets": [ + 285324608, + 285326912 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.norm1.weight": { + "data_offsets": [ + 1735132192, + 1735134496 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.norm2.bias": { + "data_offsets": [ + 285313088, + 285315392 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.10.norm2.weight": { + "data_offsets": [ + 285315392, + 285317696 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.attn.proj.bias": { + "data_offsets": [ + 238746208, + 238748512 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.attn.proj.biases": { + "data_offsets": [ + 238872928, + 238914400 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.attn.proj.scales": { + "data_offsets": [ + 1079041440, + 1079082912 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.attn.proj.weight": { + "data_offsets": [ + 238914400, + 239577952 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.attn.qkv.bias": { + "data_offsets": [ + 1623560288, + 1623567200 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.attn.qkv.biases": { + "data_offsets": [ + 239577952, + 239702368 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.attn.qkv.scales": { + "data_offsets": [ + 347304992, + 347429408 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.attn.qkv.weight": { + "data_offsets": [ + 239702368, + 241693024 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.mlp.linear_fc1.bias": { + "data_offsets": [ + 2265549536, + 2265558144 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.mlp.linear_fc1.biases": { + "data_offsets": [ + 1876515008, + 1876669952 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.mlp.linear_fc1.scales": { + "data_offsets": [ + 238591264, + 238746208 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.mlp.linear_fc1.weight": { + "data_offsets": [ + 1082040640, + 1084519744 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.mlp.linear_fc2.bias": { + "data_offsets": [ + 1086528640, + 1086530944 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.mlp.linear_fc2.weight": { + "data_offsets": [ + 4220472352, + 4230388768 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.norm1.bias": { + "data_offsets": [ + 241695328, + 241697632 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.norm1.weight": { + "data_offsets": [ + 373014080, + 373016384 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.norm2.bias": { + "data_offsets": [ + 241693024, + 241695328 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.11.norm2.weight": { + "data_offsets": [ + 4276890240, + 4276892544 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.attn.proj.bias": { + "data_offsets": [ + 2143928320, + 2143930624 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.attn.proj.biases": { + "data_offsets": [ + 235759648, + 235801120 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.attn.proj.scales": { + "data_offsets": [ + 422439840, + 422481312 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.attn.proj.weight": { + "data_offsets": [ + 235801120, + 236464672 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.attn.qkv.bias": { + "data_offsets": [ + 236464672, + 236471584 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.attn.qkv.biases": { + "data_offsets": [ + 236471584, + 236596000 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.attn.qkv.scales": { + "data_offsets": [ + 418729248, + 418853664 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.attn.qkv.weight": { + "data_offsets": [ + 236596000, + 238586656 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.mlp.linear_fc1.bias": { + "data_offsets": [ + 5120135328, + 5120143936 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.mlp.linear_fc1.biases": { + "data_offsets": [ + 331005120, + 331160064 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.mlp.linear_fc1.scales": { + "data_offsets": [ + 3496277152, + 3496432096 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.mlp.linear_fc1.weight": { + "data_offsets": [ + 2455871840, + 2458350944 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.mlp.linear_fc2.bias": { + "data_offsets": [ + 235755040, + 235757344 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.mlp.linear_fc2.weight": { + "data_offsets": [ + 241699936, + 251616352 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.norm1.bias": { + "data_offsets": [ + 1347267872, + 1347270176 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.norm1.weight": { + "data_offsets": [ + 4497921984, + 4497924288 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.norm2.bias": { + "data_offsets": [ + 238586656, + 238588960 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.12.norm2.weight": { + "data_offsets": [ + 238588960, + 238591264 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.attn.proj.bias": { + "data_offsets": [ + 235529856, + 235532160 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.attn.proj.biases": { + "data_offsets": [ + 235532160, + 235573632 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.attn.proj.scales": { + "data_offsets": [ + 235582240, + 235623712 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.attn.proj.weight": { + "data_offsets": [ + 4615880864, + 4616544416 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.attn.qkv.bias": { + "data_offsets": [ + 1026948256, + 1026955168 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.attn.qkv.biases": { + "data_offsets": [ + 235623712, + 235748128 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.attn.qkv.scales": { + "data_offsets": [ + 3785596288, + 3785720704 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.attn.qkv.weight": { + "data_offsets": [ + 3629516416, + 3631507072 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.mlp.linear_fc1.bias": { + "data_offsets": [ + 4505288192, + 4505296800 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.mlp.linear_fc1.biases": { + "data_offsets": [ + 1026793312, + 1026948256 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.mlp.linear_fc1.scales": { + "data_offsets": [ + 4892652224, + 4892807168 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.mlp.linear_fc1.weight": { + "data_offsets": [ + 233050752, + 235529856 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.mlp.linear_fc2.bias": { + "data_offsets": [ + 223122816, + 223125120 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.mlp.linear_fc2.weight": { + "data_offsets": [ + 223134336, + 233050752 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.norm1.bias": { + "data_offsets": [ + 4841334208, + 4841336512 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.norm1.weight": { + "data_offsets": [ + 235750432, + 235752736 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.norm2.bias": { + "data_offsets": [ + 235748128, + 235750432 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.13.norm2.weight": { + "data_offsets": [ + 4846094784, + 4846097088 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.attn.proj.bias": { + "data_offsets": [ + 222897024, + 222899328 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.attn.proj.biases": { + "data_offsets": [ + 222899328, + 222940800 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.attn.proj.scales": { + "data_offsets": [ + 222940800, + 222982272 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.attn.proj.weight": { + "data_offsets": [ + 3720799872, + 3721463424 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.attn.qkv.bias": { + "data_offsets": [ + 222982272, + 222989184 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.attn.qkv.biases": { + "data_offsets": [ + 1640291680, + 1640416096 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.attn.qkv.scales": { + "data_offsets": [ + 5124716224, + 5124840640 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.attn.qkv.weight": { + "data_offsets": [ + 295398272, + 297388928 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.mlp.linear_fc1.bias": { + "data_offsets": [ + 396049888, + 396058496 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.mlp.linear_fc1.biases": { + "data_offsets": [ + 222742080, + 222897024 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.mlp.linear_fc1.scales": { + "data_offsets": [ + 312982720, + 313137664 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.mlp.linear_fc1.weight": { + "data_offsets": [ + 2074759424, + 2077238528 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.mlp.linear_fc2.bias": { + "data_offsets": [ + 3932609984, + 3932612288 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.mlp.linear_fc2.weight": { + "data_offsets": [ + 212825664, + 222742080 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.norm1.bias": { + "data_offsets": [ + 1690315456, + 1690317760 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.norm1.weight": { + "data_offsets": [ + 4792988864, + 4792991168 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.norm2.bias": { + "data_offsets": [ + 223115904, + 223118208 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.14.norm2.weight": { + "data_offsets": [ + 223120512, + 223122816 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.attn.proj.bias": { + "data_offsets": [ + 3352577696, + 3352580000 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.attn.proj.biases": { + "data_offsets": [ + 211406400, + 211447872 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.attn.proj.scales": { + "data_offsets": [ + 211447872, + 211489344 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.attn.proj.weight": { + "data_offsets": [ + 212152896, + 212816448 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.attn.qkv.bias": { + "data_offsets": [ + 4846087872, + 4846094784 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.attn.qkv.biases": { + "data_offsets": [ + 1735007776, + 1735132192 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.attn.qkv.scales": { + "data_offsets": [ + 238748512, + 238872928 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.attn.qkv.weight": { + "data_offsets": [ + 342399776, + 344390432 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.mlp.linear_fc1.bias": { + "data_offsets": [ + 1082032032, + 1082040640 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.mlp.linear_fc1.biases": { + "data_offsets": [ + 283126016, + 283280960 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.mlp.linear_fc1.scales": { + "data_offsets": [ + 872762752, + 872917696 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.mlp.linear_fc1.weight": { + "data_offsets": [ + 208927296, + 211406400 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.mlp.linear_fc2.bias": { + "data_offsets": [ + 199008576, + 199010880 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.mlp.linear_fc2.weight": { + "data_offsets": [ + 199010880, + 208927296 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.norm1.bias": { + "data_offsets": [ + 4230389312, + 4230391616 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.norm1.weight": { + "data_offsets": [ + 235757344, + 235759648 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.norm2.bias": { + "data_offsets": [ + 212816448, + 212818752 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.15.norm2.weight": { + "data_offsets": [ + 1824532512, + 1824534816 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.attn.proj.bias": { + "data_offsets": [ + 198748224, + 198750528 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.attn.proj.biases": { + "data_offsets": [ + 198833472, + 198874944 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.attn.proj.scales": { + "data_offsets": [ + 198792000, + 198833472 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.attn.proj.weight": { + "data_offsets": [ + 3727536768, + 3728200320 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.attn.qkv.bias": { + "data_offsets": [ + 198874944, + 198881856 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.attn.qkv.biases": { + "data_offsets": [ + 198881856, + 199006272 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.attn.qkv.scales": { + "data_offsets": [ + 2461156544, + 2461280960 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.attn.qkv.weight": { + "data_offsets": [ + 3332236960, + 3334227616 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.mlp.linear_fc1.bias": { + "data_offsets": [ + 198739616, + 198748224 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.mlp.linear_fc1.biases": { + "data_offsets": [ + 2464066240, + 2464221184 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.mlp.linear_fc1.scales": { + "data_offsets": [ + 285326912, + 285481856 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.mlp.linear_fc1.weight": { + "data_offsets": [ + 1645495648, + 1647974752 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.mlp.linear_fc2.bias": { + "data_offsets": [ + 960399872, + 960402176 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.mlp.linear_fc2.weight": { + "data_offsets": [ + 178904480, + 188820896 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.norm1.bias": { + "data_offsets": [ + 199006272, + 199008576 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.norm1.weight": { + "data_offsets": [ + 4153697568, + 4153699872 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.norm2.bias": { + "data_offsets": [ + 3948340928, + 3948343232 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.16.norm2.weight": { + "data_offsets": [ + 223132032, + 223134336 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.attn.proj.bias": { + "data_offsets": [ + 1690440992, + 1690443296 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.attn.proj.biases": { + "data_offsets": [ + 5141602400, + 5141643872 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.attn.proj.scales": { + "data_offsets": [ + 166165664, + 166207136 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.attn.proj.weight": { + "data_offsets": [ + 166207136, + 166870688 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.attn.qkv.bias": { + "data_offsets": [ + 212818752, + 212825664 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.attn.qkv.biases": { + "data_offsets": [ + 1876390592, + 1876515008 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.attn.qkv.scales": { + "data_offsets": [ + 176787104, + 176911520 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.attn.qkv.weight": { + "data_offsets": [ + 176911520, + 178902176 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.mlp.linear_fc1.bias": { + "data_offsets": [ + 166002112, + 166010720 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.mlp.linear_fc1.biases": { + "data_offsets": [ + 166010720, + 166165664 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.mlp.linear_fc1.scales": { + "data_offsets": [ + 1573966624, + 1574121568 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.mlp.linear_fc1.weight": { + "data_offsets": [ + 1953456896, + 1955936000 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.mlp.linear_fc2.bias": { + "data_offsets": [ + 601960256, + 601962560 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.mlp.linear_fc2.weight": { + "data_offsets": [ + 2299337088, + 2309253504 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.norm1.bias": { + "data_offsets": [ + 235752736, + 235755040 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.norm1.weight": { + "data_offsets": [ + 178902176, + 178904480 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.norm2.bias": { + "data_offsets": [ + 869847936, + 869850240 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.17.norm2.weight": { + "data_offsets": [ + 4711299840, + 4711302144 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.attn.proj.bias": { + "data_offsets": [ + 3954234208, + 3954236512 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.attn.proj.biases": { + "data_offsets": [ + 1882240512, + 1882281984 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.attn.proj.scales": { + "data_offsets": [ + 3727495296, + 3727536768 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.attn.proj.weight": { + "data_offsets": [ + 163304128, + 163967680 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.attn.qkv.bias": { + "data_offsets": [ + 285317696, + 285324608 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.attn.qkv.biases": { + "data_offsets": [ + 3830440128, + 3830564544 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.attn.qkv.scales": { + "data_offsets": [ + 3870551744, + 3870676160 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.attn.qkv.weight": { + "data_offsets": [ + 164009152, + 165999808 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.mlp.linear_fc1.bias": { + "data_offsets": [ + 163140576, + 163149184 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.mlp.linear_fc1.biases": { + "data_offsets": [ + 163149184, + 163304128 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.mlp.linear_fc1.scales": { + "data_offsets": [ + 2474186752, + 2474341696 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.mlp.linear_fc1.weight": { + "data_offsets": [ + 373721408, + 376200512 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.mlp.linear_fc2.bias": { + "data_offsets": [ + 163138272, + 163140576 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.mlp.linear_fc2.weight": { + "data_offsets": [ + 2464270336, + 2474186752 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.norm1.bias": { + "data_offsets": [ + 1468046368, + 1468048672 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.norm1.weight": { + "data_offsets": [ + 342397472, + 342399776 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.norm2.bias": { + "data_offsets": [ + 165999808, + 166002112 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.18.norm2.weight": { + "data_offsets": [ + 1566209824, + 1566212128 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.attn.proj.bias": { + "data_offsets": [ + 872760448, + 872762752 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.attn.proj.biases": { + "data_offsets": [ + 392886048, + 392927520 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.attn.proj.scales": { + "data_offsets": [ + 370452032, + 370493504 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.attn.proj.weight": { + "data_offsets": [ + 211489344, + 212152896 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.attn.qkv.bias": { + "data_offsets": [ + 160884960, + 160891872 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.attn.qkv.biases": { + "data_offsets": [ + 160891872, + 161016288 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.attn.qkv.scales": { + "data_offsets": [ + 161016288, + 161140704 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.attn.qkv.weight": { + "data_offsets": [ + 161140704, + 163131360 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.mlp.linear_fc1.bias": { + "data_offsets": [ + 864260160, + 864268768 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.mlp.linear_fc1.biases": { + "data_offsets": [ + 4709506496, + 4709661440 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.mlp.linear_fc1.scales": { + "data_offsets": [ + 3582011712, + 3582166656 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.mlp.linear_fc1.weight": { + "data_offsets": [ + 1464584224, + 1467063328 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.mlp.linear_fc2.bias": { + "data_offsets": [ + 380475008, + 380477312 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.mlp.linear_fc2.weight": { + "data_offsets": [ + 778145600, + 788062016 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.norm1.bias": { + "data_offsets": [ + 163133664, + 163135968 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.norm1.weight": { + "data_offsets": [ + 163135968, + 163138272 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.norm2.bias": { + "data_offsets": [ + 4649565792, + 4649568096 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.19.norm2.weight": { + "data_offsets": [ + 163131360, + 163133664 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.attn.proj.bias": { + "data_offsets": [ + 421642656, + 421644960 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.attn.proj.biases": { + "data_offsets": [ + 1084519744, + 1084561216 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.attn.proj.scales": { + "data_offsets": [ + 1199085696, + 1199127168 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.attn.proj.weight": { + "data_offsets": [ + 421644960, + 422308512 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.attn.qkv.bias": { + "data_offsets": [ + 422308512, + 422315424 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.attn.qkv.biases": { + "data_offsets": [ + 422315424, + 422439840 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.attn.qkv.scales": { + "data_offsets": [ + 422481312, + 422605728 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.attn.qkv.weight": { + "data_offsets": [ + 4890661568, + 4892652224 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.mlp.linear_fc1.bias": { + "data_offsets": [ + 418720640, + 418729248 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.mlp.linear_fc1.biases": { + "data_offsets": [ + 418853664, + 419008608 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.mlp.linear_fc1.scales": { + "data_offsets": [ + 419008608, + 419163552 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.mlp.linear_fc1.weight": { + "data_offsets": [ + 419163552, + 421642656 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.mlp.linear_fc2.bias": { + "data_offsets": [ + 408801920, + 408804224 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.mlp.linear_fc2.weight": { + "data_offsets": [ + 408804224, + 418720640 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.norm1.bias": { + "data_offsets": [ + 5144127584, + 5144129888 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.norm1.weight": { + "data_offsets": [ + 5144125280, + 5144127584 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.norm2.bias": { + "data_offsets": [ + 4507401120, + 4507403424 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.2.norm2.weight": { + "data_offsets": [ + 5144129888, + 5144132192 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.attn.proj.bias": { + "data_offsets": [ + 158228448, + 158230752 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.attn.proj.biases": { + "data_offsets": [ + 4661041600, + 4661083072 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.attn.proj.scales": { + "data_offsets": [ + 198750528, + 198792000 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.attn.proj.weight": { + "data_offsets": [ + 158230752, + 158894304 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.attn.qkv.bias": { + "data_offsets": [ + 2299330176, + 2299337088 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.attn.qkv.biases": { + "data_offsets": [ + 222991488, + 223115904 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.attn.qkv.scales": { + "data_offsets": [ + 2144169792, + 2144294208 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.attn.qkv.weight": { + "data_offsets": [ + 158894304, + 160884960 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.mlp.linear_fc1.bias": { + "data_offsets": [ + 1299909504, + 1299918112 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.mlp.linear_fc1.biases": { + "data_offsets": [ + 157916256, + 158071200 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.mlp.linear_fc1.scales": { + "data_offsets": [ + 158071200, + 158226144 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.mlp.linear_fc1.weight": { + "data_offsets": [ + 316119584, + 318598688 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.mlp.linear_fc2.bias": { + "data_offsets": [ + 2480134208, + 2480136512 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.mlp.linear_fc2.weight": { + "data_offsets": [ + 147999840, + 157916256 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.norm1.bias": { + "data_offsets": [ + 223118208, + 223120512 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.norm1.weight": { + "data_offsets": [ + 1640289376, + 1640291680 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.norm2.bias": { + "data_offsets": [ + 4771718144, + 4771720448 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.20.norm2.weight": { + "data_offsets": [ + 778143296, + 778145600 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.attn.proj.bias": { + "data_offsets": [ + 3727475072, + 3727477376 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.attn.proj.biases": { + "data_offsets": [ + 143020896, + 143062368 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.attn.proj.scales": { + "data_offsets": [ + 143062368, + 143103840 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.attn.proj.weight": { + "data_offsets": [ + 469965728, + 470629280 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.attn.qkv.bias": { + "data_offsets": [ + 223125120, + 223132032 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.attn.qkv.biases": { + "data_offsets": [ + 143767392, + 143891808 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.attn.qkv.scales": { + "data_offsets": [ + 143891808, + 144016224 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.attn.qkv.weight": { + "data_offsets": [ + 146006880, + 147997536 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.mlp.linear_fc1.bias": { + "data_offsets": [ + 142693792, + 142702400 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.mlp.linear_fc1.biases": { + "data_offsets": [ + 142702400, + 142857344 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.mlp.linear_fc1.scales": { + "data_offsets": [ + 142865952, + 143020896 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.mlp.linear_fc1.weight": { + "data_offsets": [ + 1005048576, + 1007527680 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.mlp.linear_fc2.bias": { + "data_offsets": [ + 3886404800, + 3886407104 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.mlp.linear_fc2.weight": { + "data_offsets": [ + 166870688, + 176787104 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.norm1.bias": { + "data_offsets": [ + 147997536, + 147999840 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.norm1.weight": { + "data_offsets": [ + 3282788928, + 3282791232 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.norm2.bias": { + "data_offsets": [ + 313463072, + 313465376 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.21.norm2.weight": { + "data_offsets": [ + 5060182272, + 5060184576 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.attn.proj.bias": { + "data_offsets": [ + 141735328, + 141737632 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.attn.proj.biases": { + "data_offsets": [ + 3727433600, + 3727475072 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.attn.proj.scales": { + "data_offsets": [ + 141737632, + 141779104 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.attn.proj.weight": { + "data_offsets": [ + 141779104, + 142442656 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.attn.qkv.bias": { + "data_offsets": [ + 313456160, + 313463072 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.attn.qkv.biases": { + "data_offsets": [ + 142442656, + 142567072 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.attn.qkv.scales": { + "data_offsets": [ + 142567072, + 142691488 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.attn.qkv.weight": { + "data_offsets": [ + 378482048, + 380472704 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.mlp.linear_fc1.bias": { + "data_offsets": [ + 142857344, + 142865952 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.mlp.linear_fc1.biases": { + "data_offsets": [ + 2144014848, + 2144169792 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.mlp.linear_fc1.scales": { + "data_offsets": [ + 139101280, + 139256224 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.mlp.linear_fc1.weight": { + "data_offsets": [ + 139256224, + 141735328 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.mlp.linear_fc2.bias": { + "data_offsets": [ + 129182560, + 129184864 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.mlp.linear_fc2.weight": { + "data_offsets": [ + 129184864, + 139101280 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.norm1.bias": { + "data_offsets": [ + 142691488, + 142693792 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.norm1.weight": { + "data_offsets": [ + 222989184, + 222991488 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.norm2.bias": { + "data_offsets": [ + 1903459584, + 1903461888 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.22.norm2.weight": { + "data_offsets": [ + 4023339552, + 4023341856 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.attn.proj.bias": { + "data_offsets": [ + 1463598880, + 1463601184 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.attn.proj.biases": { + "data_offsets": [ + 347256608, + 347298080 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.attn.proj.scales": { + "data_offsets": [ + 1566168352, + 1566209824 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.attn.proj.weight": { + "data_offsets": [ + 4708842944, + 4709506496 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.attn.qkv.bias": { + "data_offsets": [ + 2266551424, + 2266558336 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.attn.qkv.biases": { + "data_offsets": [ + 1573842208, + 1573966624 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.attn.qkv.scales": { + "data_offsets": [ + 872636032, + 872760448 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.attn.qkv.weight": { + "data_offsets": [ + 127189600, + 129180256 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.mlp.linear_fc1.bias": { + "data_offsets": [ + 235573632, + 235582240 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.mlp.linear_fc1.biases": { + "data_offsets": [ + 124555552, + 124710496 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.mlp.linear_fc1.scales": { + "data_offsets": [ + 5123727424, + 5123882368 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.mlp.linear_fc1.weight": { + "data_offsets": [ + 124710496, + 127189600 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.mlp.linear_fc2.bias": { + "data_offsets": [ + 1908726272, + 1908728576 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.mlp.linear_fc2.weight": { + "data_offsets": [ + 918491840, + 928408256 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.norm1.bias": { + "data_offsets": [ + 198737312, + 198739616 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.norm1.weight": { + "data_offsets": [ + 129180256, + 129182560 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.norm2.bias": { + "data_offsets": [ + 4094120736, + 4094123040 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.23.norm2.weight": { + "data_offsets": [ + 2144012544, + 2144014848 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.attn.proj.bias": { + "data_offsets": [ + 121763104, + 121765408 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.attn.proj.biases": { + "data_offsets": [ + 163967680, + 164009152 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.attn.proj.scales": { + "data_offsets": [ + 347212832, + 347254304 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.attn.proj.weight": { + "data_offsets": [ + 121765408, + 122428960 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.attn.qkv.bias": { + "data_offsets": [ + 3332230048, + 3332236960 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.attn.qkv.biases": { + "data_offsets": [ + 959150080, + 959274496 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.attn.qkv.scales": { + "data_offsets": [ + 122431264, + 122555680 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.attn.qkv.weight": { + "data_offsets": [ + 122555680, + 124546336 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.mlp.linear_fc1.bias": { + "data_offsets": [ + 121754496, + 121763104 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.mlp.linear_fc1.biases": { + "data_offsets": [ + 1876235648, + 1876390592 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.mlp.linear_fc1.scales": { + "data_offsets": [ + 932698816, + 932853760 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.mlp.linear_fc1.weight": { + "data_offsets": [ + 5141646176, + 5144125280 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.mlp.linear_fc2.bias": { + "data_offsets": [ + 121752192, + 121754496 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.mlp.linear_fc2.weight": { + "data_offsets": [ + 4622799712, + 4632716128 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.norm1.bias": { + "data_offsets": [ + 124550944, + 124553248 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.norm1.weight": { + "data_offsets": [ + 124553248, + 124555552 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.norm2.bias": { + "data_offsets": [ + 124546336, + 124548640 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.24.norm2.weight": { + "data_offsets": [ + 124548640, + 124550944 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.attn.proj.bias": { + "data_offsets": [ + 120796032, + 120798336 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.attn.proj.biases": { + "data_offsets": [ + 1364579104, + 1364620576 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.attn.proj.scales": { + "data_offsets": [ + 120798336, + 120839808 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.attn.proj.weight": { + "data_offsets": [ + 120839808, + 121503360 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.attn.qkv.bias": { + "data_offsets": [ + 1088497120, + 1088504032 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.attn.qkv.biases": { + "data_offsets": [ + 121503360, + 121627776 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.attn.qkv.scales": { + "data_offsets": [ + 121627776, + 121752192 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.attn.qkv.weight": { + "data_offsets": [ + 3948351840, + 3950342496 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.mlp.linear_fc1.bias": { + "data_offsets": [ + 869839328, + 869847936 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.mlp.linear_fc1.biases": { + "data_offsets": [ + 120641088, + 120796032 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.mlp.linear_fc1.scales": { + "data_offsets": [ + 393237408, + 393392352 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.mlp.linear_fc1.weight": { + "data_offsets": [ + 1685051072, + 1687530176 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.mlp.linear_fc2.bias": { + "data_offsets": [ + 120628544, + 120630848 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.mlp.linear_fc2.weight": { + "data_offsets": [ + 592036160, + 601952576 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.norm1.bias": { + "data_offsets": [ + 2283591552, + 2283593856 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.norm1.weight": { + "data_offsets": [ + 1297121920, + 1297124224 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.norm2.bias": { + "data_offsets": [ + 122428960, + 122431264 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.25.norm2.weight": { + "data_offsets": [ + 347479488, + 347481792 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.attn.proj.bias": { + "data_offsets": [ + 316114976, + 316117280 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.attn.proj.biases": { + "data_offsets": [ + 119611456, + 119652928 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.attn.proj.scales": { + "data_offsets": [ + 119652928, + 119694400 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.attn.proj.weight": { + "data_offsets": [ + 119694400, + 120357952 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.attn.qkv.bias": { + "data_offsets": [ + 120357952, + 120364864 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.attn.qkv.biases": { + "data_offsets": [ + 120364864, + 120489280 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.attn.qkv.scales": { + "data_offsets": [ + 120489280, + 120613696 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.attn.qkv.weight": { + "data_offsets": [ + 144016224, + 146006880 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.mlp.linear_fc1.bias": { + "data_offsets": [ + 348350976, + 348359584 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.mlp.linear_fc1.biases": { + "data_offsets": [ + 119456512, + 119611456 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.mlp.linear_fc1.scales": { + "data_offsets": [ + 348189120, + 348344064 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.mlp.linear_fc1.weight": { + "data_offsets": [ + 2402092800, + 2404571904 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.mlp.linear_fc2.bias": { + "data_offsets": [ + 119454208, + 119456512 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.mlp.linear_fc2.weight": { + "data_offsets": [ + 188820896, + 198737312 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.norm1.bias": { + "data_offsets": [ + 120616000, + 120618304 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.norm1.weight": { + "data_offsets": [ + 158226144, + 158228448 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.norm2.bias": { + "data_offsets": [ + 347302688, + 347304992 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.26.norm2.weight": { + "data_offsets": [ + 120613696, + 120616000 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.attn.proj.bias": { + "data_offsets": [ + 395881696, + 395884000 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.attn.proj.biases": { + "data_offsets": [ + 5124840640, + 5124882112 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.attn.proj.scales": { + "data_offsets": [ + 396008416, + 396049888 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.attn.proj.weight": { + "data_offsets": [ + 3356479904, + 3357143456 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.attn.qkv.bias": { + "data_offsets": [ + 396058496, + 396065408 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.attn.qkv.biases": { + "data_offsets": [ + 406682240, + 406806656 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.attn.qkv.scales": { + "data_offsets": [ + 1347434016, + 1347558432 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.attn.qkv.weight": { + "data_offsets": [ + 406806656, + 408797312 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.mlp.linear_fc1.bias": { + "data_offsets": [ + 392877440, + 392886048 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.mlp.linear_fc1.biases": { + "data_offsets": [ + 392927520, + 393082464 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.mlp.linear_fc1.scales": { + "data_offsets": [ + 393082464, + 393237408 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.mlp.linear_fc1.weight": { + "data_offsets": [ + 393392352, + 395871456 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.mlp.linear_fc2.bias": { + "data_offsets": [ + 382958720, + 382961024 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.mlp.linear_fc2.weight": { + "data_offsets": [ + 382961024, + 392877440 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.norm1.bias": { + "data_offsets": [ + 408797312, + 408799616 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.norm1.weight": { + "data_offsets": [ + 408799616, + 408801920 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.norm2.bias": { + "data_offsets": [ + 2458361280, + 2458363584 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.3.norm2.weight": { + "data_offsets": [ + 4281010048, + 4281012352 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.attn.proj.bias": { + "data_offsets": [ + 3514956256, + 3514958560 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.attn.proj.biases": { + "data_offsets": [ + 372972608, + 373014080 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.attn.proj.scales": { + "data_offsets": [ + 373016384, + 373057856 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.attn.proj.weight": { + "data_offsets": [ + 373057856, + 373721408 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.attn.qkv.bias": { + "data_offsets": [ + 376200512, + 376207424 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.attn.qkv.biases": { + "data_offsets": [ + 376207424, + 376331840 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.attn.qkv.scales": { + "data_offsets": [ + 1735134496, + 1735258912 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.attn.qkv.weight": { + "data_offsets": [ + 376489088, + 378479744 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.mlp.linear_fc1.bias": { + "data_offsets": [ + 370443424, + 370452032 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.mlp.linear_fc1.biases": { + "data_offsets": [ + 376334144, + 376489088 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.mlp.linear_fc1.scales": { + "data_offsets": [ + 4500709568, + 4500864512 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.mlp.linear_fc1.weight": { + "data_offsets": [ + 370493504, + 372972608 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.mlp.linear_fc2.bias": { + "data_offsets": [ + 2476963136, + 2476965440 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.mlp.linear_fc2.weight": { + "data_offsets": [ + 360527008, + 370443424 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.norm1.bias": { + "data_offsets": [ + 3517744096, + 3517746400 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.norm1.weight": { + "data_offsets": [ + 380477312, + 380479616 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.norm2.bias": { + "data_offsets": [ + 378479744, + 378482048 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.4.norm2.weight": { + "data_offsets": [ + 380472704, + 380475008 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.attn.proj.bias": { + "data_offsets": [ + 835411776, + 835414080 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.attn.proj.biases": { + "data_offsets": [ + 347438016, + 347479488 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.attn.proj.scales": { + "data_offsets": [ + 347481792, + 347523264 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.attn.proj.weight": { + "data_offsets": [ + 347525568, + 348189120 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.attn.qkv.bias": { + "data_offsets": [ + 348344064, + 348350976 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.attn.qkv.biases": { + "data_offsets": [ + 348359584, + 348484000 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.attn.qkv.scales": { + "data_offsets": [ + 1071519648, + 1071644064 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.attn.qkv.weight": { + "data_offsets": [ + 348484000, + 350474656 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.mlp.linear_fc1.bias": { + "data_offsets": [ + 347429408, + 347438016 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.mlp.linear_fc1.biases": { + "data_offsets": [ + 588850176, + 589005120 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.mlp.linear_fc1.scales": { + "data_offsets": [ + 2383416256, + 2383571200 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.mlp.linear_fc1.weight": { + "data_offsets": [ + 1105215808, + 1107694912 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.mlp.linear_fc2.bias": { + "data_offsets": [ + 347300384, + 347302688 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.mlp.linear_fc2.weight": { + "data_offsets": [ + 350481568, + 360397984 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.norm1.bias": { + "data_offsets": [ + 1348676640, + 1348678944 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.norm1.weight": { + "data_offsets": [ + 4102478880, + 4102481184 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.norm2.bias": { + "data_offsets": [ + 360397984, + 360400288 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.5.norm2.weight": { + "data_offsets": [ + 360524704, + 360527008 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.attn.proj.bias": { + "data_offsets": [ + 342231328, + 342233632 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.attn.proj.biases": { + "data_offsets": [ + 344390432, + 344431904 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.attn.proj.scales": { + "data_offsets": [ + 1682224320, + 1682265792 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.attn.proj.weight": { + "data_offsets": [ + 344431904, + 345095456 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.attn.qkv.bias": { + "data_offsets": [ + 1650760032, + 1650766944 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.attn.qkv.biases": { + "data_offsets": [ + 1348541984, + 1348666400 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.attn.qkv.scales": { + "data_offsets": [ + 345095456, + 345219872 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.attn.qkv.weight": { + "data_offsets": [ + 345219872, + 347210528 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.mlp.linear_fc1.bias": { + "data_offsets": [ + 341912832, + 341921440 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.mlp.linear_fc1.biases": { + "data_offsets": [ + 341921440, + 342076384 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.mlp.linear_fc1.scales": { + "data_offsets": [ + 342076384, + 342231328 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.mlp.linear_fc1.weight": { + "data_offsets": [ + 1487495456, + 1489974560 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.mlp.linear_fc2.bias": { + "data_offsets": [ + 331994112, + 331996416 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.mlp.linear_fc2.weight": { + "data_offsets": [ + 331996416, + 341912832 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.norm1.bias": { + "data_offsets": [ + 347254304, + 347256608 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.norm1.weight": { + "data_offsets": [ + 347298080, + 347300384 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.norm2.bias": { + "data_offsets": [ + 347210528, + 347212832 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.6.norm2.weight": { + "data_offsets": [ + 2213919712, + 2213922016 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.attn.proj.bias": { + "data_offsets": [ + 331160064, + 331162368 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.attn.proj.biases": { + "data_offsets": [ + 4511296928, + 4511338400 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.attn.proj.scales": { + "data_offsets": [ + 331162368, + 331203840 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.attn.proj.weight": { + "data_offsets": [ + 331203840, + 331867392 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.attn.qkv.bias": { + "data_offsets": [ + 2077238528, + 2077245440 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.attn.qkv.biases": { + "data_offsets": [ + 360400288, + 360524704 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.attn.qkv.scales": { + "data_offsets": [ + 331867392, + 331991808 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.attn.qkv.weight": { + "data_offsets": [ + 3999056160, + 4001046816 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.mlp.linear_fc1.bias": { + "data_offsets": [ + 328517408, + 328526016 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.mlp.linear_fc1.biases": { + "data_offsets": [ + 2270326656, + 2270481600 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.mlp.linear_fc1.scales": { + "data_offsets": [ + 3785720704, + 3785875648 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.mlp.linear_fc1.weight": { + "data_offsets": [ + 328526016, + 331005120 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.mlp.linear_fc2.bias": { + "data_offsets": [ + 4649568096, + 4649570400 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.mlp.linear_fc2.weight": { + "data_offsets": [ + 318600992, + 328517408 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.norm1.bias": { + "data_offsets": [ + 2276871552, + 2276873856 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.norm1.weight": { + "data_offsets": [ + 331991808, + 331994112 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.norm2.bias": { + "data_offsets": [ + 4649570400, + 4649572704 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.7.norm2.weight": { + "data_offsets": [ + 1568178208, + 1568180512 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.attn.proj.bias": { + "data_offsets": [ + 313465376, + 313467680 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.attn.proj.biases": { + "data_offsets": [ + 1882282752, + 1882324224 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.attn.proj.scales": { + "data_offsets": [ + 315946784, + 315988256 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.attn.proj.weight": { + "data_offsets": [ + 1364620576, + 1365284128 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.attn.qkv.bias": { + "data_offsets": [ + 917489088, + 917496000 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.attn.qkv.biases": { + "data_offsets": [ + 2382301120, + 2382425536 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.attn.qkv.scales": { + "data_offsets": [ + 1828474656, + 1828599072 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.attn.qkv.weight": { + "data_offsets": [ + 1074265504, + 1076256160 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.mlp.linear_fc1.bias": { + "data_offsets": [ + 313137664, + 313146272 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.mlp.linear_fc1.biases": { + "data_offsets": [ + 313146272, + 313301216 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.mlp.linear_fc1.scales": { + "data_offsets": [ + 313301216, + 313456160 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.mlp.linear_fc1.weight": { + "data_offsets": [ + 313467680, + 315946784 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.mlp.linear_fc2.bias": { + "data_offsets": [ + 4511294624, + 4511296928 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.mlp.linear_fc2.weight": { + "data_offsets": [ + 303066304, + 312982720 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.norm1.bias": { + "data_offsets": [ + 2190652032, + 2190654336 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.norm1.weight": { + "data_offsets": [ + 318598688, + 318600992 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.norm2.bias": { + "data_offsets": [ + 316112672, + 316114976 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.8.norm2.weight": { + "data_offsets": [ + 316117280, + 316119584 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.attn.proj.bias": { + "data_offsets": [ + 347523264, + 347525568 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.attn.proj.biases": { + "data_offsets": [ + 300329152, + 300370624 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.attn.proj.scales": { + "data_offsets": [ + 300370624, + 300412096 + ], + "dtype": "BF16", + "shape": [ + 1152, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.attn.proj.weight": { + "data_offsets": [ + 300412096, + 301075648 + ], + "dtype": "U32", + "shape": [ + 1152, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.attn.qkv.bias": { + "data_offsets": [ + 2381023104, + 2381030016 + ], + "dtype": "BF16", + "shape": [ + 3456 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.attn.qkv.biases": { + "data_offsets": [ + 315988256, + 316112672 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.attn.qkv.scales": { + "data_offsets": [ + 5118020256, + 5118144672 + ], + "dtype": "BF16", + "shape": [ + 3456, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.attn.qkv.weight": { + "data_offsets": [ + 301075648, + 303066304 + ], + "dtype": "U32", + "shape": [ + 3456, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.mlp.linear_fc1.bias": { + "data_offsets": [ + 5116285696, + 5116294304 + ], + "dtype": "BF16", + "shape": [ + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.mlp.linear_fc1.biases": { + "data_offsets": [ + 297388928, + 297543872 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.mlp.linear_fc1.scales": { + "data_offsets": [ + 2382135936, + 2382290880 + ], + "dtype": "BF16", + "shape": [ + 4304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.mlp.linear_fc1.weight": { + "data_offsets": [ + 4278530944, + 4281010048 + ], + "dtype": "U32", + "shape": [ + 4304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.mlp.linear_fc2.bias": { + "data_offsets": [ + 4358398144, + 4358400448 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.mlp.linear_fc2.weight": { + "data_offsets": [ + 285481856, + 295398272 + ], + "dtype": "BF16", + "shape": [ + 1152, + 4304 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.norm1.bias": { + "data_offsets": [ + 376331840, + 376334144 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.norm1.weight": { + "data_offsets": [ + 4632716128, + 4632718432 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.norm2.bias": { + "data_offsets": [ + 4632718688, + 4632720992 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.blocks.9.norm2.weight": { + "data_offsets": [ + 2479927104, + 2479929408 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.linear_fc1.bias": { + "data_offsets": [ + 1573751072, + 1573760288 + ], + "dtype": "BF16", + "shape": [ + 4608 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.linear_fc1.biases": { + "data_offsets": [ + 1563538720, + 1564202272 + ], + "dtype": "BF16", + "shape": [ + 4608, + 72 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.linear_fc1.scales": { + "data_offsets": [ + 143103840, + 143767392 + ], + "dtype": "BF16", + "shape": [ + 4608, + 72 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.linear_fc1.weight": { + "data_offsets": [ + 396065408, + 406682240 + ], + "dtype": "U32", + "shape": [ + 4608, + 576 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.linear_fc2.bias": { + "data_offsets": [ + 5123884672, + 5123894912 + ], + "dtype": "BF16", + "shape": [ + 5120 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.linear_fc2.biases": { + "data_offsets": [ + 117975040, + 118712320 + ], + "dtype": "BF16", + "shape": [ + 5120, + 72 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.linear_fc2.scales": { + "data_offsets": [ + 118712320, + 119449600 + ], + "dtype": "BF16", + "shape": [ + 5120, + 72 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.linear_fc2.weight": { + "data_offsets": [ + 5104489216, + 5116285696 + ], + "dtype": "U32", + "shape": [ + 5120, + 576 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.norm.bias": { + "data_offsets": [ + 119449600, + 119451904 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.merger.norm.weight": { + "data_offsets": [ + 119451904, + 119454208 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.patch_embed.proj.bias": { + "data_offsets": [ + 5116294304, + 5116296608 + ], + "dtype": "BF16", + "shape": [ + 1152 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.patch_embed.proj.weight": { + "data_offsets": [ + 5100950272, + 5104489216 + ], + "dtype": "BF16", + "shape": [ + 1152, + 2, + 16, + 16, + 3 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.pos_embed.biases": { + "data_offsets": [ + 5116296608, + 5116379552 + ], + "dtype": "BF16", + "shape": [ + 2304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.pos_embed.scales": { + "data_offsets": [ + 4149838624, + 4149921568 + ], + "dtype": "BF16", + "shape": [ + 2304, + 18 + ], + "shard": "model-00003-of-00003.safetensors" + }, + "visual.pos_embed.weight": { + "data_offsets": [ + 2047865600, + 2049192704 + ], + "dtype": "U32", + "shape": [ + 2304, + 144 + ], + "shard": "model-00003-of-00003.safetensors" + } +}