{ "draft": [ { "bytes": 34324480, "path": "draft/layer-0.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "attention_convolution", "offset": 32768, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "attention_dynamic", "offset": 49152, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "qkv", "offset": 638976, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "bytes": 256, "dtype": "bf16", "name": "query_norm", "offset": 7716864, "shape": [ 128 ] }, { "bytes": 256, "dtype": "bf16", "name": "key_norm", "offset": 7733248, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 7749632, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 12468224, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "mlp_convolution", "offset": 12484608, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "mlp_dynamic", "offset": 12500992, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_gate_proj", "offset": 13090816, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_up_proj", "offset": 20168704, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_down_proj", "offset": 27246592, "shape": [ 2048, 6144 ], "storage_n": 256 } ] }, { "bytes": 34324480, "path": "draft/layer-1.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "attention_convolution", "offset": 32768, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "attention_dynamic", "offset": 49152, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "qkv", "offset": 638976, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "bytes": 256, "dtype": "bf16", "name": "query_norm", "offset": 7716864, "shape": [ 128 ] }, { "bytes": 256, "dtype": "bf16", "name": "key_norm", "offset": 7733248, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 7749632, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 12468224, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "mlp_convolution", "offset": 12484608, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "mlp_dynamic", "offset": 12500992, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_gate_proj", "offset": 13090816, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_up_proj", "offset": 20168704, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_down_proj", "offset": 27246592, "shape": [ 2048, 6144 ], "storage_n": 256 } ] }, { "bytes": 34324480, "path": "draft/layer-2.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "attention_convolution", "offset": 32768, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "attention_dynamic", "offset": 49152, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "qkv", "offset": 638976, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "bytes": 256, "dtype": "bf16", "name": "query_norm", "offset": 7716864, "shape": [ 128 ] }, { "bytes": 256, "dtype": "bf16", "name": "key_norm", "offset": 7733248, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 7749632, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 12468224, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "mlp_convolution", "offset": 12484608, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "mlp_dynamic", "offset": 12500992, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_gate_proj", "offset": 13090816, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_up_proj", "offset": 20168704, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_down_proj", "offset": 27246592, "shape": [ 2048, 6144 ], "storage_n": 256 } ] }, { "bytes": 34324480, "path": "draft/layer-3.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "attention_convolution", "offset": 32768, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "attention_dynamic", "offset": 49152, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "qkv", "offset": 638976, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "bytes": 256, "dtype": "bf16", "name": "query_norm", "offset": 7716864, "shape": [ 128 ] }, { "bytes": 256, "dtype": "bf16", "name": "key_norm", "offset": 7733248, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 7749632, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 12468224, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "mlp_convolution", "offset": 12484608, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "mlp_dynamic", "offset": 12500992, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_gate_proj", "offset": 13090816, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_up_proj", "offset": 20168704, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_down_proj", "offset": 27246592, "shape": [ 2048, 6144 ], "storage_n": 256 } ] }, { "bytes": 34324480, "path": "draft/layer-4.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "attention_convolution", "offset": 32768, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "attention_dynamic", "offset": 49152, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "qkv", "offset": 638976, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "bytes": 256, "dtype": "bf16", "name": "query_norm", "offset": 7716864, "shape": [ 128 ] }, { "bytes": 256, "dtype": "bf16", "name": "key_norm", "offset": 7733248, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 7749632, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 12468224, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "mlp_convolution", "offset": 12484608, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "mlp_dynamic", "offset": 12500992, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_gate_proj", "offset": 13090816, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_up_proj", "offset": 20168704, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_down_proj", "offset": 27246592, "shape": [ 2048, 6144 ], "storage_n": 256 } ] }, { "bytes": 34324480, "path": "draft/layer-5.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "attention_convolution", "offset": 32768, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "attention_dynamic", "offset": 49152, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "qkv", "offset": 638976, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "bytes": 256, "dtype": "bf16", "name": "query_norm", "offset": 7716864, "shape": [ 128 ] }, { "bytes": 256, "dtype": "bf16", "name": "key_norm", "offset": 7733248, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 7749632, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 12468224, "shape": [ 2048 ] }, { "bytes": 16384, "dtype": "bf16", "name": "mlp_convolution", "offset": 12484608, "shape": [ 2, 2, 2048 ] }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "mlp_dynamic", "offset": 12500992, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_gate_proj", "offset": 13090816, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 6144, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_up_proj", "offset": 20168704, "shape": [ 6144, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 7077888, "dtype": "q4_affine", "group_size": 64, "name": "mlp_down_proj", "offset": 27246592, "shape": [ 2048, 6144 ], "storage_n": 256 } ] }, { "bytes": 273498112, "path": "draft/model.bin", "sections": [ { "actual_output": 2048, "bytes": 18874368, "dtype": "q4_affine", "group_size": 64, "name": "context_projection", "offset": 16384, "shape": [ 2048, 16384 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "hidden_norm", "offset": 18890752, "shape": [ 2048 ] }, { "bytes": 4096, "dtype": "bf16", "name": "final_norm", "offset": 18907136, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 294912, "dtype": "q4_affine", "group_size": 64, "name": "selector_projection", "offset": 18923520, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 127139840, "dtype": "bf16", "name": "predecessor_codebook", "offset": 19218432, "shape": [ 248320, 256 ] }, { "bytes": 127139840, "dtype": "bf16", "name": "successor_codebook", "offset": 146358272, "shape": [ 248320, 256 ] } ] } ], "target": [ { "bytes": 475201536, "path": "target/layer-0.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-1.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-2.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-3.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-4.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-5.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-6.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-7.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-8.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-9.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-10.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-11.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-12.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-13.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-14.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-15.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-16.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-17.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-18.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-19.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-20.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-21.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-22.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-23.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-24.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-25.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-26.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-27.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-28.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-29.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-30.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-31.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-32.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-33.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-34.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-35.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-36.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-37.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 475201536, "path": "target/layer-38.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 12352, "bytes": 14450688, "dtype": "q4_affine", "group_size": 64, "name": "gdn_input", "offset": 32768, "shape": [ 12544, 2048 ], "storage_n": 256 }, { "bytes": 65536, "dtype": "bf16", "name": "gdn_convolution", "offset": 14483456, "shape": [ 8192, 4, 1 ] }, { "bytes": 128, "dtype": "f32", "name": "gdn_decay", "offset": 14548992, "shape": [ 32 ] }, { "bytes": 64, "dtype": "bf16", "name": "gdn_time_bias", "offset": 14565376, "shape": [ 32 ] }, { "bytes": 256, "dtype": "bf16", "name": "gdn_norm", "offset": 14581760, "shape": [ 128 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "gdn_output", "offset": 14598144, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 19316736, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 19333120, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 19890176, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 170885120, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 321880064, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 472875008, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 473464832, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 474054656, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 474644480, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 471285760, "path": "target/layer-39.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "input_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 9216, "bytes": 10616832, "dtype": "q4_affine", "group_size": 64, "name": "attention_input", "offset": 32768, "shape": [ 9216, 2048 ], "storage_n": 256 }, { "bytes": 512, "dtype": "bf16", "name": "query_norm", "offset": 10649600, "shape": [ 256 ] }, { "bytes": 512, "dtype": "bf16", "name": "key_norm", "offset": 10665984, "shape": [ 256 ] }, { "actual_output": 2048, "bytes": 4718592, "dtype": "q4_affine", "group_size": 64, "name": "attention_output", "offset": 10682368, "shape": [ 2048, 4096 ], "storage_n": 256 }, { "bytes": 4096, "dtype": "bf16", "name": "post_attention_norm", "offset": 15400960, "shape": [ 2048 ] }, { "actual_output": 256, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "router", "offset": 15417344, "shape": [ 256, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.gate_proj", "offset": 15974400, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.up_proj", "offset": 166969344, "shape": [ 256, 512, 2048 ], "storage_n": 256 }, { "bytes": 150994944, "dtype": "q4_affine", "expert_stride_bytes": 589824, "group_size": 64, "name": "experts.down_proj", "offset": 317964288, "shape": [ 256, 2048, 512 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.gate_proj", "offset": 468959232, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 512, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.up_proj", "offset": 469549056, "shape": [ 512, 2048 ], "storage_n": 256 }, { "actual_output": 2048, "bytes": 589824, "dtype": "q4_affine", "group_size": 64, "name": "shared_expert.down_proj", "offset": 470138880, "shape": [ 2048, 512 ], "storage_n": 256 }, { "actual_output": 1, "bytes": 557056, "dtype": "q8_affine", "group_size": 64, "name": "shared_expert_gate", "offset": 470728704, "shape": [ 256, 2048 ], "storage_n": 256 } ] }, { "bytes": 286097408, "path": "target/head.bin", "sections": [ { "bytes": 4096, "dtype": "bf16", "name": "final_norm", "offset": 16384, "shape": [ 2048 ] }, { "actual_output": 248320, "bytes": 286064640, "dtype": "q4_affine", "group_size": 64, "name": "logits", "offset": 32768, "shape": [ 248320, 2048 ], "storage_n": 256 } ] }, { "bytes": 286081024, "path": "target/embedding.bin", "sections": [ { "bytes": 254279680, "dtype": "q4_affine_weights", "group_size": 64, "layout": "vocabulary_row_major", "name": "embedding.weights", "offset": 16384, "shape": [ 248320, 2048 ] }, { "bytes": 15892480, "dtype": "bf16", "name": "embedding.scales", "offset": 254296064, "shape": [ 248320, 32 ] }, { "bytes": 15892480, "dtype": "bf16", "name": "embedding.biases", "offset": 270188544, "shape": [ 248320, 32 ] } ] } ], "vision": { "bytes": 901939200, "path": "vision/model.bin", "sections": [ { "bytes": 3538944, "dtype": "bf16", "name": "patch_embed.weight", "offset": 16384, "shape": [ 1152, 1536 ] }, { "bytes": 2304, "dtype": "bf16", "name": "patch_embed.bias", "offset": 3555328, "shape": [ 1152 ] }, { "bytes": 5308416, "dtype": "bf16", "name": "position_table", "offset": 3571712, "shape": [ 2304, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.0.norm1.weight", "offset": 8880128, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.0.norm1.bias", "offset": 8896512, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.0.qkv.weight", "offset": 8912896, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.0.qkv.bias", "offset": 16875520, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.0.projection.weight", "offset": 16891904, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.0.projection.bias", "offset": 19546112, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.0.norm2.weight", "offset": 19562496, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.0.norm2.bias", "offset": 19578880, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.0.fc1.weight", "offset": 19595264, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.0.fc1.bias", "offset": 29622272, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.0.fc2.weight", "offset": 29638656, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.0.fc2.bias", "offset": 39665664, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.1.norm1.weight", "offset": 39682048, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.1.norm1.bias", "offset": 39698432, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.1.qkv.weight", "offset": 39714816, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.1.qkv.bias", "offset": 47677440, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.1.projection.weight", "offset": 47693824, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.1.projection.bias", "offset": 50348032, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.1.norm2.weight", "offset": 50364416, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.1.norm2.bias", "offset": 50380800, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.1.fc1.weight", "offset": 50397184, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.1.fc1.bias", "offset": 60424192, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.1.fc2.weight", "offset": 60440576, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.1.fc2.bias", "offset": 70467584, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.2.norm1.weight", "offset": 70483968, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.2.norm1.bias", "offset": 70500352, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.2.qkv.weight", "offset": 70516736, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.2.qkv.bias", "offset": 78479360, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.2.projection.weight", "offset": 78495744, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.2.projection.bias", "offset": 81149952, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.2.norm2.weight", "offset": 81166336, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.2.norm2.bias", "offset": 81182720, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.2.fc1.weight", "offset": 81199104, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.2.fc1.bias", "offset": 91226112, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.2.fc2.weight", "offset": 91242496, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.2.fc2.bias", "offset": 101269504, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.3.norm1.weight", "offset": 101285888, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.3.norm1.bias", "offset": 101302272, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.3.qkv.weight", "offset": 101318656, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.3.qkv.bias", "offset": 109281280, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.3.projection.weight", "offset": 109297664, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.3.projection.bias", "offset": 111951872, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.3.norm2.weight", "offset": 111968256, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.3.norm2.bias", "offset": 111984640, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.3.fc1.weight", "offset": 112001024, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.3.fc1.bias", "offset": 122028032, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.3.fc2.weight", "offset": 122044416, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.3.fc2.bias", "offset": 132071424, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.4.norm1.weight", "offset": 132087808, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.4.norm1.bias", "offset": 132104192, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.4.qkv.weight", "offset": 132120576, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.4.qkv.bias", "offset": 140083200, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.4.projection.weight", "offset": 140099584, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.4.projection.bias", "offset": 142753792, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.4.norm2.weight", "offset": 142770176, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.4.norm2.bias", "offset": 142786560, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.4.fc1.weight", "offset": 142802944, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.4.fc1.bias", "offset": 152829952, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.4.fc2.weight", "offset": 152846336, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.4.fc2.bias", "offset": 162873344, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.5.norm1.weight", "offset": 162889728, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.5.norm1.bias", "offset": 162906112, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.5.qkv.weight", "offset": 162922496, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.5.qkv.bias", "offset": 170885120, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.5.projection.weight", "offset": 170901504, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.5.projection.bias", "offset": 173555712, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.5.norm2.weight", "offset": 173572096, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.5.norm2.bias", "offset": 173588480, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.5.fc1.weight", "offset": 173604864, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.5.fc1.bias", "offset": 183631872, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.5.fc2.weight", "offset": 183648256, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.5.fc2.bias", "offset": 193675264, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.6.norm1.weight", "offset": 193691648, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.6.norm1.bias", "offset": 193708032, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.6.qkv.weight", "offset": 193724416, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.6.qkv.bias", "offset": 201687040, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.6.projection.weight", "offset": 201703424, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.6.projection.bias", "offset": 204357632, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.6.norm2.weight", "offset": 204374016, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.6.norm2.bias", "offset": 204390400, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.6.fc1.weight", "offset": 204406784, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.6.fc1.bias", "offset": 214433792, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.6.fc2.weight", "offset": 214450176, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.6.fc2.bias", "offset": 224477184, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.7.norm1.weight", "offset": 224493568, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.7.norm1.bias", "offset": 224509952, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.7.qkv.weight", "offset": 224526336, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.7.qkv.bias", "offset": 232488960, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.7.projection.weight", "offset": 232505344, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.7.projection.bias", "offset": 235159552, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.7.norm2.weight", "offset": 235175936, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.7.norm2.bias", "offset": 235192320, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.7.fc1.weight", "offset": 235208704, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.7.fc1.bias", "offset": 245235712, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.7.fc2.weight", "offset": 245252096, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.7.fc2.bias", "offset": 255279104, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.8.norm1.weight", "offset": 255295488, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.8.norm1.bias", "offset": 255311872, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.8.qkv.weight", "offset": 255328256, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.8.qkv.bias", "offset": 263290880, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.8.projection.weight", "offset": 263307264, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.8.projection.bias", "offset": 265961472, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.8.norm2.weight", "offset": 265977856, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.8.norm2.bias", "offset": 265994240, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.8.fc1.weight", "offset": 266010624, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.8.fc1.bias", "offset": 276037632, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.8.fc2.weight", "offset": 276054016, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.8.fc2.bias", "offset": 286081024, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.9.norm1.weight", "offset": 286097408, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.9.norm1.bias", "offset": 286113792, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.9.qkv.weight", "offset": 286130176, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.9.qkv.bias", "offset": 294092800, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.9.projection.weight", "offset": 294109184, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.9.projection.bias", "offset": 296763392, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.9.norm2.weight", "offset": 296779776, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.9.norm2.bias", "offset": 296796160, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.9.fc1.weight", "offset": 296812544, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.9.fc1.bias", "offset": 306839552, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.9.fc2.weight", "offset": 306855936, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.9.fc2.bias", "offset": 316882944, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.10.norm1.weight", "offset": 316899328, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.10.norm1.bias", "offset": 316915712, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.10.qkv.weight", "offset": 316932096, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.10.qkv.bias", "offset": 324894720, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.10.projection.weight", "offset": 324911104, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.10.projection.bias", "offset": 327565312, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.10.norm2.weight", "offset": 327581696, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.10.norm2.bias", "offset": 327598080, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.10.fc1.weight", "offset": 327614464, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.10.fc1.bias", "offset": 337641472, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.10.fc2.weight", "offset": 337657856, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.10.fc2.bias", "offset": 347684864, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.11.norm1.weight", "offset": 347701248, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.11.norm1.bias", "offset": 347717632, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.11.qkv.weight", "offset": 347734016, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.11.qkv.bias", "offset": 355696640, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.11.projection.weight", "offset": 355713024, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.11.projection.bias", "offset": 358367232, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.11.norm2.weight", "offset": 358383616, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.11.norm2.bias", "offset": 358400000, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.11.fc1.weight", "offset": 358416384, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.11.fc1.bias", "offset": 368443392, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.11.fc2.weight", "offset": 368459776, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.11.fc2.bias", "offset": 378486784, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.12.norm1.weight", "offset": 378503168, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.12.norm1.bias", "offset": 378519552, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.12.qkv.weight", "offset": 378535936, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.12.qkv.bias", "offset": 386498560, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.12.projection.weight", "offset": 386514944, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.12.projection.bias", "offset": 389169152, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.12.norm2.weight", "offset": 389185536, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.12.norm2.bias", "offset": 389201920, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.12.fc1.weight", "offset": 389218304, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.12.fc1.bias", "offset": 399245312, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.12.fc2.weight", "offset": 399261696, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.12.fc2.bias", "offset": 409288704, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.13.norm1.weight", "offset": 409305088, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.13.norm1.bias", "offset": 409321472, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.13.qkv.weight", "offset": 409337856, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.13.qkv.bias", "offset": 417300480, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.13.projection.weight", "offset": 417316864, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.13.projection.bias", "offset": 419971072, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.13.norm2.weight", "offset": 419987456, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.13.norm2.bias", "offset": 420003840, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.13.fc1.weight", "offset": 420020224, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.13.fc1.bias", "offset": 430047232, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.13.fc2.weight", "offset": 430063616, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.13.fc2.bias", "offset": 440090624, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.14.norm1.weight", "offset": 440107008, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.14.norm1.bias", "offset": 440123392, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.14.qkv.weight", "offset": 440139776, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.14.qkv.bias", "offset": 448102400, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.14.projection.weight", "offset": 448118784, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.14.projection.bias", "offset": 450772992, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.14.norm2.weight", "offset": 450789376, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.14.norm2.bias", "offset": 450805760, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.14.fc1.weight", "offset": 450822144, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.14.fc1.bias", "offset": 460849152, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.14.fc2.weight", "offset": 460865536, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.14.fc2.bias", "offset": 470892544, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.15.norm1.weight", "offset": 470908928, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.15.norm1.bias", "offset": 470925312, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.15.qkv.weight", "offset": 470941696, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.15.qkv.bias", "offset": 478904320, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.15.projection.weight", "offset": 478920704, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.15.projection.bias", "offset": 481574912, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.15.norm2.weight", "offset": 481591296, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.15.norm2.bias", "offset": 481607680, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.15.fc1.weight", "offset": 481624064, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.15.fc1.bias", "offset": 491651072, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.15.fc2.weight", "offset": 491667456, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.15.fc2.bias", "offset": 501694464, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.16.norm1.weight", "offset": 501710848, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.16.norm1.bias", "offset": 501727232, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.16.qkv.weight", "offset": 501743616, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.16.qkv.bias", "offset": 509706240, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.16.projection.weight", "offset": 509722624, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.16.projection.bias", "offset": 512376832, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.16.norm2.weight", "offset": 512393216, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.16.norm2.bias", "offset": 512409600, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.16.fc1.weight", "offset": 512425984, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.16.fc1.bias", "offset": 522452992, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.16.fc2.weight", "offset": 522469376, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.16.fc2.bias", "offset": 532496384, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.17.norm1.weight", "offset": 532512768, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.17.norm1.bias", "offset": 532529152, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.17.qkv.weight", "offset": 532545536, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.17.qkv.bias", "offset": 540508160, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.17.projection.weight", "offset": 540524544, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.17.projection.bias", "offset": 543178752, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.17.norm2.weight", "offset": 543195136, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.17.norm2.bias", "offset": 543211520, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.17.fc1.weight", "offset": 543227904, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.17.fc1.bias", "offset": 553254912, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.17.fc2.weight", "offset": 553271296, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.17.fc2.bias", "offset": 563298304, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.18.norm1.weight", "offset": 563314688, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.18.norm1.bias", "offset": 563331072, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.18.qkv.weight", "offset": 563347456, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.18.qkv.bias", "offset": 571310080, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.18.projection.weight", "offset": 571326464, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.18.projection.bias", "offset": 573980672, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.18.norm2.weight", "offset": 573997056, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.18.norm2.bias", "offset": 574013440, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.18.fc1.weight", "offset": 574029824, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.18.fc1.bias", "offset": 584056832, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.18.fc2.weight", "offset": 584073216, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.18.fc2.bias", "offset": 594100224, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.19.norm1.weight", "offset": 594116608, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.19.norm1.bias", "offset": 594132992, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.19.qkv.weight", "offset": 594149376, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.19.qkv.bias", "offset": 602112000, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.19.projection.weight", "offset": 602128384, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.19.projection.bias", "offset": 604782592, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.19.norm2.weight", "offset": 604798976, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.19.norm2.bias", "offset": 604815360, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.19.fc1.weight", "offset": 604831744, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.19.fc1.bias", "offset": 614858752, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.19.fc2.weight", "offset": 614875136, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.19.fc2.bias", "offset": 624902144, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.20.norm1.weight", "offset": 624918528, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.20.norm1.bias", "offset": 624934912, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.20.qkv.weight", "offset": 624951296, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.20.qkv.bias", "offset": 632913920, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.20.projection.weight", "offset": 632930304, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.20.projection.bias", "offset": 635584512, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.20.norm2.weight", "offset": 635600896, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.20.norm2.bias", "offset": 635617280, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.20.fc1.weight", "offset": 635633664, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.20.fc1.bias", "offset": 645660672, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.20.fc2.weight", "offset": 645677056, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.20.fc2.bias", "offset": 655704064, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.21.norm1.weight", "offset": 655720448, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.21.norm1.bias", "offset": 655736832, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.21.qkv.weight", "offset": 655753216, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.21.qkv.bias", "offset": 663715840, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.21.projection.weight", "offset": 663732224, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.21.projection.bias", "offset": 666386432, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.21.norm2.weight", "offset": 666402816, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.21.norm2.bias", "offset": 666419200, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.21.fc1.weight", "offset": 666435584, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.21.fc1.bias", "offset": 676462592, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.21.fc2.weight", "offset": 676478976, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.21.fc2.bias", "offset": 686505984, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.22.norm1.weight", "offset": 686522368, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.22.norm1.bias", "offset": 686538752, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.22.qkv.weight", "offset": 686555136, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.22.qkv.bias", "offset": 694517760, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.22.projection.weight", "offset": 694534144, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.22.projection.bias", "offset": 697188352, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.22.norm2.weight", "offset": 697204736, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.22.norm2.bias", "offset": 697221120, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.22.fc1.weight", "offset": 697237504, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.22.fc1.bias", "offset": 707264512, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.22.fc2.weight", "offset": 707280896, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.22.fc2.bias", "offset": 717307904, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.23.norm1.weight", "offset": 717324288, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.23.norm1.bias", "offset": 717340672, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.23.qkv.weight", "offset": 717357056, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.23.qkv.bias", "offset": 725319680, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.23.projection.weight", "offset": 725336064, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.23.projection.bias", "offset": 727990272, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.23.norm2.weight", "offset": 728006656, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.23.norm2.bias", "offset": 728023040, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.23.fc1.weight", "offset": 728039424, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.23.fc1.bias", "offset": 738066432, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.23.fc2.weight", "offset": 738082816, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.23.fc2.bias", "offset": 748109824, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.24.norm1.weight", "offset": 748126208, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.24.norm1.bias", "offset": 748142592, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.24.qkv.weight", "offset": 748158976, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.24.qkv.bias", "offset": 756121600, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.24.projection.weight", "offset": 756137984, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.24.projection.bias", "offset": 758792192, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.24.norm2.weight", "offset": 758808576, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.24.norm2.bias", "offset": 758824960, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.24.fc1.weight", "offset": 758841344, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.24.fc1.bias", "offset": 768868352, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.24.fc2.weight", "offset": 768884736, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.24.fc2.bias", "offset": 778911744, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.25.norm1.weight", "offset": 778928128, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.25.norm1.bias", "offset": 778944512, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.25.qkv.weight", "offset": 778960896, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.25.qkv.bias", "offset": 786923520, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.25.projection.weight", "offset": 786939904, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.25.projection.bias", "offset": 789594112, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.25.norm2.weight", "offset": 789610496, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.25.norm2.bias", "offset": 789626880, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.25.fc1.weight", "offset": 789643264, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.25.fc1.bias", "offset": 799670272, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.25.fc2.weight", "offset": 799686656, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.25.fc2.bias", "offset": 809713664, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.26.norm1.weight", "offset": 809730048, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.26.norm1.bias", "offset": 809746432, "shape": [ 1152 ] }, { "bytes": 7962624, "dtype": "bf16", "name": "blocks.26.qkv.weight", "offset": 809762816, "shape": [ 3456, 1152 ] }, { "bytes": 6912, "dtype": "bf16", "name": "blocks.26.qkv.bias", "offset": 817725440, "shape": [ 3456 ] }, { "bytes": 2654208, "dtype": "bf16", "name": "blocks.26.projection.weight", "offset": 817741824, "shape": [ 1152, 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.26.projection.bias", "offset": 820396032, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.26.norm2.weight", "offset": 820412416, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.26.norm2.bias", "offset": 820428800, "shape": [ 1152 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.26.fc1.weight", "offset": 820445184, "shape": [ 4352, 1152 ] }, { "bytes": 8704, "dtype": "bf16", "name": "blocks.26.fc1.bias", "offset": 830472192, "shape": [ 4352 ] }, { "bytes": 10027008, "dtype": "bf16", "name": "blocks.26.fc2.weight", "offset": 830488576, "shape": [ 1152, 4352 ] }, { "bytes": 2304, "dtype": "bf16", "name": "blocks.26.fc2.bias", "offset": 840515584, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "merger.norm.weight", "offset": 840531968, "shape": [ 1152 ] }, { "bytes": 2304, "dtype": "bf16", "name": "merger.norm.bias", "offset": 840548352, "shape": [ 1152 ] }, { "bytes": 42467328, "dtype": "bf16", "name": "merger.fc1.weight", "offset": 840564736, "shape": [ 4608, 4608 ] }, { "bytes": 9216, "dtype": "bf16", "name": "merger.fc1.bias", "offset": 883032064, "shape": [ 4608 ] }, { "bytes": 18874368, "dtype": "bf16", "name": "merger.fc2.weight", "offset": 883048448, "shape": [ 2048, 4608 ] }, { "bytes": 4096, "dtype": "bf16", "name": "merger.fc2.bias", "offset": 901922816, "shape": [ 2048 ] } ] } }