{ "schema": "trellis-webgpu-pack", "version": 1, "stage": "structuredLatentFlow", "upstreamRevision": "25e0d31ffbebe4b5a97464dd851910efc3002d96", "shards": [ { "file": "stage-000.bin", "bytes": 62871552, "sha256": "a6ac40ddadb92cef5709e04f4c6d1fc6a01d2ce3d69f792a8d614db332795da9" }, { "file": "stage-001.bin", "bytes": 62859264, "sha256": "02b714b0a26ab144b7d0dc7b4ff5486f51a04c0c130ec230afaea5fd694b8b5d" }, { "file": "stage-002.bin", "bytes": 62859264, "sha256": "f5cbcab39328cead3600c197dc45e0cd57ee1e6d6ed0dd44666818ba195f3c78" }, { "file": "stage-003.bin", "bytes": 62859264, "sha256": "466f5fd821976ba5f75519aa5a0dbee0cc569d597325d862e7b4a42d9607f2e2" }, { "file": "stage-004.bin", "bytes": 62859264, "sha256": "9e539c8d15a21fc1a53f5841a01a1b496318488f589858f0ba31c6a1c9d09f36" }, { "file": "stage-005.bin", "bytes": 62859264, "sha256": "eb2bc0de5b2df7ebc844856fce3c5187d22e94900e750ba1397324089d01cdc0" }, { "file": "stage-006.bin", "bytes": 62859264, "sha256": "3cb705673e389da3fac48567e065726c63471a1cf436651dbe66bc7e2bb21db9" }, { "file": "stage-007.bin", "bytes": 67099136, "sha256": "f80598c459842fcb90d044e89d6897228c3015831e937cad4245d17178fb21e3" }, { "file": "stage-008.bin", "bytes": 37651968, "sha256": "2e5bb4b2683e0db04556f746f16f4b4e47b24934f264486b863fce4afcb72a35" } ], "tensors": { "blocks.0.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 0, "bytes": 12288 } }, "blocks.0.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 0, "offset": 12288, "bytes": 5505024 }, "scales": { "shard": 0, "offset": 5517312, "bytes": 196608 } }, "blocks.0.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 5713920, "bytes": 4096 } }, "blocks.0.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 0, "offset": 5718016, "bytes": 1835008 }, "scales": { "shard": 0, "offset": 7553024, "bytes": 65536 } }, "blocks.0.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 7618560, "bytes": 2048 } }, "blocks.0.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 7620608, "bytes": 917504 }, "scales": { "shard": 0, "offset": 8538112, "bytes": 32768 } }, "blocks.0.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 8570880, "bytes": 2048 } }, "blocks.0.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 8572928, "bytes": 917504 }, "scales": { "shard": 0, "offset": 9490432, "bytes": 32768 } }, "blocks.0.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 9523200, "bytes": 8192 } }, "blocks.0.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 0, "offset": 9531392, "bytes": 3670016 }, "scales": { "shard": 0, "offset": 13201408, "bytes": 131072 } }, "blocks.0.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 13332480, "bytes": 2048 } }, "blocks.0.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 0, "offset": 13334528, "bytes": 3670016 }, "scales": { "shard": 0, "offset": 17004544, "bytes": 131072 } }, "blocks.0.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 17135616, "bytes": 2048 } }, "blocks.0.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 17137664, "bytes": 2048 } }, "blocks.0.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 17139712, "bytes": 2048 } }, "blocks.0.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 17141760, "bytes": 2048 } }, "blocks.0.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 17143808, "bytes": 2048 } }, "blocks.0.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 17145856, "bytes": 917504 }, "scales": { "shard": 0, "offset": 18063360, "bytes": 32768 } }, "blocks.0.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 18096128, "bytes": 6144 } }, "blocks.0.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 0, "offset": 18102272, "bytes": 2752512 }, "scales": { "shard": 0, "offset": 20854784, "bytes": 98304 } }, "blocks.1.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 20953088, "bytes": 12288 } }, "blocks.1.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 0, "offset": 20965376, "bytes": 5505024 }, "scales": { "shard": 0, "offset": 26470400, "bytes": 196608 } }, "blocks.1.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 26667008, "bytes": 4096 } }, "blocks.1.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 0, "offset": 26671104, "bytes": 1835008 }, "scales": { "shard": 0, "offset": 28506112, "bytes": 65536 } }, "blocks.1.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 28571648, "bytes": 2048 } }, "blocks.1.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 28573696, "bytes": 917504 }, "scales": { "shard": 0, "offset": 29491200, "bytes": 32768 } }, "blocks.1.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 29523968, "bytes": 2048 } }, "blocks.1.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 29526016, "bytes": 917504 }, "scales": { "shard": 0, "offset": 30443520, "bytes": 32768 } }, "blocks.1.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 30476288, "bytes": 8192 } }, "blocks.1.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 0, "offset": 30484480, "bytes": 3670016 }, "scales": { "shard": 0, "offset": 34154496, "bytes": 131072 } }, "blocks.1.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 34285568, "bytes": 2048 } }, "blocks.1.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 0, "offset": 34287616, "bytes": 3670016 }, "scales": { "shard": 0, "offset": 37957632, "bytes": 131072 } }, "blocks.1.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 38088704, "bytes": 2048 } }, "blocks.1.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 38090752, "bytes": 2048 } }, "blocks.1.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 38092800, "bytes": 2048 } }, "blocks.1.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 38094848, "bytes": 2048 } }, "blocks.1.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 38096896, "bytes": 2048 } }, "blocks.1.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 38098944, "bytes": 917504 }, "scales": { "shard": 0, "offset": 39016448, "bytes": 32768 } }, "blocks.1.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 39049216, "bytes": 6144 } }, "blocks.1.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 0, "offset": 39055360, "bytes": 2752512 }, "scales": { "shard": 0, "offset": 41807872, "bytes": 98304 } }, "blocks.10.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 41906176, "bytes": 12288 } }, "blocks.10.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 0, "offset": 41918464, "bytes": 5505024 }, "scales": { "shard": 0, "offset": 47423488, "bytes": 196608 } }, "blocks.10.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 47620096, "bytes": 4096 } }, "blocks.10.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 0, "offset": 47624192, "bytes": 1835008 }, "scales": { "shard": 0, "offset": 49459200, "bytes": 65536 } }, "blocks.10.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 49524736, "bytes": 2048 } }, "blocks.10.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 49526784, "bytes": 917504 }, "scales": { "shard": 0, "offset": 50444288, "bytes": 32768 } }, "blocks.10.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 50477056, "bytes": 2048 } }, "blocks.10.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 50479104, "bytes": 917504 }, "scales": { "shard": 0, "offset": 51396608, "bytes": 32768 } }, "blocks.10.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 51429376, "bytes": 8192 } }, "blocks.10.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 0, "offset": 51437568, "bytes": 3670016 }, "scales": { "shard": 0, "offset": 55107584, "bytes": 131072 } }, "blocks.10.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 55238656, "bytes": 2048 } }, "blocks.10.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 0, "offset": 55240704, "bytes": 3670016 }, "scales": { "shard": 0, "offset": 58910720, "bytes": 131072 } }, "blocks.10.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 59041792, "bytes": 2048 } }, "blocks.10.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 59043840, "bytes": 2048 } }, "blocks.10.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 59045888, "bytes": 2048 } }, "blocks.10.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 59047936, "bytes": 2048 } }, "blocks.10.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 59049984, "bytes": 2048 } }, "blocks.10.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 59052032, "bytes": 917504 }, "scales": { "shard": 0, "offset": 59969536, "bytes": 32768 } }, "blocks.10.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 60002304, "bytes": 6144 } }, "blocks.10.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 0, "offset": 60008448, "bytes": 2752512 }, "scales": { "shard": 0, "offset": 62760960, "bytes": 98304 } }, "blocks.11.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 62859264, "bytes": 12288 } }, "blocks.11.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 1, "offset": 0, "bytes": 5505024 }, "scales": { "shard": 1, "offset": 5505024, "bytes": 196608 } }, "blocks.11.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 5701632, "bytes": 4096 } }, "blocks.11.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 1, "offset": 5705728, "bytes": 1835008 }, "scales": { "shard": 1, "offset": 7540736, "bytes": 65536 } }, "blocks.11.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 7606272, "bytes": 2048 } }, "blocks.11.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 7608320, "bytes": 917504 }, "scales": { "shard": 1, "offset": 8525824, "bytes": 32768 } }, "blocks.11.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 8558592, "bytes": 2048 } }, "blocks.11.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 8560640, "bytes": 917504 }, "scales": { "shard": 1, "offset": 9478144, "bytes": 32768 } }, "blocks.11.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 9510912, "bytes": 8192 } }, "blocks.11.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 1, "offset": 9519104, "bytes": 3670016 }, "scales": { "shard": 1, "offset": 13189120, "bytes": 131072 } }, "blocks.11.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 13320192, "bytes": 2048 } }, "blocks.11.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 1, "offset": 13322240, "bytes": 3670016 }, "scales": { "shard": 1, "offset": 16992256, "bytes": 131072 } }, "blocks.11.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 17123328, "bytes": 2048 } }, "blocks.11.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 17125376, "bytes": 2048 } }, "blocks.11.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 17127424, "bytes": 2048 } }, "blocks.11.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 17129472, "bytes": 2048 } }, "blocks.11.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 17131520, "bytes": 2048 } }, "blocks.11.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 17133568, "bytes": 917504 }, "scales": { "shard": 1, "offset": 18051072, "bytes": 32768 } }, "blocks.11.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 18083840, "bytes": 6144 } }, "blocks.11.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 1, "offset": 18089984, "bytes": 2752512 }, "scales": { "shard": 1, "offset": 20842496, "bytes": 98304 } }, "blocks.12.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 20940800, "bytes": 12288 } }, "blocks.12.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 1, "offset": 20953088, "bytes": 5505024 }, "scales": { "shard": 1, "offset": 26458112, "bytes": 196608 } }, "blocks.12.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 26654720, "bytes": 4096 } }, "blocks.12.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 1, "offset": 26658816, "bytes": 1835008 }, "scales": { "shard": 1, "offset": 28493824, "bytes": 65536 } }, "blocks.12.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 28559360, "bytes": 2048 } }, "blocks.12.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 28561408, "bytes": 917504 }, "scales": { "shard": 1, "offset": 29478912, "bytes": 32768 } }, "blocks.12.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 29511680, "bytes": 2048 } }, "blocks.12.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 29513728, "bytes": 917504 }, "scales": { "shard": 1, "offset": 30431232, "bytes": 32768 } }, "blocks.12.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 30464000, "bytes": 8192 } }, "blocks.12.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 1, "offset": 30472192, "bytes": 3670016 }, "scales": { "shard": 1, "offset": 34142208, "bytes": 131072 } }, "blocks.12.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 34273280, "bytes": 2048 } }, "blocks.12.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 1, "offset": 34275328, "bytes": 3670016 }, "scales": { "shard": 1, "offset": 37945344, "bytes": 131072 } }, "blocks.12.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 38076416, "bytes": 2048 } }, "blocks.12.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 38078464, "bytes": 2048 } }, "blocks.12.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 38080512, "bytes": 2048 } }, "blocks.12.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 38082560, "bytes": 2048 } }, "blocks.12.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 38084608, "bytes": 2048 } }, "blocks.12.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 38086656, "bytes": 917504 }, "scales": { "shard": 1, "offset": 39004160, "bytes": 32768 } }, "blocks.12.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 39036928, "bytes": 6144 } }, "blocks.12.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 1, "offset": 39043072, "bytes": 2752512 }, "scales": { "shard": 1, "offset": 41795584, "bytes": 98304 } }, "blocks.13.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 41893888, "bytes": 12288 } }, "blocks.13.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 1, "offset": 41906176, "bytes": 5505024 }, "scales": { "shard": 1, "offset": 47411200, "bytes": 196608 } }, "blocks.13.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 47607808, "bytes": 4096 } }, "blocks.13.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 1, "offset": 47611904, "bytes": 1835008 }, "scales": { "shard": 1, "offset": 49446912, "bytes": 65536 } }, "blocks.13.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 49512448, "bytes": 2048 } }, "blocks.13.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 49514496, "bytes": 917504 }, "scales": { "shard": 1, "offset": 50432000, "bytes": 32768 } }, "blocks.13.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 50464768, "bytes": 2048 } }, "blocks.13.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 50466816, "bytes": 917504 }, "scales": { "shard": 1, "offset": 51384320, "bytes": 32768 } }, "blocks.13.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 51417088, "bytes": 8192 } }, "blocks.13.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 1, "offset": 51425280, "bytes": 3670016 }, "scales": { "shard": 1, "offset": 55095296, "bytes": 131072 } }, "blocks.13.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 55226368, "bytes": 2048 } }, "blocks.13.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 1, "offset": 55228416, "bytes": 3670016 }, "scales": { "shard": 1, "offset": 58898432, "bytes": 131072 } }, "blocks.13.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 59029504, "bytes": 2048 } }, "blocks.13.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 59031552, "bytes": 2048 } }, "blocks.13.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 59033600, "bytes": 2048 } }, "blocks.13.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 59035648, "bytes": 2048 } }, "blocks.13.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 59037696, "bytes": 2048 } }, "blocks.13.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 59039744, "bytes": 917504 }, "scales": { "shard": 1, "offset": 59957248, "bytes": 32768 } }, "blocks.13.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 59990016, "bytes": 6144 } }, "blocks.13.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 1, "offset": 59996160, "bytes": 2752512 }, "scales": { "shard": 1, "offset": 62748672, "bytes": 98304 } }, "blocks.14.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 62846976, "bytes": 12288 } }, "blocks.14.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 2, "offset": 0, "bytes": 5505024 }, "scales": { "shard": 2, "offset": 5505024, "bytes": 196608 } }, "blocks.14.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 5701632, "bytes": 4096 } }, "blocks.14.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 2, "offset": 5705728, "bytes": 1835008 }, "scales": { "shard": 2, "offset": 7540736, "bytes": 65536 } }, "blocks.14.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 7606272, "bytes": 2048 } }, "blocks.14.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 7608320, "bytes": 917504 }, "scales": { "shard": 2, "offset": 8525824, "bytes": 32768 } }, "blocks.14.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 8558592, "bytes": 2048 } }, "blocks.14.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 8560640, "bytes": 917504 }, "scales": { "shard": 2, "offset": 9478144, "bytes": 32768 } }, "blocks.14.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 9510912, "bytes": 8192 } }, "blocks.14.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 2, "offset": 9519104, "bytes": 3670016 }, "scales": { "shard": 2, "offset": 13189120, "bytes": 131072 } }, "blocks.14.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 13320192, "bytes": 2048 } }, "blocks.14.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 2, "offset": 13322240, "bytes": 3670016 }, "scales": { "shard": 2, "offset": 16992256, "bytes": 131072 } }, "blocks.14.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 17123328, "bytes": 2048 } }, "blocks.14.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 17125376, "bytes": 2048 } }, "blocks.14.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 17127424, "bytes": 2048 } }, "blocks.14.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 17129472, "bytes": 2048 } }, "blocks.14.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 17131520, "bytes": 2048 } }, "blocks.14.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 17133568, "bytes": 917504 }, "scales": { "shard": 2, "offset": 18051072, "bytes": 32768 } }, "blocks.14.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 18083840, "bytes": 6144 } }, "blocks.14.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 2, "offset": 18089984, "bytes": 2752512 }, "scales": { "shard": 2, "offset": 20842496, "bytes": 98304 } }, "blocks.15.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 20940800, "bytes": 12288 } }, "blocks.15.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 2, "offset": 20953088, "bytes": 5505024 }, "scales": { "shard": 2, "offset": 26458112, "bytes": 196608 } }, "blocks.15.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 26654720, "bytes": 4096 } }, "blocks.15.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 2, "offset": 26658816, "bytes": 1835008 }, "scales": { "shard": 2, "offset": 28493824, "bytes": 65536 } }, "blocks.15.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 28559360, "bytes": 2048 } }, "blocks.15.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 28561408, "bytes": 917504 }, "scales": { "shard": 2, "offset": 29478912, "bytes": 32768 } }, "blocks.15.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 29511680, "bytes": 2048 } }, "blocks.15.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 29513728, "bytes": 917504 }, "scales": { "shard": 2, "offset": 30431232, "bytes": 32768 } }, "blocks.15.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 30464000, "bytes": 8192 } }, "blocks.15.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 2, "offset": 30472192, "bytes": 3670016 }, "scales": { "shard": 2, "offset": 34142208, "bytes": 131072 } }, "blocks.15.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 34273280, "bytes": 2048 } }, "blocks.15.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 2, "offset": 34275328, "bytes": 3670016 }, "scales": { "shard": 2, "offset": 37945344, "bytes": 131072 } }, "blocks.15.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 38076416, "bytes": 2048 } }, "blocks.15.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 38078464, "bytes": 2048 } }, "blocks.15.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 38080512, "bytes": 2048 } }, "blocks.15.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 38082560, "bytes": 2048 } }, "blocks.15.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 38084608, "bytes": 2048 } }, "blocks.15.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 38086656, "bytes": 917504 }, "scales": { "shard": 2, "offset": 39004160, "bytes": 32768 } }, "blocks.15.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 39036928, "bytes": 6144 } }, "blocks.15.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 2, "offset": 39043072, "bytes": 2752512 }, "scales": { "shard": 2, "offset": 41795584, "bytes": 98304 } }, "blocks.16.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 41893888, "bytes": 12288 } }, "blocks.16.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 2, "offset": 41906176, "bytes": 5505024 }, "scales": { "shard": 2, "offset": 47411200, "bytes": 196608 } }, "blocks.16.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 47607808, "bytes": 4096 } }, "blocks.16.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 2, "offset": 47611904, "bytes": 1835008 }, "scales": { "shard": 2, "offset": 49446912, "bytes": 65536 } }, "blocks.16.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 49512448, "bytes": 2048 } }, "blocks.16.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 49514496, "bytes": 917504 }, "scales": { "shard": 2, "offset": 50432000, "bytes": 32768 } }, "blocks.16.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 50464768, "bytes": 2048 } }, "blocks.16.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 50466816, "bytes": 917504 }, "scales": { "shard": 2, "offset": 51384320, "bytes": 32768 } }, "blocks.16.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 51417088, "bytes": 8192 } }, "blocks.16.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 2, "offset": 51425280, "bytes": 3670016 }, "scales": { "shard": 2, "offset": 55095296, "bytes": 131072 } }, "blocks.16.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 55226368, "bytes": 2048 } }, "blocks.16.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 2, "offset": 55228416, "bytes": 3670016 }, "scales": { "shard": 2, "offset": 58898432, "bytes": 131072 } }, "blocks.16.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 59029504, "bytes": 2048 } }, "blocks.16.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 59031552, "bytes": 2048 } }, "blocks.16.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 59033600, "bytes": 2048 } }, "blocks.16.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 59035648, "bytes": 2048 } }, "blocks.16.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 59037696, "bytes": 2048 } }, "blocks.16.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 59039744, "bytes": 917504 }, "scales": { "shard": 2, "offset": 59957248, "bytes": 32768 } }, "blocks.16.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 59990016, "bytes": 6144 } }, "blocks.16.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 2, "offset": 59996160, "bytes": 2752512 }, "scales": { "shard": 2, "offset": 62748672, "bytes": 98304 } }, "blocks.17.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 62846976, "bytes": 12288 } }, "blocks.17.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 3, "offset": 0, "bytes": 5505024 }, "scales": { "shard": 3, "offset": 5505024, "bytes": 196608 } }, "blocks.17.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 5701632, "bytes": 4096 } }, "blocks.17.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 3, "offset": 5705728, "bytes": 1835008 }, "scales": { "shard": 3, "offset": 7540736, "bytes": 65536 } }, "blocks.17.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 7606272, "bytes": 2048 } }, "blocks.17.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 7608320, "bytes": 917504 }, "scales": { "shard": 3, "offset": 8525824, "bytes": 32768 } }, "blocks.17.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 8558592, "bytes": 2048 } }, "blocks.17.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 8560640, "bytes": 917504 }, "scales": { "shard": 3, "offset": 9478144, "bytes": 32768 } }, "blocks.17.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 9510912, "bytes": 8192 } }, "blocks.17.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 3, "offset": 9519104, "bytes": 3670016 }, "scales": { "shard": 3, "offset": 13189120, "bytes": 131072 } }, "blocks.17.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 13320192, "bytes": 2048 } }, "blocks.17.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 3, "offset": 13322240, "bytes": 3670016 }, "scales": { "shard": 3, "offset": 16992256, "bytes": 131072 } }, "blocks.17.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 17123328, "bytes": 2048 } }, "blocks.17.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 17125376, "bytes": 2048 } }, "blocks.17.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 17127424, "bytes": 2048 } }, "blocks.17.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 17129472, "bytes": 2048 } }, "blocks.17.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 17131520, "bytes": 2048 } }, "blocks.17.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 17133568, "bytes": 917504 }, "scales": { "shard": 3, "offset": 18051072, "bytes": 32768 } }, "blocks.17.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 18083840, "bytes": 6144 } }, "blocks.17.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 3, "offset": 18089984, "bytes": 2752512 }, "scales": { "shard": 3, "offset": 20842496, "bytes": 98304 } }, "blocks.18.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 20940800, "bytes": 12288 } }, "blocks.18.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 3, "offset": 20953088, "bytes": 5505024 }, "scales": { "shard": 3, "offset": 26458112, "bytes": 196608 } }, "blocks.18.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 26654720, "bytes": 4096 } }, "blocks.18.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 3, "offset": 26658816, "bytes": 1835008 }, "scales": { "shard": 3, "offset": 28493824, "bytes": 65536 } }, "blocks.18.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 28559360, "bytes": 2048 } }, "blocks.18.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 28561408, "bytes": 917504 }, "scales": { "shard": 3, "offset": 29478912, "bytes": 32768 } }, "blocks.18.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 29511680, "bytes": 2048 } }, "blocks.18.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 29513728, "bytes": 917504 }, "scales": { "shard": 3, "offset": 30431232, "bytes": 32768 } }, "blocks.18.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 30464000, "bytes": 8192 } }, "blocks.18.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 3, "offset": 30472192, "bytes": 3670016 }, "scales": { "shard": 3, "offset": 34142208, "bytes": 131072 } }, "blocks.18.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 34273280, "bytes": 2048 } }, "blocks.18.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 3, "offset": 34275328, "bytes": 3670016 }, "scales": { "shard": 3, "offset": 37945344, "bytes": 131072 } }, "blocks.18.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 38076416, "bytes": 2048 } }, "blocks.18.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 38078464, "bytes": 2048 } }, "blocks.18.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 38080512, "bytes": 2048 } }, "blocks.18.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 38082560, "bytes": 2048 } }, "blocks.18.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 38084608, "bytes": 2048 } }, "blocks.18.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 38086656, "bytes": 917504 }, "scales": { "shard": 3, "offset": 39004160, "bytes": 32768 } }, "blocks.18.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 39036928, "bytes": 6144 } }, "blocks.18.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 3, "offset": 39043072, "bytes": 2752512 }, "scales": { "shard": 3, "offset": 41795584, "bytes": 98304 } }, "blocks.19.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 41893888, "bytes": 12288 } }, "blocks.19.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 3, "offset": 41906176, "bytes": 5505024 }, "scales": { "shard": 3, "offset": 47411200, "bytes": 196608 } }, "blocks.19.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 47607808, "bytes": 4096 } }, "blocks.19.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 3, "offset": 47611904, "bytes": 1835008 }, "scales": { "shard": 3, "offset": 49446912, "bytes": 65536 } }, "blocks.19.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 49512448, "bytes": 2048 } }, "blocks.19.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 49514496, "bytes": 917504 }, "scales": { "shard": 3, "offset": 50432000, "bytes": 32768 } }, "blocks.19.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 50464768, "bytes": 2048 } }, "blocks.19.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 50466816, "bytes": 917504 }, "scales": { "shard": 3, "offset": 51384320, "bytes": 32768 } }, "blocks.19.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 51417088, "bytes": 8192 } }, "blocks.19.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 3, "offset": 51425280, "bytes": 3670016 }, "scales": { "shard": 3, "offset": 55095296, "bytes": 131072 } }, "blocks.19.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 55226368, "bytes": 2048 } }, "blocks.19.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 3, "offset": 55228416, "bytes": 3670016 }, "scales": { "shard": 3, "offset": 58898432, "bytes": 131072 } }, "blocks.19.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 59029504, "bytes": 2048 } }, "blocks.19.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 59031552, "bytes": 2048 } }, "blocks.19.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 59033600, "bytes": 2048 } }, "blocks.19.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 59035648, "bytes": 2048 } }, "blocks.19.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 59037696, "bytes": 2048 } }, "blocks.19.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 59039744, "bytes": 917504 }, "scales": { "shard": 3, "offset": 59957248, "bytes": 32768 } }, "blocks.19.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 59990016, "bytes": 6144 } }, "blocks.19.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 3, "offset": 59996160, "bytes": 2752512 }, "scales": { "shard": 3, "offset": 62748672, "bytes": 98304 } }, "blocks.2.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 62846976, "bytes": 12288 } }, "blocks.2.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 4, "offset": 0, "bytes": 5505024 }, "scales": { "shard": 4, "offset": 5505024, "bytes": 196608 } }, "blocks.2.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 5701632, "bytes": 4096 } }, "blocks.2.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 4, "offset": 5705728, "bytes": 1835008 }, "scales": { "shard": 4, "offset": 7540736, "bytes": 65536 } }, "blocks.2.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 7606272, "bytes": 2048 } }, "blocks.2.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 7608320, "bytes": 917504 }, "scales": { "shard": 4, "offset": 8525824, "bytes": 32768 } }, "blocks.2.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 8558592, "bytes": 2048 } }, "blocks.2.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 8560640, "bytes": 917504 }, "scales": { "shard": 4, "offset": 9478144, "bytes": 32768 } }, "blocks.2.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 9510912, "bytes": 8192 } }, "blocks.2.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 4, "offset": 9519104, "bytes": 3670016 }, "scales": { "shard": 4, "offset": 13189120, "bytes": 131072 } }, "blocks.2.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 13320192, "bytes": 2048 } }, "blocks.2.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 4, "offset": 13322240, "bytes": 3670016 }, "scales": { "shard": 4, "offset": 16992256, "bytes": 131072 } }, "blocks.2.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 17123328, "bytes": 2048 } }, "blocks.2.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 17125376, "bytes": 2048 } }, "blocks.2.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 17127424, "bytes": 2048 } }, "blocks.2.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 17129472, "bytes": 2048 } }, "blocks.2.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 17131520, "bytes": 2048 } }, "blocks.2.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 17133568, "bytes": 917504 }, "scales": { "shard": 4, "offset": 18051072, "bytes": 32768 } }, "blocks.2.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 18083840, "bytes": 6144 } }, "blocks.2.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 4, "offset": 18089984, "bytes": 2752512 }, "scales": { "shard": 4, "offset": 20842496, "bytes": 98304 } }, "blocks.20.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 20940800, "bytes": 12288 } }, "blocks.20.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 4, "offset": 20953088, "bytes": 5505024 }, "scales": { "shard": 4, "offset": 26458112, "bytes": 196608 } }, "blocks.20.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 26654720, "bytes": 4096 } }, "blocks.20.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 4, "offset": 26658816, "bytes": 1835008 }, "scales": { "shard": 4, "offset": 28493824, "bytes": 65536 } }, "blocks.20.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 28559360, "bytes": 2048 } }, "blocks.20.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 28561408, "bytes": 917504 }, "scales": { "shard": 4, "offset": 29478912, "bytes": 32768 } }, "blocks.20.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 29511680, "bytes": 2048 } }, "blocks.20.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 29513728, "bytes": 917504 }, "scales": { "shard": 4, "offset": 30431232, "bytes": 32768 } }, "blocks.20.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 30464000, "bytes": 8192 } }, "blocks.20.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 4, "offset": 30472192, "bytes": 3670016 }, "scales": { "shard": 4, "offset": 34142208, "bytes": 131072 } }, "blocks.20.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 34273280, "bytes": 2048 } }, "blocks.20.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 4, "offset": 34275328, "bytes": 3670016 }, "scales": { "shard": 4, "offset": 37945344, "bytes": 131072 } }, "blocks.20.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 38076416, "bytes": 2048 } }, "blocks.20.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 38078464, "bytes": 2048 } }, "blocks.20.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 38080512, "bytes": 2048 } }, "blocks.20.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 38082560, "bytes": 2048 } }, "blocks.20.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 38084608, "bytes": 2048 } }, "blocks.20.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 38086656, "bytes": 917504 }, "scales": { "shard": 4, "offset": 39004160, "bytes": 32768 } }, "blocks.20.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 39036928, "bytes": 6144 } }, "blocks.20.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 4, "offset": 39043072, "bytes": 2752512 }, "scales": { "shard": 4, "offset": 41795584, "bytes": 98304 } }, "blocks.21.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 41893888, "bytes": 12288 } }, "blocks.21.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 4, "offset": 41906176, "bytes": 5505024 }, "scales": { "shard": 4, "offset": 47411200, "bytes": 196608 } }, "blocks.21.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 47607808, "bytes": 4096 } }, "blocks.21.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 4, "offset": 47611904, "bytes": 1835008 }, "scales": { "shard": 4, "offset": 49446912, "bytes": 65536 } }, "blocks.21.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 49512448, "bytes": 2048 } }, "blocks.21.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 49514496, "bytes": 917504 }, "scales": { "shard": 4, "offset": 50432000, "bytes": 32768 } }, "blocks.21.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 50464768, "bytes": 2048 } }, "blocks.21.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 50466816, "bytes": 917504 }, "scales": { "shard": 4, "offset": 51384320, "bytes": 32768 } }, "blocks.21.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 51417088, "bytes": 8192 } }, "blocks.21.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 4, "offset": 51425280, "bytes": 3670016 }, "scales": { "shard": 4, "offset": 55095296, "bytes": 131072 } }, "blocks.21.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 55226368, "bytes": 2048 } }, "blocks.21.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 4, "offset": 55228416, "bytes": 3670016 }, "scales": { "shard": 4, "offset": 58898432, "bytes": 131072 } }, "blocks.21.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59029504, "bytes": 2048 } }, "blocks.21.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59031552, "bytes": 2048 } }, "blocks.21.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59033600, "bytes": 2048 } }, "blocks.21.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59035648, "bytes": 2048 } }, "blocks.21.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59037696, "bytes": 2048 } }, "blocks.21.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 59039744, "bytes": 917504 }, "scales": { "shard": 4, "offset": 59957248, "bytes": 32768 } }, "blocks.21.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59990016, "bytes": 6144 } }, "blocks.21.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 4, "offset": 59996160, "bytes": 2752512 }, "scales": { "shard": 4, "offset": 62748672, "bytes": 98304 } }, "blocks.22.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 62846976, "bytes": 12288 } }, "blocks.22.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 5, "offset": 0, "bytes": 5505024 }, "scales": { "shard": 5, "offset": 5505024, "bytes": 196608 } }, "blocks.22.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 5701632, "bytes": 4096 } }, "blocks.22.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 5, "offset": 5705728, "bytes": 1835008 }, "scales": { "shard": 5, "offset": 7540736, "bytes": 65536 } }, "blocks.22.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 7606272, "bytes": 2048 } }, "blocks.22.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 7608320, "bytes": 917504 }, "scales": { "shard": 5, "offset": 8525824, "bytes": 32768 } }, "blocks.22.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 8558592, "bytes": 2048 } }, "blocks.22.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 8560640, "bytes": 917504 }, "scales": { "shard": 5, "offset": 9478144, "bytes": 32768 } }, "blocks.22.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 9510912, "bytes": 8192 } }, "blocks.22.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 5, "offset": 9519104, "bytes": 3670016 }, "scales": { "shard": 5, "offset": 13189120, "bytes": 131072 } }, "blocks.22.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 13320192, "bytes": 2048 } }, "blocks.22.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 5, "offset": 13322240, "bytes": 3670016 }, "scales": { "shard": 5, "offset": 16992256, "bytes": 131072 } }, "blocks.22.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 17123328, "bytes": 2048 } }, "blocks.22.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 17125376, "bytes": 2048 } }, "blocks.22.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 17127424, "bytes": 2048 } }, "blocks.22.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 17129472, "bytes": 2048 } }, "blocks.22.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 17131520, "bytes": 2048 } }, "blocks.22.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 17133568, "bytes": 917504 }, "scales": { "shard": 5, "offset": 18051072, "bytes": 32768 } }, "blocks.22.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 18083840, "bytes": 6144 } }, "blocks.22.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 5, "offset": 18089984, "bytes": 2752512 }, "scales": { "shard": 5, "offset": 20842496, "bytes": 98304 } }, "blocks.23.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 20940800, "bytes": 12288 } }, "blocks.23.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 5, "offset": 20953088, "bytes": 5505024 }, "scales": { "shard": 5, "offset": 26458112, "bytes": 196608 } }, "blocks.23.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 26654720, "bytes": 4096 } }, "blocks.23.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 5, "offset": 26658816, "bytes": 1835008 }, "scales": { "shard": 5, "offset": 28493824, "bytes": 65536 } }, "blocks.23.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 28559360, "bytes": 2048 } }, "blocks.23.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 28561408, "bytes": 917504 }, "scales": { "shard": 5, "offset": 29478912, "bytes": 32768 } }, "blocks.23.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 29511680, "bytes": 2048 } }, "blocks.23.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 29513728, "bytes": 917504 }, "scales": { "shard": 5, "offset": 30431232, "bytes": 32768 } }, "blocks.23.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 30464000, "bytes": 8192 } }, "blocks.23.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 5, "offset": 30472192, "bytes": 3670016 }, "scales": { "shard": 5, "offset": 34142208, "bytes": 131072 } }, "blocks.23.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 34273280, "bytes": 2048 } }, "blocks.23.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 5, "offset": 34275328, "bytes": 3670016 }, "scales": { "shard": 5, "offset": 37945344, "bytes": 131072 } }, "blocks.23.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 38076416, "bytes": 2048 } }, "blocks.23.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 38078464, "bytes": 2048 } }, "blocks.23.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 38080512, "bytes": 2048 } }, "blocks.23.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 38082560, "bytes": 2048 } }, "blocks.23.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 38084608, "bytes": 2048 } }, "blocks.23.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 38086656, "bytes": 917504 }, "scales": { "shard": 5, "offset": 39004160, "bytes": 32768 } }, "blocks.23.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 39036928, "bytes": 6144 } }, "blocks.23.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 5, "offset": 39043072, "bytes": 2752512 }, "scales": { "shard": 5, "offset": 41795584, "bytes": 98304 } }, "blocks.3.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 41893888, "bytes": 12288 } }, "blocks.3.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 5, "offset": 41906176, "bytes": 5505024 }, "scales": { "shard": 5, "offset": 47411200, "bytes": 196608 } }, "blocks.3.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 47607808, "bytes": 4096 } }, "blocks.3.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 5, "offset": 47611904, "bytes": 1835008 }, "scales": { "shard": 5, "offset": 49446912, "bytes": 65536 } }, "blocks.3.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 49512448, "bytes": 2048 } }, "blocks.3.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 49514496, "bytes": 917504 }, "scales": { "shard": 5, "offset": 50432000, "bytes": 32768 } }, "blocks.3.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 50464768, "bytes": 2048 } }, "blocks.3.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 50466816, "bytes": 917504 }, "scales": { "shard": 5, "offset": 51384320, "bytes": 32768 } }, "blocks.3.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 51417088, "bytes": 8192 } }, "blocks.3.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 5, "offset": 51425280, "bytes": 3670016 }, "scales": { "shard": 5, "offset": 55095296, "bytes": 131072 } }, "blocks.3.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 55226368, "bytes": 2048 } }, "blocks.3.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 5, "offset": 55228416, "bytes": 3670016 }, "scales": { "shard": 5, "offset": 58898432, "bytes": 131072 } }, "blocks.3.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 59029504, "bytes": 2048 } }, "blocks.3.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 59031552, "bytes": 2048 } }, "blocks.3.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 59033600, "bytes": 2048 } }, "blocks.3.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 59035648, "bytes": 2048 } }, "blocks.3.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 59037696, "bytes": 2048 } }, "blocks.3.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 59039744, "bytes": 917504 }, "scales": { "shard": 5, "offset": 59957248, "bytes": 32768 } }, "blocks.3.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 59990016, "bytes": 6144 } }, "blocks.3.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 5, "offset": 59996160, "bytes": 2752512 }, "scales": { "shard": 5, "offset": 62748672, "bytes": 98304 } }, "blocks.4.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 62846976, "bytes": 12288 } }, "blocks.4.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 6, "offset": 0, "bytes": 5505024 }, "scales": { "shard": 6, "offset": 5505024, "bytes": 196608 } }, "blocks.4.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 5701632, "bytes": 4096 } }, "blocks.4.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 6, "offset": 5705728, "bytes": 1835008 }, "scales": { "shard": 6, "offset": 7540736, "bytes": 65536 } }, "blocks.4.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 7606272, "bytes": 2048 } }, "blocks.4.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 7608320, "bytes": 917504 }, "scales": { "shard": 6, "offset": 8525824, "bytes": 32768 } }, "blocks.4.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 8558592, "bytes": 2048 } }, "blocks.4.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 8560640, "bytes": 917504 }, "scales": { "shard": 6, "offset": 9478144, "bytes": 32768 } }, "blocks.4.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 9510912, "bytes": 8192 } }, "blocks.4.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 6, "offset": 9519104, "bytes": 3670016 }, "scales": { "shard": 6, "offset": 13189120, "bytes": 131072 } }, "blocks.4.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 13320192, "bytes": 2048 } }, "blocks.4.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 6, "offset": 13322240, "bytes": 3670016 }, "scales": { "shard": 6, "offset": 16992256, "bytes": 131072 } }, "blocks.4.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 17123328, "bytes": 2048 } }, "blocks.4.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 17125376, "bytes": 2048 } }, "blocks.4.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 17127424, "bytes": 2048 } }, "blocks.4.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 17129472, "bytes": 2048 } }, "blocks.4.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 17131520, "bytes": 2048 } }, "blocks.4.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 17133568, "bytes": 917504 }, "scales": { "shard": 6, "offset": 18051072, "bytes": 32768 } }, "blocks.4.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 18083840, "bytes": 6144 } }, "blocks.4.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 6, "offset": 18089984, "bytes": 2752512 }, "scales": { "shard": 6, "offset": 20842496, "bytes": 98304 } }, "blocks.5.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 20940800, "bytes": 12288 } }, "blocks.5.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 6, "offset": 20953088, "bytes": 5505024 }, "scales": { "shard": 6, "offset": 26458112, "bytes": 196608 } }, "blocks.5.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 26654720, "bytes": 4096 } }, "blocks.5.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 6, "offset": 26658816, "bytes": 1835008 }, "scales": { "shard": 6, "offset": 28493824, "bytes": 65536 } }, "blocks.5.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 28559360, "bytes": 2048 } }, "blocks.5.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 28561408, "bytes": 917504 }, "scales": { "shard": 6, "offset": 29478912, "bytes": 32768 } }, "blocks.5.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 29511680, "bytes": 2048 } }, "blocks.5.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 29513728, "bytes": 917504 }, "scales": { "shard": 6, "offset": 30431232, "bytes": 32768 } }, "blocks.5.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 30464000, "bytes": 8192 } }, "blocks.5.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 6, "offset": 30472192, "bytes": 3670016 }, "scales": { "shard": 6, "offset": 34142208, "bytes": 131072 } }, "blocks.5.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 34273280, "bytes": 2048 } }, "blocks.5.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 6, "offset": 34275328, "bytes": 3670016 }, "scales": { "shard": 6, "offset": 37945344, "bytes": 131072 } }, "blocks.5.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 38076416, "bytes": 2048 } }, "blocks.5.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 38078464, "bytes": 2048 } }, "blocks.5.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 38080512, "bytes": 2048 } }, "blocks.5.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 38082560, "bytes": 2048 } }, "blocks.5.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 38084608, "bytes": 2048 } }, "blocks.5.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 38086656, "bytes": 917504 }, "scales": { "shard": 6, "offset": 39004160, "bytes": 32768 } }, "blocks.5.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 39036928, "bytes": 6144 } }, "blocks.5.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 6, "offset": 39043072, "bytes": 2752512 }, "scales": { "shard": 6, "offset": 41795584, "bytes": 98304 } }, "blocks.6.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 41893888, "bytes": 12288 } }, "blocks.6.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 6, "offset": 41906176, "bytes": 5505024 }, "scales": { "shard": 6, "offset": 47411200, "bytes": 196608 } }, "blocks.6.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 47607808, "bytes": 4096 } }, "blocks.6.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 6, "offset": 47611904, "bytes": 1835008 }, "scales": { "shard": 6, "offset": 49446912, "bytes": 65536 } }, "blocks.6.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 49512448, "bytes": 2048 } }, "blocks.6.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 49514496, "bytes": 917504 }, "scales": { "shard": 6, "offset": 50432000, "bytes": 32768 } }, "blocks.6.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 50464768, "bytes": 2048 } }, "blocks.6.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 50466816, "bytes": 917504 }, "scales": { "shard": 6, "offset": 51384320, "bytes": 32768 } }, "blocks.6.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 51417088, "bytes": 8192 } }, "blocks.6.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 6, "offset": 51425280, "bytes": 3670016 }, "scales": { "shard": 6, "offset": 55095296, "bytes": 131072 } }, "blocks.6.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 55226368, "bytes": 2048 } }, "blocks.6.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 6, "offset": 55228416, "bytes": 3670016 }, "scales": { "shard": 6, "offset": 58898432, "bytes": 131072 } }, "blocks.6.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 59029504, "bytes": 2048 } }, "blocks.6.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 59031552, "bytes": 2048 } }, "blocks.6.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 59033600, "bytes": 2048 } }, "blocks.6.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 59035648, "bytes": 2048 } }, "blocks.6.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 59037696, "bytes": 2048 } }, "blocks.6.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 59039744, "bytes": 917504 }, "scales": { "shard": 6, "offset": 59957248, "bytes": 32768 } }, "blocks.6.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 59990016, "bytes": 6144 } }, "blocks.6.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 6, "offset": 59996160, "bytes": 2752512 }, "scales": { "shard": 6, "offset": 62748672, "bytes": 98304 } }, "blocks.7.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 62846976, "bytes": 12288 } }, "blocks.7.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 7, "offset": 0, "bytes": 5505024 }, "scales": { "shard": 7, "offset": 5505024, "bytes": 196608 } }, "blocks.7.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 5701632, "bytes": 4096 } }, "blocks.7.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 7, "offset": 5705728, "bytes": 1835008 }, "scales": { "shard": 7, "offset": 7540736, "bytes": 65536 } }, "blocks.7.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 7606272, "bytes": 2048 } }, "blocks.7.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 7608320, "bytes": 917504 }, "scales": { "shard": 7, "offset": 8525824, "bytes": 32768 } }, "blocks.7.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 8558592, "bytes": 2048 } }, "blocks.7.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 8560640, "bytes": 917504 }, "scales": { "shard": 7, "offset": 9478144, "bytes": 32768 } }, "blocks.7.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 9510912, "bytes": 8192 } }, "blocks.7.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 7, "offset": 9519104, "bytes": 3670016 }, "scales": { "shard": 7, "offset": 13189120, "bytes": 131072 } }, "blocks.7.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 13320192, "bytes": 2048 } }, "blocks.7.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 7, "offset": 13322240, "bytes": 3670016 }, "scales": { "shard": 7, "offset": 16992256, "bytes": 131072 } }, "blocks.7.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 17123328, "bytes": 2048 } }, "blocks.7.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 17125376, "bytes": 2048 } }, "blocks.7.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 17127424, "bytes": 2048 } }, "blocks.7.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 17129472, "bytes": 2048 } }, "blocks.7.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 17131520, "bytes": 2048 } }, "blocks.7.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 17133568, "bytes": 917504 }, "scales": { "shard": 7, "offset": 18051072, "bytes": 32768 } }, "blocks.7.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 18083840, "bytes": 6144 } }, "blocks.7.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 7, "offset": 18089984, "bytes": 2752512 }, "scales": { "shard": 7, "offset": 20842496, "bytes": 98304 } }, "blocks.8.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 20940800, "bytes": 12288 } }, "blocks.8.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 7, "offset": 20953088, "bytes": 5505024 }, "scales": { "shard": 7, "offset": 26458112, "bytes": 196608 } }, "blocks.8.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 26654720, "bytes": 4096 } }, "blocks.8.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 7, "offset": 26658816, "bytes": 1835008 }, "scales": { "shard": 7, "offset": 28493824, "bytes": 65536 } }, "blocks.8.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 28559360, "bytes": 2048 } }, "blocks.8.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 28561408, "bytes": 917504 }, "scales": { "shard": 7, "offset": 29478912, "bytes": 32768 } }, "blocks.8.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 29511680, "bytes": 2048 } }, "blocks.8.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 29513728, "bytes": 917504 }, "scales": { "shard": 7, "offset": 30431232, "bytes": 32768 } }, "blocks.8.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 30464000, "bytes": 8192 } }, "blocks.8.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 7, "offset": 30472192, "bytes": 3670016 }, "scales": { "shard": 7, "offset": 34142208, "bytes": 131072 } }, "blocks.8.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 34273280, "bytes": 2048 } }, "blocks.8.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 7, "offset": 34275328, "bytes": 3670016 }, "scales": { "shard": 7, "offset": 37945344, "bytes": 131072 } }, "blocks.8.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 38076416, "bytes": 2048 } }, "blocks.8.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 38078464, "bytes": 2048 } }, "blocks.8.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 38080512, "bytes": 2048 } }, "blocks.8.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 38082560, "bytes": 2048 } }, "blocks.8.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 38084608, "bytes": 2048 } }, "blocks.8.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 38086656, "bytes": 917504 }, "scales": { "shard": 7, "offset": 39004160, "bytes": 32768 } }, "blocks.8.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 39036928, "bytes": 6144 } }, "blocks.8.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 7, "offset": 39043072, "bytes": 2752512 }, "scales": { "shard": 7, "offset": 41795584, "bytes": 98304 } }, "blocks.9.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 41893888, "bytes": 12288 } }, "blocks.9.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 7, "offset": 41906176, "bytes": 5505024 }, "scales": { "shard": 7, "offset": 47411200, "bytes": 196608 } }, "blocks.9.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 47607808, "bytes": 4096 } }, "blocks.9.cross_attn.to_kv.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 7, "offset": 47611904, "bytes": 1835008 }, "scales": { "shard": 7, "offset": 49446912, "bytes": 65536 } }, "blocks.9.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 49512448, "bytes": 2048 } }, "blocks.9.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 49514496, "bytes": 917504 }, "scales": { "shard": 7, "offset": 50432000, "bytes": 32768 } }, "blocks.9.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 50464768, "bytes": 2048 } }, "blocks.9.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 50466816, "bytes": 917504 }, "scales": { "shard": 7, "offset": 51384320, "bytes": 32768 } }, "blocks.9.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 51417088, "bytes": 8192 } }, "blocks.9.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 7, "offset": 51425280, "bytes": 3670016 }, "scales": { "shard": 7, "offset": 55095296, "bytes": 131072 } }, "blocks.9.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 55226368, "bytes": 2048 } }, "blocks.9.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 7, "offset": 55228416, "bytes": 3670016 }, "scales": { "shard": 7, "offset": 58898432, "bytes": 131072 } }, "blocks.9.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 59029504, "bytes": 2048 } }, "blocks.9.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 59031552, "bytes": 2048 } }, "blocks.9.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 59033600, "bytes": 2048 } }, "blocks.9.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 59035648, "bytes": 2048 } }, "blocks.9.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 59037696, "bytes": 2048 } }, "blocks.9.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 59039744, "bytes": 917504 }, "scales": { "shard": 7, "offset": 59957248, "bytes": 32768 } }, "blocks.9.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 59990016, "bytes": 6144 } }, "blocks.9.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 7, "offset": 59996160, "bytes": 2752512 }, "scales": { "shard": 7, "offset": 62748672, "bytes": 98304 } }, "input_blocks.0.conv1.conv.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 62846976, "bytes": 256 } }, "input_blocks.0.conv1.conv.weight": { "shape": [ 128, 3, 3, 3, 128 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 128, "columns": 3456, "padded_rows": 128, "padded_columns": 3456, "data": { "shard": 7, "offset": 62847232, "bytes": 387072 }, "scales": { "shard": 7, "offset": 63234304, "bytes": 13824 } }, "input_blocks.0.conv2.conv.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 63248128, "bytes": 256 } }, "input_blocks.0.conv2.conv.weight": { "shape": [ 128, 3, 3, 3, 128 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 128, "columns": 3456, "padded_rows": 128, "padded_columns": 3456, "data": { "shard": 7, "offset": 63248384, "bytes": 387072 }, "scales": { "shard": 7, "offset": 63635456, "bytes": 13824 } }, "input_blocks.0.emb_layers.1.bias": { "shape": [ 256 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 63649280, "bytes": 512 } }, "input_blocks.0.emb_layers.1.weight": { "shape": [ 256, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 256, "columns": 1024, "padded_rows": 256, "padded_columns": 1024, "data": { "shard": 7, "offset": 63649792, "bytes": 229376 }, "scales": { "shard": 7, "offset": 63879168, "bytes": 8192 } }, "input_blocks.0.norm1.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 63887360, "bytes": 256 } }, "input_blocks.0.norm1.weight": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 63887616, "bytes": 256 } }, "input_blocks.1.conv1.conv.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 63887872, "bytes": 2048 } }, "input_blocks.1.conv1.conv.weight": { "shape": [ 1024, 3, 3, 3, 128 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 3456, "padded_rows": 1024, "padded_columns": 3456, "data": { "shard": 7, "offset": 63889920, "bytes": 3096576 }, "scales": { "shard": 7, "offset": 66986496, "bytes": 110592 } }, "input_blocks.1.conv2.conv.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 67097088, "bytes": 2048 } }, "input_blocks.1.conv2.conv.weight": { "shape": [ 1024, 3, 3, 3, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 27648, "padded_rows": 1024, "padded_columns": 27648, "data": { "shard": 8, "offset": 0, "bytes": 24772608 }, "scales": { "shard": 8, "offset": 24772608, "bytes": 884736 } }, "input_blocks.1.emb_layers.1.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 25657344, "bytes": 4096 } }, "input_blocks.1.emb_layers.1.weight": { "shape": [ 2048, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 1024, "padded_rows": 2048, "padded_columns": 1024, "data": { "shard": 8, "offset": 25661440, "bytes": 1835008 }, "scales": { "shard": 8, "offset": 27496448, "bytes": 65536 } }, "input_blocks.1.norm1.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 27561984, "bytes": 256 } }, "input_blocks.1.norm1.weight": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 27562240, "bytes": 256 } }, "input_blocks.1.skip_connection.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 27562496, "bytes": 2048 } }, "input_blocks.1.skip_connection.weight": { "shape": [ 1024, 128 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 128, "padded_rows": 1024, "padded_columns": 128, "data": { "shard": 8, "offset": 27564544, "bytes": 114688 }, "scales": { "shard": 8, "offset": 27679232, "bytes": 4096 } }, "input_layer.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 27683328, "bytes": 256 } }, "input_layer.weight": { "shape": [ 128, 8 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 27683584, "bytes": 2048 } }, "out_blocks.0.conv1.conv.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 27685632, "bytes": 256 } }, "out_blocks.0.conv1.conv.weight": { "shape": [ 128, 3, 3, 3, 2048 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 128, "columns": 55296, "padded_rows": 128, "padded_columns": 55296, "data": { "shard": 8, "offset": 27685888, "bytes": 6193152 }, "scales": { "shard": 8, "offset": 33879040, "bytes": 221184 } }, "out_blocks.0.conv2.conv.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 34100224, "bytes": 256 } }, "out_blocks.0.conv2.conv.weight": { "shape": [ 128, 3, 3, 3, 128 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 128, "columns": 3456, "padded_rows": 128, "padded_columns": 3456, "data": { "shard": 8, "offset": 34100480, "bytes": 387072 }, "scales": { "shard": 8, "offset": 34487552, "bytes": 13824 } }, "out_blocks.0.emb_layers.1.bias": { "shape": [ 256 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 34501376, "bytes": 512 } }, "out_blocks.0.emb_layers.1.weight": { "shape": [ 256, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 256, "columns": 1024, "padded_rows": 256, "padded_columns": 1024, "data": { "shard": 8, "offset": 34501888, "bytes": 229376 }, "scales": { "shard": 8, "offset": 34731264, "bytes": 8192 } }, "out_blocks.0.norm1.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 34739456, "bytes": 4096 } }, "out_blocks.0.norm1.weight": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 34743552, "bytes": 4096 } }, "out_blocks.0.skip_connection.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 34747648, "bytes": 256 } }, "out_blocks.0.skip_connection.weight": { "shape": [ 128, 2048 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 128, "columns": 2048, "padded_rows": 128, "padded_columns": 2048, "data": { "shard": 8, "offset": 34747904, "bytes": 229376 }, "scales": { "shard": 8, "offset": 34977280, "bytes": 8192 } }, "out_blocks.1.conv1.conv.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 34985472, "bytes": 256 } }, "out_blocks.1.conv1.conv.weight": { "shape": [ 128, 3, 3, 3, 256 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 128, "columns": 6912, "padded_rows": 128, "padded_columns": 6912, "data": { "shard": 8, "offset": 34985728, "bytes": 774144 }, "scales": { "shard": 8, "offset": 35759872, "bytes": 27648 } }, "out_blocks.1.conv2.conv.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 35787520, "bytes": 256 } }, "out_blocks.1.conv2.conv.weight": { "shape": [ 128, 3, 3, 3, 128 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 128, "columns": 3456, "padded_rows": 128, "padded_columns": 3456, "data": { "shard": 8, "offset": 35787776, "bytes": 387072 }, "scales": { "shard": 8, "offset": 36174848, "bytes": 13824 } }, "out_blocks.1.emb_layers.1.bias": { "shape": [ 256 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36188672, "bytes": 512 } }, "out_blocks.1.emb_layers.1.weight": { "shape": [ 256, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 256, "columns": 1024, "padded_rows": 256, "padded_columns": 1024, "data": { "shard": 8, "offset": 36189184, "bytes": 229376 }, "scales": { "shard": 8, "offset": 36418560, "bytes": 8192 } }, "out_blocks.1.norm1.bias": { "shape": [ 256 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36426752, "bytes": 512 } }, "out_blocks.1.norm1.weight": { "shape": [ 256 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36427264, "bytes": 512 } }, "out_blocks.1.skip_connection.bias": { "shape": [ 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36427776, "bytes": 256 } }, "out_blocks.1.skip_connection.weight": { "shape": [ 128, 256 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 128, "columns": 256, "padded_rows": 128, "padded_columns": 256, "data": { "shard": 8, "offset": 36428032, "bytes": 28672 }, "scales": { "shard": 8, "offset": 36456704, "bytes": 1024 } }, "out_layer.bias": { "shape": [ 8 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36457728, "bytes": 16 } }, "out_layer.weight": { "shape": [ 8, 128 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36457984, "bytes": 2048 } }, "t_embedder.mlp.0.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36460032, "bytes": 2048 } }, "t_embedder.mlp.0.weight": { "shape": [ 1024, 256 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 256, "padded_rows": 1024, "padded_columns": 256, "data": { "shard": 8, "offset": 36462080, "bytes": 229376 }, "scales": { "shard": 8, "offset": 36691456, "bytes": 8192 } }, "t_embedder.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36699648, "bytes": 2048 } }, "t_embedder.mlp.2.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 7, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 8, "offset": 36701696, "bytes": 917504 }, "scales": { "shard": 8, "offset": 37619200, "bytes": 32768 } } } }