diff --git "a/refiner/sdnq_experts.json" "b/refiner/sdnq_experts.json" new file mode 100644--- /dev/null +++ "b/refiner/sdnq_experts.json" @@ -0,0 +1,8460 @@ +{ + "entries": [ + { + "module": "blocks.0.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.1.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.2.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.3.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.4.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.5.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.6.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.7.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.8.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.9.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.10.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.11.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.12.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.13.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.14.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.15.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.16.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.17.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.18.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.19.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.20.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.21.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.22.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.23.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.24.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.25.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.26.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.27.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.28.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.29.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.30.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.31.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.32.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.33.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.34.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.35.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.36.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.37.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.38.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.39.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.40.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.41.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.42.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.43.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.44.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.45.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.46.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + }, + { + "module": "blocks.47.ffn.experts", + "weights": { + "w1": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + }, + "w2": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 2048, + 768 + ], + "original_stride": [ + 1572864, + 768, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 2048, + 24, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 2048, + 768 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 2048, + 24, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 2048, + 24, + 1 + ] + }, + "w3": { + "group_size": 32, + "hadamard_group_size": 256, + "layer_class_name": null, + "logical_numel": 201326592, + "original_bytes": 402653184, + "original_numel": 201326592, + "original_shape": [ + 128, + 768, + 2048 + ], + "original_stride": [ + 1572864, + 2048, + 1 + ], + "quantized_matmul_dtype": "int8", + "quantized_weight_shape": [ + 128, + 768, + 64, + 32 + ], + "re_quantize_for_matmul": true, + "result_dtype": "bfloat16", + "result_shape": [ + 128, + 768, + 2048 + ], + "scale_dtype": "bfloat16", + "scale_shape": [ + 128, + 768, + 64, + 1 + ], + "stored_bytes": 125829120, + "stored_weight_dtype": "uint8", + "stored_weight_shape": [ + 100663296 + ], + "svd_rank": 32, + "svd_steps": 8, + "use_hadamard": false, + "use_quantized_matmul": false, + "use_stochastic_rounding": false, + "weights_dtype": "uint4", + "zero_point_dtype": "bfloat16", + "zero_point_shape": [ + 128, + 768, + 64, + 1 + ] + } + } + } + ], + "format": "lingbot-video-sdnq-experts", + "group_size": 0, + "quant_conv": false, + "quant_embedding": false, + "use_dynamic_quantization": false, + "use_svd": false, + "version": "1", + "weights_dtype": "uint4" +}