{ "format_version": 1, "base_model": "lmsys/longchat-7b-v1.5-32k", "vllm_pin": "0.10.1.1", "profiles": { "balanced": { "layers": [ { "layer_idx": 0, "k_head_ranks": [ 55, 48, 47, 64, 62, 52, 64, 64, 50, 61, 53, 59, 64, 60, 50, 64, 50, 64, 64, 59, 50, 64, 64, 63, 64, 64, 51, 61, 51, 64, 61, 55 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 1, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 2, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 3, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 4, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 5, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 6, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 7, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 8, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 9, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 10, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 11, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 12, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 13, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 14, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 15, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 16, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 17, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 18, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 19, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 20, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 21, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 22, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 23, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 24, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 25, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 26, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 27, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 28, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 29, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 30, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } }, { "layer_idx": 31, "k_head_ranks": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "k_padded_rank": 64, "v_joint_rank": 64, "quant": { "top_fraction": 0.25, "top_bits": 4, "bottom_bits": 3 } } ] } }, "active_profile": "balanced", "calibration": { "dataset": "HuggingFaceFW/fineweb-edu (data/CC-MAIN-2013-20/train-00000-of-00014.parquet)", "samples": 64, "seq_len": 256, "paper_reference_samples": 3000, "deviation_note": "scaled down from paper's 3000-sample/6-GPU-hour budget for single-session feasibility" } }