Hugging Face's logo

tenosra
/
longchat-7b-starkv-proxima

Text Generation
Transformers
Safetensors
llama
vllm
kv-cache-compression
low-rank
long-context
text-generation-inference
Model card Files
xet
 Community
iampoppyxx commited on
Commit
d46a292
·
verified ·
1 Parent(s): dfe3d14

Upload STAR-KV checkpoint (lmsys/longchat-7b-v1.5-32k base)

Browse files
Files changed (10) hide show
  1. config.json +31 -0
  2. generation_config.json +9 -0
  3. model-00001-of-00002.safetensors +3 -0
  4. model-00002-of-00002.safetensors +3 -0
  5. model.safetensors.index.json +330 -0
  6. special_tokens_map.json +24 -0
  7. star_kv/weights.safetensors +3 -0
  8. star_kv_config.json +1427 -0
  9. tokenizer.model +3 -0
  10. tokenizer_config.json +35 -0
config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "longchat-v2-7b-32k",
3
+ "architectures": [
4
+ "ProximaStarKVLlamaForCausalLM"
5
+ ],
6
+ "bos_token_id": 1,
7
+ "eos_token_id": 2,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 4096,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 11008,
12
+ "max_sequence_length": 32768,
13
+ "max_position_embeddings": 4096,
14
+ "model_type": "llama",
15
+ "num_attention_heads": 32,
16
+ "num_hidden_layers": 32,
17
+ "num_key_value_heads": 32,
18
+ "pad_token_id": 0,
19
+ "pretraining_tp": 1,
20
+ "rms_norm_eps": 1e-05,
21
+ "rope_condense_ratio": 8,
22
+ "rope_scaling": {
23
+ "factor": 8.0,
24
+ "type": "linear"
25
+ },
26
+ "tie_word_embeddings": false,
27
+ "torch_dtype": "float16",
28
+ "transformers_version": "4.28.0",
29
+ "use_cache": true,
30
+ "vocab_size": 32000
31
+ }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 32000,
6
+ "temperature": 0.9,
7
+ "top_p": 0.6,
8
+ "transformers_version": "4.28.0"
9
+ }
model-00001-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9c2da934af2f4bf6ec7217c3216d10c6a0e68f4c68dbcd0162c3e87f6d761eb
3
+ size 9976578928
model-00002-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a0640cdb8a4461424f05c31e85d42ef045e7ef5f53da4564d5548f33663349e
3
+ size 3500297344
model.safetensors.index.json ADDED
@@ -0,0 +1,330 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 13476839424
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "model-00002-of-00002.safetensors",
7
+ "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
8
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
9
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
10
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
11
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
12
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
13
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
14
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
15
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
16
+ "model.layers.0.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
17
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
18
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
19
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
20
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
21
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
22
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
23
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
24
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
25
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
26
+ "model.layers.1.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
27
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
28
+ "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
29
+ "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
30
+ "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
31
+ "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
32
+ "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
33
+ "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
34
+ "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
35
+ "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
36
+ "model.layers.10.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
37
+ "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
38
+ "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
39
+ "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
40
+ "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
41
+ "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
42
+ "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
43
+ "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
44
+ "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
45
+ "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
46
+ "model.layers.11.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
47
+ "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
48
+ "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
49
+ "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
50
+ "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
51
+ "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
52
+ "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
53
+ "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
54
+ "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
55
+ "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
56
+ "model.layers.12.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
57
+ "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
58
+ "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
59
+ "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
60
+ "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
61
+ "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
62
+ "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
63
+ "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
64
+ "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
65
+ "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
66
+ "model.layers.13.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
67
+ "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
68
+ "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
69
+ "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
70
+ "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
71
+ "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
72
+ "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
73
+ "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
74
+ "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
75
+ "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
76
+ "model.layers.14.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
77
+ "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
78
+ "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
79
+ "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
80
+ "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
81
+ "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
82
+ "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
83
+ "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
84
+ "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
85
+ "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
86
+ "model.layers.15.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
87
+ "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
88
+ "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
89
+ "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
90
+ "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
91
+ "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
92
+ "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
93
+ "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
94
+ "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
95
+ "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
96
+ "model.layers.16.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
97
+ "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
98
+ "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
99
+ "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
100
+ "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
101
+ "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
102
+ "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
103
+ "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
104
+ "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
105
+ "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
106
+ "model.layers.17.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
107
+ "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
108
+ "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
109
+ "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
110
+ "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
111
+ "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
112
+ "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
113
+ "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
114
+ "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
115
+ "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
116
+ "model.layers.18.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
117
+ "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
118
+ "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
119
+ "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
120
+ "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
121
+ "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
122
+ "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
123
+ "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
124
+ "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
125
+ "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
126
+ "model.layers.19.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
127
+ "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
128
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
129
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
130
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
131
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
132
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
133
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
134
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
135
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
136
+ "model.layers.2.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
137
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
138
+ "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
139
+ "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
140
+ "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
141
+ "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
142
+ "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
143
+ "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
144
+ "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
145
+ "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
146
+ "model.layers.20.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
147
+ "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
148
+ "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
149
+ "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
150
+ "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
151
+ "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
152
+ "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
153
+ "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
154
+ "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
155
+ "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
156
+ "model.layers.21.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
157
+ "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
158
+ "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
159
+ "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
160
+ "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
161
+ "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
162
+ "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
163
+ "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
164
+ "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
165
+ "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
166
+ "model.layers.22.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
167
+ "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
168
+ "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
169
+ "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
170
+ "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
171
+ "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
172
+ "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
173
+ "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
174
+ "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
175
+ "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
176
+ "model.layers.23.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
177
+ "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
178
+ "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
179
+ "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
180
+ "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
181
+ "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
182
+ "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
183
+ "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
184
+ "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
185
+ "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
186
+ "model.layers.24.self_attn.rotary_emb.inv_freq": "model-00002-of-00002.safetensors",
187
+ "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
188
+ "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
189
+ "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
190
+ "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
191
+ "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
192
+ "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
193
+ "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
194
+ "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
195
+ "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
196
+ "model.layers.25.self_attn.rotary_emb.inv_freq": "model-00002-of-00002.safetensors",
197
+ "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
198
+ "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
199
+ "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
200
+ "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
201
+ "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
202
+ "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
203
+ "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
204
+ "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
205
+ "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
206
+ "model.layers.26.self_attn.rotary_emb.inv_freq": "model-00002-of-00002.safetensors",
207
+ "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
208
+ "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
209
+ "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
210
+ "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
211
+ "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
212
+ "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
213
+ "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
214
+ "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
215
+ "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
216
+ "model.layers.27.self_attn.rotary_emb.inv_freq": "model-00002-of-00002.safetensors",
217
+ "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
218
+ "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
219
+ "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
220
+ "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
221
+ "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
222
+ "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
223
+ "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
224
+ "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
225
+ "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
226
+ "model.layers.28.self_attn.rotary_emb.inv_freq": "model-00002-of-00002.safetensors",
227
+ "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
228
+ "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
229
+ "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
230
+ "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
231
+ "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
232
+ "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
233
+ "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
234
+ "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
235
+ "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
236
+ "model.layers.29.self_attn.rotary_emb.inv_freq": "model-00002-of-00002.safetensors",
237
+ "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
238
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
239
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
240
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
241
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
242
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
243
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
244
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
245
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
246
+ "model.layers.3.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
247
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
248
+ "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
249
+ "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
250
+ "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
251
+ "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
252
+ "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
253
+ "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
254
+ "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
255
+ "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
256
+ "model.layers.30.self_attn.rotary_emb.inv_freq": "model-00002-of-00002.safetensors",
257
+ "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
258
+ "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
259
+ "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
260
+ "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
261
+ "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
262
+ "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
263
+ "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
264
+ "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
265
+ "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
266
+ "model.layers.31.self_attn.rotary_emb.inv_freq": "model-00002-of-00002.safetensors",
267
+ "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
268
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
269
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
270
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
271
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
272
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
273
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
274
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
275
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
276
+ "model.layers.4.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
277
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
278
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
279
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
280
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
281
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
282
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
283
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
284
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
285
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
286
+ "model.layers.5.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
287
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
288
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
289
+ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
290
+ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
291
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
292
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
293
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
294
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
295
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
296
+ "model.layers.6.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
297
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
298
+ "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
299
+ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
300
+ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
301
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
302
+ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
303
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
304
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
305
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
306
+ "model.layers.7.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
307
+ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
308
+ "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
309
+ "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
310
+ "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
311
+ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
312
+ "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
313
+ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
314
+ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
315
+ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
316
+ "model.layers.8.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
317
+ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
318
+ "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
319
+ "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
320
+ "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
321
+ "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
322
+ "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
323
+ "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
324
+ "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
325
+ "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
326
+ "model.layers.9.self_attn.rotary_emb.inv_freq": "model-00001-of-00002.safetensors",
327
+ "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
328
+ "model.norm.weight": "model-00002-of-00002.safetensors"
329
+ }
330
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<unk>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": true,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
star_kv/weights.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2f1e85461f0b4a9198b4a6a6a5150a6045ae4ac6ce1cb829f6f19abbfdbe7956
3
+ size 1426074312
star_kv_config.json ADDED
@@ -0,0 +1,1427 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format_version": 1,
3
+ "base_model": "lmsys/longchat-7b-v1.5-32k",
4
+ "vllm_pin": "0.10.1.1",
5
+ "profiles": {
6
+ "balanced": {
7
+ "layers": [
8
+ {
9
+ "layer_idx": 0,
10
+ "k_head_ranks": [
11
+ 55,
12
+ 48,
13
+ 47,
14
+ 64,
15
+ 62,
16
+ 52,
17
+ 64,
18
+ 64,
19
+ 50,
20
+ 61,
21
+ 53,
22
+ 59,
23
+ 64,
24
+ 60,
25
+ 50,
26
+ 64,
27
+ 50,
28
+ 64,
29
+ 64,
30
+ 59,
31
+ 50,
32
+ 64,
33
+ 64,
34
+ 63,
35
+ 64,
36
+ 64,
37
+ 51,
38
+ 61,
39
+ 51,
40
+ 64,
41
+ 61,
42
+ 55
43
+ ],
44
+ "k_padded_rank": 64,
45
+ "v_joint_rank": 64,
46
+ "quant": {
47
+ "top_fraction": 0.25,
48
+ "top_bits": 4,
49
+ "bottom_bits": 3
50
+ }
51
+ },
52
+ {
53
+ "layer_idx": 1,
54
+ "k_head_ranks": [
55
+ 64,
56
+ 64,
57
+ 64,
58
+ 64,
59
+ 64,
60
+ 64,
61
+ 64,
62
+ 64,
63
+ 64,
64
+ 64,
65
+ 64,
66
+ 64,
67
+ 64,
68
+ 64,
69
+ 64,
70
+ 64,
71
+ 64,
72
+ 64,
73
+ 64,
74
+ 64,
75
+ 64,
76
+ 64,
77
+ 64,
78
+ 64,
79
+ 64,
80
+ 64,
81
+ 64,
82
+ 64,
83
+ 64,
84
+ 64,
85
+ 64,
86
+ 64
87
+ ],
88
+ "k_padded_rank": 64,
89
+ "v_joint_rank": 64,
90
+ "quant": {
91
+ "top_fraction": 0.25,
92
+ "top_bits": 4,
93
+ "bottom_bits": 3
94
+ }
95
+ },
96
+ {
97
+ "layer_idx": 2,
98
+ "k_head_ranks": [
99
+ 64,
100
+ 64,
101
+ 64,
102
+ 64,
103
+ 64,
104
+ 64,
105
+ 64,
106
+ 64,
107
+ 64,
108
+ 64,
109
+ 64,
110
+ 64,
111
+ 64,
112
+ 64,
113
+ 64,
114
+ 64,
115
+ 64,
116
+ 64,
117
+ 64,
118
+ 64,
119
+ 64,
120
+ 64,
121
+ 64,
122
+ 64,
123
+ 64,
124
+ 64,
125
+ 64,
126
+ 64,
127
+ 64,
128
+ 64,
129
+ 64,
130
+ 64
131
+ ],
132
+ "k_padded_rank": 64,
133
+ "v_joint_rank": 64,
134
+ "quant": {
135
+ "top_fraction": 0.25,
136
+ "top_bits": 4,
137
+ "bottom_bits": 3
138
+ }
139
+ },
140
+ {
141
+ "layer_idx": 3,
142
+ "k_head_ranks": [
143
+ 64,
144
+ 64,
145
+ 64,
146
+ 64,
147
+ 64,
148
+ 64,
149
+ 64,
150
+ 64,
151
+ 64,
152
+ 64,
153
+ 64,
154
+ 64,
155
+ 64,
156
+ 64,
157
+ 64,
158
+ 64,
159
+ 64,
160
+ 64,
161
+ 64,
162
+ 64,
163
+ 64,
164
+ 64,
165
+ 64,
166
+ 64,
167
+ 64,
168
+ 64,
169
+ 64,
170
+ 64,
171
+ 64,
172
+ 64,
173
+ 64,
174
+ 64
175
+ ],
176
+ "k_padded_rank": 64,
177
+ "v_joint_rank": 64,
178
+ "quant": {
179
+ "top_fraction": 0.25,
180
+ "top_bits": 4,
181
+ "bottom_bits": 3
182
+ }
183
+ },
184
+ {
185
+ "layer_idx": 4,
186
+ "k_head_ranks": [
187
+ 64,
188
+ 64,
189
+ 64,
190
+ 64,
191
+ 64,
192
+ 64,
193
+ 64,
194
+ 64,
195
+ 64,
196
+ 64,
197
+ 64,
198
+ 64,
199
+ 64,
200
+ 64,
201
+ 64,
202
+ 64,
203
+ 64,
204
+ 64,
205
+ 64,
206
+ 64,
207
+ 64,
208
+ 64,
209
+ 64,
210
+ 64,
211
+ 64,
212
+ 64,
213
+ 64,
214
+ 64,
215
+ 64,
216
+ 64,
217
+ 64,
218
+ 64
219
+ ],
220
+ "k_padded_rank": 64,
221
+ "v_joint_rank": 64,
222
+ "quant": {
223
+ "top_fraction": 0.25,
224
+ "top_bits": 4,
225
+ "bottom_bits": 3
226
+ }
227
+ },
228
+ {
229
+ "layer_idx": 5,
230
+ "k_head_ranks": [
231
+ 64,
232
+ 64,
233
+ 64,
234
+ 64,
235
+ 64,
236
+ 64,
237
+ 64,
238
+ 64,
239
+ 64,
240
+ 64,
241
+ 64,
242
+ 64,
243
+ 64,
244
+ 64,
245
+ 64,
246
+ 64,
247
+ 64,
248
+ 64,
249
+ 64,
250
+ 64,
251
+ 64,
252
+ 64,
253
+ 64,
254
+ 64,
255
+ 64,
256
+ 64,
257
+ 64,
258
+ 64,
259
+ 64,
260
+ 64,
261
+ 64,
262
+ 64
263
+ ],
264
+ "k_padded_rank": 64,
265
+ "v_joint_rank": 64,
266
+ "quant": {
267
+ "top_fraction": 0.25,
268
+ "top_bits": 4,
269
+ "bottom_bits": 3
270
+ }
271
+ },
272
+ {
273
+ "layer_idx": 6,
274
+ "k_head_ranks": [
275
+ 64,
276
+ 64,
277
+ 64,
278
+ 64,
279
+ 64,
280
+ 64,
281
+ 64,
282
+ 64,
283
+ 64,
284
+ 64,
285
+ 64,
286
+ 64,
287
+ 64,
288
+ 64,
289
+ 64,
290
+ 64,
291
+ 64,
292
+ 64,
293
+ 64,
294
+ 64,
295
+ 64,
296
+ 64,
297
+ 64,
298
+ 64,
299
+ 64,
300
+ 64,
301
+ 64,
302
+ 64,
303
+ 64,
304
+ 64,
305
+ 64,
306
+ 64
307
+ ],
308
+ "k_padded_rank": 64,
309
+ "v_joint_rank": 64,
310
+ "quant": {
311
+ "top_fraction": 0.25,
312
+ "top_bits": 4,
313
+ "bottom_bits": 3
314
+ }
315
+ },
316
+ {
317
+ "layer_idx": 7,
318
+ "k_head_ranks": [
319
+ 64,
320
+ 64,
321
+ 64,
322
+ 64,
323
+ 64,
324
+ 64,
325
+ 64,
326
+ 64,
327
+ 64,
328
+ 64,
329
+ 64,
330
+ 64,
331
+ 64,
332
+ 64,
333
+ 64,
334
+ 64,
335
+ 64,
336
+ 64,
337
+ 64,
338
+ 64,
339
+ 64,
340
+ 64,
341
+ 64,
342
+ 64,
343
+ 64,
344
+ 64,
345
+ 64,
346
+ 64,
347
+ 64,
348
+ 64,
349
+ 64,
350
+ 64
351
+ ],
352
+ "k_padded_rank": 64,
353
+ "v_joint_rank": 64,
354
+ "quant": {
355
+ "top_fraction": 0.25,
356
+ "top_bits": 4,
357
+ "bottom_bits": 3
358
+ }
359
+ },
360
+ {
361
+ "layer_idx": 8,
362
+ "k_head_ranks": [
363
+ 64,
364
+ 64,
365
+ 64,
366
+ 64,
367
+ 64,
368
+ 64,
369
+ 64,
370
+ 64,
371
+ 64,
372
+ 64,
373
+ 64,
374
+ 64,
375
+ 64,
376
+ 64,
377
+ 64,
378
+ 64,
379
+ 64,
380
+ 64,
381
+ 64,
382
+ 64,
383
+ 64,
384
+ 64,
385
+ 64,
386
+ 64,
387
+ 64,
388
+ 64,
389
+ 64,
390
+ 64,
391
+ 64,
392
+ 64,
393
+ 64,
394
+ 64
395
+ ],
396
+ "k_padded_rank": 64,
397
+ "v_joint_rank": 64,
398
+ "quant": {
399
+ "top_fraction": 0.25,
400
+ "top_bits": 4,
401
+ "bottom_bits": 3
402
+ }
403
+ },
404
+ {
405
+ "layer_idx": 9,
406
+ "k_head_ranks": [
407
+ 64,
408
+ 64,
409
+ 64,
410
+ 64,
411
+ 64,
412
+ 64,
413
+ 64,
414
+ 64,
415
+ 64,
416
+ 64,
417
+ 64,
418
+ 64,
419
+ 64,
420
+ 64,
421
+ 64,
422
+ 64,
423
+ 64,
424
+ 64,
425
+ 64,
426
+ 64,
427
+ 64,
428
+ 64,
429
+ 64,
430
+ 64,
431
+ 64,
432
+ 64,
433
+ 64,
434
+ 64,
435
+ 64,
436
+ 64,
437
+ 64,
438
+ 64
439
+ ],
440
+ "k_padded_rank": 64,
441
+ "v_joint_rank": 64,
442
+ "quant": {
443
+ "top_fraction": 0.25,
444
+ "top_bits": 4,
445
+ "bottom_bits": 3
446
+ }
447
+ },
448
+ {
449
+ "layer_idx": 10,
450
+ "k_head_ranks": [
451
+ 64,
452
+ 64,
453
+ 64,
454
+ 64,
455
+ 64,
456
+ 64,
457
+ 64,
458
+ 64,
459
+ 64,
460
+ 64,
461
+ 64,
462
+ 64,
463
+ 64,
464
+ 64,
465
+ 64,
466
+ 64,
467
+ 64,
468
+ 64,
469
+ 64,
470
+ 64,
471
+ 64,
472
+ 64,
473
+ 64,
474
+ 64,
475
+ 64,
476
+ 64,
477
+ 64,
478
+ 64,
479
+ 64,
480
+ 64,
481
+ 64,
482
+ 64
483
+ ],
484
+ "k_padded_rank": 64,
485
+ "v_joint_rank": 64,
486
+ "quant": {
487
+ "top_fraction": 0.25,
488
+ "top_bits": 4,
489
+ "bottom_bits": 3
490
+ }
491
+ },
492
+ {
493
+ "layer_idx": 11,
494
+ "k_head_ranks": [
495
+ 64,
496
+ 64,
497
+ 64,
498
+ 64,
499
+ 64,
500
+ 64,
501
+ 64,
502
+ 64,
503
+ 64,
504
+ 64,
505
+ 64,
506
+ 64,
507
+ 64,
508
+ 64,
509
+ 64,
510
+ 64,
511
+ 64,
512
+ 64,
513
+ 64,
514
+ 64,
515
+ 64,
516
+ 64,
517
+ 64,
518
+ 64,
519
+ 64,
520
+ 64,
521
+ 64,
522
+ 64,
523
+ 64,
524
+ 64,
525
+ 64,
526
+ 64
527
+ ],
528
+ "k_padded_rank": 64,
529
+ "v_joint_rank": 64,
530
+ "quant": {
531
+ "top_fraction": 0.25,
532
+ "top_bits": 4,
533
+ "bottom_bits": 3
534
+ }
535
+ },
536
+ {
537
+ "layer_idx": 12,
538
+ "k_head_ranks": [
539
+ 64,
540
+ 64,
541
+ 64,
542
+ 64,
543
+ 64,
544
+ 64,
545
+ 64,
546
+ 64,
547
+ 64,
548
+ 64,
549
+ 64,
550
+ 64,
551
+ 64,
552
+ 64,
553
+ 64,
554
+ 64,
555
+ 64,
556
+ 64,
557
+ 64,
558
+ 64,
559
+ 64,
560
+ 64,
561
+ 64,
562
+ 64,
563
+ 64,
564
+ 64,
565
+ 64,
566
+ 64,
567
+ 64,
568
+ 64,
569
+ 64,
570
+ 64
571
+ ],
572
+ "k_padded_rank": 64,
573
+ "v_joint_rank": 64,
574
+ "quant": {
575
+ "top_fraction": 0.25,
576
+ "top_bits": 4,
577
+ "bottom_bits": 3
578
+ }
579
+ },
580
+ {
581
+ "layer_idx": 13,
582
+ "k_head_ranks": [
583
+ 64,
584
+ 64,
585
+ 64,
586
+ 64,
587
+ 64,
588
+ 64,
589
+ 64,
590
+ 64,
591
+ 64,
592
+ 64,
593
+ 64,
594
+ 64,
595
+ 64,
596
+ 64,
597
+ 64,
598
+ 64,
599
+ 64,
600
+ 64,
601
+ 64,
602
+ 64,
603
+ 64,
604
+ 64,
605
+ 64,
606
+ 64,
607
+ 64,
608
+ 64,
609
+ 64,
610
+ 64,
611
+ 64,
612
+ 64,
613
+ 64,
614
+ 64
615
+ ],
616
+ "k_padded_rank": 64,
617
+ "v_joint_rank": 64,
618
+ "quant": {
619
+ "top_fraction": 0.25,
620
+ "top_bits": 4,
621
+ "bottom_bits": 3
622
+ }
623
+ },
624
+ {
625
+ "layer_idx": 14,
626
+ "k_head_ranks": [
627
+ 64,
628
+ 64,
629
+ 64,
630
+ 64,
631
+ 64,
632
+ 64,
633
+ 64,
634
+ 64,
635
+ 64,
636
+ 64,
637
+ 64,
638
+ 64,
639
+ 64,
640
+ 64,
641
+ 64,
642
+ 64,
643
+ 64,
644
+ 64,
645
+ 64,
646
+ 64,
647
+ 64,
648
+ 64,
649
+ 64,
650
+ 64,
651
+ 64,
652
+ 64,
653
+ 64,
654
+ 64,
655
+ 64,
656
+ 64,
657
+ 64,
658
+ 64
659
+ ],
660
+ "k_padded_rank": 64,
661
+ "v_joint_rank": 64,
662
+ "quant": {
663
+ "top_fraction": 0.25,
664
+ "top_bits": 4,
665
+ "bottom_bits": 3
666
+ }
667
+ },
668
+ {
669
+ "layer_idx": 15,
670
+ "k_head_ranks": [
671
+ 64,
672
+ 64,
673
+ 64,
674
+ 64,
675
+ 64,
676
+ 64,
677
+ 64,
678
+ 64,
679
+ 64,
680
+ 64,
681
+ 64,
682
+ 64,
683
+ 64,
684
+ 64,
685
+ 64,
686
+ 64,
687
+ 64,
688
+ 64,
689
+ 64,
690
+ 64,
691
+ 64,
692
+ 64,
693
+ 64,
694
+ 64,
695
+ 64,
696
+ 64,
697
+ 64,
698
+ 64,
699
+ 64,
700
+ 64,
701
+ 64,
702
+ 64
703
+ ],
704
+ "k_padded_rank": 64,
705
+ "v_joint_rank": 64,
706
+ "quant": {
707
+ "top_fraction": 0.25,
708
+ "top_bits": 4,
709
+ "bottom_bits": 3
710
+ }
711
+ },
712
+ {
713
+ "layer_idx": 16,
714
+ "k_head_ranks": [
715
+ 64,
716
+ 64,
717
+ 64,
718
+ 64,
719
+ 64,
720
+ 64,
721
+ 64,
722
+ 64,
723
+ 64,
724
+ 64,
725
+ 64,
726
+ 64,
727
+ 64,
728
+ 64,
729
+ 64,
730
+ 64,
731
+ 64,
732
+ 64,
733
+ 64,
734
+ 64,
735
+ 64,
736
+ 64,
737
+ 64,
738
+ 64,
739
+ 64,
740
+ 64,
741
+ 64,
742
+ 64,
743
+ 64,
744
+ 64,
745
+ 64,
746
+ 64
747
+ ],
748
+ "k_padded_rank": 64,
749
+ "v_joint_rank": 64,
750
+ "quant": {
751
+ "top_fraction": 0.25,
752
+ "top_bits": 4,
753
+ "bottom_bits": 3
754
+ }
755
+ },
756
+ {
757
+ "layer_idx": 17,
758
+ "k_head_ranks": [
759
+ 64,
760
+ 64,
761
+ 64,
762
+ 64,
763
+ 64,
764
+ 64,
765
+ 64,
766
+ 64,
767
+ 64,
768
+ 64,
769
+ 64,
770
+ 64,
771
+ 64,
772
+ 64,
773
+ 64,
774
+ 64,
775
+ 64,
776
+ 64,
777
+ 64,
778
+ 64,
779
+ 64,
780
+ 64,
781
+ 64,
782
+ 64,
783
+ 64,
784
+ 64,
785
+ 64,
786
+ 64,
787
+ 64,
788
+ 64,
789
+ 64,
790
+ 64
791
+ ],
792
+ "k_padded_rank": 64,
793
+ "v_joint_rank": 64,
794
+ "quant": {
795
+ "top_fraction": 0.25,
796
+ "top_bits": 4,
797
+ "bottom_bits": 3
798
+ }
799
+ },
800
+ {
801
+ "layer_idx": 18,
802
+ "k_head_ranks": [
803
+ 64,
804
+ 64,
805
+ 64,
806
+ 64,
807
+ 64,
808
+ 64,
809
+ 64,
810
+ 64,
811
+ 64,
812
+ 64,
813
+ 64,
814
+ 64,
815
+ 64,
816
+ 64,
817
+ 64,
818
+ 64,
819
+ 64,
820
+ 64,
821
+ 64,
822
+ 64,
823
+ 64,
824
+ 64,
825
+ 64,
826
+ 64,
827
+ 64,
828
+ 64,
829
+ 64,
830
+ 64,
831
+ 64,
832
+ 64,
833
+ 64,
834
+ 64
835
+ ],
836
+ "k_padded_rank": 64,
837
+ "v_joint_rank": 64,
838
+ "quant": {
839
+ "top_fraction": 0.25,
840
+ "top_bits": 4,
841
+ "bottom_bits": 3
842
+ }
843
+ },
844
+ {
845
+ "layer_idx": 19,
846
+ "k_head_ranks": [
847
+ 64,
848
+ 64,
849
+ 64,
850
+ 64,
851
+ 64,
852
+ 64,
853
+ 64,
854
+ 64,
855
+ 64,
856
+ 64,
857
+ 64,
858
+ 64,
859
+ 64,
860
+ 64,
861
+ 64,
862
+ 64,
863
+ 64,
864
+ 64,
865
+ 64,
866
+ 64,
867
+ 64,
868
+ 64,
869
+ 64,
870
+ 64,
871
+ 64,
872
+ 64,
873
+ 64,
874
+ 64,
875
+ 64,
876
+ 64,
877
+ 64,
878
+ 64
879
+ ],
880
+ "k_padded_rank": 64,
881
+ "v_joint_rank": 64,
882
+ "quant": {
883
+ "top_fraction": 0.25,
884
+ "top_bits": 4,
885
+ "bottom_bits": 3
886
+ }
887
+ },
888
+ {
889
+ "layer_idx": 20,
890
+ "k_head_ranks": [
891
+ 64,
892
+ 64,
893
+ 64,
894
+ 64,
895
+ 64,
896
+ 64,
897
+ 64,
898
+ 64,
899
+ 64,
900
+ 64,
901
+ 64,
902
+ 64,
903
+ 64,
904
+ 64,
905
+ 64,
906
+ 64,
907
+ 64,
908
+ 64,
909
+ 64,
910
+ 64,
911
+ 64,
912
+ 64,
913
+ 64,
914
+ 64,
915
+ 64,
916
+ 64,
917
+ 64,
918
+ 64,
919
+ 64,
920
+ 64,
921
+ 64,
922
+ 64
923
+ ],
924
+ "k_padded_rank": 64,
925
+ "v_joint_rank": 64,
926
+ "quant": {
927
+ "top_fraction": 0.25,
928
+ "top_bits": 4,
929
+ "bottom_bits": 3
930
+ }
931
+ },
932
+ {
933
+ "layer_idx": 21,
934
+ "k_head_ranks": [
935
+ 64,
936
+ 64,
937
+ 64,
938
+ 64,
939
+ 64,
940
+ 64,
941
+ 64,
942
+ 64,
943
+ 64,
944
+ 64,
945
+ 64,
946
+ 64,
947
+ 64,
948
+ 64,
949
+ 64,
950
+ 64,
951
+ 64,
952
+ 64,
953
+ 64,
954
+ 64,
955
+ 64,
956
+ 64,
957
+ 64,
958
+ 64,
959
+ 64,
960
+ 64,
961
+ 64,
962
+ 64,
963
+ 64,
964
+ 64,
965
+ 64,
966
+ 64
967
+ ],
968
+ "k_padded_rank": 64,
969
+ "v_joint_rank": 64,
970
+ "quant": {
971
+ "top_fraction": 0.25,
972
+ "top_bits": 4,
973
+ "bottom_bits": 3
974
+ }
975
+ },
976
+ {
977
+ "layer_idx": 22,
978
+ "k_head_ranks": [
979
+ 64,
980
+ 64,
981
+ 64,
982
+ 64,
983
+ 64,
984
+ 64,
985
+ 64,
986
+ 64,
987
+ 64,
988
+ 64,
989
+ 64,
990
+ 64,
991
+ 64,
992
+ 64,
993
+ 64,
994
+ 64,
995
+ 64,
996
+ 64,
997
+ 64,
998
+ 64,
999
+ 64,
1000
+ 64,
1001
+ 64,
1002
+ 64,
1003
+ 64,
1004
+ 64,
1005
+ 64,
1006
+ 64,
1007
+ 64,
1008
+ 64,
1009
+ 64,
1010
+ 64
1011
+ ],
1012
+ "k_padded_rank": 64,
1013
+ "v_joint_rank": 64,
1014
+ "quant": {
1015
+ "top_fraction": 0.25,
1016
+ "top_bits": 4,
1017
+ "bottom_bits": 3
1018
+ }
1019
+ },
1020
+ {
1021
+ "layer_idx": 23,
1022
+ "k_head_ranks": [
1023
+ 64,
1024
+ 64,
1025
+ 64,
1026
+ 64,
1027
+ 64,
1028
+ 64,
1029
+ 64,
1030
+ 64,
1031
+ 64,
1032
+ 64,
1033
+ 64,
1034
+ 64,
1035
+ 64,
1036
+ 64,
1037
+ 64,
1038
+ 64,
1039
+ 64,
1040
+ 64,
1041
+ 64,
1042
+ 64,
1043
+ 64,
1044
+ 64,
1045
+ 64,
1046
+ 64,
1047
+ 64,
1048
+ 64,
1049
+ 64,
1050
+ 64,
1051
+ 64,
1052
+ 64,
1053
+ 64,
1054
+ 64
1055
+ ],
1056
+ "k_padded_rank": 64,
1057
+ "v_joint_rank": 64,
1058
+ "quant": {
1059
+ "top_fraction": 0.25,
1060
+ "top_bits": 4,
1061
+ "bottom_bits": 3
1062
+ }
1063
+ },
1064
+ {
1065
+ "layer_idx": 24,
1066
+ "k_head_ranks": [
1067
+ 64,
1068
+ 64,
1069
+ 64,
1070
+ 64,
1071
+ 64,
1072
+ 64,
1073
+ 64,
1074
+ 64,
1075
+ 64,
1076
+ 64,
1077
+ 64,
1078
+ 64,
1079
+ 64,
1080
+ 64,
1081
+ 64,
1082
+ 64,
1083
+ 64,
1084
+ 64,
1085
+ 64,
1086
+ 64,
1087
+ 64,
1088
+ 64,
1089
+ 64,
1090
+ 64,
1091
+ 64,
1092
+ 64,
1093
+ 64,
1094
+ 64,
1095
+ 64,
1096
+ 64,
1097
+ 64,
1098
+ 64
1099
+ ],
1100
+ "k_padded_rank": 64,
1101
+ "v_joint_rank": 64,
1102
+ "quant": {
1103
+ "top_fraction": 0.25,
1104
+ "top_bits": 4,
1105
+ "bottom_bits": 3
1106
+ }
1107
+ },
1108
+ {
1109
+ "layer_idx": 25,
1110
+ "k_head_ranks": [
1111
+ 64,
1112
+ 64,
1113
+ 64,
1114
+ 64,
1115
+ 64,
1116
+ 64,
1117
+ 64,
1118
+ 64,
1119
+ 64,
1120
+ 64,
1121
+ 64,
1122
+ 64,
1123
+ 64,
1124
+ 64,
1125
+ 64,
1126
+ 64,
1127
+ 64,
1128
+ 64,
1129
+ 64,
1130
+ 64,
1131
+ 64,
1132
+ 64,
1133
+ 64,
1134
+ 64,
1135
+ 64,
1136
+ 64,
1137
+ 64,
1138
+ 64,
1139
+ 64,
1140
+ 64,
1141
+ 64,
1142
+ 64
1143
+ ],
1144
+ "k_padded_rank": 64,
1145
+ "v_joint_rank": 64,
1146
+ "quant": {
1147
+ "top_fraction": 0.25,
1148
+ "top_bits": 4,
1149
+ "bottom_bits": 3
1150
+ }
1151
+ },
1152
+ {
1153
+ "layer_idx": 26,
1154
+ "k_head_ranks": [
1155
+ 64,
1156
+ 64,
1157
+ 64,
1158
+ 64,
1159
+ 64,
1160
+ 64,
1161
+ 64,
1162
+ 64,
1163
+ 64,
1164
+ 64,
1165
+ 64,
1166
+ 64,
1167
+ 64,
1168
+ 64,
1169
+ 64,
1170
+ 64,
1171
+ 64,
1172
+ 64,
1173
+ 64,
1174
+ 64,
1175
+ 64,
1176
+ 64,
1177
+ 64,
1178
+ 64,
1179
+ 64,
1180
+ 64,
1181
+ 64,
1182
+ 64,
1183
+ 64,
1184
+ 64,
1185
+ 64,
1186
+ 64
1187
+ ],
1188
+ "k_padded_rank": 64,
1189
+ "v_joint_rank": 64,
1190
+ "quant": {
1191
+ "top_fraction": 0.25,
1192
+ "top_bits": 4,
1193
+ "bottom_bits": 3
1194
+ }
1195
+ },
1196
+ {
1197
+ "layer_idx": 27,
1198
+ "k_head_ranks": [
1199
+ 64,
1200
+ 64,
1201
+ 64,
1202
+ 64,
1203
+ 64,
1204
+ 64,
1205
+ 64,
1206
+ 64,
1207
+ 64,
1208
+ 64,
1209
+ 64,
1210
+ 64,
1211
+ 64,
1212
+ 64,
1213
+ 64,
1214
+ 64,
1215
+ 64,
1216
+ 64,
1217
+ 64,
1218
+ 64,
1219
+ 64,
1220
+ 64,
1221
+ 64,
1222
+ 64,
1223
+ 64,
1224
+ 64,
1225
+ 64,
1226
+ 64,
1227
+ 64,
1228
+ 64,
1229
+ 64,
1230
+ 64
1231
+ ],
1232
+ "k_padded_rank": 64,
1233
+ "v_joint_rank": 64,
1234
+ "quant": {
1235
+ "top_fraction": 0.25,
1236
+ "top_bits": 4,
1237
+ "bottom_bits": 3
1238
+ }
1239
+ },
1240
+ {
1241
+ "layer_idx": 28,
1242
+ "k_head_ranks": [
1243
+ 64,
1244
+ 64,
1245
+ 64,
1246
+ 64,
1247
+ 64,
1248
+ 64,
1249
+ 64,
1250
+ 64,
1251
+ 64,
1252
+ 64,
1253
+ 64,
1254
+ 64,
1255
+ 64,
1256
+ 64,
1257
+ 64,
1258
+ 64,
1259
+ 64,
1260
+ 64,
1261
+ 64,
1262
+ 64,
1263
+ 64,
1264
+ 64,
1265
+ 64,
1266
+ 64,
1267
+ 64,
1268
+ 64,
1269
+ 64,
1270
+ 64,
1271
+ 64,
1272
+ 64,
1273
+ 64,
1274
+ 64
1275
+ ],
1276
+ "k_padded_rank": 64,
1277
+ "v_joint_rank": 64,
1278
+ "quant": {
1279
+ "top_fraction": 0.25,
1280
+ "top_bits": 4,
1281
+ "bottom_bits": 3
1282
+ }
1283
+ },
1284
+ {
1285
+ "layer_idx": 29,
1286
+ "k_head_ranks": [
1287
+ 64,
1288
+ 64,
1289
+ 64,
1290
+ 64,
1291
+ 64,
1292
+ 64,
1293
+ 64,
1294
+ 64,
1295
+ 64,
1296
+ 64,
1297
+ 64,
1298
+ 64,
1299
+ 64,
1300
+ 64,
1301
+ 64,
1302
+ 64,
1303
+ 64,
1304
+ 64,
1305
+ 64,
1306
+ 64,
1307
+ 64,
1308
+ 64,
1309
+ 64,
1310
+ 64,
1311
+ 64,
1312
+ 64,
1313
+ 64,
1314
+ 64,
1315
+ 64,
1316
+ 64,
1317
+ 64,
1318
+ 64
1319
+ ],
1320
+ "k_padded_rank": 64,
1321
+ "v_joint_rank": 64,
1322
+ "quant": {
1323
+ "top_fraction": 0.25,
1324
+ "top_bits": 4,
1325
+ "bottom_bits": 3
1326
+ }
1327
+ },
1328
+ {
1329
+ "layer_idx": 30,
1330
+ "k_head_ranks": [
1331
+ 64,
1332
+ 64,
1333
+ 64,
1334
+ 64,
1335
+ 64,
1336
+ 64,
1337
+ 64,
1338
+ 64,
1339
+ 64,
1340
+ 64,
1341
+ 64,
1342
+ 64,
1343
+ 64,
1344
+ 64,
1345
+ 64,
1346
+ 64,
1347
+ 64,
1348
+ 64,
1349
+ 64,
1350
+ 64,
1351
+ 64,
1352
+ 64,
1353
+ 64,
1354
+ 64,
1355
+ 64,
1356
+ 64,
1357
+ 64,
1358
+ 64,
1359
+ 64,
1360
+ 64,
1361
+ 64,
1362
+ 64
1363
+ ],
1364
+ "k_padded_rank": 64,
1365
+ "v_joint_rank": 64,
1366
+ "quant": {
1367
+ "top_fraction": 0.25,
1368
+ "top_bits": 4,
1369
+ "bottom_bits": 3
1370
+ }
1371
+ },
1372
+ {
1373
+ "layer_idx": 31,
1374
+ "k_head_ranks": [
1375
+ 64,
1376
+ 64,
1377
+ 64,
1378
+ 64,
1379
+ 64,
1380
+ 64,
1381
+ 64,
1382
+ 64,
1383
+ 64,
1384
+ 64,
1385
+ 64,
1386
+ 64,
1387
+ 64,
1388
+ 64,
1389
+ 64,
1390
+ 64,
1391
+ 64,
1392
+ 64,
1393
+ 64,
1394
+ 64,
1395
+ 64,
1396
+ 64,
1397
+ 64,
1398
+ 64,
1399
+ 64,
1400
+ 64,
1401
+ 64,
1402
+ 64,
1403
+ 64,
1404
+ 64,
1405
+ 64,
1406
+ 64
1407
+ ],
1408
+ "k_padded_rank": 64,
1409
+ "v_joint_rank": 64,
1410
+ "quant": {
1411
+ "top_fraction": 0.25,
1412
+ "top_bits": 4,
1413
+ "bottom_bits": 3
1414
+ }
1415
+ }
1416
+ ]
1417
+ }
1418
+ },
1419
+ "active_profile": "balanced",
1420
+ "calibration": {
1421
+ "dataset": "HuggingFaceFW/fineweb-edu (data/CC-MAIN-2013-20/train-00000-of-00014.parquet)",
1422
+ "samples": 64,
1423
+ "seq_len": 256,
1424
+ "paper_reference_samples": 3000,
1425
+ "deviation_note": "scaled down from paper's 3000-sample/6-GPU-hour budget for single-session feasibility"
1426
+ }
1427
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "bos_token": {
5
+ "__type": "AddedToken",
6
+ "content": "<s>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "clean_up_tokenization_spaces": false,
13
+ "eos_token": {
14
+ "__type": "AddedToken",
15
+ "content": "</s>",
16
+ "lstrip": false,
17
+ "normalized": true,
18
+ "rstrip": false,
19
+ "single_word": false
20
+ },
21
+ "legacy": false,
22
+ "model_max_length": 32768,
23
+ "pad_token": null,
24
+ "padding_side": "right",
25
+ "sp_model_kwargs": {},
26
+ "tokenizer_class": "LlamaTokenizer",
27
+ "unk_token": {
28
+ "__type": "AddedToken",
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": true,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ }
35
+ }