Hugging Face's logo

redashes
/
QwenPaw-Flash-9B-heretic-INT4-AWQ-MTP

Text Generation
Transformers
Safetensors
English
Chinese
qwen3_5
image-text-to-text
qwen
qwen3.5
awq
4-bit precision
int4
quantized
compressed-tensors
sglang
conversational
Model card Files
xet
 Community
redashes commited on
Commit
72c471c
·
verified ·
1 Parent(s): e6e249a

Upload config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +344 -0
config.json ADDED
@@ -0,0 +1,344 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3_5ForConditionalGeneration"
4
+ ],
5
+ "model_type": "qwen3_5",
6
+ "image_token_id": 248056,
7
+ "video_token_id": 248057,
8
+ "vision_start_token_id": 248053,
9
+ "vision_end_token_id": 248054,
10
+ "tie_word_embeddings": false,
11
+ "transformers_version": "5.12.1",
12
+ "text_config": {
13
+ "attention_bias": false,
14
+ "attention_dropout": 0.0,
15
+ "attn_output_gate": true,
16
+ "bos_token_id": null,
17
+ "dtype": "bfloat16",
18
+ "eos_token_id": 248044,
19
+ "full_attention_interval": 4,
20
+ "head_dim": 256,
21
+ "hidden_act": "silu",
22
+ "hidden_size": 4096,
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 12288,
25
+ "layer_types": [
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "linear_attention",
29
+ "full_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "linear_attention",
33
+ "full_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "linear_attention",
37
+ "full_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "linear_attention",
41
+ "full_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "linear_attention",
45
+ "full_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "linear_attention",
49
+ "full_attention",
50
+ "linear_attention",
51
+ "linear_attention",
52
+ "linear_attention",
53
+ "full_attention",
54
+ "linear_attention",
55
+ "linear_attention",
56
+ "linear_attention",
57
+ "full_attention"
58
+ ],
59
+ "linear_conv_kernel_dim": 4,
60
+ "linear_key_head_dim": 128,
61
+ "linear_num_key_heads": 16,
62
+ "linear_num_value_heads": 32,
63
+ "linear_value_head_dim": 128,
64
+ "mamba_ssm_dtype": "float32",
65
+ "max_position_embeddings": 262144,
66
+ "mlp_only_layers": [],
67
+ "mtp_num_hidden_layers": 1,
68
+ "mtp_use_dedicated_embeddings": false,
69
+ "num_attention_heads": 16,
70
+ "num_hidden_layers": 32,
71
+ "num_key_value_heads": 4,
72
+ "pad_token_id": null,
73
+ "partial_rotary_factor": 0.25,
74
+ "rms_norm_eps": 1e-06,
75
+ "rope_parameters": {
76
+ "mrope_interleaved": true,
77
+ "mrope_section": [
78
+ 11,
79
+ 11,
80
+ 10
81
+ ],
82
+ "partial_rotary_factor": 0.25,
83
+ "rope_theta": 10000000,
84
+ "rope_type": "default"
85
+ },
86
+ "tie_word_embeddings": false,
87
+ "use_cache": true,
88
+ "vocab_size": 248320
89
+ },
90
+ "vision_config": {
91
+ "deepstack_visual_indexes": [],
92
+ "depth": 27,
93
+ "dtype": "float32",
94
+ "hidden_act": "gelu_pytorch_tanh",
95
+ "hidden_size": 1152,
96
+ "in_channels": 3,
97
+ "initializer_range": 0.02,
98
+ "intermediate_size": 4304,
99
+ "model_type": "qwen3_5_vision",
100
+ "num_heads": 16,
101
+ "num_position_embeddings": 2304,
102
+ "out_hidden_size": 4096,
103
+ "patch_size": 16,
104
+ "spatial_merge_size": 2,
105
+ "temporal_patch_size": 2
106
+ },
107
+ "quantization_config": {
108
+ "config_groups": {
109
+ "group_0": {
110
+ "format": "pack-quantized",
111
+ "input_activations": null,
112
+ "output_activations": null,
113
+ "targets": [
114
+ "Linear"
115
+ ],
116
+ "weights": {
117
+ "actorder": null,
118
+ "block_structure": null,
119
+ "dynamic": false,
120
+ "group_size": 128,
121
+ "num_bits": 4,
122
+ "observer": "mse",
123
+ "observer_kwargs": {},
124
+ "scale_dtype": null,
125
+ "strategy": "group",
126
+ "symmetric": true,
127
+ "type": "int",
128
+ "zp_dtype": null
129
+ }
130
+ }
131
+ },
132
+ "format": "pack-quantized",
133
+ "global_compression_ratio": null,
134
+ "ignore": [
135
+ "model.language_model.layers.0.linear_attn",
136
+ "model.language_model.layers.0.linear_attn.norm",
137
+ "model.language_model.layers.0.linear_attn.out_proj",
138
+ "model.language_model.layers.0.linear_attn.in_proj_qkv",
139
+ "model.language_model.layers.0.linear_attn.in_proj_z",
140
+ "model.language_model.layers.0.linear_attn.in_proj_b",
141
+ "model.language_model.layers.0.linear_attn.in_proj_a",
142
+ "model.language_model.layers.1.linear_attn",
143
+ "model.language_model.layers.1.linear_attn.norm",
144
+ "model.language_model.layers.1.linear_attn.out_proj",
145
+ "model.language_model.layers.1.linear_attn.in_proj_qkv",
146
+ "model.language_model.layers.1.linear_attn.in_proj_z",
147
+ "model.language_model.layers.1.linear_attn.in_proj_b",
148
+ "model.language_model.layers.1.linear_attn.in_proj_a",
149
+ "model.language_model.layers.2.linear_attn",
150
+ "model.language_model.layers.2.linear_attn.norm",
151
+ "model.language_model.layers.2.linear_attn.out_proj",
152
+ "model.language_model.layers.2.linear_attn.in_proj_qkv",
153
+ "model.language_model.layers.2.linear_attn.in_proj_z",
154
+ "model.language_model.layers.2.linear_attn.in_proj_b",
155
+ "model.language_model.layers.2.linear_attn.in_proj_a",
156
+ "model.language_model.layers.3.self_attn.q_proj",
157
+ "model.language_model.layers.3.self_attn.k_proj",
158
+ "model.language_model.layers.3.self_attn.v_proj",
159
+ "model.language_model.layers.3.self_attn.o_proj",
160
+ "model.language_model.layers.4.linear_attn",
161
+ "model.language_model.layers.4.linear_attn.norm",
162
+ "model.language_model.layers.4.linear_attn.out_proj",
163
+ "model.language_model.layers.4.linear_attn.in_proj_qkv",
164
+ "model.language_model.layers.4.linear_attn.in_proj_z",
165
+ "model.language_model.layers.4.linear_attn.in_proj_b",
166
+ "model.language_model.layers.4.linear_attn.in_proj_a",
167
+ "model.language_model.layers.5.linear_attn",
168
+ "model.language_model.layers.5.linear_attn.norm",
169
+ "model.language_model.layers.5.linear_attn.out_proj",
170
+ "model.language_model.layers.5.linear_attn.in_proj_qkv",
171
+ "model.language_model.layers.5.linear_attn.in_proj_z",
172
+ "model.language_model.layers.5.linear_attn.in_proj_b",
173
+ "model.language_model.layers.5.linear_attn.in_proj_a",
174
+ "model.language_model.layers.6.linear_attn",
175
+ "model.language_model.layers.6.linear_attn.norm",
176
+ "model.language_model.layers.6.linear_attn.out_proj",
177
+ "model.language_model.layers.6.linear_attn.in_proj_qkv",
178
+ "model.language_model.layers.6.linear_attn.in_proj_z",
179
+ "model.language_model.layers.6.linear_attn.in_proj_b",
180
+ "model.language_model.layers.6.linear_attn.in_proj_a",
181
+ "model.language_model.layers.7.self_attn.q_proj",
182
+ "model.language_model.layers.7.self_attn.k_proj",
183
+ "model.language_model.layers.7.self_attn.v_proj",
184
+ "model.language_model.layers.7.self_attn.o_proj",
185
+ "model.language_model.layers.8.linear_attn",
186
+ "model.language_model.layers.8.linear_attn.norm",
187
+ "model.language_model.layers.8.linear_attn.out_proj",
188
+ "model.language_model.layers.8.linear_attn.in_proj_qkv",
189
+ "model.language_model.layers.8.linear_attn.in_proj_z",
190
+ "model.language_model.layers.8.linear_attn.in_proj_b",
191
+ "model.language_model.layers.8.linear_attn.in_proj_a",
192
+ "model.language_model.layers.9.linear_attn",
193
+ "model.language_model.layers.9.linear_attn.norm",
194
+ "model.language_model.layers.9.linear_attn.out_proj",
195
+ "model.language_model.layers.9.linear_attn.in_proj_qkv",
196
+ "model.language_model.layers.9.linear_attn.in_proj_z",
197
+ "model.language_model.layers.9.linear_attn.in_proj_b",
198
+ "model.language_model.layers.9.linear_attn.in_proj_a",
199
+ "model.language_model.layers.10.linear_attn",
200
+ "model.language_model.layers.10.linear_attn.norm",
201
+ "model.language_model.layers.10.linear_attn.out_proj",
202
+ "model.language_model.layers.10.linear_attn.in_proj_qkv",
203
+ "model.language_model.layers.10.linear_attn.in_proj_z",
204
+ "model.language_model.layers.10.linear_attn.in_proj_b",
205
+ "model.language_model.layers.10.linear_attn.in_proj_a",
206
+ "model.language_model.layers.11.self_attn.q_proj",
207
+ "model.language_model.layers.11.self_attn.k_proj",
208
+ "model.language_model.layers.11.self_attn.v_proj",
209
+ "model.language_model.layers.11.self_attn.o_proj",
210
+ "model.language_model.layers.12.linear_attn",
211
+ "model.language_model.layers.12.linear_attn.norm",
212
+ "model.language_model.layers.12.linear_attn.out_proj",
213
+ "model.language_model.layers.12.linear_attn.in_proj_qkv",
214
+ "model.language_model.layers.12.linear_attn.in_proj_z",
215
+ "model.language_model.layers.12.linear_attn.in_proj_b",
216
+ "model.language_model.layers.12.linear_attn.in_proj_a",
217
+ "model.language_model.layers.13.linear_attn",
218
+ "model.language_model.layers.13.linear_attn.norm",
219
+ "model.language_model.layers.13.linear_attn.out_proj",
220
+ "model.language_model.layers.13.linear_attn.in_proj_qkv",
221
+ "model.language_model.layers.13.linear_attn.in_proj_z",
222
+ "model.language_model.layers.13.linear_attn.in_proj_b",
223
+ "model.language_model.layers.13.linear_attn.in_proj_a",
224
+ "model.language_model.layers.14.linear_attn",
225
+ "model.language_model.layers.14.linear_attn.norm",
226
+ "model.language_model.layers.14.linear_attn.out_proj",
227
+ "model.language_model.layers.14.linear_attn.in_proj_qkv",
228
+ "model.language_model.layers.14.linear_attn.in_proj_z",
229
+ "model.language_model.layers.14.linear_attn.in_proj_b",
230
+ "model.language_model.layers.14.linear_attn.in_proj_a",
231
+ "model.language_model.layers.15.self_attn.q_proj",
232
+ "model.language_model.layers.15.self_attn.k_proj",
233
+ "model.language_model.layers.15.self_attn.v_proj",
234
+ "model.language_model.layers.15.self_attn.o_proj",
235
+ "model.language_model.layers.16.linear_attn",
236
+ "model.language_model.layers.16.linear_attn.norm",
237
+ "model.language_model.layers.16.linear_attn.out_proj",
238
+ "model.language_model.layers.16.linear_attn.in_proj_qkv",
239
+ "model.language_model.layers.16.linear_attn.in_proj_z",
240
+ "model.language_model.layers.16.linear_attn.in_proj_b",
241
+ "model.language_model.layers.16.linear_attn.in_proj_a",
242
+ "model.language_model.layers.17.linear_attn",
243
+ "model.language_model.layers.17.linear_attn.norm",
244
+ "model.language_model.layers.17.linear_attn.out_proj",
245
+ "model.language_model.layers.17.linear_attn.in_proj_qkv",
246
+ "model.language_model.layers.17.linear_attn.in_proj_z",
247
+ "model.language_model.layers.17.linear_attn.in_proj_b",
248
+ "model.language_model.layers.17.linear_attn.in_proj_a",
249
+ "model.language_model.layers.18.linear_attn",
250
+ "model.language_model.layers.18.linear_attn.norm",
251
+ "model.language_model.layers.18.linear_attn.out_proj",
252
+ "model.language_model.layers.18.linear_attn.in_proj_qkv",
253
+ "model.language_model.layers.18.linear_attn.in_proj_z",
254
+ "model.language_model.layers.18.linear_attn.in_proj_b",
255
+ "model.language_model.layers.18.linear_attn.in_proj_a",
256
+ "model.language_model.layers.19.self_attn.q_proj",
257
+ "model.language_model.layers.19.self_attn.k_proj",
258
+ "model.language_model.layers.19.self_attn.v_proj",
259
+ "model.language_model.layers.19.self_attn.o_proj",
260
+ "model.language_model.layers.20.linear_attn",
261
+ "model.language_model.layers.20.linear_attn.norm",
262
+ "model.language_model.layers.20.linear_attn.out_proj",
263
+ "model.language_model.layers.20.linear_attn.in_proj_qkv",
264
+ "model.language_model.layers.20.linear_attn.in_proj_z",
265
+ "model.language_model.layers.20.linear_attn.in_proj_b",
266
+ "model.language_model.layers.20.linear_attn.in_proj_a",
267
+ "model.language_model.layers.21.linear_attn",
268
+ "model.language_model.layers.21.linear_attn.norm",
269
+ "model.language_model.layers.21.linear_attn.out_proj",
270
+ "model.language_model.layers.21.linear_attn.in_proj_qkv",
271
+ "model.language_model.layers.21.linear_attn.in_proj_z",
272
+ "model.language_model.layers.21.linear_attn.in_proj_b",
273
+ "model.language_model.layers.21.linear_attn.in_proj_a",
274
+ "model.language_model.layers.22.linear_attn",
275
+ "model.language_model.layers.22.linear_attn.norm",
276
+ "model.language_model.layers.22.linear_attn.out_proj",
277
+ "model.language_model.layers.22.linear_attn.in_proj_qkv",
278
+ "model.language_model.layers.22.linear_attn.in_proj_z",
279
+ "model.language_model.layers.22.linear_attn.in_proj_b",
280
+ "model.language_model.layers.22.linear_attn.in_proj_a",
281
+ "model.language_model.layers.23.self_attn.q_proj",
282
+ "model.language_model.layers.23.self_attn.k_proj",
283
+ "model.language_model.layers.23.self_attn.v_proj",
284
+ "model.language_model.layers.23.self_attn.o_proj",
285
+ "model.language_model.layers.24.linear_attn",
286
+ "model.language_model.layers.24.linear_attn.norm",
287
+ "model.language_model.layers.24.linear_attn.out_proj",
288
+ "model.language_model.layers.24.linear_attn.in_proj_qkv",
289
+ "model.language_model.layers.24.linear_attn.in_proj_z",
290
+ "model.language_model.layers.24.linear_attn.in_proj_b",
291
+ "model.language_model.layers.24.linear_attn.in_proj_a",
292
+ "model.language_model.layers.25.linear_attn",
293
+ "model.language_model.layers.25.linear_attn.norm",
294
+ "model.language_model.layers.25.linear_attn.out_proj",
295
+ "model.language_model.layers.25.linear_attn.in_proj_qkv",
296
+ "model.language_model.layers.25.linear_attn.in_proj_z",
297
+ "model.language_model.layers.25.linear_attn.in_proj_b",
298
+ "model.language_model.layers.25.linear_attn.in_proj_a",
299
+ "model.language_model.layers.26.linear_attn",
300
+ "model.language_model.layers.26.linear_attn.norm",
301
+ "model.language_model.layers.26.linear_attn.out_proj",
302
+ "model.language_model.layers.26.linear_attn.in_proj_qkv",
303
+ "model.language_model.layers.26.linear_attn.in_proj_z",
304
+ "model.language_model.layers.26.linear_attn.in_proj_b",
305
+ "model.language_model.layers.26.linear_attn.in_proj_a",
306
+ "model.language_model.layers.27.self_attn.q_proj",
307
+ "model.language_model.layers.27.self_attn.k_proj",
308
+ "model.language_model.layers.27.self_attn.v_proj",
309
+ "model.language_model.layers.27.self_attn.o_proj",
310
+ "model.language_model.layers.28.linear_attn",
311
+ "model.language_model.layers.28.linear_attn.norm",
312
+ "model.language_model.layers.28.linear_attn.out_proj",
313
+ "model.language_model.layers.28.linear_attn.in_proj_qkv",
314
+ "model.language_model.layers.28.linear_attn.in_proj_z",
315
+ "model.language_model.layers.28.linear_attn.in_proj_b",
316
+ "model.language_model.layers.28.linear_attn.in_proj_a",
317
+ "model.language_model.layers.29.linear_attn",
318
+ "model.language_model.layers.29.linear_attn.norm",
319
+ "model.language_model.layers.29.linear_attn.out_proj",
320
+ "model.language_model.layers.29.linear_attn.in_proj_qkv",
321
+ "model.language_model.layers.29.linear_attn.in_proj_z",
322
+ "model.language_model.layers.29.linear_attn.in_proj_b",
323
+ "model.language_model.layers.29.linear_attn.in_proj_a",
324
+ "model.language_model.layers.30.linear_attn",
325
+ "model.language_model.layers.30.linear_attn.norm",
326
+ "model.language_model.layers.30.linear_attn.out_proj",
327
+ "model.language_model.layers.30.linear_attn.in_proj_qkv",
328
+ "model.language_model.layers.30.linear_attn.in_proj_z",
329
+ "model.language_model.layers.30.linear_attn.in_proj_b",
330
+ "model.language_model.layers.30.linear_attn.in_proj_a",
331
+ "model.language_model.layers.31.self_attn.q_proj",
332
+ "model.language_model.layers.31.self_attn.k_proj",
333
+ "model.language_model.layers.31.self_attn.v_proj",
334
+ "model.language_model.layers.31.self_attn.o_proj",
335
+ "lm_head"
336
+ ],
337
+ "kv_cache_scheme": null,
338
+ "quant_method": "compressed-tensors",
339
+ "quantization_status": "compressed",
340
+ "sparsity_config": {},
341
+ "transform_config": {},
342
+ "version": "0.17.2.a20260618"
343
+ }
344
+ }