Hugging Face's logo

trymirai
/
Qwen3.6-27B-L

Text Generation
Safetensors
uzu
mirai
8-bit precision
Model card Files
xet
 Community
Roman Knyazhitskiy commited on
Commit
4d21578
·
1 Parent(s): 9700580

Fuse gate

Browse files
Files changed (4) hide show
  1. config.json +73 -65
  2. model.safetensors +2 -2
  3. speculator/config.json +20 -20
  4. speculator/model.safetensors +2 -2
config.json CHANGED
@@ -10,7 +10,15 @@
10
  "eos_token": "<|im_end|>",
11
  "bos_token": null,
12
  "end_of_thinking_tag": "\n</think>",
13
- "default_system_prompt": null
 
 
 
 
 
 
 
 
14
  },
15
  "decoder_config": {
16
  "embedding_config": {
@@ -206,7 +214,7 @@
206
  },
207
  "mixer_config": {
208
  "type": "AttentionConfig",
209
- "qkv_projection_config": {},
210
  "out_projection_config": {},
211
  "query_norm_config": {
212
  "epsilon": 1e-06,
@@ -232,11 +240,11 @@
232
  "sliding_window_size": null,
233
  "logit_soft_cap": null,
234
  "has_sinks": false,
235
- "has_qkv_biases": false,
236
  "has_out_biases": false,
237
- "gate_projection_config": {},
238
  "normalize_values": false,
239
- "is_kv_sharing": false
 
240
  },
241
  "post_mixer_norm_config": null,
242
  "pre_mlp_norm_config": {
@@ -456,7 +464,7 @@
456
  },
457
  "mixer_config": {
458
  "type": "AttentionConfig",
459
- "qkv_projection_config": {},
460
  "out_projection_config": {},
461
  "query_norm_config": {
462
  "epsilon": 1e-06,
@@ -482,11 +490,11 @@
482
  "sliding_window_size": null,
483
  "logit_soft_cap": null,
484
  "has_sinks": false,
485
- "has_qkv_biases": false,
486
  "has_out_biases": false,
487
- "gate_projection_config": {},
488
  "normalize_values": false,
489
- "is_kv_sharing": false
 
490
  },
491
  "post_mixer_norm_config": null,
492
  "pre_mlp_norm_config": {
@@ -706,7 +714,7 @@
706
  },
707
  "mixer_config": {
708
  "type": "AttentionConfig",
709
- "qkv_projection_config": {},
710
  "out_projection_config": {},
711
  "query_norm_config": {
712
  "epsilon": 1e-06,
@@ -732,11 +740,11 @@
732
  "sliding_window_size": null,
733
  "logit_soft_cap": null,
734
  "has_sinks": false,
735
- "has_qkv_biases": false,
736
  "has_out_biases": false,
737
- "gate_projection_config": {},
738
  "normalize_values": false,
739
- "is_kv_sharing": false
 
740
  },
741
  "post_mixer_norm_config": null,
742
  "pre_mlp_norm_config": {
@@ -956,7 +964,7 @@
956
  },
957
  "mixer_config": {
958
  "type": "AttentionConfig",
959
- "qkv_projection_config": {},
960
  "out_projection_config": {},
961
  "query_norm_config": {
962
  "epsilon": 1e-06,
@@ -982,11 +990,11 @@
982
  "sliding_window_size": null,
983
  "logit_soft_cap": null,
984
  "has_sinks": false,
985
- "has_qkv_biases": false,
986
  "has_out_biases": false,
987
- "gate_projection_config": {},
988
  "normalize_values": false,
989
- "is_kv_sharing": false
 
990
  },
991
  "post_mixer_norm_config": null,
992
  "pre_mlp_norm_config": {
@@ -1206,7 +1214,7 @@
1206
  },
1207
  "mixer_config": {
1208
  "type": "AttentionConfig",
1209
- "qkv_projection_config": {},
1210
  "out_projection_config": {},
1211
  "query_norm_config": {
1212
  "epsilon": 1e-06,
@@ -1232,11 +1240,11 @@
1232
  "sliding_window_size": null,
1233
  "logit_soft_cap": null,
1234
  "has_sinks": false,
1235
- "has_qkv_biases": false,
1236
  "has_out_biases": false,
1237
- "gate_projection_config": {},
1238
  "normalize_values": false,
1239
- "is_kv_sharing": false
 
1240
  },
1241
  "post_mixer_norm_config": null,
1242
  "pre_mlp_norm_config": {
@@ -1456,7 +1464,7 @@
1456
  },
1457
  "mixer_config": {
1458
  "type": "AttentionConfig",
1459
- "qkv_projection_config": {},
1460
  "out_projection_config": {},
1461
  "query_norm_config": {
1462
  "epsilon": 1e-06,
@@ -1482,11 +1490,11 @@
1482
  "sliding_window_size": null,
1483
  "logit_soft_cap": null,
1484
  "has_sinks": false,
1485
- "has_qkv_biases": false,
1486
  "has_out_biases": false,
1487
- "gate_projection_config": {},
1488
  "normalize_values": false,
1489
- "is_kv_sharing": false
 
1490
  },
1491
  "post_mixer_norm_config": null,
1492
  "pre_mlp_norm_config": {
@@ -1706,7 +1714,7 @@
1706
  },
1707
  "mixer_config": {
1708
  "type": "AttentionConfig",
1709
- "qkv_projection_config": {},
1710
  "out_projection_config": {},
1711
  "query_norm_config": {
1712
  "epsilon": 1e-06,
@@ -1732,11 +1740,11 @@
1732
  "sliding_window_size": null,
1733
  "logit_soft_cap": null,
1734
  "has_sinks": false,
1735
- "has_qkv_biases": false,
1736
  "has_out_biases": false,
1737
- "gate_projection_config": {},
1738
  "normalize_values": false,
1739
- "is_kv_sharing": false
 
1740
  },
1741
  "post_mixer_norm_config": null,
1742
  "pre_mlp_norm_config": {
@@ -1956,7 +1964,7 @@
1956
  },
1957
  "mixer_config": {
1958
  "type": "AttentionConfig",
1959
- "qkv_projection_config": {},
1960
  "out_projection_config": {},
1961
  "query_norm_config": {
1962
  "epsilon": 1e-06,
@@ -1982,11 +1990,11 @@
1982
  "sliding_window_size": null,
1983
  "logit_soft_cap": null,
1984
  "has_sinks": false,
1985
- "has_qkv_biases": false,
1986
  "has_out_biases": false,
1987
- "gate_projection_config": {},
1988
  "normalize_values": false,
1989
- "is_kv_sharing": false
 
1990
  },
1991
  "post_mixer_norm_config": null,
1992
  "pre_mlp_norm_config": {
@@ -2206,7 +2214,7 @@
2206
  },
2207
  "mixer_config": {
2208
  "type": "AttentionConfig",
2209
- "qkv_projection_config": {},
2210
  "out_projection_config": {},
2211
  "query_norm_config": {
2212
  "epsilon": 1e-06,
@@ -2232,11 +2240,11 @@
2232
  "sliding_window_size": null,
2233
  "logit_soft_cap": null,
2234
  "has_sinks": false,
2235
- "has_qkv_biases": false,
2236
  "has_out_biases": false,
2237
- "gate_projection_config": {},
2238
  "normalize_values": false,
2239
- "is_kv_sharing": false
 
2240
  },
2241
  "post_mixer_norm_config": null,
2242
  "pre_mlp_norm_config": {
@@ -2456,7 +2464,7 @@
2456
  },
2457
  "mixer_config": {
2458
  "type": "AttentionConfig",
2459
- "qkv_projection_config": {},
2460
  "out_projection_config": {},
2461
  "query_norm_config": {
2462
  "epsilon": 1e-06,
@@ -2482,11 +2490,11 @@
2482
  "sliding_window_size": null,
2483
  "logit_soft_cap": null,
2484
  "has_sinks": false,
2485
- "has_qkv_biases": false,
2486
  "has_out_biases": false,
2487
- "gate_projection_config": {},
2488
  "normalize_values": false,
2489
- "is_kv_sharing": false
 
2490
  },
2491
  "post_mixer_norm_config": null,
2492
  "pre_mlp_norm_config": {
@@ -2706,7 +2714,7 @@
2706
  },
2707
  "mixer_config": {
2708
  "type": "AttentionConfig",
2709
- "qkv_projection_config": {},
2710
  "out_projection_config": {},
2711
  "query_norm_config": {
2712
  "epsilon": 1e-06,
@@ -2732,11 +2740,11 @@
2732
  "sliding_window_size": null,
2733
  "logit_soft_cap": null,
2734
  "has_sinks": false,
2735
- "has_qkv_biases": false,
2736
  "has_out_biases": false,
2737
- "gate_projection_config": {},
2738
  "normalize_values": false,
2739
- "is_kv_sharing": false
 
2740
  },
2741
  "post_mixer_norm_config": null,
2742
  "pre_mlp_norm_config": {
@@ -2956,7 +2964,7 @@
2956
  },
2957
  "mixer_config": {
2958
  "type": "AttentionConfig",
2959
- "qkv_projection_config": {},
2960
  "out_projection_config": {},
2961
  "query_norm_config": {
2962
  "epsilon": 1e-06,
@@ -2982,11 +2990,11 @@
2982
  "sliding_window_size": null,
2983
  "logit_soft_cap": null,
2984
  "has_sinks": false,
2985
- "has_qkv_biases": false,
2986
  "has_out_biases": false,
2987
- "gate_projection_config": {},
2988
  "normalize_values": false,
2989
- "is_kv_sharing": false
 
2990
  },
2991
  "post_mixer_norm_config": null,
2992
  "pre_mlp_norm_config": {
@@ -3206,7 +3214,7 @@
3206
  },
3207
  "mixer_config": {
3208
  "type": "AttentionConfig",
3209
- "qkv_projection_config": {},
3210
  "out_projection_config": {},
3211
  "query_norm_config": {
3212
  "epsilon": 1e-06,
@@ -3232,11 +3240,11 @@
3232
  "sliding_window_size": null,
3233
  "logit_soft_cap": null,
3234
  "has_sinks": false,
3235
- "has_qkv_biases": false,
3236
  "has_out_biases": false,
3237
- "gate_projection_config": {},
3238
  "normalize_values": false,
3239
- "is_kv_sharing": false
 
3240
  },
3241
  "post_mixer_norm_config": null,
3242
  "pre_mlp_norm_config": {
@@ -3456,7 +3464,7 @@
3456
  },
3457
  "mixer_config": {
3458
  "type": "AttentionConfig",
3459
- "qkv_projection_config": {},
3460
  "out_projection_config": {},
3461
  "query_norm_config": {
3462
  "epsilon": 1e-06,
@@ -3482,11 +3490,11 @@
3482
  "sliding_window_size": null,
3483
  "logit_soft_cap": null,
3484
  "has_sinks": false,
3485
- "has_qkv_biases": false,
3486
  "has_out_biases": false,
3487
- "gate_projection_config": {},
3488
  "normalize_values": false,
3489
- "is_kv_sharing": false
 
3490
  },
3491
  "post_mixer_norm_config": null,
3492
  "pre_mlp_norm_config": {
@@ -3706,7 +3714,7 @@
3706
  },
3707
  "mixer_config": {
3708
  "type": "AttentionConfig",
3709
- "qkv_projection_config": {},
3710
  "out_projection_config": {},
3711
  "query_norm_config": {
3712
  "epsilon": 1e-06,
@@ -3732,11 +3740,11 @@
3732
  "sliding_window_size": null,
3733
  "logit_soft_cap": null,
3734
  "has_sinks": false,
3735
- "has_qkv_biases": false,
3736
  "has_out_biases": false,
3737
- "gate_projection_config": {},
3738
  "normalize_values": false,
3739
- "is_kv_sharing": false
 
3740
  },
3741
  "post_mixer_norm_config": null,
3742
  "pre_mlp_norm_config": {
@@ -3956,7 +3964,7 @@
3956
  },
3957
  "mixer_config": {
3958
  "type": "AttentionConfig",
3959
- "qkv_projection_config": {},
3960
  "out_projection_config": {},
3961
  "query_norm_config": {
3962
  "epsilon": 1e-06,
@@ -3982,11 +3990,11 @@
3982
  "sliding_window_size": null,
3983
  "logit_soft_cap": null,
3984
  "has_sinks": false,
3985
- "has_qkv_biases": false,
3986
  "has_out_biases": false,
3987
- "gate_projection_config": {},
3988
  "normalize_values": false,
3989
- "is_kv_sharing": false
 
3990
  },
3991
  "post_mixer_norm_config": null,
3992
  "pre_mlp_norm_config": {
 
10
  "eos_token": "<|im_end|>",
11
  "bos_token": null,
12
  "end_of_thinking_tag": "\n</think>",
13
+ "default_system_prompt": null,
14
+ "reasoning_config": {
15
+ "default_reasoning_effort": "medium",
16
+ "field_name": "enable_thinking",
17
+ "reasoning_effort_to_field_value": {
18
+ "medium": true,
19
+ "no_reasoning": false
20
+ }
21
+ }
22
  },
23
  "decoder_config": {
24
  "embedding_config": {
 
214
  },
215
  "mixer_config": {
216
  "type": "AttentionConfig",
217
+ "qkvg_projection_config": {},
218
  "out_projection_config": {},
219
  "query_norm_config": {
220
  "epsilon": 1e-06,
 
240
  "sliding_window_size": null,
241
  "logit_soft_cap": null,
242
  "has_sinks": false,
243
+ "has_qkvg_biases": false,
244
  "has_out_biases": false,
 
245
  "normalize_values": false,
246
+ "is_kv_sharing": false,
247
+ "has_gate": true
248
  },
249
  "post_mixer_norm_config": null,
250
  "pre_mlp_norm_config": {
 
464
  },
465
  "mixer_config": {
466
  "type": "AttentionConfig",
467
+ "qkvg_projection_config": {},
468
  "out_projection_config": {},
469
  "query_norm_config": {
470
  "epsilon": 1e-06,
 
490
  "sliding_window_size": null,
491
  "logit_soft_cap": null,
492
  "has_sinks": false,
493
+ "has_qkvg_biases": false,
494
  "has_out_biases": false,
 
495
  "normalize_values": false,
496
+ "is_kv_sharing": false,
497
+ "has_gate": true
498
  },
499
  "post_mixer_norm_config": null,
500
  "pre_mlp_norm_config": {
 
714
  },
715
  "mixer_config": {
716
  "type": "AttentionConfig",
717
+ "qkvg_projection_config": {},
718
  "out_projection_config": {},
719
  "query_norm_config": {
720
  "epsilon": 1e-06,
 
740
  "sliding_window_size": null,
741
  "logit_soft_cap": null,
742
  "has_sinks": false,
743
+ "has_qkvg_biases": false,
744
  "has_out_biases": false,
 
745
  "normalize_values": false,
746
+ "is_kv_sharing": false,
747
+ "has_gate": true
748
  },
749
  "post_mixer_norm_config": null,
750
  "pre_mlp_norm_config": {
 
964
  },
965
  "mixer_config": {
966
  "type": "AttentionConfig",
967
+ "qkvg_projection_config": {},
968
  "out_projection_config": {},
969
  "query_norm_config": {
970
  "epsilon": 1e-06,
 
990
  "sliding_window_size": null,
991
  "logit_soft_cap": null,
992
  "has_sinks": false,
993
+ "has_qkvg_biases": false,
994
  "has_out_biases": false,
 
995
  "normalize_values": false,
996
+ "is_kv_sharing": false,
997
+ "has_gate": true
998
  },
999
  "post_mixer_norm_config": null,
1000
  "pre_mlp_norm_config": {
 
1214
  },
1215
  "mixer_config": {
1216
  "type": "AttentionConfig",
1217
+ "qkvg_projection_config": {},
1218
  "out_projection_config": {},
1219
  "query_norm_config": {
1220
  "epsilon": 1e-06,
 
1240
  "sliding_window_size": null,
1241
  "logit_soft_cap": null,
1242
  "has_sinks": false,
1243
+ "has_qkvg_biases": false,
1244
  "has_out_biases": false,
 
1245
  "normalize_values": false,
1246
+ "is_kv_sharing": false,
1247
+ "has_gate": true
1248
  },
1249
  "post_mixer_norm_config": null,
1250
  "pre_mlp_norm_config": {
 
1464
  },
1465
  "mixer_config": {
1466
  "type": "AttentionConfig",
1467
+ "qkvg_projection_config": {},
1468
  "out_projection_config": {},
1469
  "query_norm_config": {
1470
  "epsilon": 1e-06,
 
1490
  "sliding_window_size": null,
1491
  "logit_soft_cap": null,
1492
  "has_sinks": false,
1493
+ "has_qkvg_biases": false,
1494
  "has_out_biases": false,
 
1495
  "normalize_values": false,
1496
+ "is_kv_sharing": false,
1497
+ "has_gate": true
1498
  },
1499
  "post_mixer_norm_config": null,
1500
  "pre_mlp_norm_config": {
 
1714
  },
1715
  "mixer_config": {
1716
  "type": "AttentionConfig",
1717
+ "qkvg_projection_config": {},
1718
  "out_projection_config": {},
1719
  "query_norm_config": {
1720
  "epsilon": 1e-06,
 
1740
  "sliding_window_size": null,
1741
  "logit_soft_cap": null,
1742
  "has_sinks": false,
1743
+ "has_qkvg_biases": false,
1744
  "has_out_biases": false,
 
1745
  "normalize_values": false,
1746
+ "is_kv_sharing": false,
1747
+ "has_gate": true
1748
  },
1749
  "post_mixer_norm_config": null,
1750
  "pre_mlp_norm_config": {
 
1964
  },
1965
  "mixer_config": {
1966
  "type": "AttentionConfig",
1967
+ "qkvg_projection_config": {},
1968
  "out_projection_config": {},
1969
  "query_norm_config": {
1970
  "epsilon": 1e-06,
 
1990
  "sliding_window_size": null,
1991
  "logit_soft_cap": null,
1992
  "has_sinks": false,
1993
+ "has_qkvg_biases": false,
1994
  "has_out_biases": false,
 
1995
  "normalize_values": false,
1996
+ "is_kv_sharing": false,
1997
+ "has_gate": true
1998
  },
1999
  "post_mixer_norm_config": null,
2000
  "pre_mlp_norm_config": {
 
2214
  },
2215
  "mixer_config": {
2216
  "type": "AttentionConfig",
2217
+ "qkvg_projection_config": {},
2218
  "out_projection_config": {},
2219
  "query_norm_config": {
2220
  "epsilon": 1e-06,
 
2240
  "sliding_window_size": null,
2241
  "logit_soft_cap": null,
2242
  "has_sinks": false,
2243
+ "has_qkvg_biases": false,
2244
  "has_out_biases": false,
 
2245
  "normalize_values": false,
2246
+ "is_kv_sharing": false,
2247
+ "has_gate": true
2248
  },
2249
  "post_mixer_norm_config": null,
2250
  "pre_mlp_norm_config": {
 
2464
  },
2465
  "mixer_config": {
2466
  "type": "AttentionConfig",
2467
+ "qkvg_projection_config": {},
2468
  "out_projection_config": {},
2469
  "query_norm_config": {
2470
  "epsilon": 1e-06,
 
2490
  "sliding_window_size": null,
2491
  "logit_soft_cap": null,
2492
  "has_sinks": false,
2493
+ "has_qkvg_biases": false,
2494
  "has_out_biases": false,
 
2495
  "normalize_values": false,
2496
+ "is_kv_sharing": false,
2497
+ "has_gate": true
2498
  },
2499
  "post_mixer_norm_config": null,
2500
  "pre_mlp_norm_config": {
 
2714
  },
2715
  "mixer_config": {
2716
  "type": "AttentionConfig",
2717
+ "qkvg_projection_config": {},
2718
  "out_projection_config": {},
2719
  "query_norm_config": {
2720
  "epsilon": 1e-06,
 
2740
  "sliding_window_size": null,
2741
  "logit_soft_cap": null,
2742
  "has_sinks": false,
2743
+ "has_qkvg_biases": false,
2744
  "has_out_biases": false,
 
2745
  "normalize_values": false,
2746
+ "is_kv_sharing": false,
2747
+ "has_gate": true
2748
  },
2749
  "post_mixer_norm_config": null,
2750
  "pre_mlp_norm_config": {
 
2964
  },
2965
  "mixer_config": {
2966
  "type": "AttentionConfig",
2967
+ "qkvg_projection_config": {},
2968
  "out_projection_config": {},
2969
  "query_norm_config": {
2970
  "epsilon": 1e-06,
 
2990
  "sliding_window_size": null,
2991
  "logit_soft_cap": null,
2992
  "has_sinks": false,
2993
+ "has_qkvg_biases": false,
2994
  "has_out_biases": false,
 
2995
  "normalize_values": false,
2996
+ "is_kv_sharing": false,
2997
+ "has_gate": true
2998
  },
2999
  "post_mixer_norm_config": null,
3000
  "pre_mlp_norm_config": {
 
3214
  },
3215
  "mixer_config": {
3216
  "type": "AttentionConfig",
3217
+ "qkvg_projection_config": {},
3218
  "out_projection_config": {},
3219
  "query_norm_config": {
3220
  "epsilon": 1e-06,
 
3240
  "sliding_window_size": null,
3241
  "logit_soft_cap": null,
3242
  "has_sinks": false,
3243
+ "has_qkvg_biases": false,
3244
  "has_out_biases": false,
 
3245
  "normalize_values": false,
3246
+ "is_kv_sharing": false,
3247
+ "has_gate": true
3248
  },
3249
  "post_mixer_norm_config": null,
3250
  "pre_mlp_norm_config": {
 
3464
  },
3465
  "mixer_config": {
3466
  "type": "AttentionConfig",
3467
+ "qkvg_projection_config": {},
3468
  "out_projection_config": {},
3469
  "query_norm_config": {
3470
  "epsilon": 1e-06,
 
3490
  "sliding_window_size": null,
3491
  "logit_soft_cap": null,
3492
  "has_sinks": false,
3493
+ "has_qkvg_biases": false,
3494
  "has_out_biases": false,
 
3495
  "normalize_values": false,
3496
+ "is_kv_sharing": false,
3497
+ "has_gate": true
3498
  },
3499
  "post_mixer_norm_config": null,
3500
  "pre_mlp_norm_config": {
 
3714
  },
3715
  "mixer_config": {
3716
  "type": "AttentionConfig",
3717
+ "qkvg_projection_config": {},
3718
  "out_projection_config": {},
3719
  "query_norm_config": {
3720
  "epsilon": 1e-06,
 
3740
  "sliding_window_size": null,
3741
  "logit_soft_cap": null,
3742
  "has_sinks": false,
3743
+ "has_qkvg_biases": false,
3744
  "has_out_biases": false,
 
3745
  "normalize_values": false,
3746
+ "is_kv_sharing": false,
3747
+ "has_gate": true
3748
  },
3749
  "post_mixer_norm_config": null,
3750
  "pre_mlp_norm_config": {
 
3964
  },
3965
  "mixer_config": {
3966
  "type": "AttentionConfig",
3967
+ "qkvg_projection_config": {},
3968
  "out_projection_config": {},
3969
  "query_norm_config": {
3970
  "epsilon": 1e-06,
 
3990
  "sliding_window_size": null,
3991
  "logit_soft_cap": null,
3992
  "has_sinks": false,
3993
+ "has_qkvg_biases": false,
3994
  "has_out_biases": false,
 
3995
  "normalize_values": false,
3996
+ "is_kv_sharing": false,
3997
+ "has_gate": true
3998
  },
3999
  "post_mixer_norm_config": null,
4000
  "pre_mlp_norm_config": {
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:28f82e20ac74caaea82971b2221e4ad841cdb1ea239a039195223370088f3896
3
- size 27769349616
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49e977b8de6256266ddc545a5eaae5833129c819d3fb5b29005adb804110cdbf
3
+ size 27768990416
speculator/config.json CHANGED
@@ -43,7 +43,7 @@
43
  },
44
  "mixer_config": {
45
  "type": "AttentionConfig",
46
- "qkv_projection_config": {},
47
  "out_projection_config": {},
48
  "query_norm_config": {
49
  "epsilon": 1e-06,
@@ -69,11 +69,11 @@
69
  "sliding_window_size": 4095,
70
  "logit_soft_cap": null,
71
  "has_sinks": false,
72
- "has_qkv_biases": false,
73
  "has_out_biases": false,
74
- "gate_projection_config": null,
75
  "normalize_values": false,
76
- "is_kv_sharing": false
 
77
  },
78
  "post_mixer_norm_config": null,
79
  "pre_mlp_norm_config": {
@@ -119,7 +119,7 @@
119
  },
120
  "mixer_config": {
121
  "type": "AttentionConfig",
122
- "qkv_projection_config": {},
123
  "out_projection_config": {},
124
  "query_norm_config": {
125
  "epsilon": 1e-06,
@@ -145,11 +145,11 @@
145
  "sliding_window_size": 4095,
146
  "logit_soft_cap": null,
147
  "has_sinks": false,
148
- "has_qkv_biases": false,
149
  "has_out_biases": false,
150
- "gate_projection_config": null,
151
  "normalize_values": false,
152
- "is_kv_sharing": false
 
153
  },
154
  "post_mixer_norm_config": null,
155
  "pre_mlp_norm_config": {
@@ -195,7 +195,7 @@
195
  },
196
  "mixer_config": {
197
  "type": "AttentionConfig",
198
- "qkv_projection_config": {},
199
  "out_projection_config": {},
200
  "query_norm_config": {
201
  "epsilon": 1e-06,
@@ -221,11 +221,11 @@
221
  "sliding_window_size": 4095,
222
  "logit_soft_cap": null,
223
  "has_sinks": false,
224
- "has_qkv_biases": false,
225
  "has_out_biases": false,
226
- "gate_projection_config": null,
227
  "normalize_values": false,
228
- "is_kv_sharing": false
 
229
  },
230
  "post_mixer_norm_config": null,
231
  "pre_mlp_norm_config": {
@@ -271,7 +271,7 @@
271
  },
272
  "mixer_config": {
273
  "type": "AttentionConfig",
274
- "qkv_projection_config": {},
275
  "out_projection_config": {},
276
  "query_norm_config": {
277
  "epsilon": 1e-06,
@@ -297,11 +297,11 @@
297
  "sliding_window_size": 4095,
298
  "logit_soft_cap": null,
299
  "has_sinks": false,
300
- "has_qkv_biases": false,
301
  "has_out_biases": false,
302
- "gate_projection_config": null,
303
  "normalize_values": false,
304
- "is_kv_sharing": false
 
305
  },
306
  "post_mixer_norm_config": null,
307
  "pre_mlp_norm_config": {
@@ -347,7 +347,7 @@
347
  },
348
  "mixer_config": {
349
  "type": "AttentionConfig",
350
- "qkv_projection_config": {},
351
  "out_projection_config": {},
352
  "query_norm_config": {
353
  "epsilon": 1e-06,
@@ -373,11 +373,11 @@
373
  "sliding_window_size": null,
374
  "logit_soft_cap": null,
375
  "has_sinks": false,
376
- "has_qkv_biases": false,
377
  "has_out_biases": false,
378
- "gate_projection_config": null,
379
  "normalize_values": false,
380
- "is_kv_sharing": false
 
381
  },
382
  "post_mixer_norm_config": null,
383
  "pre_mlp_norm_config": {
 
43
  },
44
  "mixer_config": {
45
  "type": "AttentionConfig",
46
+ "qkvg_projection_config": {},
47
  "out_projection_config": {},
48
  "query_norm_config": {
49
  "epsilon": 1e-06,
 
69
  "sliding_window_size": 4095,
70
  "logit_soft_cap": null,
71
  "has_sinks": false,
72
+ "has_qkvg_biases": false,
73
  "has_out_biases": false,
 
74
  "normalize_values": false,
75
+ "is_kv_sharing": false,
76
+ "has_gate": false
77
  },
78
  "post_mixer_norm_config": null,
79
  "pre_mlp_norm_config": {
 
119
  },
120
  "mixer_config": {
121
  "type": "AttentionConfig",
122
+ "qkvg_projection_config": {},
123
  "out_projection_config": {},
124
  "query_norm_config": {
125
  "epsilon": 1e-06,
 
145
  "sliding_window_size": 4095,
146
  "logit_soft_cap": null,
147
  "has_sinks": false,
148
+ "has_qkvg_biases": false,
149
  "has_out_biases": false,
 
150
  "normalize_values": false,
151
+ "is_kv_sharing": false,
152
+ "has_gate": false
153
  },
154
  "post_mixer_norm_config": null,
155
  "pre_mlp_norm_config": {
 
195
  },
196
  "mixer_config": {
197
  "type": "AttentionConfig",
198
+ "qkvg_projection_config": {},
199
  "out_projection_config": {},
200
  "query_norm_config": {
201
  "epsilon": 1e-06,
 
221
  "sliding_window_size": 4095,
222
  "logit_soft_cap": null,
223
  "has_sinks": false,
224
+ "has_qkvg_biases": false,
225
  "has_out_biases": false,
 
226
  "normalize_values": false,
227
+ "is_kv_sharing": false,
228
+ "has_gate": false
229
  },
230
  "post_mixer_norm_config": null,
231
  "pre_mlp_norm_config": {
 
271
  },
272
  "mixer_config": {
273
  "type": "AttentionConfig",
274
+ "qkvg_projection_config": {},
275
  "out_projection_config": {},
276
  "query_norm_config": {
277
  "epsilon": 1e-06,
 
297
  "sliding_window_size": 4095,
298
  "logit_soft_cap": null,
299
  "has_sinks": false,
300
+ "has_qkvg_biases": false,
301
  "has_out_biases": false,
 
302
  "normalize_values": false,
303
+ "is_kv_sharing": false,
304
+ "has_gate": false
305
  },
306
  "post_mixer_norm_config": null,
307
  "pre_mlp_norm_config": {
 
347
  },
348
  "mixer_config": {
349
  "type": "AttentionConfig",
350
+ "qkvg_projection_config": {},
351
  "out_projection_config": {},
352
  "query_norm_config": {
353
  "epsilon": 1e-06,
 
373
  "sliding_window_size": null,
374
  "logit_soft_cap": null,
375
  "has_sinks": false,
376
+ "has_qkvg_biases": false,
377
  "has_out_biases": false,
 
378
  "normalize_values": false,
379
+ "is_kv_sharing": false,
380
+ "has_gate": false
381
  },
382
  "post_mixer_norm_config": null,
383
  "pre_mlp_norm_config": {
speculator/model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:612fb2afc047de7877733cec47c8229ccb326a49f203ff6fd3ac581966df09ff
3
- size 998365376
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97415c53ece1d87663c07a36780b4456a3135fddd4ee5665867fd2e179528c3a
3
+ size 998363888