Hugging Face's logo

Quazim0t0
/
Escarda-86M-Base

Text Generation
Transformers
Safetensors
English
spike_whale
feature-extraction
small-models
base-model
mla
jepa
experimental
custom_code
Model card Files
xet
 Community
1
Quazim0t0 commited on
Commit
0d80f80
·
verified ·
1 Parent(s): 18b1ee4

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +204 -198
tokenizer_config.json CHANGED
@@ -1,198 +1,204 @@
1
- {
2
- "added_tokens_decoder": {
3
- "0": {
4
- "content": "<pad>",
5
- "lstrip": false,
6
- "normalized": false,
7
- "rstrip": false,
8
- "single_word": false,
9
- "special": true
10
- },
11
- "1": {
12
- "content": "<unk>",
13
- "lstrip": false,
14
- "normalized": false,
15
- "rstrip": false,
16
- "single_word": false,
17
- "special": true
18
- },
19
- "2": {
20
- "content": "<bos>",
21
- "lstrip": false,
22
- "normalized": false,
23
- "rstrip": false,
24
- "single_word": false,
25
- "special": true
26
- },
27
- "3": {
28
- "content": "<eos>",
29
- "lstrip": false,
30
- "normalized": false,
31
- "rstrip": false,
32
- "single_word": false,
33
- "special": true
34
- },
35
- "16384": {
36
- "content": "<|im_start|>",
37
- "lstrip": false,
38
- "normalized": false,
39
- "rstrip": false,
40
- "single_word": false,
41
- "special": true
42
- },
43
- "16385": {
44
- "content": "<|im_end|>",
45
- "lstrip": false,
46
- "normalized": false,
47
- "rstrip": false,
48
- "single_word": false,
49
- "special": true
50
- },
51
- "16386": {
52
- "content": "<think>",
53
- "lstrip": false,
54
- "normalized": false,
55
- "rstrip": false,
56
- "single_word": false,
57
- "special": true
58
- },
59
- "16387": {
60
- "content": "</think>",
61
- "lstrip": false,
62
- "normalized": false,
63
- "rstrip": false,
64
- "single_word": false,
65
- "special": true
66
- },
67
- "16388": {
68
- "content": "<begin_solution>",
69
- "lstrip": false,
70
- "normalized": false,
71
- "rstrip": false,
72
- "single_word": false,
73
- "special": true
74
- },
75
- "16389": {
76
- "content": "<end_solution>",
77
- "lstrip": false,
78
- "normalized": false,
79
- "rstrip": false,
80
- "single_word": false,
81
- "special": true
82
- },
83
- "16390": {
84
- "content": "<tool_call>",
85
- "lstrip": false,
86
- "normalized": false,
87
- "rstrip": false,
88
- "single_word": false,
89
- "special": true
90
- },
91
- "16391": {
92
- "content": "</tool_call>",
93
- "lstrip": false,
94
- "normalized": false,
95
- "rstrip": false,
96
- "single_word": false,
97
- "special": true
98
- },
99
- "16392": {
100
- "content": "<tool_response>",
101
- "lstrip": false,
102
- "normalized": false,
103
- "rstrip": false,
104
- "single_word": false,
105
- "special": true
106
- },
107
- "16393": {
108
- "content": "</tool_response>",
109
- "lstrip": false,
110
- "normalized": false,
111
- "rstrip": false,
112
- "single_word": false,
113
- "special": true
114
- },
115
- "16394": {
116
- "content": "<|system|>",
117
- "lstrip": false,
118
- "normalized": false,
119
- "rstrip": false,
120
- "single_word": false,
121
- "special": true
122
- },
123
- "16395": {
124
- "content": "<|user|>",
125
- "lstrip": false,
126
- "normalized": false,
127
- "rstrip": false,
128
- "single_word": false,
129
- "special": true
130
- },
131
- "16396": {
132
- "content": "<|assistant|>",
133
- "lstrip": false,
134
- "normalized": false,
135
- "rstrip": false,
136
- "single_word": false,
137
- "special": true
138
- },
139
- "16397": {
140
- "content": "<|fim_prefix|>",
141
- "lstrip": false,
142
- "normalized": false,
143
- "rstrip": false,
144
- "single_word": false,
145
- "special": true
146
- },
147
- "16398": {
148
- "content": "<|fim_middle|>",
149
- "lstrip": false,
150
- "normalized": false,
151
- "rstrip": false,
152
- "single_word": false,
153
- "special": true
154
- },
155
- "16399": {
156
- "content": "<|fim_suffix|>",
157
- "lstrip": false,
158
- "normalized": false,
159
- "rstrip": false,
160
- "single_word": false,
161
- "special": true
162
- },
163
- "16400": {
164
- "content": "<|endoftext|>",
165
- "lstrip": false,
166
- "normalized": false,
167
- "rstrip": false,
168
- "single_word": false,
169
- "special": true
170
- }
171
- },
172
- "backend": "custom",
173
- "bos_token": "<bos>",
174
- "eos_token": "<eos>",
175
- "extra_special_tokens": [
176
- "</think>",
177
- "</tool_call>",
178
- "</tool_response>",
179
- "<begin_solution>",
180
- "<end_solution>",
181
- "<think>",
182
- "<tool_call>",
183
- "<tool_response>",
184
- "<|assistant|>",
185
- "<|endoftext|>",
186
- "<|fim_middle|>",
187
- "<|fim_prefix|>",
188
- "<|fim_suffix|>",
189
- "<|im_end|>",
190
- "<|im_start|>",
191
- "<|system|>",
192
- "<|user|>"
193
- ],
194
- "model_max_length": 1000000000000000019884624838656,
195
- "pad_token": "<pad>",
196
- "tokenizer_class": "SpikeTokenizer",
197
- "unk_token": "<unk>"
198
- }
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<pad>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<unk>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<bos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<eos>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "16384": {
36
+ "content": "<|im_start|>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "16385": {
44
+ "content": "<|im_end|>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "16386": {
52
+ "content": "<think>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "16387": {
60
+ "content": "</think>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "16388": {
68
+ "content": "<begin_solution>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "16389": {
76
+ "content": "<end_solution>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "16390": {
84
+ "content": "<tool_call>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "16391": {
92
+ "content": "</tool_call>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "16392": {
100
+ "content": "<tool_response>",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "16393": {
108
+ "content": "</tool_response>",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "16394": {
116
+ "content": "<|system|>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "16395": {
124
+ "content": "<|user|>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "16396": {
132
+ "content": "<|assistant|>",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": true
138
+ },
139
+ "16397": {
140
+ "content": "<|fim_prefix|>",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": true
146
+ },
147
+ "16398": {
148
+ "content": "<|fim_middle|>",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": true
154
+ },
155
+ "16399": {
156
+ "content": "<|fim_suffix|>",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": true
162
+ },
163
+ "16400": {
164
+ "content": "<|endoftext|>",
165
+ "lstrip": false,
166
+ "normalized": false,
167
+ "rstrip": false,
168
+ "single_word": false,
169
+ "special": true
170
+ }
171
+ },
172
+ "backend": "custom",
173
+ "bos_token": "<bos>",
174
+ "eos_token": "<eos>",
175
+ "extra_special_tokens": [
176
+ "</think>",
177
+ "</tool_call>",
178
+ "</tool_response>",
179
+ "<begin_solution>",
180
+ "<end_solution>",
181
+ "<think>",
182
+ "<tool_call>",
183
+ "<tool_response>",
184
+ "<|assistant|>",
185
+ "<|endoftext|>",
186
+ "<|fim_middle|>",
187
+ "<|fim_prefix|>",
188
+ "<|fim_suffix|>",
189
+ "<|im_end|>",
190
+ "<|im_start|>",
191
+ "<|system|>",
192
+ "<|user|>"
193
+ ],
194
+ "model_max_length": 1000000000000000019884624838656,
195
+ "pad_token": "<pad>",
196
+ "tokenizer_class": "SpikeTokenizer",
197
+ "unk_token": "<unk>",
198
+ "auto_map": {
199
+ "AutoTokenizer": [
200
+ "spike_tokenizer.SpikeTokenizer",
201
+ null
202
+ ]
203
+ }
204
+ }