Hugging Face's logo

Xerv-AI
/
CRAYON-tokenizer

Text Generation
English
generic
tokenizers
hardware-accelerated
avx2
cuda
rocm
double-array-trie
crayon
Model card Files
xet
 Community
CRAYON-tokenizer
File size: 10,519 Bytes
708f4a3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
[
  {
    "impl": "crayon:cpu:lite",
    "case": "code",
    "n": 10,
    "tokens_per_sec_mean": 32543163.61544498,
    "tokens_per_sec_std": 1660418.469066782,
    "cold_load_time_ms_mean": 8.423077899999498,
    "cold_load_time_ms_std": 0.184571569644575,
    "warm_load_time_ms_mean": 7.978830799999059,
    "warm_load_time_ms_std": 0.21588091628542938,
    "mb_per_sec_mean": 58.49871784489161,
    "mb_per_sec_std": 2.984723694173529,
    "tokens_produced_mean": 556000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "crayon:cpu:lite",
    "case": "english",
    "n": 10,
    "tokens_per_sec_mean": 18605171.456859484,
    "tokens_per_sec_std": 1031803.966051145,
    "cold_load_time_ms_mean": 12.301843200023654,
    "cold_load_time_ms_std": 0.9205535110367989,
    "warm_load_time_ms_mean": 8.13174449996552,
    "warm_load_time_ms_std": 0.31186301379951287,
    "mb_per_sec_mean": 91.18008129352394,
    "mb_per_sec_std": 5.05665694732621,
    "tokens_produced_mean": 144001.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "crayon:cpu:lite",
    "case": "mixed",
    "n": 10,
    "tokens_per_sec_mean": 26267093.78252945,
    "tokens_per_sec_std": 1602078.7282628026,
    "cold_load_time_ms_mean": 8.81386069999337,
    "cold_load_time_ms_std": 1.2445905519076692,
    "warm_load_time_ms_mean": 8.083804200018108,
    "warm_load_time_ms_std": 0.28357285227868667,
    "mb_per_sec_mean": 54.69957524699704,
    "mb_per_sec_std": 3.336228464167218,
    "tokens_produced_mean": 640000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "crayon:cpu:lite",
    "case": "unicode",
    "n": 10,
    "tokens_per_sec_mean": 42149609.116450116,
    "tokens_per_sec_std": 1563828.113027747,
    "cold_load_time_ms_mean": 8.860825800007888,
    "cold_load_time_ms_std": 1.2318979023066992,
    "warm_load_time_ms_mean": 7.961432899998044,
    "warm_load_time_ms_std": 0.20027127594089345,
    "mb_per_sec_mean": 55.97050585697889,
    "mb_per_sec_std": 2.0766088320702214,
    "tokens_produced_mean": 474000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "crayon:cpu:standard",
    "case": "code",
    "n": 10,
    "tokens_per_sec_mean": 18187056.770267364,
    "tokens_per_sec_std": 1211012.2796866265,
    "cold_load_time_ms_mean": 55.81563290000986,
    "cold_load_time_ms_std": 1.5913846515152064,
    "warm_load_time_ms_mean": 56.87761500000761,
    "warm_load_time_ms_std": 2.2221049765449994,
    "mb_per_sec_mean": 58.25982795385274,
    "mb_per_sec_std": 3.879317470427001,
    "tokens_produced_mean": 312000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "crayon:cpu:standard",
    "case": "english",
    "n": 10,
    "tokens_per_sec_mean": 17873251.748225935,
    "tokens_per_sec_std": 793733.3382264881,
    "cold_load_time_ms_mean": 57.222686999995176,
    "cold_load_time_ms_std": 0.9429351112404759,
    "warm_load_time_ms_mean": 56.375195000009626,
    "warm_load_time_ms_std": 0.8928807337267614,
    "mb_per_sec_mean": 92.74559191314898,
    "mb_per_sec_std": 4.11873951712912,
    "tokens_produced_mean": 136001.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "crayon:cpu:standard",
    "case": "mixed",
    "n": 10,
    "tokens_per_sec_mean": 16543894.171191728,
    "tokens_per_sec_std": 597306.6951312533,
    "cold_load_time_ms_mean": 53.648192899987635,
    "cold_load_time_ms_std": 1.2538254564721554,
    "warm_load_time_ms_mean": 57.10128319998091,
    "warm_load_time_ms_std": 1.90463315356463,
    "mb_per_sec_mean": 59.19188950569539,
    "mb_per_sec_std": 2.1370852311656443,
    "tokens_produced_mean": 372501.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "crayon:cpu:standard",
    "case": "unicode",
    "n": 10,
    "tokens_per_sec_mean": 28951524.000394344,
    "tokens_per_sec_std": 1390934.9083544528,
    "cold_load_time_ms_mean": 53.00248129999261,
    "cold_load_time_ms_std": 1.3924670036990465,
    "warm_load_time_ms_mean": 56.75235230000908,
    "warm_load_time_ms_std": 2.0804969789494345,
    "mb_per_sec_mean": 84.3644909403051,
    "mb_per_sec_std": 4.053172312201081,
    "tokens_produced_mean": 216001.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:cl100k_base",
    "case": "code",
    "n": 10,
    "tokens_per_sec_mean": 1433910.8675132527,
    "tokens_per_sec_std": 50377.42553661942,
    "cold_load_time_ms_mean": 0.009309600019946629,
    "cold_load_time_ms_std": 0.010288662787986066,
    "warm_load_time_ms_mean": 0.0004852999950344383,
    "warm_load_time_ms_std": 4.36578980047418e-05,
    "mb_per_sec_mean": 4.652997392040999,
    "mb_per_sec_std": 0.16347322204632306,
    "tokens_produced_mean": 308000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:cl100k_base",
    "case": "english",
    "n": 10,
    "tokens_per_sec_mean": 1796095.8496349081,
    "tokens_per_sec_std": 52793.25030888532,
    "cold_load_time_ms_mean": 79.61717379999413,
    "cold_load_time_ms_std": 251.7495307024531,
    "warm_load_time_ms_mean": 0.0009670000054029515,
    "warm_load_time_ms_std": 0.001478137342961715,
    "mb_per_sec_mean": 9.053785018567714,
    "mb_per_sec_std": 0.26612095274605746,
    "tokens_produced_mean": 140001.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:cl100k_base",
    "case": "mixed",
    "n": 10,
    "tokens_per_sec_mean": 1694892.820124155,
    "tokens_per_sec_std": 17249.8561346943,
    "cold_load_time_ms_mean": 0.007002899999974943,
    "cold_load_time_ms_std": 0.0024695621825708276,
    "warm_load_time_ms_mean": 0.00046349999536232644,
    "warm_load_time_ms_std": 7.491810746504971e-05,
    "mb_per_sec_mean": 5.509475885073746,
    "mb_per_sec_std": 0.056072965361979525,
    "tokens_produced_mean": 410000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:cl100k_base",
    "case": "unicode",
    "n": 10,
    "tokens_per_sec_mean": 2401549.827809613,
    "tokens_per_sec_std": 288884.55917785416,
    "cold_load_time_ms_mean": 0.0068406999957915104,
    "cold_load_time_ms_std": 0.001325396877696408,
    "warm_load_time_ms_mean": 0.00044320001961750677,
    "warm_load_time_ms_std": 3.8574022529242686e-05,
    "mb_per_sec_mean": 4.939854959582487,
    "mb_per_sec_std": 0.5942195351837,
    "tokens_produced_mean": 306000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:o200k_base",
    "case": "code",
    "n": 10,
    "tokens_per_sec_mean": 873023.1351143563,
    "tokens_per_sec_std": 99348.24078521204,
    "cold_load_time_ms_mean": 0.006607900024846458,
    "cold_load_time_ms_std": 0.00213868227735025,
    "warm_load_time_ms_mean": 0.0004508999893459986,
    "warm_load_time_ms_std": 3.718258233207919e-05,
    "mb_per_sec_mean": 2.796614003792315,
    "mb_per_sec_std": 0.31824893322633546,
    "tokens_produced_mean": 312000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:o200k_base",
    "case": "english",
    "n": 10,
    "tokens_per_sec_mean": 943924.5584751449,
    "tokens_per_sec_std": 10877.634765897466,
    "cold_load_time_ms_mean": 112.38300849999519,
    "cold_load_time_ms_std": 355.36493424763796,
    "warm_load_time_ms_mean": 0.0012451999992890705,
    "warm_load_time_ms_std": 0.0023487978392401484,
    "mb_per_sec_mean": 4.758148084311633,
    "mb_per_sec_std": 0.05483213309632247,
    "tokens_produced_mean": 140001.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:o200k_base",
    "case": "mixed",
    "n": 10,
    "tokens_per_sec_mean": 960701.105688441,
    "tokens_per_sec_std": 11606.046915111614,
    "cold_load_time_ms_mean": 0.006605300001183423,
    "cold_load_time_ms_std": 0.0021411037255844156,
    "warm_load_time_ms_mean": 0.0004602999979397282,
    "warm_load_time_ms_std": 8.956942613561185e-05,
    "mb_per_sec_mean": 3.437272129953874,
    "mb_per_sec_std": 0.041525029339549595,
    "tokens_produced_mean": 372500.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:o200k_base",
    "case": "unicode",
    "n": 10,
    "tokens_per_sec_mean": 1218811.126541087,
    "tokens_per_sec_std": 133843.5003360904,
    "cold_load_time_ms_mean": 0.0061763999866570884,
    "cold_load_time_ms_std": 0.0007053232757927612,
    "warm_load_time_ms_mean": 0.00047830000653448224,
    "warm_load_time_ms_std": 4.933797594218068e-05,
    "mb_per_sec_mean": 3.6530964420732652,
    "mb_per_sec_std": 0.4011640558779565,
    "tokens_produced_mean": 210000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:p50k_base",
    "case": "code",
    "n": 10,
    "tokens_per_sec_mean": 1874283.4220600787,
    "tokens_per_sec_std": 214791.74774445157,
    "cold_load_time_ms_mean": 0.006904799991502841,
    "cold_load_time_ms_std": 0.002517008202014823,
    "warm_load_time_ms_mean": 0.00047970000025543413,
    "warm_load_time_ms_std": 6.630576200775165e-05,
    "mb_per_sec_mean": 4.460128208158045,
    "mb_per_sec_std": 0.5111279978892581,
    "tokens_produced_mean": 420000.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:p50k_base",
    "case": "english",
    "n": 10,
    "tokens_per_sec_mean": 2079144.465738733,
    "tokens_per_sec_std": 132466.7738651459,
    "cold_load_time_ms_mean": 87.16504709999242,
    "cold_load_time_ms_std": 275.62101757793516,
    "warm_load_time_ms_mean": 0.0012856999774157885,
    "warm_load_time_ms_std": 0.002490739036019967,
    "mb_per_sec_mean": 10.189455219298377,
    "mb_per_sec_std": 0.6491921473403963,
    "tokens_produced_mean": 144001.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:p50k_base",
    "case": "mixed",
    "n": 10,
    "tokens_per_sec_mean": 2174162.5156145184,
    "tokens_per_sec_std": 21587.89759563529,
    "cold_load_time_ms_mean": 0.005877099997064761,
    "cold_load_time_ms_std": 0.0006585560836289539,
    "warm_load_time_ms_mean": 0.0004788999973470709,
    "warm_load_time_ms_std": 8.320984769302155e-05,
    "mb_per_sec_mean": 5.5992976305151485,
    "mb_per_sec_std": 0.055597069210292546,
    "tokens_produced_mean": 517500.0,
    "tokens_produced_std": 0.0
  },
  {
    "impl": "tiktoken:p50k_base",
    "case": "unicode",
    "n": 10,
    "tokens_per_sec_mean": 3512371.3360246257,
    "tokens_per_sec_std": 32909.21781789151,
    "cold_load_time_ms_mean": 0.007314200007613181,
    "cold_load_time_ms_std": 0.00328893151723234,
    "warm_load_time_ms_mean": 0.0005098000059433616,
    "warm_load_time_ms_std": 0.00010494317397068813,
    "mb_per_sec_mean": 5.848609787512716,
    "mb_per_sec_std": 0.054798640296089184,
    "tokens_produced_mean": 378000.0,
    "tokens_produced_std": 0.0
  }
]