MostafaMaroof commited on
Commit
32ac63a
·
verified ·
1 Parent(s): feccaae

Upload processor

Browse files
Files changed (3) hide show
  1. added_tokens.json +2 -2
  2. tokenizer_config.json +145 -49
  3. vocab.json +77 -54
added_tokens.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "</s>": 60,
3
- "<s>": 59
4
  }
 
1
  {
2
+ "</s>": 83,
3
+ "<s>": 82
4
  }
tokenizer_config.json CHANGED
@@ -17,7 +17,7 @@
17
  "special": false
18
  },
19
  "6": {
20
- "content": "^^",
21
  "lstrip": true,
22
  "normalized": false,
23
  "rstrip": true,
@@ -25,7 +25,7 @@
25
  "special": false
26
  },
27
  "8": {
28
- "content": "a'",
29
  "lstrip": true,
30
  "normalized": false,
31
  "rstrip": true,
@@ -33,7 +33,7 @@
33
  "special": false
34
  },
35
  "9": {
36
- "content": "aa",
37
  "lstrip": true,
38
  "normalized": false,
39
  "rstrip": true,
@@ -41,39 +41,39 @@
41
  "special": false
42
  },
43
  "10": {
44
- "content": "aa'",
45
  "lstrip": true,
46
  "normalized": false,
47
  "rstrip": true,
48
  "single_word": false,
49
  "special": false
50
  },
51
- "11": {
52
- "content": "ah",
53
  "lstrip": true,
54
  "normalized": false,
55
  "rstrip": true,
56
  "single_word": false,
57
  "special": false
58
  },
59
- "15": {
60
- "content": "ee",
61
  "lstrip": true,
62
  "normalized": false,
63
  "rstrip": true,
64
  "single_word": false,
65
  "special": false
66
  },
67
- "20": {
68
- "content": "i0",
69
  "lstrip": true,
70
  "normalized": false,
71
  "rstrip": true,
72
  "single_word": false,
73
  "special": false
74
  },
75
- "21": {
76
- "content": "i0'",
77
  "lstrip": true,
78
  "normalized": false,
79
  "rstrip": true,
@@ -81,7 +81,7 @@
81
  "special": false
82
  },
83
  "22": {
84
- "content": "i1",
85
  "lstrip": true,
86
  "normalized": false,
87
  "rstrip": true,
@@ -89,95 +89,95 @@
89
  "special": false
90
  },
91
  "23": {
92
- "content": "i1'",
93
  "lstrip": true,
94
  "normalized": false,
95
  "rstrip": true,
96
  "single_word": false,
97
  "special": false
98
  },
99
- "24": {
100
- "content": "ii0",
101
  "lstrip": true,
102
  "normalized": false,
103
  "rstrip": true,
104
  "single_word": false,
105
  "special": false
106
  },
107
- "25": {
108
- "content": "ii0'",
109
  "lstrip": true,
110
  "normalized": false,
111
  "rstrip": true,
112
  "single_word": false,
113
  "special": false
114
  },
115
- "26": {
116
- "content": "ii1",
117
  "lstrip": true,
118
  "normalized": false,
119
  "rstrip": true,
120
  "single_word": false,
121
  "special": false
122
  },
123
- "27": {
124
- "content": "ii1'",
125
  "lstrip": true,
126
  "normalized": false,
127
  "rstrip": true,
128
  "single_word": false,
129
  "special": false
130
  },
131
- "34": {
132
- "content": "pp",
133
  "lstrip": true,
134
  "normalized": false,
135
  "rstrip": true,
136
  "single_word": false,
137
  "special": false
138
  },
139
- "38": {
140
- "content": "sh",
141
  "lstrip": true,
142
  "normalized": false,
143
  "rstrip": true,
144
  "single_word": false,
145
  "special": false
146
  },
147
- "39": {
148
- "content": "sil",
149
  "lstrip": true,
150
  "normalized": false,
151
  "rstrip": true,
152
  "single_word": false,
153
  "special": false
154
  },
155
- "41": {
156
- "content": "th",
157
  "lstrip": true,
158
  "normalized": false,
159
  "rstrip": true,
160
  "single_word": false,
161
  "special": false
162
  },
163
- "43": {
164
- "content": "u0",
165
  "lstrip": true,
166
  "normalized": false,
167
  "rstrip": true,
168
  "single_word": false,
169
  "special": false
170
  },
171
- "44": {
172
- "content": "u0'",
173
  "lstrip": true,
174
  "normalized": false,
175
  "rstrip": true,
176
  "single_word": false,
177
  "special": false
178
  },
179
- "45": {
180
- "content": "u1",
181
  "lstrip": true,
182
  "normalized": false,
183
  "rstrip": true,
@@ -185,39 +185,39 @@
185
  "special": false
186
  },
187
  "46": {
188
- "content": "u1'",
189
  "lstrip": true,
190
  "normalized": false,
191
  "rstrip": true,
192
  "single_word": false,
193
  "special": false
194
  },
195
- "47": {
196
- "content": "uu0",
197
  "lstrip": true,
198
  "normalized": false,
199
  "rstrip": true,
200
  "single_word": false,
201
  "special": false
202
  },
203
- "48": {
204
- "content": "uu0'",
205
  "lstrip": true,
206
  "normalized": false,
207
  "rstrip": true,
208
  "single_word": false,
209
  "special": false
210
  },
211
- "49": {
212
- "content": "uu1",
213
  "lstrip": true,
214
  "normalized": false,
215
  "rstrip": true,
216
  "single_word": false,
217
  "special": false
218
  },
219
- "50": {
220
- "content": "uu1'",
221
  "lstrip": true,
222
  "normalized": false,
223
  "rstrip": true,
@@ -225,6 +225,102 @@
225
  "special": false
226
  },
227
  "56": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
228
  "content": "|",
229
  "lstrip": false,
230
  "normalized": false,
@@ -232,7 +328,7 @@
232
  "single_word": false,
233
  "special": true
234
  },
235
- "57": {
236
  "content": "[UNK]",
237
  "lstrip": true,
238
  "normalized": false,
@@ -240,7 +336,7 @@
240
  "single_word": false,
241
  "special": false
242
  },
243
- "58": {
244
  "content": "[PAD]",
245
  "lstrip": true,
246
  "normalized": false,
@@ -248,7 +344,7 @@
248
  "single_word": false,
249
  "special": false
250
  },
251
- "59": {
252
  "content": "<s>",
253
  "lstrip": false,
254
  "normalized": false,
@@ -256,7 +352,7 @@
256
  "single_word": false,
257
  "special": true
258
  },
259
- "60": {
260
  "content": "</s>",
261
  "lstrip": false,
262
  "normalized": false,
 
17
  "special": false
18
  },
19
  "6": {
20
+ "content": "<<",
21
  "lstrip": true,
22
  "normalized": false,
23
  "rstrip": true,
 
25
  "special": false
26
  },
27
  "8": {
28
+ "content": "AA",
29
  "lstrip": true,
30
  "normalized": false,
31
  "rstrip": true,
 
33
  "special": false
34
  },
35
  "9": {
36
+ "content": "AH",
37
  "lstrip": true,
38
  "normalized": false,
39
  "rstrip": true,
 
41
  "special": false
42
  },
43
  "10": {
44
+ "content": "Ah",
45
  "lstrip": true,
46
  "normalized": false,
47
  "rstrip": true,
48
  "single_word": false,
49
  "special": false
50
  },
51
+ "12": {
52
+ "content": "DD",
53
  "lstrip": true,
54
  "normalized": false,
55
  "rstrip": true,
56
  "single_word": false,
57
  "special": false
58
  },
59
+ "14": {
60
+ "content": "EE",
61
  "lstrip": true,
62
  "normalized": false,
63
  "rstrip": true,
64
  "single_word": false,
65
  "special": false
66
  },
67
+ "17": {
68
+ "content": "HH",
69
  "lstrip": true,
70
  "normalized": false,
71
  "rstrip": true,
72
  "single_word": false,
73
  "special": false
74
  },
75
+ "19": {
76
+ "content": "II",
77
  "lstrip": true,
78
  "normalized": false,
79
  "rstrip": true,
 
81
  "special": false
82
  },
83
  "22": {
84
+ "content": "SH",
85
  "lstrip": true,
86
  "normalized": false,
87
  "rstrip": true,
 
89
  "special": false
90
  },
91
  "23": {
92
+ "content": "SS",
93
  "lstrip": true,
94
  "normalized": false,
95
  "rstrip": true,
96
  "single_word": false,
97
  "special": false
98
  },
99
+ "25": {
100
+ "content": "TH",
101
  "lstrip": true,
102
  "normalized": false,
103
  "rstrip": true,
104
  "single_word": false,
105
  "special": false
106
  },
107
+ "26": {
108
+ "content": "TT",
109
  "lstrip": true,
110
  "normalized": false,
111
  "rstrip": true,
112
  "single_word": false,
113
  "special": false
114
  },
115
+ "28": {
116
+ "content": "UU",
117
  "lstrip": true,
118
  "normalized": false,
119
  "rstrip": true,
120
  "single_word": false,
121
  "special": false
122
  },
123
+ "30": {
124
+ "content": "ZZ",
125
  "lstrip": true,
126
  "normalized": false,
127
  "rstrip": true,
128
  "single_word": false,
129
  "special": false
130
  },
131
+ "32": {
132
+ "content": "^^",
133
  "lstrip": true,
134
  "normalized": false,
135
  "rstrip": true,
136
  "single_word": false,
137
  "special": false
138
  },
139
+ "34": {
140
+ "content": "aa",
141
  "lstrip": true,
142
  "normalized": false,
143
  "rstrip": true,
144
  "single_word": false,
145
  "special": false
146
  },
147
+ "36": {
148
+ "content": "bb",
149
  "lstrip": true,
150
  "normalized": false,
151
  "rstrip": true,
152
  "single_word": false,
153
  "special": false
154
  },
155
+ "38": {
156
+ "content": "dd",
157
  "lstrip": true,
158
  "normalized": false,
159
  "rstrip": true,
160
  "single_word": false,
161
  "special": false
162
  },
163
+ "40": {
164
+ "content": "ff",
165
  "lstrip": true,
166
  "normalized": false,
167
  "rstrip": true,
168
  "single_word": false,
169
  "special": false
170
  },
171
+ "42": {
172
+ "content": "gg",
173
  "lstrip": true,
174
  "normalized": false,
175
  "rstrip": true,
176
  "single_word": false,
177
  "special": false
178
  },
179
+ "44": {
180
+ "content": "hh",
181
  "lstrip": true,
182
  "normalized": false,
183
  "rstrip": true,
 
185
  "special": false
186
  },
187
  "46": {
188
+ "content": "ii",
189
  "lstrip": true,
190
  "normalized": false,
191
  "rstrip": true,
192
  "single_word": false,
193
  "special": false
194
  },
195
+ "48": {
196
+ "content": "jj",
197
  "lstrip": true,
198
  "normalized": false,
199
  "rstrip": true,
200
  "single_word": false,
201
  "special": false
202
  },
203
+ "50": {
204
+ "content": "kk",
205
  "lstrip": true,
206
  "normalized": false,
207
  "rstrip": true,
208
  "single_word": false,
209
  "special": false
210
  },
211
+ "52": {
212
+ "content": "ll",
213
  "lstrip": true,
214
  "normalized": false,
215
  "rstrip": true,
216
  "single_word": false,
217
  "special": false
218
  },
219
+ "54": {
220
+ "content": "mm",
221
  "lstrip": true,
222
  "normalized": false,
223
  "rstrip": true,
 
225
  "special": false
226
  },
227
  "56": {
228
+ "content": "nn",
229
+ "lstrip": true,
230
+ "normalized": false,
231
+ "rstrip": true,
232
+ "single_word": false,
233
+ "special": false
234
+ },
235
+ "58": {
236
+ "content": "pp",
237
+ "lstrip": true,
238
+ "normalized": false,
239
+ "rstrip": true,
240
+ "single_word": false,
241
+ "special": false
242
+ },
243
+ "60": {
244
+ "content": "qq",
245
+ "lstrip": true,
246
+ "normalized": false,
247
+ "rstrip": true,
248
+ "single_word": false,
249
+ "special": false
250
+ },
251
+ "62": {
252
+ "content": "rr",
253
+ "lstrip": true,
254
+ "normalized": false,
255
+ "rstrip": true,
256
+ "single_word": false,
257
+ "special": false
258
+ },
259
+ "64": {
260
+ "content": "sil",
261
+ "lstrip": true,
262
+ "normalized": false,
263
+ "rstrip": true,
264
+ "single_word": false,
265
+ "special": false
266
+ },
267
+ "65": {
268
+ "content": "ss",
269
+ "lstrip": true,
270
+ "normalized": false,
271
+ "rstrip": true,
272
+ "single_word": false,
273
+ "special": false
274
+ },
275
+ "67": {
276
+ "content": "tt",
277
+ "lstrip": true,
278
+ "normalized": false,
279
+ "rstrip": true,
280
+ "single_word": false,
281
+ "special": false
282
+ },
283
+ "69": {
284
+ "content": "uu",
285
+ "lstrip": true,
286
+ "normalized": false,
287
+ "rstrip": true,
288
+ "single_word": false,
289
+ "special": false
290
+ },
291
+ "72": {
292
+ "content": "ww",
293
+ "lstrip": true,
294
+ "normalized": false,
295
+ "rstrip": true,
296
+ "single_word": false,
297
+ "special": false
298
+ },
299
+ "74": {
300
+ "content": "xx",
301
+ "lstrip": true,
302
+ "normalized": false,
303
+ "rstrip": true,
304
+ "single_word": false,
305
+ "special": false
306
+ },
307
+ "76": {
308
+ "content": "yy",
309
+ "lstrip": true,
310
+ "normalized": false,
311
+ "rstrip": true,
312
+ "single_word": false,
313
+ "special": false
314
+ },
315
+ "78": {
316
+ "content": "zz",
317
+ "lstrip": true,
318
+ "normalized": false,
319
+ "rstrip": true,
320
+ "single_word": false,
321
+ "special": false
322
+ },
323
+ "79": {
324
  "content": "|",
325
  "lstrip": false,
326
  "normalized": false,
 
328
  "single_word": false,
329
  "special": true
330
  },
331
+ "80": {
332
  "content": "[UNK]",
333
  "lstrip": true,
334
  "normalized": false,
 
336
  "single_word": false,
337
  "special": false
338
  },
339
+ "81": {
340
  "content": "[PAD]",
341
  "lstrip": true,
342
  "normalized": false,
 
344
  "single_word": false,
345
  "special": false
346
  },
347
+ "82": {
348
  "content": "<s>",
349
  "lstrip": false,
350
  "normalized": false,
 
352
  "single_word": false,
353
  "special": true
354
  },
355
+ "83": {
356
  "content": "</s>",
357
  "lstrip": false,
358
  "normalized": false,
vocab.json CHANGED
@@ -4,58 +4,81 @@
4
  "*": 2,
5
  "**": 3,
6
  "-": 4,
7
- "[PAD]": 58,
8
- "[UNK]": 57,
9
- "^": 5,
10
- "^^": 6,
11
- "a": 7,
12
- "a'": 8,
13
- "aa": 9,
14
- "aa'": 10,
15
- "ah": 11,
16
- "b": 12,
17
- "d": 13,
18
- "e": 14,
19
- "ee": 15,
20
- "f": 16,
21
- "g": 17,
22
- "h": 18,
23
- "i": 19,
24
- "i0": 20,
25
- "i0'": 21,
26
- "i1": 22,
27
- "i1'": 23,
28
- "ii0": 24,
29
- "ii0'": 25,
30
- "ii1": 26,
31
- "ii1'": 27,
32
- "j": 28,
33
- "k": 29,
34
- "l": 30,
35
- "m": 31,
36
- "n": 32,
37
- "p": 33,
38
- "pp": 34,
39
- "q": 35,
40
- "r": 36,
41
- "s": 37,
42
- "sh": 38,
43
- "sil": 39,
44
- "t": 40,
45
- "th": 41,
46
- "u": 42,
47
- "u0": 43,
48
- "u0'": 44,
49
- "u1": 45,
50
- "u1'": 46,
51
- "uu0": 47,
52
- "uu0'": 48,
53
- "uu1": 49,
54
- "uu1'": 50,
55
- "v": 51,
56
- "w": 52,
57
- "x": 53,
58
- "y": 54,
59
- "z": 55,
60
- "|": 56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  }
 
4
  "*": 2,
5
  "**": 3,
6
  "-": 4,
7
+ "<": 5,
8
+ "<<": 6,
9
+ "A": 7,
10
+ "AA": 8,
11
+ "AH": 9,
12
+ "Ah": 10,
13
+ "D": 11,
14
+ "DD": 12,
15
+ "E": 13,
16
+ "EE": 14,
17
+ "G": 15,
18
+ "H": 16,
19
+ "HH": 17,
20
+ "I": 18,
21
+ "II": 19,
22
+ "J": 20,
23
+ "S": 21,
24
+ "SH": 22,
25
+ "SS": 23,
26
+ "T": 24,
27
+ "TH": 25,
28
+ "TT": 26,
29
+ "U": 27,
30
+ "UU": 28,
31
+ "Z": 29,
32
+ "ZZ": 30,
33
+ "[PAD]": 81,
34
+ "[UNK]": 80,
35
+ "^": 31,
36
+ "^^": 32,
37
+ "a": 33,
38
+ "aa": 34,
39
+ "b": 35,
40
+ "bb": 36,
41
+ "d": 37,
42
+ "dd": 38,
43
+ "f": 39,
44
+ "ff": 40,
45
+ "g": 41,
46
+ "gg": 42,
47
+ "h": 43,
48
+ "hh": 44,
49
+ "i": 45,
50
+ "ii": 46,
51
+ "j": 47,
52
+ "jj": 48,
53
+ "k": 49,
54
+ "kk": 50,
55
+ "l": 51,
56
+ "ll": 52,
57
+ "m": 53,
58
+ "mm": 54,
59
+ "n": 55,
60
+ "nn": 56,
61
+ "p": 57,
62
+ "pp": 58,
63
+ "q": 59,
64
+ "qq": 60,
65
+ "r": 61,
66
+ "rr": 62,
67
+ "s": 63,
68
+ "sil": 64,
69
+ "ss": 65,
70
+ "t": 66,
71
+ "tt": 67,
72
+ "u": 68,
73
+ "uu": 69,
74
+ "v": 70,
75
+ "w": 71,
76
+ "ww": 72,
77
+ "x": 73,
78
+ "xx": 74,
79
+ "y": 75,
80
+ "yy": 76,
81
+ "z": 77,
82
+ "zz": 78,
83
+ "|": 79
84
  }