File size: 7,542 Bytes
05813f4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 | {
"name": "VietnameseTokenizer-v1",
"vocab_size": 48000,
"special_token_ids": {
"<|pad|>": 0,
"<|bos|>": 1,
"<|eos|>": 2,
"<|im_start|>": 3,
"<|im_end|>": 4,
"<|fim_prefix|>": 5,
"<|fim_middle|>": 6,
"<|fim_suffix|>": 7,
"<|reserved_0|>": 8,
"<|reserved_1|>": 9,
"<|reserved_2|>": 10,
"<|reserved_3|>": 11,
"<|reserved_4|>": 12,
"<|reserved_5|>": 13,
"<|reserved_6|>": 14,
"<|reserved_7|>": 15,
"<|reserved_8|>": 16,
"<|reserved_9|>": 17,
"<|reserved_10|>": 18,
"<|reserved_11|>": 19,
"<|reserved_12|>": 20,
"<|reserved_13|>": 21,
"<|reserved_14|>": 22,
"<|reserved_15|>": 23,
"<|reserved_16|>": 24,
"<|reserved_17|>": 25,
"<|reserved_18|>": 26,
"<|reserved_19|>": 27,
"<|reserved_20|>": 28,
"<|reserved_21|>": 29,
"<|reserved_22|>": 30,
"<|reserved_23|>": 31,
"<|reserved_24|>": 32,
"<|reserved_25|>": 33,
"<|reserved_26|>": 34,
"<|reserved_27|>": 35,
"<|reserved_28|>": 36,
"<|reserved_29|>": 37,
"<|reserved_30|>": 38,
"<|reserved_31|>": 39,
"<|reserved_32|>": 40,
"<|reserved_33|>": 41,
"<|reserved_34|>": 42,
"<|reserved_35|>": 43,
"<|reserved_36|>": 44,
"<|reserved_37|>": 45,
"<|reserved_38|>": 46,
"<|reserved_39|>": 47,
"<|reserved_40|>": 48,
"<|reserved_41|>": 49,
"<|reserved_42|>": 50,
"<|reserved_43|>": 51,
"<|reserved_44|>": 52,
"<|reserved_45|>": 53,
"<|reserved_46|>": 54,
"<|reserved_47|>": 55,
"<|reserved_48|>": 56,
"<|reserved_49|>": 57,
"<|reserved_50|>": 58,
"<|reserved_51|>": 59,
"<|reserved_52|>": 60,
"<|reserved_53|>": 61,
"<|reserved_54|>": 62,
"<|reserved_55|>": 63,
"<|reserved_56|>": 64,
"<|reserved_57|>": 65,
"<|reserved_58|>": 66,
"<|reserved_59|>": 67,
"<|reserved_60|>": 68,
"<|reserved_61|>": 69,
"<|reserved_62|>": 70,
"<|reserved_63|>": 71,
"<|reserved_64|>": 72,
"<|reserved_65|>": 73,
"<|reserved_66|>": 74,
"<|reserved_67|>": 75,
"<|reserved_68|>": 76,
"<|reserved_69|>": 77,
"<|reserved_70|>": 78,
"<|reserved_71|>": 79,
"<|reserved_72|>": 80,
"<|reserved_73|>": 81,
"<|reserved_74|>": 82,
"<|reserved_75|>": 83,
"<|reserved_76|>": 84,
"<|reserved_77|>": 85,
"<|reserved_78|>": 86,
"<|reserved_79|>": 87,
"<|reserved_80|>": 88,
"<|reserved_81|>": 89,
"<|reserved_82|>": 90,
"<|reserved_83|>": 91,
"<|reserved_84|>": 92,
"<|reserved_85|>": 93,
"<|reserved_86|>": 94,
"<|reserved_87|>": 95,
"<|reserved_88|>": 96,
"<|reserved_89|>": 97,
"<|reserved_90|>": 98,
"<|reserved_91|>": 99,
"<|reserved_92|>": 100,
"<|reserved_93|>": 101,
"<|reserved_94|>": 102,
"<|reserved_95|>": 103,
"<|reserved_96|>": 104,
"<|reserved_97|>": 105,
"<|reserved_98|>": 106,
"<|reserved_99|>": 107,
"<|reserved_100|>": 108,
"<|reserved_101|>": 109,
"<|reserved_102|>": 110,
"<|reserved_103|>": 111,
"<|reserved_104|>": 112,
"<|reserved_105|>": 113,
"<|reserved_106|>": 114,
"<|reserved_107|>": 115,
"<|reserved_108|>": 116,
"<|reserved_109|>": 117,
"<|reserved_110|>": 118,
"<|reserved_111|>": 119,
"<|reserved_112|>": 120,
"<|reserved_113|>": 121,
"<|reserved_114|>": 122,
"<|reserved_115|>": 123,
"<|reserved_116|>": 124,
"<|reserved_117|>": 125,
"<|reserved_118|>": 126,
"<|reserved_119|>": 127,
"<|reserved_120|>": 128,
"<|reserved_121|>": 129,
"<|reserved_122|>": 130,
"<|reserved_123|>": 131,
"<|reserved_124|>": 132,
"<|reserved_125|>": 133,
"<|reserved_126|>": 134,
"<|reserved_127|>": 135,
"<|reserved_128|>": 136,
"<|reserved_129|>": 137,
"<|reserved_130|>": 138,
"<|reserved_131|>": 139,
"<|reserved_132|>": 140,
"<|reserved_133|>": 141,
"<|reserved_134|>": 142,
"<|reserved_135|>": 143,
"<|reserved_136|>": 144,
"<|reserved_137|>": 145,
"<|reserved_138|>": 146,
"<|reserved_139|>": 147,
"<|reserved_140|>": 148,
"<|reserved_141|>": 149,
"<|reserved_142|>": 150,
"<|reserved_143|>": 151,
"<|reserved_144|>": 152,
"<|reserved_145|>": 153,
"<|reserved_146|>": 154,
"<|reserved_147|>": 155,
"<|reserved_148|>": 156,
"<|reserved_149|>": 157,
"<|reserved_150|>": 158,
"<|reserved_151|>": 159,
"<|reserved_152|>": 160,
"<|reserved_153|>": 161,
"<|reserved_154|>": 162,
"<|reserved_155|>": 163,
"<|reserved_156|>": 164,
"<|reserved_157|>": 165,
"<|reserved_158|>": 166,
"<|reserved_159|>": 167,
"<|reserved_160|>": 168,
"<|reserved_161|>": 169,
"<|reserved_162|>": 170,
"<|reserved_163|>": 171,
"<|reserved_164|>": 172,
"<|reserved_165|>": 173,
"<|reserved_166|>": 174,
"<|reserved_167|>": 175,
"<|reserved_168|>": 176,
"<|reserved_169|>": 177,
"<|reserved_170|>": 178,
"<|reserved_171|>": 179,
"<|reserved_172|>": 180,
"<|reserved_173|>": 181,
"<|reserved_174|>": 182,
"<|reserved_175|>": 183,
"<|reserved_176|>": 184,
"<|reserved_177|>": 185,
"<|reserved_178|>": 186,
"<|reserved_179|>": 187,
"<|reserved_180|>": 188,
"<|reserved_181|>": 189,
"<|reserved_182|>": 190,
"<|reserved_183|>": 191,
"<|reserved_184|>": 192,
"<|reserved_185|>": 193,
"<|reserved_186|>": 194,
"<|reserved_187|>": 195,
"<|reserved_188|>": 196,
"<|reserved_189|>": 197,
"<|reserved_190|>": 198,
"<|reserved_191|>": 199,
"<|reserved_192|>": 200,
"<|reserved_193|>": 201,
"<|reserved_194|>": 202,
"<|reserved_195|>": 203,
"<|reserved_196|>": 204,
"<|reserved_197|>": 205,
"<|reserved_198|>": 206,
"<|reserved_199|>": 207,
"<|reserved_200|>": 208,
"<|reserved_201|>": 209,
"<|reserved_202|>": 210,
"<|reserved_203|>": 211,
"<|reserved_204|>": 212,
"<|reserved_205|>": 213,
"<|reserved_206|>": 214,
"<|reserved_207|>": 215,
"<|reserved_208|>": 216,
"<|reserved_209|>": 217,
"<|reserved_210|>": 218,
"<|reserved_211|>": 219,
"<|reserved_212|>": 220,
"<|reserved_213|>": 221,
"<|reserved_214|>": 222,
"<|reserved_215|>": 223,
"<|reserved_216|>": 224,
"<|reserved_217|>": 225,
"<|reserved_218|>": 226,
"<|reserved_219|>": 227,
"<|reserved_220|>": 228,
"<|reserved_221|>": 229,
"<|reserved_222|>": 230,
"<|reserved_223|>": 231,
"<|reserved_224|>": 232,
"<|reserved_225|>": 233,
"<|reserved_226|>": 234,
"<|reserved_227|>": 235,
"<|reserved_228|>": 236,
"<|reserved_229|>": 237,
"<|reserved_230|>": 238,
"<|reserved_231|>": 239,
"<|reserved_232|>": 240,
"<|reserved_233|>": 241,
"<|reserved_234|>": 242,
"<|reserved_235|>": 243,
"<|reserved_236|>": 244,
"<|reserved_237|>": 245,
"<|reserved_238|>": 246,
"<|reserved_239|>": 247,
"<|reserved_240|>": 248,
"<|reserved_241|>": 249,
"<|reserved_242|>": 250,
"<|reserved_243|>": 251,
"<|reserved_244|>": 252,
"<|reserved_245|>": 253,
"<|reserved_246|>": 254,
"<|reserved_247|>": 255,
"<|reserved_248|>": 256,
"<|reserved_249|>": 257,
"<|reserved_250|>": 258,
"<|reserved_251|>": 259,
"<|reserved_252|>": 260,
"<|reserved_253|>": 261,
"<|reserved_254|>": 262,
"<|reserved_255|>": 263
},
"normalization": "NFC",
"algorithm": "Byte-level BPE"
} |