{ "name": "VietnameseTokenizer-v1", "vocab_size": 48000, "special_token_ids": { "<|pad|>": 0, "<|bos|>": 1, "<|eos|>": 2, "<|im_start|>": 3, "<|im_end|>": 4, "<|fim_prefix|>": 5, "<|fim_middle|>": 6, "<|fim_suffix|>": 7, "<|reserved_0|>": 8, "<|reserved_1|>": 9, "<|reserved_2|>": 10, "<|reserved_3|>": 11, "<|reserved_4|>": 12, "<|reserved_5|>": 13, "<|reserved_6|>": 14, "<|reserved_7|>": 15, "<|reserved_8|>": 16, "<|reserved_9|>": 17, "<|reserved_10|>": 18, "<|reserved_11|>": 19, "<|reserved_12|>": 20, "<|reserved_13|>": 21, "<|reserved_14|>": 22, "<|reserved_15|>": 23, "<|reserved_16|>": 24, "<|reserved_17|>": 25, "<|reserved_18|>": 26, "<|reserved_19|>": 27, "<|reserved_20|>": 28, "<|reserved_21|>": 29, "<|reserved_22|>": 30, "<|reserved_23|>": 31, "<|reserved_24|>": 32, "<|reserved_25|>": 33, "<|reserved_26|>": 34, "<|reserved_27|>": 35, "<|reserved_28|>": 36, "<|reserved_29|>": 37, "<|reserved_30|>": 38, "<|reserved_31|>": 39, "<|reserved_32|>": 40, "<|reserved_33|>": 41, "<|reserved_34|>": 42, "<|reserved_35|>": 43, "<|reserved_36|>": 44, "<|reserved_37|>": 45, "<|reserved_38|>": 46, "<|reserved_39|>": 47, "<|reserved_40|>": 48, "<|reserved_41|>": 49, "<|reserved_42|>": 50, "<|reserved_43|>": 51, "<|reserved_44|>": 52, "<|reserved_45|>": 53, "<|reserved_46|>": 54, "<|reserved_47|>": 55, "<|reserved_48|>": 56, "<|reserved_49|>": 57, "<|reserved_50|>": 58, "<|reserved_51|>": 59, "<|reserved_52|>": 60, "<|reserved_53|>": 61, "<|reserved_54|>": 62, "<|reserved_55|>": 63, "<|reserved_56|>": 64, "<|reserved_57|>": 65, "<|reserved_58|>": 66, "<|reserved_59|>": 67, "<|reserved_60|>": 68, "<|reserved_61|>": 69, "<|reserved_62|>": 70, "<|reserved_63|>": 71, "<|reserved_64|>": 72, "<|reserved_65|>": 73, "<|reserved_66|>": 74, "<|reserved_67|>": 75, "<|reserved_68|>": 76, "<|reserved_69|>": 77, "<|reserved_70|>": 78, "<|reserved_71|>": 79, "<|reserved_72|>": 80, "<|reserved_73|>": 81, "<|reserved_74|>": 82, "<|reserved_75|>": 83, "<|reserved_76|>": 84, "<|reserved_77|>": 85, "<|reserved_78|>": 86, "<|reserved_79|>": 87, "<|reserved_80|>": 88, "<|reserved_81|>": 89, "<|reserved_82|>": 90, "<|reserved_83|>": 91, "<|reserved_84|>": 92, "<|reserved_85|>": 93, "<|reserved_86|>": 94, "<|reserved_87|>": 95, "<|reserved_88|>": 96, "<|reserved_89|>": 97, "<|reserved_90|>": 98, "<|reserved_91|>": 99, "<|reserved_92|>": 100, "<|reserved_93|>": 101, "<|reserved_94|>": 102, "<|reserved_95|>": 103, "<|reserved_96|>": 104, "<|reserved_97|>": 105, "<|reserved_98|>": 106, "<|reserved_99|>": 107, "<|reserved_100|>": 108, "<|reserved_101|>": 109, "<|reserved_102|>": 110, "<|reserved_103|>": 111, "<|reserved_104|>": 112, "<|reserved_105|>": 113, "<|reserved_106|>": 114, "<|reserved_107|>": 115, "<|reserved_108|>": 116, "<|reserved_109|>": 117, "<|reserved_110|>": 118, "<|reserved_111|>": 119, "<|reserved_112|>": 120, "<|reserved_113|>": 121, "<|reserved_114|>": 122, "<|reserved_115|>": 123, "<|reserved_116|>": 124, "<|reserved_117|>": 125, "<|reserved_118|>": 126, "<|reserved_119|>": 127, "<|reserved_120|>": 128, "<|reserved_121|>": 129, "<|reserved_122|>": 130, "<|reserved_123|>": 131, "<|reserved_124|>": 132, "<|reserved_125|>": 133, "<|reserved_126|>": 134, "<|reserved_127|>": 135, "<|reserved_128|>": 136, "<|reserved_129|>": 137, "<|reserved_130|>": 138, "<|reserved_131|>": 139, "<|reserved_132|>": 140, "<|reserved_133|>": 141, "<|reserved_134|>": 142, "<|reserved_135|>": 143, "<|reserved_136|>": 144, "<|reserved_137|>": 145, "<|reserved_138|>": 146, "<|reserved_139|>": 147, "<|reserved_140|>": 148, "<|reserved_141|>": 149, "<|reserved_142|>": 150, "<|reserved_143|>": 151, "<|reserved_144|>": 152, "<|reserved_145|>": 153, "<|reserved_146|>": 154, "<|reserved_147|>": 155, "<|reserved_148|>": 156, "<|reserved_149|>": 157, "<|reserved_150|>": 158, "<|reserved_151|>": 159, "<|reserved_152|>": 160, "<|reserved_153|>": 161, "<|reserved_154|>": 162, "<|reserved_155|>": 163, "<|reserved_156|>": 164, "<|reserved_157|>": 165, "<|reserved_158|>": 166, "<|reserved_159|>": 167, "<|reserved_160|>": 168, "<|reserved_161|>": 169, "<|reserved_162|>": 170, "<|reserved_163|>": 171, "<|reserved_164|>": 172, "<|reserved_165|>": 173, "<|reserved_166|>": 174, "<|reserved_167|>": 175, "<|reserved_168|>": 176, "<|reserved_169|>": 177, "<|reserved_170|>": 178, "<|reserved_171|>": 179, "<|reserved_172|>": 180, "<|reserved_173|>": 181, "<|reserved_174|>": 182, "<|reserved_175|>": 183, "<|reserved_176|>": 184, "<|reserved_177|>": 185, "<|reserved_178|>": 186, "<|reserved_179|>": 187, "<|reserved_180|>": 188, "<|reserved_181|>": 189, "<|reserved_182|>": 190, "<|reserved_183|>": 191, "<|reserved_184|>": 192, "<|reserved_185|>": 193, "<|reserved_186|>": 194, "<|reserved_187|>": 195, "<|reserved_188|>": 196, "<|reserved_189|>": 197, "<|reserved_190|>": 198, "<|reserved_191|>": 199, "<|reserved_192|>": 200, "<|reserved_193|>": 201, "<|reserved_194|>": 202, "<|reserved_195|>": 203, "<|reserved_196|>": 204, "<|reserved_197|>": 205, "<|reserved_198|>": 206, "<|reserved_199|>": 207, "<|reserved_200|>": 208, "<|reserved_201|>": 209, "<|reserved_202|>": 210, "<|reserved_203|>": 211, "<|reserved_204|>": 212, "<|reserved_205|>": 213, "<|reserved_206|>": 214, "<|reserved_207|>": 215, "<|reserved_208|>": 216, "<|reserved_209|>": 217, "<|reserved_210|>": 218, "<|reserved_211|>": 219, "<|reserved_212|>": 220, "<|reserved_213|>": 221, "<|reserved_214|>": 222, "<|reserved_215|>": 223, "<|reserved_216|>": 224, "<|reserved_217|>": 225, "<|reserved_218|>": 226, "<|reserved_219|>": 227, "<|reserved_220|>": 228, "<|reserved_221|>": 229, "<|reserved_222|>": 230, "<|reserved_223|>": 231, "<|reserved_224|>": 232, "<|reserved_225|>": 233, "<|reserved_226|>": 234, "<|reserved_227|>": 235, "<|reserved_228|>": 236, "<|reserved_229|>": 237, "<|reserved_230|>": 238, "<|reserved_231|>": 239, "<|reserved_232|>": 240, "<|reserved_233|>": 241, "<|reserved_234|>": 242, "<|reserved_235|>": 243, "<|reserved_236|>": 244, "<|reserved_237|>": 245, "<|reserved_238|>": 246, "<|reserved_239|>": 247, "<|reserved_240|>": 248, "<|reserved_241|>": 249, "<|reserved_242|>": 250, "<|reserved_243|>": 251, "<|reserved_244|>": 252, "<|reserved_245|>": 253, "<|reserved_246|>": 254, "<|reserved_247|>": 255, "<|reserved_248|>": 256, "<|reserved_249|>": 257, "<|reserved_250|>": 258, "<|reserved_251|>": 259, "<|reserved_252|>": 260, "<|reserved_253|>": 261, "<|reserved_254|>": 262, "<|reserved_255|>": 263 }, "normalization": "NFC", "algorithm": "Byte-level BPE" }