| { | |
| "name": "VietnameseTokenizer-v1", | |
| "vocab_size": 48000, | |
| "special_token_ids": { | |
| "<|pad|>": 0, | |
| "<|bos|>": 1, | |
| "<|eos|>": 2, | |
| "<|im_start|>": 3, | |
| "<|im_end|>": 4, | |
| "<|fim_prefix|>": 5, | |
| "<|fim_middle|>": 6, | |
| "<|fim_suffix|>": 7, | |
| "<|reserved_0|>": 8, | |
| "<|reserved_1|>": 9, | |
| "<|reserved_2|>": 10, | |
| "<|reserved_3|>": 11, | |
| "<|reserved_4|>": 12, | |
| "<|reserved_5|>": 13, | |
| "<|reserved_6|>": 14, | |
| "<|reserved_7|>": 15, | |
| "<|reserved_8|>": 16, | |
| "<|reserved_9|>": 17, | |
| "<|reserved_10|>": 18, | |
| "<|reserved_11|>": 19, | |
| "<|reserved_12|>": 20, | |
| "<|reserved_13|>": 21, | |
| "<|reserved_14|>": 22, | |
| "<|reserved_15|>": 23, | |
| "<|reserved_16|>": 24, | |
| "<|reserved_17|>": 25, | |
| "<|reserved_18|>": 26, | |
| "<|reserved_19|>": 27, | |
| "<|reserved_20|>": 28, | |
| "<|reserved_21|>": 29, | |
| "<|reserved_22|>": 30, | |
| "<|reserved_23|>": 31, | |
| "<|reserved_24|>": 32, | |
| "<|reserved_25|>": 33, | |
| "<|reserved_26|>": 34, | |
| "<|reserved_27|>": 35, | |
| "<|reserved_28|>": 36, | |
| "<|reserved_29|>": 37, | |
| "<|reserved_30|>": 38, | |
| "<|reserved_31|>": 39, | |
| "<|reserved_32|>": 40, | |
| "<|reserved_33|>": 41, | |
| "<|reserved_34|>": 42, | |
| "<|reserved_35|>": 43, | |
| "<|reserved_36|>": 44, | |
| "<|reserved_37|>": 45, | |
| "<|reserved_38|>": 46, | |
| "<|reserved_39|>": 47, | |
| "<|reserved_40|>": 48, | |
| "<|reserved_41|>": 49, | |
| "<|reserved_42|>": 50, | |
| "<|reserved_43|>": 51, | |
| "<|reserved_44|>": 52, | |
| "<|reserved_45|>": 53, | |
| "<|reserved_46|>": 54, | |
| "<|reserved_47|>": 55, | |
| "<|reserved_48|>": 56, | |
| "<|reserved_49|>": 57, | |
| "<|reserved_50|>": 58, | |
| "<|reserved_51|>": 59, | |
| "<|reserved_52|>": 60, | |
| "<|reserved_53|>": 61, | |
| "<|reserved_54|>": 62, | |
| "<|reserved_55|>": 63, | |
| "<|reserved_56|>": 64, | |
| "<|reserved_57|>": 65, | |
| "<|reserved_58|>": 66, | |
| "<|reserved_59|>": 67, | |
| "<|reserved_60|>": 68, | |
| "<|reserved_61|>": 69, | |
| "<|reserved_62|>": 70, | |
| "<|reserved_63|>": 71, | |
| "<|reserved_64|>": 72, | |
| "<|reserved_65|>": 73, | |
| "<|reserved_66|>": 74, | |
| "<|reserved_67|>": 75, | |
| "<|reserved_68|>": 76, | |
| "<|reserved_69|>": 77, | |
| "<|reserved_70|>": 78, | |
| "<|reserved_71|>": 79, | |
| "<|reserved_72|>": 80, | |
| "<|reserved_73|>": 81, | |
| "<|reserved_74|>": 82, | |
| "<|reserved_75|>": 83, | |
| "<|reserved_76|>": 84, | |
| "<|reserved_77|>": 85, | |
| "<|reserved_78|>": 86, | |
| "<|reserved_79|>": 87, | |
| "<|reserved_80|>": 88, | |
| "<|reserved_81|>": 89, | |
| "<|reserved_82|>": 90, | |
| "<|reserved_83|>": 91, | |
| "<|reserved_84|>": 92, | |
| "<|reserved_85|>": 93, | |
| "<|reserved_86|>": 94, | |
| "<|reserved_87|>": 95, | |
| "<|reserved_88|>": 96, | |
| "<|reserved_89|>": 97, | |
| "<|reserved_90|>": 98, | |
| "<|reserved_91|>": 99, | |
| "<|reserved_92|>": 100, | |
| "<|reserved_93|>": 101, | |
| "<|reserved_94|>": 102, | |
| "<|reserved_95|>": 103, | |
| "<|reserved_96|>": 104, | |
| "<|reserved_97|>": 105, | |
| "<|reserved_98|>": 106, | |
| "<|reserved_99|>": 107, | |
| "<|reserved_100|>": 108, | |
| "<|reserved_101|>": 109, | |
| "<|reserved_102|>": 110, | |
| "<|reserved_103|>": 111, | |
| "<|reserved_104|>": 112, | |
| "<|reserved_105|>": 113, | |
| "<|reserved_106|>": 114, | |
| "<|reserved_107|>": 115, | |
| "<|reserved_108|>": 116, | |
| "<|reserved_109|>": 117, | |
| "<|reserved_110|>": 118, | |
| "<|reserved_111|>": 119, | |
| "<|reserved_112|>": 120, | |
| "<|reserved_113|>": 121, | |
| "<|reserved_114|>": 122, | |
| "<|reserved_115|>": 123, | |
| "<|reserved_116|>": 124, | |
| "<|reserved_117|>": 125, | |
| "<|reserved_118|>": 126, | |
| "<|reserved_119|>": 127, | |
| "<|reserved_120|>": 128, | |
| "<|reserved_121|>": 129, | |
| "<|reserved_122|>": 130, | |
| "<|reserved_123|>": 131, | |
| "<|reserved_124|>": 132, | |
| "<|reserved_125|>": 133, | |
| "<|reserved_126|>": 134, | |
| "<|reserved_127|>": 135, | |
| "<|reserved_128|>": 136, | |
| "<|reserved_129|>": 137, | |
| "<|reserved_130|>": 138, | |
| "<|reserved_131|>": 139, | |
| "<|reserved_132|>": 140, | |
| "<|reserved_133|>": 141, | |
| "<|reserved_134|>": 142, | |
| "<|reserved_135|>": 143, | |
| "<|reserved_136|>": 144, | |
| "<|reserved_137|>": 145, | |
| "<|reserved_138|>": 146, | |
| "<|reserved_139|>": 147, | |
| "<|reserved_140|>": 148, | |
| "<|reserved_141|>": 149, | |
| "<|reserved_142|>": 150, | |
| "<|reserved_143|>": 151, | |
| "<|reserved_144|>": 152, | |
| "<|reserved_145|>": 153, | |
| "<|reserved_146|>": 154, | |
| "<|reserved_147|>": 155, | |
| "<|reserved_148|>": 156, | |
| "<|reserved_149|>": 157, | |
| "<|reserved_150|>": 158, | |
| "<|reserved_151|>": 159, | |
| "<|reserved_152|>": 160, | |
| "<|reserved_153|>": 161, | |
| "<|reserved_154|>": 162, | |
| "<|reserved_155|>": 163, | |
| "<|reserved_156|>": 164, | |
| "<|reserved_157|>": 165, | |
| "<|reserved_158|>": 166, | |
| "<|reserved_159|>": 167, | |
| "<|reserved_160|>": 168, | |
| "<|reserved_161|>": 169, | |
| "<|reserved_162|>": 170, | |
| "<|reserved_163|>": 171, | |
| "<|reserved_164|>": 172, | |
| "<|reserved_165|>": 173, | |
| "<|reserved_166|>": 174, | |
| "<|reserved_167|>": 175, | |
| "<|reserved_168|>": 176, | |
| "<|reserved_169|>": 177, | |
| "<|reserved_170|>": 178, | |
| "<|reserved_171|>": 179, | |
| "<|reserved_172|>": 180, | |
| "<|reserved_173|>": 181, | |
| "<|reserved_174|>": 182, | |
| "<|reserved_175|>": 183, | |
| "<|reserved_176|>": 184, | |
| "<|reserved_177|>": 185, | |
| "<|reserved_178|>": 186, | |
| "<|reserved_179|>": 187, | |
| "<|reserved_180|>": 188, | |
| "<|reserved_181|>": 189, | |
| "<|reserved_182|>": 190, | |
| "<|reserved_183|>": 191, | |
| "<|reserved_184|>": 192, | |
| "<|reserved_185|>": 193, | |
| "<|reserved_186|>": 194, | |
| "<|reserved_187|>": 195, | |
| "<|reserved_188|>": 196, | |
| "<|reserved_189|>": 197, | |
| "<|reserved_190|>": 198, | |
| "<|reserved_191|>": 199, | |
| "<|reserved_192|>": 200, | |
| "<|reserved_193|>": 201, | |
| "<|reserved_194|>": 202, | |
| "<|reserved_195|>": 203, | |
| "<|reserved_196|>": 204, | |
| "<|reserved_197|>": 205, | |
| "<|reserved_198|>": 206, | |
| "<|reserved_199|>": 207, | |
| "<|reserved_200|>": 208, | |
| "<|reserved_201|>": 209, | |
| "<|reserved_202|>": 210, | |
| "<|reserved_203|>": 211, | |
| "<|reserved_204|>": 212, | |
| "<|reserved_205|>": 213, | |
| "<|reserved_206|>": 214, | |
| "<|reserved_207|>": 215, | |
| "<|reserved_208|>": 216, | |
| "<|reserved_209|>": 217, | |
| "<|reserved_210|>": 218, | |
| "<|reserved_211|>": 219, | |
| "<|reserved_212|>": 220, | |
| "<|reserved_213|>": 221, | |
| "<|reserved_214|>": 222, | |
| "<|reserved_215|>": 223, | |
| "<|reserved_216|>": 224, | |
| "<|reserved_217|>": 225, | |
| "<|reserved_218|>": 226, | |
| "<|reserved_219|>": 227, | |
| "<|reserved_220|>": 228, | |
| "<|reserved_221|>": 229, | |
| "<|reserved_222|>": 230, | |
| "<|reserved_223|>": 231, | |
| "<|reserved_224|>": 232, | |
| "<|reserved_225|>": 233, | |
| "<|reserved_226|>": 234, | |
| "<|reserved_227|>": 235, | |
| "<|reserved_228|>": 236, | |
| "<|reserved_229|>": 237, | |
| "<|reserved_230|>": 238, | |
| "<|reserved_231|>": 239, | |
| "<|reserved_232|>": 240, | |
| "<|reserved_233|>": 241, | |
| "<|reserved_234|>": 242, | |
| "<|reserved_235|>": 243, | |
| "<|reserved_236|>": 244, | |
| "<|reserved_237|>": 245, | |
| "<|reserved_238|>": 246, | |
| "<|reserved_239|>": 247, | |
| "<|reserved_240|>": 248, | |
| "<|reserved_241|>": 249, | |
| "<|reserved_242|>": 250, | |
| "<|reserved_243|>": 251, | |
| "<|reserved_244|>": 252, | |
| "<|reserved_245|>": 253, | |
| "<|reserved_246|>": 254, | |
| "<|reserved_247|>": 255, | |
| "<|reserved_248|>": 256, | |
| "<|reserved_249|>": 257, | |
| "<|reserved_250|>": 258, | |
| "<|reserved_251|>": 259, | |
| "<|reserved_252|>": 260, | |
| "<|reserved_253|>": 261, | |
| "<|reserved_254|>": 262, | |
| "<|reserved_255|>": 263 | |
| }, | |
| "normalization": "NFC", | |
| "algorithm": "Byte-level BPE" | |
| } |