Vietnamese-Tokenizer / special_tokens_map.json
PaxiAI's picture
Upload folder using huggingface_hub
05813f4 verified
Raw History Blame
6.26 kB
{
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|fim_prefix|>",
"<|fim_middle|>",
"<|fim_suffix|>",
"<|reserved_0|>",
"<|reserved_1|>",
"<|reserved_2|>",
"<|reserved_3|>",
"<|reserved_4|>",
"<|reserved_5|>",
"<|reserved_6|>",
"<|reserved_7|>",
"<|reserved_8|>",
"<|reserved_9|>",
"<|reserved_10|>",
"<|reserved_11|>",
"<|reserved_12|>",
"<|reserved_13|>",
"<|reserved_14|>",
"<|reserved_15|>",
"<|reserved_16|>",
"<|reserved_17|>",
"<|reserved_18|>",
"<|reserved_19|>",
"<|reserved_20|>",
"<|reserved_21|>",
"<|reserved_22|>",
"<|reserved_23|>",
"<|reserved_24|>",
"<|reserved_25|>",
"<|reserved_26|>",
"<|reserved_27|>",
"<|reserved_28|>",
"<|reserved_29|>",
"<|reserved_30|>",
"<|reserved_31|>",
"<|reserved_32|>",
"<|reserved_33|>",
"<|reserved_34|>",
"<|reserved_35|>",
"<|reserved_36|>",
"<|reserved_37|>",
"<|reserved_38|>",
"<|reserved_39|>",
"<|reserved_40|>",
"<|reserved_41|>",
"<|reserved_42|>",
"<|reserved_43|>",
"<|reserved_44|>",
"<|reserved_45|>",
"<|reserved_46|>",
"<|reserved_47|>",
"<|reserved_48|>",
"<|reserved_49|>",
"<|reserved_50|>",
"<|reserved_51|>",
"<|reserved_52|>",
"<|reserved_53|>",
"<|reserved_54|>",
"<|reserved_55|>",
"<|reserved_56|>",
"<|reserved_57|>",
"<|reserved_58|>",
"<|reserved_59|>",
"<|reserved_60|>",
"<|reserved_61|>",
"<|reserved_62|>",
"<|reserved_63|>",
"<|reserved_64|>",
"<|reserved_65|>",
"<|reserved_66|>",
"<|reserved_67|>",
"<|reserved_68|>",
"<|reserved_69|>",
"<|reserved_70|>",
"<|reserved_71|>",
"<|reserved_72|>",
"<|reserved_73|>",
"<|reserved_74|>",
"<|reserved_75|>",
"<|reserved_76|>",
"<|reserved_77|>",
"<|reserved_78|>",
"<|reserved_79|>",
"<|reserved_80|>",
"<|reserved_81|>",
"<|reserved_82|>",
"<|reserved_83|>",
"<|reserved_84|>",
"<|reserved_85|>",
"<|reserved_86|>",
"<|reserved_87|>",
"<|reserved_88|>",
"<|reserved_89|>",
"<|reserved_90|>",
"<|reserved_91|>",
"<|reserved_92|>",
"<|reserved_93|>",
"<|reserved_94|>",
"<|reserved_95|>",
"<|reserved_96|>",
"<|reserved_97|>",
"<|reserved_98|>",
"<|reserved_99|>",
"<|reserved_100|>",
"<|reserved_101|>",
"<|reserved_102|>",
"<|reserved_103|>",
"<|reserved_104|>",
"<|reserved_105|>",
"<|reserved_106|>",
"<|reserved_107|>",
"<|reserved_108|>",
"<|reserved_109|>",
"<|reserved_110|>",
"<|reserved_111|>",
"<|reserved_112|>",
"<|reserved_113|>",
"<|reserved_114|>",
"<|reserved_115|>",
"<|reserved_116|>",
"<|reserved_117|>",
"<|reserved_118|>",
"<|reserved_119|>",
"<|reserved_120|>",
"<|reserved_121|>",
"<|reserved_122|>",
"<|reserved_123|>",
"<|reserved_124|>",
"<|reserved_125|>",
"<|reserved_126|>",
"<|reserved_127|>",
"<|reserved_128|>",
"<|reserved_129|>",
"<|reserved_130|>",
"<|reserved_131|>",
"<|reserved_132|>",
"<|reserved_133|>",
"<|reserved_134|>",
"<|reserved_135|>",
"<|reserved_136|>",
"<|reserved_137|>",
"<|reserved_138|>",
"<|reserved_139|>",
"<|reserved_140|>",
"<|reserved_141|>",
"<|reserved_142|>",
"<|reserved_143|>",
"<|reserved_144|>",
"<|reserved_145|>",
"<|reserved_146|>",
"<|reserved_147|>",
"<|reserved_148|>",
"<|reserved_149|>",
"<|reserved_150|>",
"<|reserved_151|>",
"<|reserved_152|>",
"<|reserved_153|>",
"<|reserved_154|>",
"<|reserved_155|>",
"<|reserved_156|>",
"<|reserved_157|>",
"<|reserved_158|>",
"<|reserved_159|>",
"<|reserved_160|>",
"<|reserved_161|>",
"<|reserved_162|>",
"<|reserved_163|>",
"<|reserved_164|>",
"<|reserved_165|>",
"<|reserved_166|>",
"<|reserved_167|>",
"<|reserved_168|>",
"<|reserved_169|>",
"<|reserved_170|>",
"<|reserved_171|>",
"<|reserved_172|>",
"<|reserved_173|>",
"<|reserved_174|>",
"<|reserved_175|>",
"<|reserved_176|>",
"<|reserved_177|>",
"<|reserved_178|>",
"<|reserved_179|>",
"<|reserved_180|>",
"<|reserved_181|>",
"<|reserved_182|>",
"<|reserved_183|>",
"<|reserved_184|>",
"<|reserved_185|>",
"<|reserved_186|>",
"<|reserved_187|>",
"<|reserved_188|>",
"<|reserved_189|>",
"<|reserved_190|>",
"<|reserved_191|>",
"<|reserved_192|>",
"<|reserved_193|>",
"<|reserved_194|>",
"<|reserved_195|>",
"<|reserved_196|>",
"<|reserved_197|>",
"<|reserved_198|>",
"<|reserved_199|>",
"<|reserved_200|>",
"<|reserved_201|>",
"<|reserved_202|>",
"<|reserved_203|>",
"<|reserved_204|>",
"<|reserved_205|>",
"<|reserved_206|>",
"<|reserved_207|>",
"<|reserved_208|>",
"<|reserved_209|>",
"<|reserved_210|>",
"<|reserved_211|>",
"<|reserved_212|>",
"<|reserved_213|>",
"<|reserved_214|>",
"<|reserved_215|>",
"<|reserved_216|>",
"<|reserved_217|>",
"<|reserved_218|>",
"<|reserved_219|>",
"<|reserved_220|>",
"<|reserved_221|>",
"<|reserved_222|>",
"<|reserved_223|>",
"<|reserved_224|>",
"<|reserved_225|>",
"<|reserved_226|>",
"<|reserved_227|>",
"<|reserved_228|>",
"<|reserved_229|>",
"<|reserved_230|>",
"<|reserved_231|>",
"<|reserved_232|>",
"<|reserved_233|>",
"<|reserved_234|>",
"<|reserved_235|>",
"<|reserved_236|>",
"<|reserved_237|>",
"<|reserved_238|>",
"<|reserved_239|>",
"<|reserved_240|>",
"<|reserved_241|>",
"<|reserved_242|>",
"<|reserved_243|>",
"<|reserved_244|>",
"<|reserved_245|>",
"<|reserved_246|>",
"<|reserved_247|>",
"<|reserved_248|>",
"<|reserved_249|>",
"<|reserved_250|>",
"<|reserved_251|>",
"<|reserved_252|>",
"<|reserved_253|>",
"<|reserved_254|>",
"<|reserved_255|>"
],
"bos_token": "<|bos|>",
"eos_token": "<|eos|>",
"pad_token": "<|pad|>"
}