| { |
| "version": "1.0", |
| "truncation": null, |
| "padding": null, |
| "added_tokens": [ |
| { |
| "id": 74, |
| "content": "<|endoftext|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 75, |
| "content": "<|im_start|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 76, |
| "content": "<|im_end|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 77, |
| "content": "<|object_ref_start|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 78, |
| "content": "<|object_ref_end|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 79, |
| "content": "<|box_start|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 80, |
| "content": "<|box_end|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 81, |
| "content": "<|quad_start|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 82, |
| "content": "<|quad_end|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 83, |
| "content": "<|vision_start|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 84, |
| "content": "<|vision_end|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 85, |
| "content": "<|vision_pad|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 86, |
| "content": "<|image_pad|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 87, |
| "content": "<|video_pad|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| } |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": false, |
| "use_regex": false |
| } |
| ] |
| }, |
| "post_processor": { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": false, |
| "use_regex": false |
| }, |
| "decoder": { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": false, |
| "use_regex": false |
| }, |
| "model": { |
| "type": "BPE", |
| "dropout": null, |
| "unk_token": null, |
| "continuing_subword_prefix": "", |
| "end_of_word_suffix": "", |
| "fuse_unk": false, |
| "byte_fallback": false, |
| "ignore_merges": false, |
| "vocab": { |
| "!": 0, |
| "*": 1, |
| "+": 2, |
| ",": 3, |
| "-": 4, |
| ".": 5, |
| "/": 6, |
| "0": 7, |
| "1": 8, |
| "2": 9, |
| "3": 10, |
| "4": 11, |
| "5": 12, |
| "6": 13, |
| "7": 14, |
| "8": 15, |
| "9": 16, |
| ":": 17, |
| ";": 18, |
| "=": 19, |
| "?": 20, |
| "A": 21, |
| "B": 22, |
| "C": 23, |
| "D": 24, |
| "E": 25, |
| "F": 26, |
| "G": 27, |
| "H": 28, |
| "I": 29, |
| "J": 30, |
| "K": 31, |
| "L": 32, |
| "M": 33, |
| "N": 34, |
| "O": 35, |
| "P": 36, |
| "Q": 37, |
| "R": 38, |
| "S": 39, |
| "T": 40, |
| "U": 41, |
| "V": 42, |
| "W": 43, |
| "X": 44, |
| "Y": 45, |
| "Z": 46, |
| "a": 47, |
| "b": 48, |
| "c": 49, |
| "d": 50, |
| "e": 51, |
| "f": 52, |
| "g": 53, |
| "h": 54, |
| "i": 55, |
| "j": 56, |
| "k": 57, |
| "l": 58, |
| "m": 59, |
| "n": 60, |
| "o": 61, |
| "p": 62, |
| "q": 63, |
| "r": 64, |
| "s": 65, |
| "t": 66, |
| "u": 67, |
| "v": 68, |
| "w": 69, |
| "x": 70, |
| "y": 71, |
| "z": 72, |
| "Ċ": 73 |
| }, |
| "merges": [] |
| } |
| } |
