| name: "swallow-code-8k" | |
| type: "BPE" | |
| vocab_size: 8192 | |
| dataset_config: "configs/datasets/swallow-code.yaml" | |
| val_split: 0.01 | |
| special_tokens: | |
| pad: "<pad>" | |
| eos: "<eos>" | |
| bos: "<bos>" | |
| unk: "<unk>" | |
| output_dir: "tokenizers/swallow-code-8k" | |
| tokenized_output_dir: "data/swallow-code-8k-tokenized" | |
| num_threads: 12 | |