{
  "tokenizer_class": "PreTrainedTokenizerFast",
  "bos_token": "<bos>",
  "eos_token": "<eos>",
  "unk_token": "<unk>",
  "pad_token": "<pad>",
  "model_max_length": 512,
  "vocab_size": 32000,
  "clean_up_tokenization_spaces": true,
  "_learnllm_meta": {
    "vocab_size": 32000,
    "min_frequency": 2,
    "special_tokens": [
      "<pad>",
      "<unk>",
      "<bos>",
      "<eos>"
    ],
    "sample_size_mb": 250.0,
    "seed": 42,
    "save_dir": "J:\\LearnLLM\\models\\tokenizer"
  }
}