simonko912 commited on
Commit
9de686e
·
verified ·
1 Parent(s): dd91290

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -2,11 +2,11 @@
2
  "add_prefix_space": null,
3
  "backend": "tokenizers",
4
  "bos_token": "<s>",
5
- "chat_template": null,
6
  "clean_up_tokenization_spaces": false,
7
  "eos_token": "</s>",
8
  "is_local": false,
9
- "model_max_length": 2048,
10
  "pad_token": "</s>",
11
  "sp_model_kwargs": {},
12
  "tokenizer_class": "LlamaTokenizer",
 
2
  "add_prefix_space": null,
3
  "backend": "tokenizers",
4
  "bos_token": "<s>",
5
+ "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ message['content'] }} [/INST]{% elif message['role'] == 'assistant' %}{{ message['content'] }}{% if not loop.last %} [INST]{% endif %}{% endif %}{% endfor %}",
6
  "clean_up_tokenization_spaces": false,
7
  "eos_token": "</s>",
8
  "is_local": false,
9
+ "model_max_length": 4096,
10
  "pad_token": "</s>",
11
  "sp_model_kwargs": {},
12
  "tokenizer_class": "LlamaTokenizer",