Update tokenization_chatglm.py
Browse files- tokenization_chatglm.py +1 -1
tokenization_chatglm.py
CHANGED
|
@@ -73,12 +73,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
| 73 |
kwargs.pop("eos_token")
|
| 74 |
kwargs.pop("pad_token")
|
| 75 |
kwargs.pop("unk_token")
|
| 76 |
-
super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
|
| 77 |
self.special_tokens = {
|
| 78 |
"<bos>": self.tokenizer.bos_id,
|
| 79 |
"<eos>": self.tokenizer.eos_id,
|
| 80 |
"<pad>": self.tokenizer.pad_id
|
| 81 |
}
|
|
|
|
| 82 |
|
| 83 |
def get_command(self, token):
|
| 84 |
if token in self.special_tokens:
|
|
|
|
| 73 |
kwargs.pop("eos_token")
|
| 74 |
kwargs.pop("pad_token")
|
| 75 |
kwargs.pop("unk_token")
|
|
|
|
| 76 |
self.special_tokens = {
|
| 77 |
"<bos>": self.tokenizer.bos_id,
|
| 78 |
"<eos>": self.tokenizer.eos_id,
|
| 79 |
"<pad>": self.tokenizer.pad_id
|
| 80 |
}
|
| 81 |
+
super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
|
| 82 |
|
| 83 |
def get_command(self, token):
|
| 84 |
if token in self.special_tokens:
|