duzx16
commited on
Commit
•
4e38bef
1
Parent(s):
8eb45c8
Fix clean_up_tokenization_spaces
Browse files- tokenization_chatglm.py +2 -2
tokenization_chatglm.py
CHANGED
@@ -65,8 +65,8 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
65 |
|
66 |
model_input_names = ["input_ids", "attention_mask", "position_ids"]
|
67 |
|
68 |
-
def __init__(self, vocab_file, padding_side="left", **kwargs):
|
69 |
-
super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=
|
70 |
self.name = "GLMTokenizer"
|
71 |
|
72 |
self.vocab_file = vocab_file
|
|
|
65 |
|
66 |
model_input_names = ["input_ids", "attention_mask", "position_ids"]
|
67 |
|
68 |
+
def __init__(self, vocab_file, padding_side="left", clean_up_tokenization_spaces=False, **kwargs):
|
69 |
+
super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
|
70 |
self.name = "GLMTokenizer"
|
71 |
|
72 |
self.vocab_file = vocab_file
|