duzx16
commited on
Commit
•
c0203e1
1
Parent(s):
a6d54fa
Add vocab_file in tokenizer
Browse files- tokenization_chatglm.py +1 -0
tokenization_chatglm.py
CHANGED
@@ -69,6 +69,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
69 |
super().__init__(padding_side=padding_side, **kwargs)
|
70 |
self.name = "GLMTokenizer"
|
71 |
|
|
|
72 |
self.tokenizer = SPTokenizer(vocab_file)
|
73 |
self.special_tokens = {
|
74 |
"<bos>": self.tokenizer.bos_id,
|
|
|
69 |
super().__init__(padding_side=padding_side, **kwargs)
|
70 |
self.name = "GLMTokenizer"
|
71 |
|
72 |
+
self.vocab_file = vocab_file
|
73 |
self.tokenizer = SPTokenizer(vocab_file)
|
74 |
self.special_tokens = {
|
75 |
"<bos>": self.tokenizer.bos_id,
|