duzx16 commited on
Commit
c0203e1
1 Parent(s): a6d54fa

Add vocab_file in tokenizer

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +1 -0
tokenization_chatglm.py CHANGED
@@ -69,6 +69,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
69
  super().__init__(padding_side=padding_side, **kwargs)
70
  self.name = "GLMTokenizer"
71
 
 
72
  self.tokenizer = SPTokenizer(vocab_file)
73
  self.special_tokens = {
74
  "<bos>": self.tokenizer.bos_id,
 
69
  super().__init__(padding_side=padding_side, **kwargs)
70
  self.name = "GLMTokenizer"
71
 
72
+ self.vocab_file = vocab_file
73
  self.tokenizer = SPTokenizer(vocab_file)
74
  self.special_tokens = {
75
  "<bos>": self.tokenizer.bos_id,