beomi commited on
Commit
56c1f44
1 Parent(s): bb873ac

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +4 -0
tokenizer_config.json CHANGED
@@ -1,4 +1,8 @@
1
  {
 
 
 
 
2
  "model_max_length": 1000000000000000019884624838656,
3
  "tokenizer_class": "PreTrainedTokenizerFast"
4
  }
 
1
  {
2
+ "name_or_path": "beomi/KoLLAMA",
3
+ "eos_token": "<|endoftext|>",
4
+ "bos_token": "<|sep|>",
5
+ "pad_token": "<|endoftext|>",
6
  "model_max_length": 1000000000000000019884624838656,
7
  "tokenizer_class": "PreTrainedTokenizerFast"
8
  }