TheBloke commited on
Commit
8199cc3
1 Parent(s): ad1f9cc

Fix tokenizer files

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +1 -3
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -34,11 +34,9 @@
34
  "legacy": true,
35
  "model_max_length": 1000000000000000019884624838656,
36
  "pad_token": null,
37
- "padding_side": "right",
38
  "sp_model_kwargs": {},
39
  "spaces_between_special_tokens": false,
40
  "tokenizer_class": "LlamaTokenizer",
41
- "tokenizer_file": "/workspace/mistral-7b/tokenizer.json",
42
  "unk_token": "<unk>",
43
  "use_default_system_prompt": true
44
- }
 
34
  "legacy": true,
35
  "model_max_length": 1000000000000000019884624838656,
36
  "pad_token": null,
 
37
  "sp_model_kwargs": {},
38
  "spaces_between_special_tokens": false,
39
  "tokenizer_class": "LlamaTokenizer",
 
40
  "unk_token": "<unk>",
41
  "use_default_system_prompt": true
42
+ }