Upload tokenizer

#9
by ArthurZ HF staff - opened
Files changed (1) hide show
  1. tokenizer_config.json +7 -0
tokenizer_config.json CHANGED
@@ -1,6 +1,7 @@
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
 
4
  "bos_token": {
5
  "__type": "AddedToken",
6
  "content": "<s>",
@@ -18,10 +19,16 @@
18
  "rstrip": false,
19
  "single_word": false
20
  },
 
 
21
  "legacy": null,
 
22
  "model_max_length": 1000000000000000019884624838656,
23
  "pad_token": null,
 
24
  "sp_model_kwargs": {},
 
 
25
  "tokenizer_class": "CodeLlamaTokenizer",
26
  "unk_token": {
27
  "__type": "AddedToken",
 
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
4
+ "additional_special_tokens": [],
5
  "bos_token": {
6
  "__type": "AddedToken",
7
  "content": "<s>",
 
19
  "rstrip": false,
20
  "single_word": false
21
  },
22
+ "eot_token": null,
23
+ "fill_token": "<FILL_ME>",
24
  "legacy": null,
25
+ "middle_token": null,
26
  "model_max_length": 1000000000000000019884624838656,
27
  "pad_token": null,
28
+ "prefix_token": null,
29
  "sp_model_kwargs": {},
30
+ "suffix_first": false,
31
+ "suffix_token": null,
32
  "tokenizer_class": "CodeLlamaTokenizer",
33
  "unk_token": {
34
  "__type": "AddedToken",