Hugging Face's logo

lhallee
/
CodonBERT

Transformers
Safetensors
bert
pretraining
Inference Endpoints
Model card Files Community
lhallee commited on
Commit
1778256
•
1 Parent(s): 4b5b1fd

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +8 -3
tokenizer_config.json CHANGED
@@ -43,11 +43,16 @@
43
  },
44
  "clean_up_tokenization_spaces": true,
45
  "cls_token": "[CLS]",
46
- "do_basic_tokenize": true,
47
- "do_lower_case": true,
48
  "mask_token": "[MASK]",
49
  "model_max_length": 1000000000000000019884624838656,
50
- "never_split": null,
 
 
 
 
 
51
  "pad_token": "[PAD]",
52
  "sep_token": "[SEP]",
53
  "strip_accents": null,
 
43
  },
44
  "clean_up_tokenization_spaces": true,
45
  "cls_token": "[CLS]",
46
+ "do_basic_tokenize": false,
47
+ "do_lower_case": false,
48
  "mask_token": "[MASK]",
49
  "model_max_length": 1000000000000000019884624838656,
50
+ "never_split": [
51
+ "A",
52
+ "U",
53
+ "G",
54
+ "C"
55
+ ],
56
  "pad_token": "[PAD]",
57
  "sep_token": "[SEP]",
58
  "strip_accents": null,