ibraheemmoosa commited on
Commit
43f0d9d
1 Parent(s): f3b7bed

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -8
tokenizer_config.json CHANGED
@@ -4,14 +4,7 @@
4
  "do_lower_case": true,
5
  "eos_token": "<sep>",
6
  "keep_accents": false,
7
- "mask_token": {
8
- "__type": "AddedToken",
9
- "content": "<mask>",
10
- "lstrip": true,
11
- "normalized": true,
12
- "rstrip": false,
13
- "single_word": false
14
- },
15
  "model_max_length": 512,
16
  "pad_token": "<pad>",
17
  "remove_space": true,
 
4
  "do_lower_case": true,
5
  "eos_token": "<sep>",
6
  "keep_accents": false,
7
+ "mask_token": "<mask>",
 
 
 
 
 
 
 
8
  "model_max_length": 512,
9
  "pad_token": "<pad>",
10
  "remove_space": true,