0x7o commited on
Commit
acdb4b6
1 Parent(s): 31dfe76

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +2 -1
  2. tokenizer_config.json +1 -0
special_tokens_map.json CHANGED
@@ -17,5 +17,6 @@
17
  "normalized": false,
18
  "rstrip": false,
19
  "single_word": false
20
- }
 
21
  }
 
17
  "normalized": false,
18
  "rstrip": false,
19
  "single_word": false
20
+ },
21
+ "pad_token": "<|end_of_text|>"
22
  }
tokenizer_config.json CHANGED
@@ -2102,5 +2102,6 @@
2102
  "attention_mask"
2103
  ],
2104
  "model_max_length": 1000000000000000019884624838656,
 
2105
  "tokenizer_class": "PreTrainedTokenizerFast"
2106
  }
 
2102
  "attention_mask"
2103
  ],
2104
  "model_max_length": 1000000000000000019884624838656,
2105
+ "pad_token": "<|end_of_text|>",
2106
  "tokenizer_class": "PreTrainedTokenizerFast"
2107
  }