r-terada commited on
Commit
fb1ba61
1 Parent(s): ea834a1

Upload tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +13 -0
tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "ElectraSudachipyTokenizer",
3
+ "do_lower_case": false,
4
+ "do_word_tokenize": true,
5
+ "do_subword_tokenize": true,
6
+ "word_tokenizer_type": "sudachipy",
7
+ "word_form_type": "dictionary_and_surface",
8
+ "subword_tokenizer_type": "wordpiece",
9
+ "model_max_length": 512,
10
+ "sudachipy_kwargs": {"split_mode":"A","dict_type":"core"},
11
+ "auto_map": {"AutoTokenizer": ["modeling.ElectraSudachipyTokenizer", null]}
12
+ }
13
+