tokenizer_config.json

{
 "tokenizer_class": "BertJapaneseTokenizer",
 "do_lower_case": false,
 "do_word_tokenize": true,
 "do_subword_tokenize": true,
 "word_tokenizer_type": "sudachi",
 "subword_tokenizer_type": "wordpiece",
 "model_max_length": 512,
 "sudachi_kwargs": {"sudachi_split_mode":"A","sudachi_dict_type":"core"}
}