{ | |
"additional_special_tokens": [ | |
"[bib_ref]", | |
"[/bib_ref]", | |
"[fig_ref]", | |
"[/fig_ref]", | |
"[bib]", | |
"[/bib]", | |
"[fig]", | |
"[/fig]", | |
"[table]", | |
"[/table]", | |
"[formula]", | |
"[/formula]", | |
"<|im_start|>", | |
"<|im_end|>" | |
], | |
"bos_token": { | |
"content": "<s>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false | |
}, | |
"cls_token": "<CLS>", | |
"eos_token": "<|im_end|>", | |
"mask_token": "<MASK>", | |
"pad_token": "<PAD>", | |
"sep_token": "<SEP>", | |
"unk_token": { | |
"content": "<unk>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false | |
} | |
} | |