prism / special_tokens_map.json
DaryaTereshchenko
add additional_special_tokens
fcdd642
raw
history blame
576 Bytes
{
"bos_token": "<s>",
"eos_token": "</s>",
"pad_token": "<pad>",
"sep_token": "</s>",
"unk_token": "<unk>",
"additional_special_tokens": ["<ar>",
"<bg>",
"<bn>",
"<ca>",
"<cs>",
"<da>",
"<de>",
"<el>",
"<en>",
"<es>",
"<et>",
"<eo>",
"<fi>",
"<fr>",
"<he>",
"<hr>",
"<hu>",
"<id>",
"<it>",
"<ja>",
"<kk>",
"<lt>",
"<lv>",
"<mk>",
"<nl>",
"<no>",
"<pl>",
"<pt>",
"<ro>",
"<ru>",
"<sk>",
"<sl>",
"<sq>",
"<sr>",
"<sv>",
"<tr>",
"<uk>",
"<vi>",
"<zh>"]
}