BauyrjanQ's picture
Upload tokenizer
92c8682
{
"kaz": {
"\t": 0,
"\n": 1,
"\r": 2,
"#": 4,
"&": 5,
"(": 6,
")": 7,
"+": 8,
"/": 9,
"0": 10,
"1": 11,
"2": 12,
"3": 13,
"4": 14,
"5": 15,
"6": 16,
"7": 17,
"8": 18,
"9": 19,
"[": 20,
"[PAD]": 140,
"[UNK]": 139,
"\\": 21,
"]": 22,
"_": 23,
"a": 24,
"b": 25,
"c": 26,
"e": 27,
"f": 28,
"i": 29,
"k": 30,
"m": 31,
"n": 32,
"o": 33,
"p": 34,
"r": 35,
"s": 36,
"t": 37,
"y": 38,
"z": 39,
"{": 40,
"|": 3,
"}": 41,
"~": 42,
" ": 43,
"¥": 44,
"©": 45,
"«": 46,
"¬": 47,
"­": 48,
"·": 49,
"»": 50,
"ë": 51,
"ŋ": 52,
"ɕ": 53,
"ə": 54,
"ɵ": 55,
"ʨ": 56,
"̀": 57,
"́": 58,
"̃": 59,
"̆": 60,
"̇": 61,
"̈": 62,
"̜": 63,
"̟": 64,
"̥": 65,
"̩": 66,
"̰": 67,
"̶": 68,
"̹": 69,
"υ": 70,
"а": 71,
"б": 72,
"в": 73,
"г": 74,
"д": 75,
"е": 76,
"ж": 77,
"з": 78,
"и": 79,
"й": 80,
"к": 81,
"л": 82,
"м": 83,
"н": 84,
"о": 85,
"п": 86,
"р": 87,
"с": 88,
"т": 89,
"у": 90,
"ф": 91,
"х": 92,
"ц": 93,
"ч": 94,
"ш": 95,
"щ": 96,
"ъ": 97,
"ы": 98,
"ь": 99,
"э": 100,
"ю": 101,
"я": 102,
"ё": 103,
"і": 104,
"ї": 105,
"ў": 106,
"ѳ": 107,
"ғ": 108,
"қ": 109,
"ң": 110,
"ү": 111,
"ұ": 112,
"һ": 113,
"ӊ": 114,
"ӏ": 115,
"ә": 116,
"ө": 117,
"​": 118,
"‎": 119,
"‐": 120,
"‒": 121,
"–": 122,
"—": 123,
"―": 124,
"’": 125,
"‚": 126,
"„": 127,
"…": 128,
"№": 129,
"−": 130,
"─": 131,
"●": 132,
"☺": 133,
"♥": 134,
"⠀": 135,
"": 136,
"": 137,
"-": 138
}
}