add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 606, "</s>": 607}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "special_tokens_map_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -0,0 +1 @@
+{"가": 1, "각": 2, "간": 3, "갈": 4, "감": 5, "갑": 6, "값": 7, "갔": 8, "강": 9, "같": 10, "개": 11, "걀": 12, "거": 13, "건": 14, "걸": 15, "검": 16, "겁": 17, "것": 18, "게": 19, "겠": 20, "겨": 21, "격": 22, "결": 23, "겼": 24, "경": 25, "계": 26, "고": 27, "곧": 28, "곱": 29, "곳": 30, "공": 31, "과": 32, "관": 33, "괄": 34, "광": 35, "괜": 36, "교": 37, "구": 38, "궁": 39, "궈": 40, "궜": 41, "귄": 42, "균": 43, "그": 44, "근": 45, "글": 46, "금": 47, "급": 48, "기": 49, "긴": 50, "길": 51, "깃": 52, "까": 53, "깐": 54, "깜": 55, "깥": 56, "깨": 57, "꺼": 58, "껐": 59, "꼈": 60, "꼽": 61, "꽂": 62, "꽉": 63, "꾸": 64, "꾼": 65, "꿀": 66, "꿔": 67, "꿨": 68, "뀌": 69, "끄": 70, "끈": 71, "끊": 72, "끌": 73, "끓": 74, "끔": 75, "끕": 76, "끝": 77, "끼": 78, "낀": 79, "나": 80, "난": 81, "날": 82, "남": 83, "낫": 84, "났": 85, "낮": 86, "내": 87, "낼": 88, "냄": 89, "냉": 90, "냐": 91, "냥": 92, "너": 93, "널": 94, "넓": 95, "넘": 96, "넣": 97, "네": 98, "녁": 99, "년": 100, "노": 101, "녹": 102, "놀": 103, "높": 104, "놓": 105, "놔": 106, "놨": 107, "누": 108, "눈": 109, "눌": 110, "느": 111, "는": 112, "늘": 113, "능": 114, "늦": 115, "니": 116, "닌": 117, "님": 118, "다": 119, "닥": 120, "닦": 121, "단": 122, "닫": 123, "달": 124, "담": 125, "답": 126, "당": 127, "닿": 128, "대": 129, "더": 130, "던": 131, "덜": 132, "덟": 133, "덥": 134, "덩": 135, "데": 136, "도": 137, "돌": 138, "동": 139, "돼": 140, "됐": 141, "되": 142, "된": 143, "될": 144, "두": 145, "둔": 146, "둘": 147, "둡": 148, "둬": 149, "뒀": 150, "뒤": 151, "뒷": 152, "드": 153, "득": 154, "든": 155, "듣": 156, "들": 157, "듯": 158, "등": 159, "디": 160, "딘": 161, "딨": 162, "따": 163, "딱": 164, "땀": 165, "때": 166, "떤": 167, "떨": 168, "떻": 169, "또": 170, "똑": 171, "뜨": 172, "뜩": 173, "뜻": 174, "띄": 175, "라": 176, "락": 177, "란": 178, "랄": 179, "람": 180, "랍": 181, "랑": 182, "래": 183, "랙": 184, "량": 185, "러": 186, "럭": 187, "런": 188, "럼": 189, "렁": 190, "레": 191, "렉": 192, "려": 193, "력": 194, "렴": 195, "렸": 196, "령": 197, "로": 198, "록": 199, "롤": 200, "료": 201, "루": 202, "룩": 203, "르": 204, "른": 205, "를": 206, "름": 207, "리": 208, "린": 209, "릴": 210, "림": 211, "마": 212, "막": 213, "만": 214, "많": 215, "말": 216, "맑": 217, "망": 218, "맞": 219, "매": 220, "맥": 221, "머": 222, "먼": 223, "멀": 224, "멈": 225, "메": 226, "며": 227, "면": 228, "멸": 229, "명": 230, "몇": 231, "모": 232, "목": 233, "못": 234, "무": 235, "문": 236, "묻": 237, "물": 238, "뭐": 239, "뭔": 240, "뭘": 241, "미": 242, "밀": 243, "밋": 244, "밍": 245, "밑": 246, "바": 247, "박": 248, "밖": 249, "반": 250, "받": 251, "발": 252, "밝": 253, "밤": 254, "방": 255, "배": 256, "밸": 257, "버": 258, "번": 259, "벌": 260, "범": 261, "베": 262, "벨": 263, "벽": 264, "변": 265, "별": 266, "보": 267, "복": 268, "본": 269, "볼": 270, "봄": 271, "봇": 272, "봐": 273, "부": 274, "분": 275, "불": 276, "브": 277, "블": 278, "비": 279, "빗": 280, "빙": 281, "빛": 282, "빠": 283, "빡": 284, "빨": 285, "빵": 286, "빼": 287, "뽑": 288, "사": 289, "삭": 290, "산": 291, "살": 292, "삼": 293, "샀": 294, "상": 295, "새": 296, "색": 297, "생": 298, "서": 299, "석": 300, "선": 301, "설": 302, "섯": 303, "성": 304, "세": 305, "센": 306, "셀": 307, "셔": 308, "션": 309, "셨": 310, "소": 311, "속": 312, "솟": 313, "송": 314, "쇄": 315, "수": 316, "순": 317, "쉬": 318, "스": 319, "슨": 320, "슬": 321, "습": 322, "승": 323, "시": 324, "식": 325, "신": 326, "실": 327, "싫": 328, "심": 329, "십": 330, "싶": 331, "싹": 332, "싼": 333, "쌀": 334, "써": 335, "썼": 336, "쎄": 337, "쐬": 338, "쓰": 339, "쓴": 340, "쓸": 341, "씨": 342, "씩": 343, "아": 344, "악": 345, "안": 346, "앉": 347, "않": 348, "알": 349, "암": 350, "았": 351, "앙": 352, "앞": 353, "애": 354, "야": 355, "약": 356, "양": 357, "얘": 358, "어": 359, "억": 360, "언": 361, "얼": 362, "엄": 363, "업": 364, "없": 365, "었": 366, "엉": 367, "엌": 368, "에": 369, "엔": 370, "여": 371, "역": 372, "연": 373, "열": 374, "였": 375, "영": 376, "옆": 377, "예": 378, "오": 379, "온": 380, "올": 381, "와": 382, "완": 383, "왔": 384, "왜": 385, "외": 386, "왼": 387, "요": 388, "용": 389, "우": 390, "운": 391, "울": 392, "움": 393, "워": 394, "원": 395, "월": 396, "웠": 397, "위": 398, "윈": 399, "유": 400, "육": 401, "으": 402, "은": 403, "을": 404, "음": 405, "의": 406, "이": 407, "인": 408, "일": 409, "읽": 410, "임": 411, "입": 412, "있": 413, "자": 414, "작": 415, "잔": 416, "잘": 417, "잠": 418, "잡": 419, "장": 420, "재": 421, "저": 422, "적": 423, "전": 424, "절": 425, "점": 426, "접": 427, "정": 428, "젖": 429, "제": 430, "젠": 431, "져": 432, "졌": 433, "조": 434, "족": 435, "졸": 436, "좀": 437, "종": 438, "좋": 439, "좌": 440, "죄": 441, "죠": 442, "주": 443, "준": 444, "줄": 445, "중": 446, "줘": 447, "줬": 448, "즈": 449, "증": 450, "지": 451, "직": 452, "진": 453, "질": 454, "짐": 455, "집": 456, "짜": 457, "짝": 458, "째": 459, "쪽": 460, "쭉": 461, "쯤": 462, "찍": 463, "차": 464, "착": 465, "찮": 466, "참": 467, "찼": 468, "창": 469, "찾": 470, "채": 471, "처": 472, "척": 473, "천": 474, "첫": 475, "청": 476, "체": 477, "쳐": 478, "초": 479, "총": 480, "촬": 481, "최": 482, "추": 483, "출": 484, "춤": 485, "춥": 486, "충": 487, "춰": 488, "취": 489, "측": 490, "층": 491, "치": 492, "친": 493, "칟": 494, "칠": 495, "침": 496, "카": 497, "칸": 498, "캘": 499, "커": 500, "컨": 501, "컴": 502, "케": 503, "켜": 504, "켠": 505, "켤": 506, "켭": 507, "켯": 508, "켰": 509, "코": 510, "콘": 511, "쾌": 512, "크": 513, "큰": 514, "클": 515, "큼": 516, "키": 517, "킨": 518, "킬": 519, "킹": 520, "타": 521, "탁": 522, "탈": 523, "탐": 524, "탑": 525, "태": 526, "택": 527, "탠": 528, "터": 529, "턴": 530, "테": 531, "텐": 532, "토": 533, "톨": 534, "톱": 535, "통": 536, "퇴": 537, "트": 538, "특": 539, "튼": 540, "틀": 541, "티": 542, "팅": 543, "파": 544, "팎": 545, "판": 546, "팔": 547, "퍼": 548, "펑": 549, "펴": 550, "편": 551, "평": 552, "폐": 553, "포": 554, "폰": 555, "표": 556, "풀": 557, "품": 558, "풍": 559, "퓨": 560, "프": 561, "픈": 562, "플": 563, "피": 564, "필": 565, "핑": 566, "하": 567, "학": 568, "한": 569, "할": 570, "합": 571, "항": 572, "해": 573, "핸": 574, "햇": 575, "했": 576, "행": 577, "향": 578, "험": 579, "혀": 580, "현": 581, "혔": 582, "형": 583, "호": 584, "혹": 585, "홈": 586, "홉": 587, "홍": 588, "화": 589, "확": 590, "환": 591, "활": 592, "황": 593, "회": 594, "후": 595, "훈": 596, "휘": 597, "휴": 598, "흡": 599, "희": 600, "히": 601, "힌": 602, "힐": 603, "|": 0, "[UNK]": 604, "[PAD]": 605}