Thunpitcha commited on
Commit
a7a8578
1 Parent(s): bee669a

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +2 -3
  2. tokenizer_config.json +4 -4
  3. vocab.json +83 -98
added_tokens.json CHANGED
@@ -1,5 +1,4 @@
1
  {
2
- "</s>": 100,
3
- "<s>": 99,
4
- "[PAD]": 98
5
  }
 
1
  {
2
+ "</s>": 85,
3
+ "<s>": 84
 
4
  }
tokenizer_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "added_tokens_decoder": {
3
- "97": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
@@ -8,7 +8,7 @@
8
  "single_word": false,
9
  "special": false
10
  },
11
- "98": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
@@ -16,7 +16,7 @@
16
  "single_word": false,
17
  "special": false
18
  },
19
- "99": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": true,
@@ -24,7 +24,7 @@
24
  "single_word": false,
25
  "special": true
26
  },
27
- "100": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": true,
 
1
  {
2
  "added_tokens_decoder": {
3
+ "82": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
 
8
  "single_word": false,
9
  "special": false
10
  },
11
+ "83": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
 
16
  "single_word": false,
17
  "special": false
18
  },
19
+ "84": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": true,
 
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "85": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": true,
vocab.json CHANGED
@@ -1,101 +1,86 @@
1
  {
2
- "'": 3,
3
- "[PAD]": 98,
4
- "[UNK]": 97,
5
- "_": 85,
6
- "a": 32,
7
- "b": 16,
8
- "c": 43,
9
- "e": 29,
10
- "f": 71,
11
- "g": 74,
12
- "h": 38,
13
- "i": 22,
14
- "j": 70,
15
- "k": 36,
16
- "l": 87,
17
- "m": 66,
18
- "n": 34,
19
- "o": 84,
20
- "p": 50,
21
- "r": 75,
22
- "s": 37,
23
- "t": 57,
24
- "u": 44,
25
- "w": 68,
26
- "x": 1,
27
- "y": 10,
28
- "|": 78,
29
- "~": 11,
30
  "ก": 0,
31
- "ข": 92,
32
- "": 7,
33
- "": 88,
34
- "": 59,
35
- "": 19,
36
- "": 89,
37
- "": 65,
38
- "": 25,
39
- "": 17,
40
- "": 61,
41
- "": 13,
42
- "": 63,
43
- "": 48,
44
- "": 81,
45
- "": 62,
46
- "": 40,
47
- "": 49,
48
- "": 96,
49
- "": 83,
50
- "": 45,
51
- "": 2,
52
- "": 46,
53
- "": 64,
54
- "": 8,
55
- "": 60,
56
- "": 97,
57
- "": 56,
58
- "": 69,
59
- "": 21,
60
- "": 47,
61
- "": 30,
62
- "": 14,
63
- "": 79,
64
- "": 93,
65
- "": 73,
66
- "": 95,
67
- "": 82,
68
- "": 27,
69
- "": 94,
70
- "": 80,
71
- "": 51,
72
- "": 86,
73
- "": 67,
74
- "": 91,
75
- "": 31,
76
- "": 58,
77
- "": 55,
78
- "": 33,
79
- "": 12,
80
- "": 77,
81
- "": 15,
82
- "": 20,
83
- "": 6,
84
- "": 23,
85
- "": 42,
86
- "": 72,
87
- "": 24,
88
- "": 35,
89
- "": 54,
90
- "": 90,
91
- "": 28,
92
- "": 9,
93
- "": 4,
94
- "": 39,
95
- "": 53,
96
- "": 5,
97
- "": 76,
98
- "ํ": 52,
99
- "—": 26,
100
- "’": 41
101
  }
 
1
  {
2
+ "'": 4,
3
+ "[PAD]": 83,
4
+ "[UNK]": 82,
5
+ "_": 57,
6
+ "c": 74,
7
+ "e": 47,
8
+ "i": 35,
9
+ "k": 63,
10
+ "l": 62,
11
+ "m": 30,
12
+ "n": 60,
13
+ "o": 56,
14
+ "r": 42,
15
+ "y": 15,
16
+ "|": 48,
17
+ "~": 16,
 
 
 
 
 
 
 
 
 
 
 
 
18
  "ก": 0,
19
+ "ข": 67,
20
+ "": 64,
21
+ "": 20,
22
+ "": 31,
23
+ "": 65,
24
+ "": 29,
25
+ "": 39,
26
+ "": 26,
27
+ "": 24,
28
+ "": 18,
29
+ "": 27,
30
+ "": 81,
31
+ "": 51,
32
+ "": 25,
33
+ "": 70,
34
+ "": 1,
35
+ "": 76,
36
+ "": 53,
37
+ "": 78,
38
+ "": 3,
39
+ "": 79,
40
+ "": 28,
41
+ "": 13,
42
+ "": 21,
43
+ "": 77,
44
+ "": 12,
45
+ "": 38,
46
+ "": 34,
47
+ "": 80,
48
+ "": 54,
49
+ "": 22,
50
+ "": 49,
51
+ "": 68,
52
+ "": 41,
53
+ "": 75,
54
+ "": 52,
55
+ "": 44,
56
+ "": 72,
57
+ "": 50,
58
+ "": 2,
59
+ "": 59,
60
+ "": 32,
61
+ "": 55,
62
+ "": 19,
63
+ "": 10,
64
+ "": 58,
65
+ "": 17,
66
+ "": 45,
67
+ "": 23,
68
+ "": 33,
69
+ "": 11,
70
+ "": 36,
71
+ "": 73,
72
+ "": 40,
73
+ "": 37,
74
+ "": 61,
75
+ "": 8,
76
+ "": 66,
77
+ "": 46,
78
+ "": 14,
79
+ "": 6,
80
+ "": 69,
81
+ "": 7,
82
+ "": 9,
83
+ "": 43,
84
+ "": 5,
85
+ "": 71
 
 
 
86
  }