Thunpitcha commited on
Commit
bee669a
1 Parent(s): fc938cf

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +3 -2
  2. tokenizer_config.json +4 -4
  3. vocab.json +98 -67
added_tokens.json CHANGED
@@ -1,4 +1,5 @@
1
  {
2
- "</s>": 69,
3
- "<s>": 68
 
4
  }
 
1
  {
2
+ "</s>": 100,
3
+ "<s>": 99,
4
+ "[PAD]": 98
5
  }
tokenizer_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "added_tokens_decoder": {
3
- "66": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
@@ -8,7 +8,7 @@
8
  "single_word": false,
9
  "special": false
10
  },
11
- "67": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
@@ -16,7 +16,7 @@
16
  "single_word": false,
17
  "special": false
18
  },
19
- "68": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": true,
@@ -24,7 +24,7 @@
24
  "single_word": false,
25
  "special": true
26
  },
27
- "69": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": true,
 
1
  {
2
  "added_tokens_decoder": {
3
+ "97": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
 
8
  "single_word": false,
9
  "special": false
10
  },
11
+ "98": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
 
16
  "single_word": false,
17
  "special": false
18
  },
19
+ "99": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": true,
 
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "100": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": true,
vocab.json CHANGED
@@ -1,70 +1,101 @@
1
  {
2
- "'": 4,
3
- "[PAD]": 67,
4
- "[UNK]": 66,
5
- "|": 40,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  "ก": 0,
7
- "ข": 53,
8
- "": 51,
9
- "": 17,
10
- "": 26,
11
- "": 52,
12
- "": 25,
13
- "": 33,
14
- "": 22,
15
- "": 16,
16
- "": 23,
17
- "": 65,
18
- "": 43,
19
- "": 21,
20
- "": 56,
21
- "": 1,
22
- "": 60,
23
- "": 45,
24
- "": 62,
25
- "": 3,
26
- "": 63,
27
- "": 24,
28
- "": 12,
29
- "": 18,
30
- "": 61,
31
- "": 11,
32
- "": 31,
33
- "": 29,
34
- "": 64,
35
- "": 46,
36
- "": 19,
37
- "": 41,
38
- "": 54,
39
- "": 35,
40
- "": 59,
41
- "": 44,
42
- "": 37,
43
- "": 57,
44
- "": 42,
45
- "": 2,
46
- "": 49,
47
- "": 27,
48
- "": 47,
49
- "": 15,
50
- "": 9,
51
- "": 48,
52
- "": 14,
53
- "": 38,
54
- "": 20,
55
- "": 28,
56
- "": 10,
57
- "": 30,
58
- "": 58,
59
- "": 34,
60
- "": 32,
61
- "": 50,
62
- "": 6,
63
- "": 39,
64
- "": 13,
65
- "": 5,
66
- "": 55,
67
- "": 7,
68
- "": 8,
69
- "": 36
 
 
 
 
 
 
 
70
  }
 
1
  {
2
+ "'": 3,
3
+ "[PAD]": 98,
4
+ "[UNK]": 97,
5
+ "_": 85,
6
+ "a": 32,
7
+ "b": 16,
8
+ "c": 43,
9
+ "e": 29,
10
+ "f": 71,
11
+ "g": 74,
12
+ "h": 38,
13
+ "i": 22,
14
+ "j": 70,
15
+ "k": 36,
16
+ "l": 87,
17
+ "m": 66,
18
+ "n": 34,
19
+ "o": 84,
20
+ "p": 50,
21
+ "r": 75,
22
+ "s": 37,
23
+ "t": 57,
24
+ "u": 44,
25
+ "w": 68,
26
+ "x": 1,
27
+ "y": 10,
28
+ "|": 78,
29
+ "~": 11,
30
  "ก": 0,
31
+ "ข": 92,
32
+ "": 7,
33
+ "": 88,
34
+ "": 59,
35
+ "": 19,
36
+ "": 89,
37
+ "": 65,
38
+ "": 25,
39
+ "": 17,
40
+ "": 61,
41
+ "": 13,
42
+ "": 63,
43
+ "": 48,
44
+ "": 81,
45
+ "": 62,
46
+ "": 40,
47
+ "": 49,
48
+ "": 96,
49
+ "": 83,
50
+ "": 45,
51
+ "": 2,
52
+ "": 46,
53
+ "": 64,
54
+ "": 8,
55
+ "": 60,
56
+ "": 97,
57
+ "": 56,
58
+ "": 69,
59
+ "": 21,
60
+ "": 47,
61
+ "": 30,
62
+ "": 14,
63
+ "": 79,
64
+ "": 93,
65
+ "": 73,
66
+ "": 95,
67
+ "": 82,
68
+ "": 27,
69
+ "": 94,
70
+ "": 80,
71
+ "": 51,
72
+ "": 86,
73
+ "": 67,
74
+ "": 91,
75
+ "": 31,
76
+ "": 58,
77
+ "": 55,
78
+ "": 33,
79
+ "": 12,
80
+ "": 77,
81
+ "": 15,
82
+ "": 20,
83
+ "": 6,
84
+ "": 23,
85
+ "": 42,
86
+ "": 72,
87
+ "": 24,
88
+ "": 35,
89
+ "": 54,
90
+ "": 90,
91
+ "": 28,
92
+ "": 9,
93
+ "": 4,
94
+ "้": 39,
95
+ "๊": 53,
96
+ "๋": 5,
97
+ "์": 76,
98
+ "ํ": 52,
99
+ "—": 26,
100
+ "’": 41
101
  }