MERTENST commited on
Commit
5e4dfff
1 Parent(s): 007d046

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +28 -28
vocab.json CHANGED
@@ -1,32 +1,32 @@
1
  {
2
- "'": 11,
3
- "A": 27,
4
- "B": 26,
5
- "C": 23,
6
- "D": 2,
7
- "E": 13,
8
- "F": 15,
9
- "G": 20,
10
- "H": 24,
11
- "I": 3,
12
- "J": 19,
13
- "K": 12,
14
- "L": 6,
15
- "M": 18,
16
- "N": 14,
17
- "O": 5,
18
- "P": 4,
19
- "Q": 0,
20
- "R": 16,
21
- "S": 21,
22
- "T": 22,
23
- "U": 25,
24
- "V": 17,
25
- "W": 1,
26
- "X": 8,
27
- "Y": 7,
28
- "Z": 10,
29
  "[PAD]": 29,
30
  "[UNK]": 28,
31
- "|": 9
32
  }
 
1
  {
2
+ "'": 2,
3
+ "A": 19,
4
+ "B": 11,
5
+ "C": 21,
6
+ "D": 24,
7
+ "E": 17,
8
+ "F": 0,
9
+ "G": 16,
10
+ "H": 18,
11
+ "I": 27,
12
+ "J": 6,
13
+ "K": 10,
14
+ "L": 9,
15
+ "M": 7,
16
+ "N": 1,
17
+ "O": 13,
18
+ "P": 8,
19
+ "Q": 3,
20
+ "R": 22,
21
+ "S": 26,
22
+ "T": 15,
23
+ "U": 20,
24
+ "V": 23,
25
+ "W": 12,
26
+ "X": 14,
27
+ "Y": 4,
28
+ "Z": 25,
29
  "[PAD]": 29,
30
  "[UNK]": 28,
31
+ "|": 5
32
  }