Hack90 commited on
Commit
b7b59a4
1 Parent(s): af62b73

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -3
tokenizer_config.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
- "0": {
5
  "content": "<|endoftext|>",
6
  "lstrip": false,
7
  "normalized": false,
@@ -9,7 +9,7 @@
9
  "single_word": false,
10
  "special": true
11
  },
12
- "1": {
13
  "content": "<|unknown|>",
14
  "lstrip": false,
15
  "normalized": false,
@@ -17,7 +17,7 @@
17
  "single_word": false,
18
  "special": true
19
  },
20
- "2": {
21
  "content": "<|beginoftext|>",
22
  "lstrip": false,
23
  "normalized": true,
 
1
  {
2
  "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
+ "1": {
5
  "content": "<|endoftext|>",
6
  "lstrip": false,
7
  "normalized": false,
 
9
  "single_word": false,
10
  "special": true
11
  },
12
+ "2": {
13
  "content": "<|unknown|>",
14
  "lstrip": false,
15
  "normalized": false,
 
17
  "single_word": false,
18
  "special": true
19
  },
20
+ "0": {
21
  "content": "<|beginoftext|>",
22
  "lstrip": false,
23
  "normalized": true,