Commit
•
6395e6f
1
Parent(s):
98c1d6e
Upload tokenizer
Browse files- tokenizer.json +27 -0
tokenizer.json
CHANGED
|
@@ -110,6 +110,33 @@
|
|
| 110 |
"rstrip": false,
|
| 111 |
"normalized": false,
|
| 112 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
}
|
| 114 |
],
|
| 115 |
"normalizer": null,
|
|
|
|
| 110 |
"rstrip": false,
|
| 111 |
"normalized": false,
|
| 112 |
"special": true
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"id": 65024,
|
| 116 |
+
"content": "<|system|>",
|
| 117 |
+
"single_word": false,
|
| 118 |
+
"lstrip": false,
|
| 119 |
+
"rstrip": false,
|
| 120 |
+
"normalized": true,
|
| 121 |
+
"special": false
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"id": 65025,
|
| 125 |
+
"content": "<|prompt|>",
|
| 126 |
+
"single_word": false,
|
| 127 |
+
"lstrip": false,
|
| 128 |
+
"rstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"special": false
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"id": 65026,
|
| 134 |
+
"content": "<|answer|>",
|
| 135 |
+
"single_word": false,
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"rstrip": false,
|
| 138 |
+
"normalized": true,
|
| 139 |
+
"special": false
|
| 140 |
}
|
| 141 |
],
|
| 142 |
"normalizer": null,
|