add tokenizer
Browse files- added_tokens.json +1 -1
- tokenizer.json +2 -2
added_tokens.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"
|
|
|
1 |
+
{"\n": 50258, "<|endoftext|>": 50257}
|
tokenizer.json
CHANGED
@@ -59,12 +59,12 @@
|
|
59 |
},
|
60 |
{
|
61 |
"id": 50258,
|
62 |
-
"special":
|
63 |
"content": "\n",
|
64 |
"single_word": false,
|
65 |
"lstrip": false,
|
66 |
"rstrip": false,
|
67 |
-
"normalized":
|
68 |
}
|
69 |
],
|
70 |
"normalizer": null,
|
|
|
59 |
},
|
60 |
{
|
61 |
"id": 50258,
|
62 |
+
"special": true,
|
63 |
"content": "\n",
|
64 |
"single_word": false,
|
65 |
"lstrip": false,
|
66 |
"rstrip": false,
|
67 |
+
"normalized": false
|
68 |
}
|
69 |
],
|
70 |
"normalizer": null,
|