Upload tokenizer

special_tokens_map.json

@@ -1,25 +1,24 @@
 {
   "additional_special_tokens": [
-    {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    }
+    "<|im_start|>",
+    "<|im_end|>"
   ],
-  "bos_token": "<|im_start|>",
-  "eos_token": "<|im_end|>",
-  "pad_token": "<|im_end|>",
-  "unk_token": {
-    "content": "<unk>",
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:970fc7f7a74bd21e2b088b042e5f5d634631f3e28686a5729969c4086dea4369
-size 17518901
+oid sha256:e782ad0f261ec58b8184650dd5030f646b1ff96add9f3af5ef16a0de6db0744a
+size 78866

tokenizer_config.json

@@ -1,9 +1,7 @@
 {
-  "add_bos_token": true,
-  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
-      "content": "<pad>",
+      "content": "<|begin_of_text|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -11,7 +9,7 @@
       "special": true
     },
     "1": {
-      "content": "<eos>",
+      "content": "<|end_of_text|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -19,7 +17,7 @@
       "special": true
     },
     "2": {
-      "content": "<bos>",
+      "content": "<|reserved_special_token_0|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -27,7 +25,7 @@
       "special": true
     },
     "3": {
-      "content": "<unk>",
+      "content": "<|reserved_special_token_1|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -35,823 +33,823 @@
       "special": true
     },
     "4": {
-      "content": "<mask>",
+      "content": "<|reserved_special_token_2|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "5": {
-      "content": "<2mass>",
+      "content": "<|reserved_special_token_3|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "6": {
-      "content": "[@BOS@]",
+      "content": "<|start_header_id|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "7": {
-      "content": "<unused0>",
+      "content": "<|end_header_id|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "8": {
-      "content": "<unused1>",
+      "content": "<|reserved_special_token_4|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "9": {
-      "content": "<unused2>",
+      "content": "<|eot_id|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "10": {
-      "content": "<unused3>",
+      "content": "<|reserved_special_token_5|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "11": {
-      "content": "<unused4>",
+      "content": "<|reserved_special_token_6|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "12": {
-      "content": "<unused5>",
+      "content": "<|reserved_special_token_7|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "13": {
-      "content": "<unused6>",
+      "content": "<|reserved_special_token_8|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "14": {
-      "content": "<unused7>",
+      "content": "<|reserved_special_token_9|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "15": {
-      "content": "<unused8>",
+      "content": "<|reserved_special_token_10|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "16": {
-      "content": "<unused9>",
+      "content": "<|reserved_special_token_11|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "17": {
-      "content": "<unused10>",
+      "content": "<|reserved_special_token_12|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "18": {
-      "content": "<unused11>",
+      "content": "<|reserved_special_token_13|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "19": {
-      "content": "<unused12>",
+      "content": "<|reserved_special_token_14|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "20": {
-      "content": "<unused13>",
+      "content": "<|reserved_special_token_15|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "21": {
-      "content": "<unused14>",
+      "content": "<|reserved_special_token_16|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "22": {
-      "content": "<unused15>",
+      "content": "<|reserved_special_token_17|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "23": {
-      "content": "<unused16>",
+      "content": "<|reserved_special_token_18|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "24": {
-      "content": "<unused17>",
+      "content": "<|reserved_special_token_19|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "25": {
-      "content": "<unused18>",
+      "content": "<|reserved_special_token_20|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "26": {
-      "content": "<unused19>",
+      "content": "<|reserved_special_token_21|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "27": {
-      "content": "<unused20>",
+      "content": "<|reserved_special_token_22|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "28": {
-      "content": "<unused21>",
+      "content": "<|reserved_special_token_23|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "29": {
-      "content": "<unused22>",
+      "content": "<|reserved_special_token_24|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "30": {
-      "content": "<unused23>",
+      "content": "<|reserved_special_token_25|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "31": {
-      "content": "<unused24>",
+      "content": "<|reserved_special_token_26|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "32": {
-      "content": "<unused25>",
+      "content": "<|reserved_special_token_27|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "33": {
-      "content": "<unused26>",
+      "content": "<|reserved_special_token_28|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "34": {
-      "content": "<unused27>",
+      "content": "<|reserved_special_token_29|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "35": {
-      "content": "<unused28>",
+      "content": "<|reserved_special_token_30|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "36": {
-      "content": "<unused29>",
+      "content": "<|reserved_special_token_31|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "37": {
-      "content": "<unused30>",
+      "content": "<|reserved_special_token_32|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "38": {
-      "content": "<unused31>",
+      "content": "<|reserved_special_token_33|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "39": {
-      "content": "<unused32>",
+      "content": "<|reserved_special_token_34|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "40": {
-      "content": "<unused33>",
+      "content": "<|reserved_special_token_35|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "41": {
-      "content": "<unused34>",
+      "content": "<|reserved_special_token_36|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "42": {
-      "content": "<unused35>",
+      "content": "<|reserved_special_token_37|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "43": {
-      "content": "<unused36>",
+      "content": "<|reserved_special_token_38|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "44": {
-      "content": "<unused37>",
+      "content": "<|reserved_special_token_39|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "45": {
-      "content": "<unused38>",
+      "content": "<|reserved_special_token_40|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "46": {
-      "content": "<unused39>",
+      "content": "<|reserved_special_token_41|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "47": {
-      "content": "<unused40>",
+      "content": "<|reserved_special_token_42|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "48": {
-      "content": "<unused41>",
+      "content": "<|reserved_special_token_43|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "49": {
-      "content": "<unused42>",
+      "content": "<|reserved_special_token_44|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50": {
-      "content": "<unused43>",
+      "content": "<|reserved_special_token_45|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "51": {
-      "content": "<unused44>",
+      "content": "<|reserved_special_token_46|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "52": {
-      "content": "<unused45>",
+      "content": "<|reserved_special_token_47|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "53": {
-      "content": "<unused46>",
+      "content": "<|reserved_special_token_48|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "54": {
-      "content": "<unused47>",
+      "content": "<|reserved_special_token_49|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "55": {
-      "content": "<unused48>",
+      "content": "<|reserved_special_token_50|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "56": {
-      "content": "<unused49>",
+      "content": "<|reserved_special_token_51|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "57": {
-      "content": "<unused50>",
+      "content": "<|reserved_special_token_52|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "58": {
-      "content": "<unused51>",
+      "content": "<|reserved_special_token_53|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "59": {
-      "content": "<unused52>",
+      "content": "<|reserved_special_token_54|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "60": {
-      "content": "<unused53>",
+      "content": "<|reserved_special_token_55|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "61": {
-      "content": "<unused54>",
+      "content": "<|reserved_special_token_56|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "62": {
-      "content": "<unused55>",
+      "content": "<|reserved_special_token_57|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "63": {
-      "content": "<unused56>",
+      "content": "<|reserved_special_token_58|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "64": {
-      "content": "<unused57>",
+      "content": "<|reserved_special_token_59|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "65": {
-      "content": "<unused58>",
+      "content": "<|reserved_special_token_60|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "66": {
-      "content": "<unused59>",
+      "content": "<|reserved_special_token_61|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "67": {
-      "content": "<unused60>",
+      "content": "<|reserved_special_token_62|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "68": {
-      "content": "<unused61>",
+      "content": "<|reserved_special_token_63|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "69": {
-      "content": "<unused62>",
+      "content": "<|reserved_special_token_64|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "70": {
-      "content": "<unused63>",
+      "content": "<|reserved_special_token_65|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "71": {
-      "content": "<unused64>",
+      "content": "<|reserved_special_token_66|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "72": {
-      "content": "<unused65>",
+      "content": "<|reserved_special_token_67|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "73": {
-      "content": "<unused66>",
+      "content": "<|reserved_special_token_68|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "74": {
-      "content": "<unused67>",
+      "content": "<|reserved_special_token_69|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "75": {
-      "content": "<unused68>",
+      "content": "<|reserved_special_token_70|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "76": {
-      "content": "<unused69>",
+      "content": "<|reserved_special_token_71|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "77": {
-      "content": "<unused70>",
+      "content": "<|reserved_special_token_72|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "78": {
-      "content": "<unused71>",
+      "content": "<|reserved_special_token_73|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "79": {
-      "content": "<unused72>",
+      "content": "<|reserved_special_token_74|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "80": {
-      "content": "<unused73>",
+      "content": "<|reserved_special_token_75|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "81": {
-      "content": "<unused74>",
+      "content": "<|reserved_special_token_76|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "82": {
-      "content": "<unused75>",
+      "content": "<|reserved_special_token_77|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "83": {
-      "content": "<unused76>",
+      "content": "<|reserved_special_token_78|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "84": {
-      "content": "<unused77>",
+      "content": "<|reserved_special_token_79|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "85": {
-      "content": "<unused78>",
+      "content": "<|reserved_special_token_80|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "86": {
-      "content": "<unused79>",
+      "content": "<|reserved_special_token_81|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "87": {
-      "content": "<unused80>",
+      "content": "<|reserved_special_token_82|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "88": {
-      "content": "<unused81>",
+      "content": "<|reserved_special_token_83|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "89": {
-      "content": "<unused82>",
+      "content": "<|reserved_special_token_84|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "90": {
-      "content": "<unused83>",
+      "content": "<|reserved_special_token_85|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "91": {
-      "content": "<unused84>",
+      "content": "<|reserved_special_token_86|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "92": {
-      "content": "<unused85>",
+      "content": "<|reserved_special_token_87|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "93": {
-      "content": "<unused86>",
+      "content": "<|reserved_special_token_88|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "94": {
-      "content": "<unused87>",
+      "content": "<|reserved_special_token_89|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "95": {
-      "content": "<unused88>",
+      "content": "<|reserved_special_token_90|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "96": {
-      "content": "<unused89>",
+      "content": "<|reserved_special_token_91|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "97": {
-      "content": "<unused90>",
+      "content": "<|reserved_special_token_92|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "98": {
-      "content": "<unused91>",
+      "content": "<|reserved_special_token_93|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "99": {
-      "content": "<unused92>",
+      "content": "<|reserved_special_token_94|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "100": {
-      "content": "<unused93>",
+      "content": "<|reserved_special_token_95|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "101": {
-      "content": "<unused94>",
+      "content": "<|reserved_special_token_96|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "102": {
-      "content": "<unused95>",
+      "content": "<|reserved_special_token_97|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "103": {
-      "content": "<unused96>",
+      "content": "<|reserved_special_token_98|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "104": {
-      "content": "<unused97>",
+      "content": "<|reserved_special_token_99|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "105": {
-      "content": "<unused98>",
+      "content": "<|reserved_special_token_100|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "106": {
-      "content": "<start_of_turn>",
+      "content": "<|reserved_special_token_101|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -859,7 +857,7 @@
       "special": true
     },
     "107": {
-      "content": "<end_of_turn>",
+      "content": "<|reserved_special_token_102|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -867,878 +865,1190 @@
       "special": true
     },
     "108": {
-      "content": "\n",
+      "content": "<|reserved_special_token_103|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "109": {
-      "content": "\n\n",
+      "content": "<|reserved_special_token_104|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "110": {
-      "content": "\n\n\n",
+      "content": "<|reserved_special_token_105|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "111": {
-      "content": "\n\n\n\n",
+      "content": "<|reserved_special_token_106|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "112": {
-      "content": "\n\n\n\n\n",
+      "content": "<|reserved_special_token_107|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "113": {
-      "content": "\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_108|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "114": {
-      "content": "\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_109|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "115": {
-      "content": "\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_110|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "116": {
-      "content": "\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_111|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "117": {
-      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_112|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "118": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_113|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "119": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_114|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "120": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_115|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "121": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_116|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "122": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_117|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "123": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_118|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "124": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_119|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "125": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_120|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "126": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_121|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "127": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_122|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "128": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_123|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "129": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_124|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "130": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_125|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "131": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_126|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "132": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_127|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "133": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_128|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "134": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_129|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "135": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_130|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "136": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_131|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "137": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_132|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "138": {
-      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "content": "<|reserved_special_token_133|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "139": {
-      "content": "▁▁",
+      "content": "<|reserved_special_token_134|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "140": {
-      "content": "▁▁▁",
+      "content": "<|reserved_special_token_135|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "141": {
-      "content": "▁▁▁▁",
+      "content": "<|reserved_special_token_136|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "142": {
-      "content": "▁▁▁▁▁",
+      "content": "<|reserved_special_token_137|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "143": {
-      "content": "▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_138|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "144": {
-      "content": "▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_139|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "145": {
-      "content": "▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_140|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "146": {
-      "content": "▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_141|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "147": {
-      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_142|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "148": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_143|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "149": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_144|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "150": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_145|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "151": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_146|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "152": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_147|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "153": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_148|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "154": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_149|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "155": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_150|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "156": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_151|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "157": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_152|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "158": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_153|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "159": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_154|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "160": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_155|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "161": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_156|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "162": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_157|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "163": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_158|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "164": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_159|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "165": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_160|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "166": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_161|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "167": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_162|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "168": {
-      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "content": "<|reserved_special_token_163|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "169": {
-      "content": "<table>",
+      "content": "<|reserved_special_token_164|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "170": {
-      "content": "<caption>",
+      "content": "<|reserved_special_token_165|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "171": {
-      "content": "<thead>",
+      "content": "<|reserved_special_token_166|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "172": {
-      "content": "<tbody>",
+      "content": "<|reserved_special_token_167|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "173": {
-      "content": "<tfoot>",
+      "content": "<|reserved_special_token_168|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "174": {
-      "content": "<tr>",
+      "content": "<|reserved_special_token_169|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "175": {
-      "content": "<th>",
+      "content": "<|reserved_special_token_170|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "176": {
-      "content": "<td>",
+      "content": "<|reserved_special_token_171|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "177": {
-      "content": "</table>",
+      "content": "<|reserved_special_token_172|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "178": {
-      "content": "</caption>",
+      "content": "<|reserved_special_token_173|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "179": {
-      "content": "</thead>",
+      "content": "<|reserved_special_token_174|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "180": {
-      "content": "</tbody>",
+      "content": "<|reserved_special_token_175|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "181": {
-      "content": "</tfoot>",
+      "content": "<|reserved_special_token_176|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "182": {
-      "content": "</tr>",
+      "content": "<|reserved_special_token_177|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "183": {
-      "content": "</th>",
+      "content": "<|reserved_special_token_178|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "184": {
-      "content": "</td>",
+      "content": "<|reserved_special_token_179|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "185": {
-      "content": "<h1>",
+      "content": "<|reserved_special_token_180|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "186": {
-      "content": "<h2>",
+      "content": "<|reserved_special_token_181|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "187": {
-      "content": "<h3>",
+      "content": "<|reserved_special_token_182|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "188": {
-      "content": "<h4>",
+      "content": "<|reserved_special_token_183|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "189": {
-      "content": "<h5>",
+      "content": "<|reserved_special_token_184|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "190": {
-      "content": "<h6>",
+      "content": "<|reserved_special_token_185|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "191": {
-      "content": "<blockquote>",
+      "content": "<|reserved_special_token_186|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "192": {
-      "content": "</h1>",
+      "content": "<|reserved_special_token_187|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "193": {
-      "content": "</h2>",
+      "content": "<|reserved_special_token_188|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "194": {
-      "content": "</h3>",
+      "content": "<|reserved_special_token_189|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "195": {
-      "content": "</h4>",
+      "content": "<|reserved_special_token_190|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "196": {
-      "content": "</h5>",
+      "content": "<|reserved_special_token_191|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "197": {
-      "content": "</h6>",
+      "content": "<|reserved_special_token_192|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "198": {
-      "content": "</blockquote>",
+      "content": "<|reserved_special_token_193|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "199": {
-      "content": "<strong>",
+      "content": "<|reserved_special_token_194|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "200": {
-      "content": "<em>",
+      "content": "<|reserved_special_token_195|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "201": {
-      "content": "<b>",
+      "content": "<|reserved_special_token_196|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "202": {
-      "content": "<i>",
+      "content": "<|reserved_special_token_197|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "203": {
-      "content": "<u>",
+      "content": "<|reserved_special_token_198|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "204": {
-      "content": "<s>",
+      "content": "<|reserved_special_token_199|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "205": {
-      "content": "<sub>",
+      "content": "<|reserved_special_token_200|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "206": {
-      "content": "<sup>",
+      "content": "<|reserved_special_token_201|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "207": {
-      "content": "<code>",
+      "content": "<|reserved_special_token_202|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "208": {
-      "content": "</strong>",
+      "content": "<|reserved_special_token_203|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "209": {
-      "content": "</em>",
+      "content": "<|reserved_special_token_204|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "210": {
-      "content": "</b>",
+      "content": "<|reserved_special_token_205|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "211": {
-      "content": "</i>",
+      "content": "<|reserved_special_token_206|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "212": {
-      "content": "</u>",
+      "content": "<|reserved_special_token_207|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "213": {
-      "content": "</s>",
+      "content": "<|reserved_special_token_208|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "214": {
-      "content": "</sub>",
+      "content": "<|reserved_special_token_209|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "215": {
-      "content": "</sup>",
+      "content": "<|reserved_special_token_210|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "216": {
-      "content": "</code>",
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "236": {
+      "content": "<|reserved_special_token_231|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "256000": {
+    "237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256": {
       "content": "<|im_start|>",
       "lstrip": false,
       "normalized": false,
@@ -1746,7 +2056,7 @@
       "single_word": false,
       "special": true
     },
-    "256001": {
+    "257": {
       "content": "<|im_end|>",
       "lstrip": false,
       "normalized": false,
@@ -1761,13 +2071,14 @@
   ],
   "bos_token": "<|im_start|>",
   "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
-  "clean_up_tokenization_spaces": false,
+  "clean_up_tokenization_spaces": true,
   "eos_token": "<|im_end|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<|im_end|>",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "GemmaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
+  "padding_side": "left",
+  "tokenizer_class": "PreTrainedTokenizerFast"
 }