Upload tokenizer

tokenizer_config.json

@@ -17,7 +17,7 @@
       "special": false
     },
     "28": {
-      "content": "n</w>",
+      "content": "g</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -25,7 +25,7 @@
       "special": false
     },
     "29": {
-      "content": "r</w>",
+      "content": "d</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -33,7 +33,7 @@
       "special": false
     },
     "30": {
-      "content": "h</w>",
+      "content": "s</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -41,7 +41,7 @@
       "special": false
     },
     "31": {
-      "content": "f</w>",
+      "content": "e</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -49,7 +49,7 @@
       "special": false
     },
     "32": {
-      "content": "u</w>",
+      "content": "t</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -57,7 +57,7 @@
       "special": false
     },
     "33": {
-      "content": "t</w>",
+      "content": "u</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -65,7 +65,7 @@
       "special": false
     },
     "34": {
-      "content": "d</w>",
+      "content": "f</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -73,7 +73,7 @@
       "special": false
     },
     "35": {
-      "content": "m</w>",
+      "content": "y</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -89,7 +89,7 @@
       "special": false
     },
     "37": {
-      "content": "o</w>",
+      "content": "i</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -97,7 +97,7 @@
       "special": false
     },
     "38": {
-      "content": "e</w>",
+      "content": "h</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -105,7 +105,7 @@
       "special": false
     },
     "39": {
-      "content": "y</w>",
+      "content": "k</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -113,7 +113,7 @@
       "special": false
     },
     "40": {
-      "content": "k</w>",
+      "content": "n</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -121,7 +121,7 @@
       "special": false
     },
     "41": {
-      "content": "s</w>",
+      "content": "r</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -129,7 +129,7 @@
       "special": false
     },
     "42": {
-      "content": "l</w>",
+      "content": "m</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -137,7 +137,7 @@
       "special": false
     },
     "43": {
-      "content": "p</w>",
+      "content": "c</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -145,7 +145,7 @@
       "special": false
     },
     "44": {
-      "content": "g</w>",
+      "content": "v</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -153,7 +153,7 @@
       "special": false
     },
     "45": {
-      "content": "v</w>",
+      "content": "l</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -161,7 +161,7 @@
       "special": false
     },
     "46": {
-      "content": "c</w>",
+      "content": "x</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -169,7 +169,7 @@
       "special": false
     },
     "47": {
-      "content": "w</w>",
+      "content": "o</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -177,7 +177,7 @@
       "special": false
     },
     "48": {
-      "content": "i</w>",
+      "content": "w</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -185,7 +185,7 @@
       "special": false
     },
     "49": {
-      "content": "x</w>",
+      "content": "p</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -193,7 +193,7 @@
       "special": false
     },
     "50": {
-      "content": "b</w>",
+      "content": "z</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -201,7 +201,7 @@
       "special": false
     },
     "51": {
-      "content": "q</w>",
+      "content": "b</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -209,7 +209,7 @@
       "special": false
     },
     "52": {
-      "content": "z</w>",
+      "content": "q</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -9113,7 +9113,7 @@
       "special": false
     },
     "1165": {
-      "content": "thr</w>",
+      "content": "tha</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -9121,7 +9121,7 @@
       "special": false
     },
     "1166": {
-      "content": "tha</w>",
+      "content": "thr</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -9897,7 +9897,7 @@
       "special": false
     },
     "1263": {
-      "content": "out</w>",
+      "content": "ous</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -9905,7 +9905,7 @@
       "special": false
     },
     "1264": {
-      "content": "ous</w>",
+      "content": "out</w>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,

vocab.json

@@ -119,7 +119,7 @@
   "aw": 245,
   "ay</w>": 290,
   "b": 3,
-  "b</w>": 50,
+  "b</w>": 51,
   "ba": 305,
   "back</w>": 332,
   "baf": 1015,
@@ -183,7 +183,7 @@
   "by</w>": 398,
   "bye</w>": 135,
   "c": 4,
-  "c</w>": 46,
+  "c</w>": 43,
   "cal": 202,
   "call": 836,
   "call</w>": 224,
@@ -286,7 +286,7 @@
   "cut</w>": 799,
   "cy</w>": 512,
   "d": 5,
-  "d</w>": 34,
+  "d</w>": 29,
   "da</w>": 616,
   "day</w>": 369,
   "de": 77,
@@ -326,7 +326,7 @@
   "du</w>": 634,
   "due</w>": 556,
   "e": 6,
-  "e</w>": 38,
+  "e</w>": 31,
   "ea": 326,
   "ear": 1221,
   "earlier</w>": 1353,
@@ -390,7 +390,7 @@
   "expedite</w>": 804,
   "expediti": 1330,
   "f": 7,
-  "f</w>": 31,
+  "f</w>": 34,
   "fa</w>": 232,
   "famili": 1140,
   "familiar</w>": 1197,
@@ -445,7 +445,7 @@
   "fusse</w>": 368,
   "futura</w>": 985,
   "g": 8,
-  "g</w>": 44,
+  "g</w>": 28,
   "ga": 617,
   "ga</w>": 840,
   "gal</w>": 816,
@@ -490,7 +490,7 @@
   "gulf</w>": 576,
   "guten</w>": 412,
   "h": 9,
-  "h</w>": 30,
+  "h</w>": 38,
   "ha": 186,
   "ha</w>": 1084,
   "hal": 1228,
@@ -533,7 +533,7 @@
   "hundred</w>": 287,
   "huss</w>": 252,
   "i": 10,
-  "i</w>": 48,
+  "i</w>": 37,
   "ia</w>": 289,
   "iber": 542,
   "iberia</w>": 546,
@@ -607,7 +607,7 @@
   "juliett</w>": 456,
   "just</w>": 825,
   "k": 12,
-  "k</w>": 40,
+  "k</w>": 39,
   "ka": 951,
   "kair</w>": 774,
   "kamas</w>": 986,
@@ -639,7 +639,7 @@
   "ks</w>": 952,
   "ky</w>": 1086,
   "l": 13,
-  "l</w>": 42,
+  "l</w>": 45,
   "la": 374,
   "la</w>": 686,
   "lan": 431,
@@ -714,7 +714,7 @@
   "ly</w>": 312,
   "lyon</w>": 1356,
   "m": 14,
-  "m</w>": 35,
+  "m</w>": 42,
   "ma": 98,
   "ma</w>": 762,
   "mach</w>": 691,
@@ -784,7 +784,7 @@
   "munich</w>": 1341,
   "my</w>": 1019,
   "n": 15,
-  "n</w>": 28,
+  "n</w>": 40,
   "na": 384,
   "na</w>": 972,
   "nato</w>": 698,
@@ -841,7 +841,7 @@
   "nu": 735,
   "number</w>": 740,
   "o": 16,
-  "o</w>": 37,
+  "o</w>": 47,
   "ob": 1238,
   "occ": 1051,
   "occupied</w>": 1074,
@@ -884,9 +884,9 @@
   "our": 302,
   "our</w>": 65,
   "ous": 386,
-  "ous</w>": 1264,
+  "ous</w>": 1263,
   "ously</w>": 1314,
-  "out</w>": 1263,
+  "out</w>": 1264,
   "outh</w>": 1094,
   "ov": 1147,
   "over</w>": 1198,
@@ -895,7 +895,7 @@
   "ow</w>": 1089,
   "own</w>": 264,
   "p": 17,
-  "p</w>": 43,
+  "p</w>": 49,
   "pa": 192,
   "pa</w>": 322,
   "pace</w>": 1300,
@@ -944,12 +944,12 @@
   "psa</w>": 709,
   "pt</w>": 721,
   "q": 18,
-  "q</w>": 51,
+  "q</w>": 52,
   "qu": 216,
   "quen": 509,
   "qui": 1302,
   "r": 19,
-  "r</w>": 29,
+  "r</w>": 41,
   "ra": 84,
   "ra</w>": 313,
   "rad": 124,
@@ -1037,7 +1037,7 @@
   "rvsm</w>": 680,
   "ry</w>": 382,
   "s": 20,
-  "s</w>": 41,
+  "s</w>": 30,
   "sa": 163,
   "sa</w>": 110,
   "sabena</w>": 215,
@@ -1145,7 +1145,7 @@
   "swiss</w>": 240,
   "switch</w>": 1175,
   "t": 21,
-  "t</w>": 33,
+  "t</w>": 32,
   "ta": 74,
   "ta</w>": 834,
   "tag</w>": 227,
@@ -1172,7 +1172,7 @@
   "testar</w>": 1295,
   "th": 55,
   "th</w>": 405,
-  "tha</w>": 1166,
+  "tha</w>": 1165,
   "than": 111,
   "than</w>": 1260,
   "thank</w>": 508,
@@ -1194,7 +1194,7 @@
   "this</w>": 894,
   "thous": 387,
   "thousand</w>": 389,
-  "thr</w>": 1165,
+  "thr</w>": 1166,
   "three</w>": 57,
   "throu": 1262,
   "through</w>": 1362,
@@ -1255,7 +1255,7 @@
   "two</w>": 63,
   "ty</w>": 744,
   "u": 22,
-  "u</w>": 32,
+  "u</w>": 33,
   "uda</w>": 650,
   "uh": 404,
   "ui": 629,
@@ -1281,7 +1281,7 @@
   "ut</w>": 736,
   "uter</w>": 1156,
   "v": 23,
-  "v</w>": 45,
+  "v</w>": 44,
   "va</w>": 507,
   "val": 1158,
   "valda</w>": 1201,
@@ -1305,7 +1305,7 @@
   "vsm</w>": 673,
   "vus</w>": 1250,
   "w": 24,
-  "w</w>": 47,
+  "w</w>": 48,
   "wal": 483,
   "wald</w>": 488,
   "warb": 1252,
@@ -1347,12 +1347,12 @@
   "wron": 955,
   "wrong</w>": 962,
   "x": 25,
-  "x</w>": 49,
+  "x</w>": 46,
   "xeui": 722,
   "xra": 956,
   "xray</w>": 963,
   "y": 26,
-  "y</w>": 39,
+  "y</w>": 35,
   "yan": 924,
   "yankee</w>": 931,
   "yd</w>": 187,
@@ -1368,7 +1368,7 @@
   "ysian</w>": 578,
   "ystar</w>": 1254,
   "z": 27,
-  "z</w>": 52,
+  "z</w>": 50,
   "zero</w>": 70,
   "zulu</w>": 1054,
   "zur": 150,