Upload tokenizer
Browse files- tokenizer.json +96 -96
- tokenizer_config.json +2 -3
tokenizer.json
CHANGED
@@ -98898,126 +98898,126 @@
|
|
98898 |
"βlivre s",
|
98899 |
"lu b",
|
98900 |
"l ub",
|
98901 |
-
"β β",
|
98902 |
"ββ ββ",
|
98903 |
-
"
|
98904 |
-
"
|
|
|
98905 |
"ββ ββββββ",
|
98906 |
-
"
|
98907 |
-
"
|
98908 |
-
"
|
98909 |
-
"βββ βββββ",
|
98910 |
-
"βββββββ β",
|
98911 |
-
"β βββββββ",
|
98912 |
-
"ββ βββ",
|
98913 |
-
"ββββ β",
|
98914 |
-
"βββ ββ",
|
98915 |
-
"β ββββ",
|
98916 |
"ββ ββββββββββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98917 |
"ββββ ββββββββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98918 |
"ββββββββ ββββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98919 |
"βββββ βββββββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
98920 |
"ββββββ ββββββββββ",
|
|
|
|
|
|
|
|
|
|
|
98921 |
"ββββββββββββ ββββ",
|
|
|
|
|
|
|
98922 |
"βββββββββββββ βββ",
|
|
|
|
|
|
|
|
|
98923 |
"ββββββββββ ββββββ",
|
|
|
|
|
98924 |
"ββββββββββββββ ββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98925 |
"βββ βββββββββββββ",
|
98926 |
-
"
|
98927 |
-
"βββββββ βββββββββ",
|
98928 |
-
"βββββββββββ βββββ",
|
98929 |
-
"βββββββββββββββ β",
|
98930 |
-
"β βββββββββββββββ",
|
98931 |
-
"ββ ββββ",
|
98932 |
-
"ββββ ββ",
|
98933 |
-
"βββββ β",
|
98934 |
"βββ βββ",
|
98935 |
-
"β βββββ",
|
98936 |
-
"ββ ββββββββββ",
|
98937 |
-
"ββββ ββββββββ",
|
98938 |
-
"ββββββββ ββββ",
|
98939 |
-
"βββββ βββββββ",
|
98940 |
-
"ββββββ ββββββ",
|
98941 |
-
"ββββββββββ ββ",
|
98942 |
"βββ βββββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98943 |
"βββββββββ βββ",
|
|
|
|
|
|
|
|
|
|
|
98944 |
"βββββββ βββββ",
|
98945 |
-
"βββββββββββ β",
|
98946 |
-
"β βββββββββββ",
|
98947 |
-
"ββ βββββββββββ",
|
98948 |
-
"ββββ βββββββββ",
|
98949 |
-
"ββββββββ βββββ",
|
98950 |
-
"βββββ ββββββββ",
|
98951 |
-
"ββββββ βββββββ",
|
98952 |
-
"ββββββββββββ β",
|
98953 |
-
"ββββββββββ βββ",
|
98954 |
-
"βββ ββββββββββ",
|
98955 |
-
"βββββββββ ββββ",
|
98956 |
"βββββββ ββββββ",
|
98957 |
-
"βββββββββββ ββ",
|
98958 |
-
"β ββββββββββββ",
|
98959 |
-
"ββ ββββββββ",
|
98960 |
-
"ββββ ββββββ",
|
98961 |
-
"ββββββββ ββ",
|
98962 |
-
"βββββ βββββ",
|
98963 |
-
"ββββββ ββββ",
|
98964 |
-
"βββ βββββββ",
|
98965 |
-
"βββββββββ β",
|
98966 |
"βββββββ βββ",
|
98967 |
-
"
|
98968 |
-
"ββ ββββββββββββ",
|
98969 |
-
"ββββ ββββββββββ",
|
98970 |
-
"ββββββββ ββββββ",
|
98971 |
-
"βββββ βββββββββ",
|
98972 |
-
"ββββββ ββββββββ",
|
98973 |
-
"ββββββββββββ ββ",
|
98974 |
-
"βββββββββββββ β",
|
98975 |
-
"ββββββββββ ββββ",
|
98976 |
-
"βββ βββββββββββ",
|
98977 |
-
"βββββββββ βββββ",
|
98978 |
"βββββββ βββββββ",
|
|
|
|
|
|
|
|
|
98979 |
"βββββββββββ βββ",
|
98980 |
-
"β
|
98981 |
-
"
|
98982 |
"β ββ",
|
98983 |
-
"
|
98984 |
-
"ββββ βββββ",
|
98985 |
-
"ββββββββ β",
|
98986 |
-
"βββββ ββββ",
|
98987 |
-
"ββββββ βββ",
|
98988 |
-
"βββ ββββββ",
|
98989 |
-
"βββββββ ββ",
|
98990 |
"β ββββββββ",
|
98991 |
-
"
|
98992 |
-
"ββββ βββ",
|
98993 |
-
"βββββ ββ",
|
98994 |
-
"ββββββ β",
|
98995 |
-
"βββ ββββ",
|
98996 |
"β ββββββ",
|
98997 |
-
"
|
98998 |
-
"
|
98999 |
-
"ββββββββ βββ",
|
99000 |
-
"βββββ ββββββ",
|
99001 |
-
"ββββββ βββββ",
|
99002 |
-
"ββββββββββ β",
|
99003 |
-
"βββ ββββββββ",
|
99004 |
-
"βββββββββ ββ",
|
99005 |
-
"βββββββ ββββ",
|
99006 |
"β ββββββββββ",
|
99007 |
-
"
|
99008 |
-
"
|
99009 |
-
"
|
99010 |
-
"
|
99011 |
-
"
|
99012 |
-
"
|
99013 |
-
"
|
99014 |
-
"ββββββββββ βββββ",
|
99015 |
-
"ββββββββββββββ β",
|
99016 |
-
"βββ ββββββββββββ",
|
99017 |
-
"βββββββββ ββββββ",
|
99018 |
-
"βββββββ ββββββββ",
|
99019 |
-
"βββββββββββ ββββ",
|
99020 |
-
"β ββββββββββββββ"
|
99021 |
]
|
99022 |
}
|
99023 |
}
|
|
|
98898 |
"βlivre s",
|
98899 |
"lu b",
|
98900 |
"l ub",
|
|
|
98901 |
"ββ ββ",
|
98902 |
+
"ββ ββββ",
|
98903 |
+
"ββ ββββββββ",
|
98904 |
+
"ββ βββββ",
|
98905 |
"ββ ββββββ",
|
98906 |
+
"ββ ββββββββββββ",
|
98907 |
+
"ββ βββββββββββββ",
|
98908 |
+
"ββ ββββββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98909 |
"ββ ββββββββββββββ",
|
98910 |
+
"ββ βββ",
|
98911 |
+
"ββ βββββββββ",
|
98912 |
+
"ββ βββββββ",
|
98913 |
+
"ββ βββββββββββ",
|
98914 |
+
"ββ β",
|
98915 |
+
"ββββ ββ",
|
98916 |
+
"ββββ ββββ",
|
98917 |
+
"ββββ ββββββββ",
|
98918 |
+
"ββββ βββββ",
|
98919 |
+
"ββββ ββββββ",
|
98920 |
"ββββ ββββββββββββ",
|
98921 |
+
"ββββ ββββββββββ",
|
98922 |
+
"ββββ βββ",
|
98923 |
+
"ββββ βββββββββ",
|
98924 |
+
"ββββ βββββββ",
|
98925 |
+
"ββββ βββββββββββ",
|
98926 |
+
"ββββ β",
|
98927 |
+
"ββββββββ ββ",
|
98928 |
+
"ββββββββ ββββ",
|
98929 |
"ββββββββ ββββββββ",
|
98930 |
+
"ββββββββ βββββ",
|
98931 |
+
"ββββββββ ββββββ",
|
98932 |
+
"ββββββββ βββ",
|
98933 |
+
"ββββββββ βββββββ",
|
98934 |
+
"ββββββββ β",
|
98935 |
+
"βββββ ββ",
|
98936 |
+
"βββββ ββββ",
|
98937 |
+
"βββββ ββββββββ",
|
98938 |
+
"βββββ βββββ",
|
98939 |
+
"βββββ ββββββ",
|
98940 |
+
"βββββ ββββββββββ",
|
98941 |
+
"βββββ βββ",
|
98942 |
+
"βββββ βββββββββ",
|
98943 |
+
"βββββ βββββββ",
|
98944 |
"βββββ βββββββββββ",
|
98945 |
+
"βββββ β",
|
98946 |
+
"ββββββ ββ",
|
98947 |
+
"ββββββ ββββ",
|
98948 |
+
"ββββββ ββββββββ",
|
98949 |
+
"ββββββ βββββ",
|
98950 |
+
"ββββββ ββββββ",
|
98951 |
"ββββββ ββββββββββ",
|
98952 |
+
"ββββββ βββ",
|
98953 |
+
"ββββββ βββββββββ",
|
98954 |
+
"ββββββ βββββββ",
|
98955 |
+
"ββββββ β",
|
98956 |
+
"ββββββββββββ ββ",
|
98957 |
"ββββββββββββ ββββ",
|
98958 |
+
"ββββββββββββ βββ",
|
98959 |
+
"ββββββββββββ β",
|
98960 |
+
"βββββββββββββ ββ",
|
98961 |
"βββββββββββββ βββ",
|
98962 |
+
"βββββββββββββ β",
|
98963 |
+
"ββββββββββ ββ",
|
98964 |
+
"ββββββββββ ββββ",
|
98965 |
+
"ββββββββββ βββββ",
|
98966 |
"ββββββββββ ββββββ",
|
98967 |
+
"ββββββββββ βββ",
|
98968 |
+
"ββββββββββ β",
|
98969 |
"ββββββββββββββ ββ",
|
98970 |
+
"ββββββββββββββ β",
|
98971 |
+
"βββ ββ",
|
98972 |
+
"βββ ββββ",
|
98973 |
+
"βββ ββββββββ",
|
98974 |
+
"βββ βββββ",
|
98975 |
+
"βββ ββββββ",
|
98976 |
+
"βββ ββββββββββββ",
|
98977 |
"βββ βββββββββββββ",
|
98978 |
+
"βββ ββββββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98979 |
"βββ βββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98980 |
"βββ βββββββββ",
|
98981 |
+
"βββ βββββββ",
|
98982 |
+
"βββ βββββββββββ",
|
98983 |
+
"βββ β",
|
98984 |
+
"βββββββββ ββ",
|
98985 |
+
"βββββββββ ββββ",
|
98986 |
+
"βββββββοΏ½οΏ½β βββββ",
|
98987 |
+
"βββββββββ ββββββ",
|
98988 |
"βββββββββ βββ",
|
98989 |
+
"βββββββββ βββββββ",
|
98990 |
+
"βββββββββ β",
|
98991 |
+
"βββββββ ββ",
|
98992 |
+
"βββββββ ββββ",
|
98993 |
+
"βββββββ ββββββββ",
|
98994 |
"βββββββ βββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98995 |
"βββββββ ββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98996 |
"βββββββ βββ",
|
98997 |
+
"βββββββ βββββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98998 |
"βββββββ βββββββ",
|
98999 |
+
"βββββββ β",
|
99000 |
+
"βββββββββββ ββ",
|
99001 |
+
"βββββββββββ ββββ",
|
99002 |
+
"βββββββββββ βββββ",
|
99003 |
"βββββββββββ βββ",
|
99004 |
+
"βββββββββββ β",
|
99005 |
+
"βββββββββββββββ β",
|
99006 |
"β ββ",
|
99007 |
+
"β ββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
99008 |
"β ββββββββ",
|
99009 |
+
"β βββββ",
|
|
|
|
|
|
|
|
|
99010 |
"β ββββββ",
|
99011 |
+
"β ββββββββββββ",
|
99012 |
+
"β βββββββββββββ",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
99013 |
"β ββββββββββ",
|
99014 |
+
"β ββββββββββββββ",
|
99015 |
+
"β βββ",
|
99016 |
+
"β βββββββββ",
|
99017 |
+
"β βββββββ",
|
99018 |
+
"β βββββββββββ",
|
99019 |
+
"β βββββββββββββββ",
|
99020 |
+
"β β"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
99021 |
]
|
99022 |
}
|
99023 |
}
|
tokenizer_config.json
CHANGED
@@ -1,6 +1,4 @@
|
|
1 |
{
|
2 |
-
"add_bos_token": true,
|
3 |
-
"add_eos_token": false,
|
4 |
"bos_token": {
|
5 |
"__type": "AddedToken",
|
6 |
"content": "<s>",
|
@@ -30,5 +28,6 @@
|
|
30 |
"normalized": true,
|
31 |
"rstrip": false,
|
32 |
"single_word": false
|
33 |
-
}
|
|
|
34 |
}
|
|
|
1 |
{
|
|
|
|
|
2 |
"bos_token": {
|
3 |
"__type": "AddedToken",
|
4 |
"content": "<s>",
|
|
|
28 |
"normalized": true,
|
29 |
"rstrip": false,
|
30 |
"single_word": false
|
31 |
+
},
|
32 |
+
"use_default_system_prompt": true
|
33 |
}
|