Kaan Apaydin
commited on
Commit
•
8fe2390
1
Parent(s):
82aa5c0
Upload model trained with Unsloth
Browse filesUpload model trained with Unsloth 2x faster
- tokenizer.json +5 -5
- tokenizer_config.json +6 -6
tokenizer.json
CHANGED
@@ -2309,7 +2309,7 @@
|
|
2309 |
},
|
2310 |
{
|
2311 |
"id": 128256,
|
2312 |
-
"content": "
|
2313 |
"single_word": false,
|
2314 |
"lstrip": false,
|
2315 |
"rstrip": false,
|
@@ -2318,7 +2318,7 @@
|
|
2318 |
},
|
2319 |
{
|
2320 |
"id": 128257,
|
2321 |
-
"content": "
|
2322 |
"single_word": false,
|
2323 |
"lstrip": false,
|
2324 |
"rstrip": false,
|
@@ -2327,7 +2327,7 @@
|
|
2327 |
},
|
2328 |
{
|
2329 |
"id": 128258,
|
2330 |
-
"content": "
|
2331 |
"single_word": false,
|
2332 |
"lstrip": false,
|
2333 |
"rstrip": false,
|
@@ -2336,7 +2336,7 @@
|
|
2336 |
},
|
2337 |
{
|
2338 |
"id": 128259,
|
2339 |
-
"content": "
|
2340 |
"single_word": false,
|
2341 |
"lstrip": false,
|
2342 |
"rstrip": false,
|
@@ -2345,7 +2345,7 @@
|
|
2345 |
},
|
2346 |
{
|
2347 |
"id": 128260,
|
2348 |
-
"content": "
|
2349 |
"single_word": false,
|
2350 |
"lstrip": false,
|
2351 |
"rstrip": false,
|
|
|
2309 |
},
|
2310 |
{
|
2311 |
"id": 128256,
|
2312 |
+
"content": "X_token",
|
2313 |
"single_word": false,
|
2314 |
"lstrip": false,
|
2315 |
"rstrip": false,
|
|
|
2318 |
},
|
2319 |
{
|
2320 |
"id": 128257,
|
2321 |
+
"content": "process_tree=",
|
2322 |
"single_word": false,
|
2323 |
"lstrip": false,
|
2324 |
"rstrip": false,
|
|
|
2327 |
},
|
2328 |
{
|
2329 |
"id": 128258,
|
2330 |
+
"content": "->_token",
|
2331 |
"single_word": false,
|
2332 |
"lstrip": false,
|
2333 |
"rstrip": false,
|
|
|
2336 |
},
|
2337 |
{
|
2338 |
"id": 128259,
|
2339 |
+
"content": "+_token",
|
2340 |
"single_word": false,
|
2341 |
"lstrip": false,
|
2342 |
"rstrip": false,
|
|
|
2345 |
},
|
2346 |
{
|
2347 |
"id": 128260,
|
2348 |
+
"content": "*_token",
|
2349 |
"single_word": false,
|
2350 |
"lstrip": false,
|
2351 |
"rstrip": false,
|
tokenizer_config.json
CHANGED
@@ -2049,7 +2049,7 @@
|
|
2049 |
"special": true
|
2050 |
},
|
2051 |
"128256": {
|
2052 |
-
"content": "
|
2053 |
"lstrip": false,
|
2054 |
"normalized": true,
|
2055 |
"rstrip": false,
|
@@ -2057,7 +2057,7 @@
|
|
2057 |
"special": false
|
2058 |
},
|
2059 |
"128257": {
|
2060 |
-
"content": "
|
2061 |
"lstrip": false,
|
2062 |
"normalized": true,
|
2063 |
"rstrip": false,
|
@@ -2065,7 +2065,7 @@
|
|
2065 |
"special": false
|
2066 |
},
|
2067 |
"128258": {
|
2068 |
-
"content": "
|
2069 |
"lstrip": false,
|
2070 |
"normalized": true,
|
2071 |
"rstrip": false,
|
@@ -2073,7 +2073,7 @@
|
|
2073 |
"special": false
|
2074 |
},
|
2075 |
"128259": {
|
2076 |
-
"content": "
|
2077 |
"lstrip": false,
|
2078 |
"normalized": true,
|
2079 |
"rstrip": false,
|
@@ -2081,7 +2081,7 @@
|
|
2081 |
"special": false
|
2082 |
},
|
2083 |
"128260": {
|
2084 |
-
"content": "
|
2085 |
"lstrip": false,
|
2086 |
"normalized": true,
|
2087 |
"rstrip": false,
|
@@ -2090,7 +2090,7 @@
|
|
2090 |
}
|
2091 |
},
|
2092 |
"bos_token": "<|begin_of_text|>",
|
2093 |
-
"chat_template": "{{ bos_token }}{% for message in messages %}{% if message['from'] == 'human' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + message['value'] | trim + '<|eot_id|>' }}{% elif message['from'] == 'gpt' %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' + message['value'] | trim + '<|eot_id|>' }}{% else %}{{ '<|start_header_id|>' + message['from'] + '<|end_header_id|>\n\n' + message['value'] | trim + '<|eot_id|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
|
2094 |
"clean_up_tokenization_spaces": true,
|
2095 |
"eos_token": "<|eot_id|>",
|
2096 |
"model_input_names": [
|
|
|
2049 |
"special": true
|
2050 |
},
|
2051 |
"128256": {
|
2052 |
+
"content": "X_token",
|
2053 |
"lstrip": false,
|
2054 |
"normalized": true,
|
2055 |
"rstrip": false,
|
|
|
2057 |
"special": false
|
2058 |
},
|
2059 |
"128257": {
|
2060 |
+
"content": "process_tree=",
|
2061 |
"lstrip": false,
|
2062 |
"normalized": true,
|
2063 |
"rstrip": false,
|
|
|
2065 |
"special": false
|
2066 |
},
|
2067 |
"128258": {
|
2068 |
+
"content": "->_token",
|
2069 |
"lstrip": false,
|
2070 |
"normalized": true,
|
2071 |
"rstrip": false,
|
|
|
2073 |
"special": false
|
2074 |
},
|
2075 |
"128259": {
|
2076 |
+
"content": "+_token",
|
2077 |
"lstrip": false,
|
2078 |
"normalized": true,
|
2079 |
"rstrip": false,
|
|
|
2081 |
"special": false
|
2082 |
},
|
2083 |
"128260": {
|
2084 |
+
"content": "*_token",
|
2085 |
"lstrip": false,
|
2086 |
"normalized": true,
|
2087 |
"rstrip": false,
|
|
|
2090 |
}
|
2091 |
},
|
2092 |
"bos_token": "<|begin_of_text|>",
|
2093 |
+
"chat_template": "{% if 'role' in messages[0] %}{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}{% elif message['role'] == 'assistant' %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}{% else %}{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}{% else %}{{ bos_token }}{% for message in messages %}{% if message['from'] == 'human' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + message['value'] | trim + '<|eot_id|>' }}{% elif message['from'] == 'gpt' %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' + message['value'] | trim + '<|eot_id|>' }}{% else %}{{ '<|start_header_id|>' + message['from'] + '<|end_header_id|>\n\n' + message['value'] | trim + '<|eot_id|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}{% endif %}",
|
2094 |
"clean_up_tokenization_spaces": true,
|
2095 |
"eos_token": "<|eot_id|>",
|
2096 |
"model_input_names": [
|