mlinmg commited on
Commit
d313109
1 Parent(s): 865aff0

Upload 4 files

Browse files

modified the tokenizer to have chat template in it

Files changed (1) hide show
  1. tokenizer_config.json +1 -2
tokenizer_config.json CHANGED
@@ -1,6 +1,4 @@
1
  {
2
- "add_bos_token": false,
3
- "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
@@ -33,6 +31,7 @@
33
  ]
34
  },
35
  "bos_token": "<s>",
 
36
  "clean_up_tokenization_spaces": false,
37
  "eos_token": "</s>",
38
  "legacy": true,
 
1
  {
 
 
2
  "added_tokens_decoder": {
3
  "0": {
4
  "content": "<unk>",
 
31
  ]
32
  },
33
  "bos_token": "<s>",
34
+ "chat_template": "{% for msg in messages %}\n {% if msg.content is not none %}\n {% if msg.role == \"user\" %}\n Human: {{ msg.content }}\n\nAssistant: \n {% elif msg.role == \"assistant\" %}\n {{ msg.content }}\n {% endif %}\n {% endif %}\n{% endfor %}\n",
35
  "clean_up_tokenization_spaces": false,
36
  "eos_token": "</s>",
37
  "legacy": true,