Update preprocessor/tokenizer_config.yaml
Browse files
preprocessor/tokenizer_config.yaml
CHANGED
|
@@ -8,13 +8,10 @@ padding_strategy: longest
|
|
| 8 |
padding_direction: right
|
| 9 |
pad_to_multiple_of: 0
|
| 10 |
pad_token_type_id: 0
|
| 11 |
-
bos_token: <|
|
| 12 |
eos_token: <|endoftext|>
|
| 13 |
unk_token: <|endoftext|>
|
| 14 |
-
sep_token: <sep>
|
| 15 |
pad_token: <|endoftext|>
|
| 16 |
-
cls_token: <cls>
|
| 17 |
-
mask_token: <mask>
|
| 18 |
additional_special_tokens:
|
| 19 |
- <|endoftext|>
|
| 20 |
- <|endoftext|>
|
|
|
|
| 8 |
padding_direction: right
|
| 9 |
pad_to_multiple_of: 0
|
| 10 |
pad_token_type_id: 0
|
| 11 |
+
bos_token: <|startoftranscript|>
|
| 12 |
eos_token: <|endoftext|>
|
| 13 |
unk_token: <|endoftext|>
|
|
|
|
| 14 |
pad_token: <|endoftext|>
|
|
|
|
|
|
|
| 15 |
additional_special_tokens:
|
| 16 |
- <|endoftext|>
|
| 17 |
- <|endoftext|>
|