Hezar: Upload tokenizer and config

preprocessor/tokenizer_config.yaml

@@ -1,18 +1,21 @@
 name: whisper_bpe_tokenizer
 config_type: preprocessor
-pretrained_path: whisper-small-fa
-max_length: 512
+max_length: 448
 truncation_strategy: longest_first
 truncation_direction: right
 stride: 0
 padding_strategy: longest
 padding_direction: right
 pad_to_multiple_of: 0
-pad_token_id: 0
-pad_token: <pad>
 pad_token_type_id: 0
+bos_token: <|endoftext|>
+eos_token: <|endoftext|>
 unk_token: <|endoftext|>
-special_tokens:
+sep_token: <sep>
+pad_token: <|endoftext|>
+cls_token: <cls>
+mask_token: <mask>
+additional_special_tokens:
 - <|endoftext|>
 - <|endoftext|>
 - <|startoftranscript|>
@@ -129,11 +132,9 @@ min_frequency: 2
 limit_alphabet: 1000
 initial_alphabet: []
 show_progress: true
-unk_token_id: 50257
-bos_token: <|startoftranscript|>
-bos_token_id: 50257
-eos_token: <|endoftext|>
-eos_token_id: 50257
+translate_token: <|translate|>
+transcribe_token: <|transcribe|>
+notimestamps_token: <|notimestamps|>
 add_prefix_space: false
 add_bos_token: false
 model_max_length: 1024