save all

finetune.sh

@@ -6,7 +6,6 @@ mkdir -p ${model_path}
 python run_common_voice.py \
     --dataloader_num_workers="8" \
     --model_name_or_path="facebook/wav2vec2-large-xlsr-53" \
-    #--overwrite_output_dir \
     --dataset_config_name="ar" \
     --output_dir=${model_path} \
     --num_train_epochs="50" \
@@ -32,6 +31,10 @@ python run_common_voice.py \
     --do_train --do_eval
 
 
+    #--model_name_or_path="facebook/wav2vec2-large-xlsr-53" \
+    #--overwrite_output_dir \
+    #--model_name_or_path="/home/othrif/projects/wav2vec2/finetune-xlsr/models/ar/msa/wav2vec2-large-xlsr-arabic" \
+
 
 
 

run_common_voice.py

@@ -319,14 +319,14 @@ def main():
         batch["text"] = re.sub(chars_to_ignore_regex, "", batch["sentence"]).lower() + " "
         return batch
 
-    train_dataset = train_dataset.map(remove_special_characters, remove_columns=["sentence"])
-    eval_dataset = eval_dataset.map(remove_special_characters, remove_columns=["sentence"])
-
     # For arabic diacritics
     cleander = tn.Tnkeeh(remove_diacritics=True)
     train_dataset = cleander.clean_hf_dataset(train_dataset, 'sentence')
     eval_dataset = cleander.clean_hf_dataset(eval_dataset, 'sentence')
 
+    train_dataset = train_dataset.map(remove_special_characters, remove_columns=["sentence"])
+    eval_dataset = eval_dataset.map(remove_special_characters, remove_columns=["sentence"])
+
     def extract_all_chars(batch):
         all_text = " ".join(batch["text"])
         vocab = list(set(all_text))