improve score

app.py

@@ -9,81 +9,134 @@ INTRODUCTION_TEXT = """
 
 ## Dataset
 
-📈 We evaluate models based on 2 datasets,
+📈 We evaluate models based on 3 datasets,
 
-1. Fleurs MY-MS test set, Malay language, https://huggingface.co/datasets/malaysia-ai/fleurs-my-ms
-2. IMDA TTS first 700 audio files, English language but with Manglish slang, https://huggingface.co/datasets/mesolitica/IMDA-TTS
+1. Malaya-Speech test set, Malay language, https://huggingface.co/datasets/huseinzol05/malaya-speech-stt-test-set/tree/main/malaya-speech
+2. Fleurs MS-MY test set, Malay language, https://huggingface.co/datasets/huseinzol05/malaya-speech-stt-test-set/tree/main/fleurs-ms-my
+3. IMDA TTS first 700 audio files, English language but with Manglish slang, https://huggingface.co/datasets/mesolitica/IMDA-TTS
 
-During test we,
+## Heavy postprocess test set
 
-1. Lowercase.
-2. Remove punctuations.
+1. We filtered test set that contain numbers because malaya-speech transducer trained on normalized numbers.
+2. We lower case because malaya-speech transducer trained on lower case.
+3. We removed punctuation because malaya-speech transducer trained without punctuation.
 """
 
 open_source = [
     {
         'model': 'openai/whisper-large-v3',
-        'Fleurs MY-MS CER': 0.027414635425413655,
-        'Fleurs MY-MS WER': 0.0912705436045907,
+        'model size FP16 (MB)': 3090,
+        'Malaya-Speech test CER': 0.0349251317825172,
+        'Malaya-Speech test WER': 0.1032828282828283,
+        'Fleurs MY-MS CER': 0.026055551396846878,
+        'Fleurs MY-MS WER': 0.07652049926522007,
         'IMDA TTS CER': 0.016648493852990828,
         'IMDA TTS WER': 0.0386282289139432,
     },
     {
         'model': 'openai/whisper-medium',
-        'Fleurs MY-MS CER': 0.045260198639505075,
-        'Fleurs MY-MS WER': 0.14913723876746685,
+        'model size FP16 (MB)': 1530,
+        'Malaya-Speech test CER': 0.05064920144820262,
+        'Malaya-Speech test WER': 0.17534205321090568,
+        'Fleurs MY-MS CER': 0.04366882208520179,
+        'Fleurs MY-MS WER': 0.13546055192128273,
         'IMDA TTS CER': 0.02065587879424904,
         'IMDA TTS WER': 0.047277690563404855,
     },
     {
         'model': 'openai/whisper-small',
-        'Fleurs MY-MS CER': 0.07028889922090295,
-        'Fleurs MY-MS WER': 0.2327510905228186,
+        'model size FP16 (MB)': 483.5,
+        'Malaya-Speech test CER': 0.07485209857268262,
+        'Malaya-Speech test WER': 0.25748516055893106,
+        'Fleurs MY-MS CER': 0.06781078047622793,
+        'Fleurs MY-MS WER': 0.21953142859857497,
         'IMDA TTS CER': 0.024812471688517194,
         'IMDA TTS WER': 0.058901277294134434,
     },
     {
         'model': 'openai/whisper-base',
-        'Fleurs MY-MS CER': 0.24820848114299138,
-        'Fleurs MY-MS WER': 0.5164123884823085,
+        'model size FP16 (MB)': 145,
+        'Malaya-Speech test CER': 0.3574879236610538,
+        'Malaya-Speech test WER': 0.8303456599563157,
+        'Fleurs MY-MS CER': 0.1319124653794061,
+        'Fleurs MY-MS WER': 0.40499286081235003,
         'IMDA TTS CER': 0.03914533450681607,
         'IMDA TTS WER': 0.08951682444539587,
     },
     {
         'model': 'openai/whisper-tiny',
-        'Fleurs MY-MS CER': 0.4569030231808184,
-        'Fleurs MY-MS WER': 0.7505351570430122,
+        'model size FP16 (MB)': 75.5,
+        'Malaya-Speech test CER': 0.26941094281472105,
+        'Malaya-Speech test WER': 0.7414099751189915,
+        'Fleurs MY-MS CER': 0.38749733168917505,
+        'Fleurs MY-MS WER': 0.812253445128297,
         'IMDA TTS CER': 0.048805770734828904,
         'IMDA TTS WER': 0.11150629529200957,
     },
     {
         'model': 'mesolitica/malaysian-whisper-medium',
-        'Fleurs MY-MS CER': 0.029782543838559473,
-        'Fleurs MY-MS WER': 0.09812949956440954,
-        'IMDA TTS CER': 0.01932580931010918,
-        'IMDA TTS WER': 0.045691970727685015,
+        'model size FP16 (MB)': 1530,
+        'Malaya-Speech test CER': 0.05622483776367814,
+        'Malaya-Speech test WER': 0.14406629724252673,
+        'Fleurs MY-MS CER': 0.025543266604368554,
+        'Fleurs MY-MS WER': 0.07940219915492629,
+        'IMDA TTS CER': 0.01971214262944062,
+        'IMDA TTS WER': 0.047223078508792794,
     },
     {
         'model': 'mesolitica/malaysian-whisper-small',
-        'Fleurs MY-MS CER': 0.03596621199151582,
-        'Fleurs MY-MS WER': 0.12024457480764372,
+        'model size FP16 (MB)': 483.5,
+        'Malaya-Speech test CER': 0.049162419174983304,
+        'Malaya-Speech test WER': 0.15926901346983313,
+        'Fleurs MY-MS CER': 0.035517572531147,
+        'Fleurs MY-MS WER': 0.10938718963023729,
         'IMDA TTS CER': 0.024228721439634855,
         'IMDA TTS WER': 0.05546294182008469,
     },
     {
         'model': 'mesolitica/malaysian-whisper-base',
-        'Fleurs MY-MS CER': 0.07478803508650385,
-        'Fleurs MY-MS WER': 0.21823941044294087,
+        'model size FP16 (MB)': 145,
+        'Malaya-Speech test CER': 0.07242006488452603,
+        'Malaya-Speech test WER': 0.22081683495617924,
+        'Fleurs MY-MS CER': 0.06639564802362424,
+        'Fleurs MY-MS WER': 0.19675812232021192,
         'IMDA TTS CER': 0.03982418421412676,
         'IMDA TTS WER': 0.08917690642690643,
     },
     {
         'model': 'mesolitica/malaysian-whisper-tiny',
+        'model size FP16 (MB)': 75.5,
+        'Malaya-Speech test CER': 0.09423990117534763,
+        'Malaya-Speech test WER': 0.295029492365558,
         'Fleurs MY-MS CER': 0.13390519685940314,
         'Fleurs MY-MS WER': 0.3461808122686204,
         'IMDA TTS CER': 0.07957313474501154,
         'IMDA TTS WER': 0.1421708648494363,
     },
+    {
+        'model': 'mesolitica/conformer-large-malay-whisper',
+        'model size FP16 (MB)': 206.5,
+        'Malaya-Speech test CER': 0.025933167255719317,
+        'Malaya-Speech test WER': 0.0912131356803488,
+        'Fleurs MY-MS CER': 0.02548791948171514,
+        'Fleurs MY-MS WER': 0.08376713097429746,
+    },
+    {
+        'model': 'mesolitica/conformer-medium-malay-whisper',
+        'model size FP16 (MB)': 121.5,
+        'Malaya-Speech test CER': 0.024955598713609053,
+        'Malaya-Speech test WER': 0.09315638444736804,
+        'Fleurs MY-MS CER': 0.029205645523910067,
+        'Fleurs MY-MS WER': 0.09253131557833799,
+    },
+    {
+        'model': 'mesolitica/conformer-medium-mixed',
+        'model size FP16 (MB)': 121.5,
+        'Malaya-Speech test CER': 0.034618711056551774,
+        'Malaya-Speech test WER': 0.11179440626161938,
+        'Fleurs MY-MS CER': 0.032894184549728075,
+        'Fleurs MY-MS WER': 0.1026977414887425,
+    },
 ]
 
 data = pd.DataFrame(open_source)