create gradio app CER,WER&Acc

app.py

@@ -0,0 +1,5 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+module = evaluate.load("cer")
+launch_gradio_widget(module)

eval_cer.py

@@ -0,0 +1,143 @@
+""" Character Error Ratio (CER) metric. """
+from typing import List
+import datasets, evaluate , jiwer
+import jiwer.transforms as tr
+from datasets.config import PY_VERSION
+from packaging import version
+
+
+if PY_VERSION < version.parse("3.8"):
+    import importlib_metadata
+else:
+    import importlib.metadata as importlib_metadata
+
+SENTENCE_DELIMITER = ""
+
+if version.parse(importlib_metadata.version("jiwer")) < version.parse("2.3.0"):
+
+    class SentencesToListOfCharacters(tr.AbstractTransform):
+        def __init__(self, sentence_delimiter: str = " "):
+            self.sentence_delimiter = sentence_delimiter
+
+        def process_string(self, s: str):
+            return list(s)
+
+        def process_list(self, inp: List[str]):
+            chars = []
+            for sent_idx, sentence in enumerate(inp):
+                chars.extend(self.process_string(sentence))
+                if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(inp) - 1:
+                    chars.append(self.sentence_delimiter)
+            return chars
+
+    cer_transform = tr.Compose(
+        [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
+    )
+else:
+    cer_transform = tr.Compose(
+        [
+            tr.RemoveMultipleSpaces(),
+            tr.Strip(),
+            tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
+            tr.ReduceToListOfListOfChars(),
+        ]
+    )
+
+
+_CITATION = """\
+@inproceedings{inproceedings,
+    author = {Morris, Andrew and Maier, Viktoria and Green, Phil},
+    year = {2004},
+    month = {01},
+    pages = {},
+    title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}
+}
+"""
+
+_DESCRIPTION = """\
+Character error rate (CER) is a common metric of the performance of an automatic speech recognition system.
+
+CER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information.
+
+Character error rate can be computed as:
+
+CER = (S + D + I) / N = (S + D + I) / (S + D + C)
+
+where
+
+S is the number of substitutions,
+D is the number of deletions,
+I is the number of insertions,
+C is the number of correct characters,
+N is the number of characters in the reference (N=S+D+C).
+
+CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the
+performance of the ASR system with a CER of 0 being a perfect score.
+"""
+
+_KWARGS_DESCRIPTION = """
+Computes CER score of transcribed segments against references.
+Args:
+    references: list of references for each speech input.
+    predictions: list of transcribtions to score.
+    concatenate_texts: Whether or not to concatenate sentences before evaluation, set to True for more accurate result.
+Returns:
+    (float): the character error rate
+
+Examples for Hungarain Languge:
+    >>> # Colab usage 
+    >>> !pip install evaluate jiwer
+    >>> import evaluate 
+    >>> from evaluate import load  
+
+    >>> predictions = ["ez a jóslat", "van egy másik minta is"]
+    >>> references = ["ez a hivatkozás", "van még egy"]
+    >>> cer = evaluate.load("cer")
+    >>> cer_score = cer.compute(predictions=predictions, references=references)
+    >>> print(cer_score)
+    >>> 0.9615384615384616
+"""
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class CER(evaluate.Metric):
+    def _info(self):
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "predictions": datasets.Value("string", id="sequence"),
+                    "references": datasets.Value("string", id="sequence"),
+                }
+            ),
+            codebase_urls=["https://github.com/jitsi/jiwer/"],
+            reference_urls=[
+                "https://en.wikipedia.org/wiki/Word_error_rate",
+                "https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates",
+            ],
+        )
+
+    def _compute(self, predictions, references, concatenate_texts=False):
+        if concatenate_texts:
+            return jiwer.compute_measures(
+                references,
+                predictions,
+                truth_transform=cer_transform,
+                hypothesis_transform=cer_transform,
+            )["wer"]
+
+        incorrect = 0
+        total = 0
+        for prediction, reference in zip(predictions, references):
+            measures = jiwer.compute_measures(
+                reference,
+                prediction,
+                truth_transform=cer_transform,
+                hypothesis_transform=cer_transform,
+            )
+            incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
+            total += measures["substitutions"] + measures["deletions"] + measures["hits"]
+
+        return incorrect / total

requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/huggingface/evaluate@8b9373dc8693ffe0244a52551ac5573cffa503aa
+jiwer

test_eval_cer.py

@@ -0,0 +1,96 @@
+import unittest
+from cer import CER
+
+cer = CER()
+class TestCER(unittest.TestCase):
+    def test_cer_case_sensitive(self):
+        refs = ["Magyar Országgyűlés"]
+        preds = ["Magyar Országgyűlés"]
+        # S = 2, D = 0, I = 0, N = 11, CER = 2 / 11
+        char_error_rate = cer.compute(predictions=preds, references=refs)
+        self.assertTrue(abs(char_error_rate - 0.1818181818) < 1e-6)
+        
+    def test_cer_whitespace(self):
+        refs = ["Farkasok voltak"]
+        preds = ["Farkasokvoltak"]
+        # S = , D = , I = 1, N = , CER = I / N
+        char_error_rate = cer.compute(predictions=preds, references=refs)
+        self.assertTrue(abs(char_error_rate - 0.) < 1e-6)
+
+        refs = ["Farkasokvoltak"]
+        preds = ["Ferkasok     voltak"]
+        # S = , D = 1, I = 0, N = 14, CER =  
+        char_error_rate = cer.compute(predictions=preds, references=refs)
+        self.assertTrue(abs(char_error_rate - 0.) < 1e-6)
+
+        # consecutive whitespaces case 1
+        refs = ["Farkasok voltak"]
+        preds = ["Farkasok               voltak"]
+        # S = 0, D = 0, I = 0, N = , CER = 0
+        char_error_rate = cer.compute(predictions=preds, references=refs)
+        self.assertTrue(abs(char_error_rate - 0.0) < 1e-6)
+
+        # consecutive whitespaces case 2
+        refs = ["Farkasok   voltak"]
+        preds = ["Farkasok               voltak"]
+        # S = 0, D = 0, I = 0, N = ?, CER = 0
+        char_error_rate = cer.compute(predictions=preds, references=refs)
+        self.assertTrue(abs(char_error_rate - 0.0) < 1e-6)
+
+    def test_cer_sub(self):
+        refs = ["Magyar"]
+        preds = ["Megyar"]
+        # S = 1, D = 0, I = 0, N = 6, CER = 0.125
+        char_error_rate = cer.compute(predictions=preds, references=refs)
+        self.assertTrue(abs(char_error_rate - 0.125) < 1e-6)
+
+    def test_cer_del(self):
+        refs = ["Farkasokvoltak"]
+        preds = ["Farkasokavoltak"]
+        # S = 0, D = 1, I = 0, N = 14, CER = 0.
+        char_error_rate = cer.compute(predictions=preds, references=refs)
+        self.assertTrue(abs(char_error_rate - 0.) < 1e-6)
+
+    def test_cer_insert(self):
+        refs = ["Farkasokvoltak"]
+        preds = ["Farkasokoltak"]
+        # S = 0, D = 0, I = 1, N = 14, CER = 0.
+        char_error_rate = cer.compute(predictions=preds, references=refs)
+        self.assertTrue(abs(char_error_rate - 0.) < 1e-6)
+
+    def test_cer_equal(self):
+        refs = ["Magyar"]
+        char_error_rate = cer.compute(predictions=refs, references=refs)
+        self.assertEqual(char_error_rate, 0.0)
+
+    def test_cer_list_of_seqs(self):
+        # ['Eötvös Loránd University','I love my daughter']
+        refs = ["Eötvös Loránd Tudományegyetem", "szeretem a lányom"] 
+        char_error_rate = cer.compute(predictions=refs, references=refs)
+        self.assertEqual(char_error_rate, 0.0)
+
+        refs = ["diák", "Az arab nyelvet könnyű megtanulni!", "autó"]
+        preds = ["dxák", "Az       arab nyelvet könnyű megtanulni!", "autó"]
+        # S = 1, D = 0, I = 0, N = 28, CER = 1 / 42
+        char_error_rate = cer.compute(predictions=preds, references=refs)
+        self.assertTrue(abs(char_error_rate - 0.0238095238) < 1e-6)
+
+    def test_correlated_sentences(self):
+        # Learn artificial intelligence to secure your future
+        # Tanuljon mesterséges intelligenciát, hogy biztosítsa jövőjét
+        refs = ["Tanuljon mesterséges intelligenciát,", " hogy biztosítsa jövőjét"]
+        preds = ["Tanuljon mesterséges intelligenciát, hogy", " biztosítsa jövőjét"]
+        # S = 0, D = 0, I = 1, N = 28, CER = 2 / 60
+        # whitespace at the front of " biztosítsa jövőjét" will be strip during preporcessing
+        # so need to insert 2 whitespaces
+        char_error_rate = cer.compute(predictions=preds, references=refs, concatenate_texts=True)
+        self.assertTrue(abs(char_error_rate - 0.03333333333) < 1e-6)
+
+    def test_cer_empty(self):
+        refs = [""]
+        preds = ["tök mindegy"]
+        with self.assertRaises(ValueError):
+            cer.compute(predictions=preds, references=refs)
+
+if __name__ == "__main__":
+    unittest.main()