Trascribe text

create conda environment.md

@@ -17,7 +17,7 @@ pip install huggingface-hub
 pip install torch
 pip install torchaudio
 <!-- pip install pysndfile==1.0.0 -->
-mamba install -y -c conda-forge libsndfile==1.0.31 pyperclip
+mamba install -y -c conda-forge libsndfile==1.0.31 pyperclip ipywidgets
 
 # Download videos
 pip install twitch-dl

requirements.txt

@@ -1,11 +1,13 @@
-fairseq2
-git+https://github.com/facebookresearch/seamless_communication
-gradio
-huggingface_hub
-torch
-torchaudio
+# fairseq2
+# git+https://github.com/facebookresearch/seamless_communication
 # libsndfile==1.0.31
 # pysndfile
+gradio
+# huggingface_hub
+torch
+# torchaudio
 twitch-dl
 pytube
 pyperclip
+transformers
+git+https://github.com/openai/whisper.git

translatube.py

@@ -4,42 +4,76 @@ import urllib.parse as urlparse
 from pytube import YouTube
 import re
 import subprocess
+import time
 from lang_list import ORIGINAL_LANGUAGE_NAME_TO_CODE, S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES
+
 import torch
-from seamless_communication.models.inference import Translator
-import time
+import whisper
+
+# get device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+# device = torch.device("cpu")
+model = whisper.load_model("large-v2", device=device)
 
 YOUTUBE = "youtube"
 TWITCH = "twitch"
-
-# Initialize a Translator object with a multitask model, vocoder on the GPU.
-# translator = Translator("seamlessM4T_large", vocoder_name_or_card="vocoder_36langs", device=torch.device("cuda:0"))
+ERROR = "error"
 
 def copy_url_from_clipboard():
     return pyperclip.paste()
 
 def clear_video_url():
-    return ""
+    visible = False
+    image = gr.Image(visible=visible, scale=1)
+    source_languaje = gr.Dropdown(visible=visible, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True)
+    target_languaje = gr.Dropdown(visible=visible, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True)
+    get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=visible)
+    transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible)
+    original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible, interactive=False)
+    original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=visible)
+    original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=visible)
+    translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=visible)
+    transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible)
+    return (
+        "",
+        image, 
+        source_languaje, 
+        target_languaje, 
+        get_audio_button, 
+        transcribe_audio_button,
+        original_audio, 
+        original_audio_transcribed, 
+        translated_audio, 
+        original_audio_translated,
+    )
 
 def get_youtube_thumbnail(video_id):
     thumbnail_url = f"https://img.youtube.com/vi/{video_id}/0.jpg"
     return thumbnail_url
 
 def get_youtube_video_id(url):
-    parsed_url = urlparse.urlparse(url)
-    video_id = urlparse.parse_qs(parsed_url.query).get('v')
-    if video_id:
-        thumbnail_url = get_youtube_thumbnail(video_id[0])
+    if "youtu.be" in url.lower():
+        yt = YouTube(url)
+        thumbnail_url = yt.thumbnail_url
         return thumbnail_url
     else:
-        return None
+        parsed_url = urlparse.urlparse(url)
+        video_id = urlparse.parse_qs(parsed_url.query).get('v')
+        if video_id:
+            thumbnail_url = get_youtube_thumbnail(video_id[0])
+            return thumbnail_url
+        else:
+            return None
 
 def is_valid_url(url):
-    button = gr.Button(size="sm", value="translate", min_width="10px", scale=0, visible=True)
-    original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=True, interactive=False)
-    translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=True)
     source_languaje = gr.Dropdown(visible=True, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True)
     target_languaje = gr.Dropdown(visible=True, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True)
+    get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=True)
+    original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=True, interactive=False)
+    original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=True)
+    original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=True)
+    translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=True)
+    transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=True)
     if "youtube" in url.lower() or "youtu.be" in url.lower():
         thumbnail = get_youtube_video_id(url)
         if thumbnail:
@@ -47,21 +81,64 @@ def is_valid_url(url):
                 gr.Image(value=thumbnail, visible=True, show_download_button=False, container=False), 
                 source_languaje,
                 target_languaje,
-                button, 
+                get_audio_button, 
+                gr.Textbox(value=YOUTUBE, label="Stream page", elem_id="stream_page", visible=False),
+                original_audio,
+                original_audio_transcribed, 
+                translated_audio,
+                transcribe_audio_button, 
+                original_audio_translated, 
+            )
+        else:
+            return (
+                gr.Image(value="assets/youtube-no-thumbnails.webp", visible=True, show_download_button=False, container=False), 
+                source_languaje,
+                target_languaje,
+                get_audio_button, 
                 gr.Textbox(value=YOUTUBE, label="Stream page", elem_id="stream_page", visible=False),
                 original_audio,
+                original_audio_transcribed, 
                 translated_audio,
-                )
+                transcribe_audio_button, 
+                original_audio_translated, 
+            )
     elif "twitch" in url.lower() or "twitch.tv" in url.lower():
         return (
             gr.Image(value="assets/twitch.webp", visible=True, show_download_button=False, container=False), 
             source_languaje,
             target_languaje,
-            button, 
+            get_audio_button, 
             gr.Textbox(value=TWITCH, label="Stream page", elem_id="stream_page", visible=False),
             original_audio,
+            original_audio_transcribed, 
             translated_audio,
-            )
+            transcribe_audio_button, 
+            original_audio_translated, 
+        )
+    else:
+        visible = False
+        image = gr.Image(value="assets/youtube_error.webp", visible=visible, show_download_button=False, container=False)
+        source_languaje = gr.Dropdown(visible=visible, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True)
+        target_languaje = gr.Dropdown(visible=visible, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True)
+        get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=visible)
+        stream_page = gr.Textbox(value=ERROR, label="Stream page", elem_id="stream_page", visible=visible)
+        original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible, interactive=False)
+        original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=visible)
+        original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=visible)
+        translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=visible)
+        transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible)
+        return (
+            image, 
+            source_languaje,
+            target_languaje,
+            get_audio_button, 
+            stream_page,
+            original_audio,
+            original_audio_transcribed, 
+            translated_audio,
+            transcribe_audio_button, 
+            original_audio_translated, 
+        )
 
 def get_audio_from_video(url, stream_page):
     if stream_page == YOUTUBE:
@@ -78,25 +155,71 @@ def get_audio_from_video(url, stream_page):
         audio_stream = audio_streams.filter(abr=abr_list[0]).first()
 
         # Download the audio
-        audio_stream.download(filename="audio.mp3")
+        filename = "audio.mp3"
+        audio_stream.download(filename=filename)
 
-        return gr.Audio("audio.mp3", label="Original audio", elem_id="original_audio", visible=True)
+        return (
+            gr.Audio(value=filename, label="Original audio", elem_id="original_audio", visible=True, interactive=False),
+            gr.Textbox(value=filename, label="Stream page", elem_id="stream_page", visible=False)
+        )
     elif stream_page == TWITCH:
         # Get the video id
         video_id = re.search("\d{10}", url).group(0)
 
         # Download the video
-        subprocess.run(["twitch-dl", "download", "--overwrite", "-q", "audio_only", "--output", "audio.mkv", video_id])
+        filename = "audio.mkv"
+        subprocess.run(["twitch-dl", "download", "--overwrite", "-q", "audio_only", "--output", filename, video_id])
+
+        return (
+            gr.Audio(value=filename, label="Original audio", elem_id="original_audio", visible=True, interactive=False),
+            gr.Textbox(value=filename, label="Stream page", elem_id="stream_page", visible=False)
+        )
 
-        return gr.Audio("audio.mkv", label="Original audio", elem_id="original_audio", visible=True)
+def trascribe_audio(audio_path):
+    audio = whisper.load_audio(audio_path)
+    audio = whisper.pad_or_trim(audio)
+    
+    mel = whisper.log_mel_spectrogram(audio).to(model.device)
+    
+    _, probs = model.detect_language(mel)
+    
+    options = whisper.DecodingOptions(fp16 = False)
+    result = whisper.decode(model, mel, options)
 
-# def translate_audio(input_audio, target_languaje):
-#     print("Translating audio...")
-#     time.sleep(5)
-#     print("Translating audio...")
-#     _, wav, _ = translator.predict(input_audio, "s2st", target_languaje)
-#     return gr.Audio(wav, label="Translated audio", elem_id="translated_audio", visible=True)
+    # Save the result to a file
+    filename = "result.txt"
+    with open(filename, "w") as f:
+        f.write(result.text)
+    
+    # Remove audio file
+    # subprocess.run(["rm", audio_path])
+    
+    return (
+        result.text,
+        gr.Textbox(value=filename, label="Original audio transcribed", elem_id="original_audio_transcribed", visible=False)
+    )
 
+def translate(original_audio_transcribed_path, source_languaje, target_languaje):
+    # Translate
+    with open(original_audio_transcribed_path, "r") as f:
+        text = f.read()
+    translated = text
+
+    # Save the result to a file
+    filename = "translated_text.txt"
+    with open(filename, "w") as f:
+        f.write(text)
+    
+    # Remove audio file
+    # subprocess.run(["rm", original_audio_transcribed_path])
+
+    return (
+        translated,
+        gr.Textbox(value=filename, label="Original audio translated", elem_id="original_audio_translated", visible=False)
+    )
+
+def tex2speech(original_audio_translated_path):
+    pass
 
 with gr.Blocks() as demo:
     with gr.Row(variant="panel"):
@@ -104,7 +227,6 @@ with gr.Blocks() as demo:
         copy_button   = gr.Button(size="sm", icon="icons/copy.svg",   value="", min_width="10px", scale=0)
         delete_button = gr.Button(size="sm", icon="icons/delete.svg", value="", min_width="10px", scale=0)
     copy_button.click(fn=copy_url_from_clipboard, outputs=url_textbox)
-    delete_button.click(fn=clear_video_url, outputs=url_textbox)
 
     stream_page = gr.Textbox(label="Stream page", elem_id="stream_page", visible=False)
     visible = False
@@ -112,14 +234,53 @@ with gr.Blocks() as demo:
         image = gr.Image(visible=visible, scale=1)
         with gr.Column():
             with gr.Row():
-                source_languaje = gr.Dropdown(visible=visible, label="Source languaje", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True)
-                target_languaje = gr.Dropdown(visible=visible, label="Target languaje", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True)
-            translate_button = gr.Button(size="lg", value="translate", min_width="10px", visible=visible)
+                source_languaje = gr.Dropdown(visible=visible, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True)
+                target_languaje = gr.Dropdown(visible=visible, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True)
+            with gr.Row():
+                get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=visible)
+                transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible)
 
-    original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible)
+    original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible, interactive=False)
+    original_audio_path = gr.Textbox(label="Stream page", elem_id="stream_page", visible=False)
+    original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=visible)
+    original_audio_transcribed_path = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", visible=False)
+    original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=visible)
+    original_audio_translated_path = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", visible=False)
     translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=visible)
-    url_textbox.change(fn=is_valid_url, inputs=url_textbox, outputs=[image, source_languaje, target_languaje, translate_button, stream_page, original_audio, translated_audio])
-    translate_button.click(fn=get_audio_from_video, inputs=[url_textbox, stream_page], outputs=original_audio)
-    # original_audio.change(fn=translate_audio, inputs=[original_audio, target_languaje], outputs=translated_audio)
+    url_textbox.change(
+        fn=is_valid_url, 
+        inputs=url_textbox, 
+        outputs=[
+            image, 
+            source_languaje, 
+            target_languaje, 
+            get_audio_button, 
+            stream_page, 
+            original_audio, 
+            original_audio_transcribed, 
+            translated_audio, 
+            transcribe_audio_button, 
+            original_audio_translated,
+        ]
+    )
+    delete_button.click(
+        fn=clear_video_url, 
+        outputs=[
+            url_textbox, 
+            image, 
+            source_languaje, 
+            target_languaje, 
+            get_audio_button, 
+            transcribe_audio_button, 
+            original_audio, 
+            original_audio_transcribed, 
+            translated_audio, 
+            original_audio_translated,
+        ]
+    )
+    get_audio_button.click(fn=get_audio_from_video, inputs=[url_textbox, stream_page], outputs=[original_audio, original_audio_path])
+    original_audio.change(fn=trascribe_audio, inputs=original_audio_path, outputs=[original_audio_transcribed, original_audio_transcribed_path])
+    original_audio_transcribed.change(fn=translate, inputs=[original_audio_transcribed_path, source_languaje, target_languaje], outputs=[original_audio_translated, original_audio_translated_path])
+    original_audio_translated.change(fn=tex2speech, inputs=original_audio_translated_path, outputs=translated_audio)
 
 demo.launch()