import gradio as gr import pyperclip import urllib.parse as urlparse from pytube import YouTube import re import subprocess import time from lang_list import ORIGINAL_LANGUAGE_NAME_TO_CODE, S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES import torch import whisper # get device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # device = torch.device("cpu") model = whisper.load_model("large-v2", device=device) YOUTUBE = "youtube" TWITCH = "twitch" ERROR = "error" def copy_url_from_clipboard(): return pyperclip.paste() def clear_video_url(): visible = False image = gr.Image(visible=visible, scale=1) source_languaje = gr.Dropdown(visible=visible, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True) target_languaje = gr.Dropdown(visible=visible, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True) get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=visible) transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible) original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible, interactive=False) original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=visible) original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=visible) translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=visible) transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible) return ( "", image, source_languaje, target_languaje, get_audio_button, transcribe_audio_button, original_audio, original_audio_transcribed, translated_audio, original_audio_translated, ) def get_youtube_thumbnail(video_id): thumbnail_url = f"https://img.youtube.com/vi/{video_id}/0.jpg" return thumbnail_url def get_youtube_video_id(url): if "youtu.be" in url.lower(): yt = YouTube(url) thumbnail_url = yt.thumbnail_url return thumbnail_url else: parsed_url = urlparse.urlparse(url) video_id = urlparse.parse_qs(parsed_url.query).get('v') if video_id: thumbnail_url = get_youtube_thumbnail(video_id[0]) return thumbnail_url else: return None def is_valid_url(url): source_languaje = gr.Dropdown(visible=True, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True) target_languaje = gr.Dropdown(visible=True, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True) get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=True) original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=True, interactive=False) original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=True) original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=True) translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=True) transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=True) if "youtube" in url.lower() or "youtu.be" in url.lower(): thumbnail = get_youtube_video_id(url) if thumbnail: return ( gr.Image(value=thumbnail, visible=True, show_download_button=False, container=False), source_languaje, target_languaje, get_audio_button, gr.Textbox(value=YOUTUBE, label="Stream page", elem_id="stream_page", visible=False), original_audio, original_audio_transcribed, translated_audio, transcribe_audio_button, original_audio_translated, ) else: return ( gr.Image(value="assets/youtube-no-thumbnails.webp", visible=True, show_download_button=False, container=False), source_languaje, target_languaje, get_audio_button, gr.Textbox(value=YOUTUBE, label="Stream page", elem_id="stream_page", visible=False), original_audio, original_audio_transcribed, translated_audio, transcribe_audio_button, original_audio_translated, ) elif "twitch" in url.lower() or "twitch.tv" in url.lower(): return ( gr.Image(value="assets/twitch.webp", visible=True, show_download_button=False, container=False), source_languaje, target_languaje, get_audio_button, gr.Textbox(value=TWITCH, label="Stream page", elem_id="stream_page", visible=False), original_audio, original_audio_transcribed, translated_audio, transcribe_audio_button, original_audio_translated, ) else: visible = False image = gr.Image(value="assets/youtube_error.webp", visible=visible, show_download_button=False, container=False) source_languaje = gr.Dropdown(visible=visible, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True) target_languaje = gr.Dropdown(visible=visible, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True) get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=visible) stream_page = gr.Textbox(value=ERROR, label="Stream page", elem_id="stream_page", visible=visible) original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible, interactive=False) original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=visible) original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=visible) translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=visible) transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible) return ( image, source_languaje, target_languaje, get_audio_button, stream_page, original_audio, original_audio_transcribed, translated_audio, transcribe_audio_button, original_audio_translated, ) def get_audio_from_video(url, stream_page): if stream_page == YOUTUBE: yt = YouTube(url) audio_streams = yt.streams.filter(mime_type="audio/mp4") # Get all available audio bitrates abr_list = [] for stream in audio_streams: abr_list.append(stream.abr) abr_list = sorted(set(abr_list)) # Get the highest audio bitrate audio_stream = audio_streams.filter(abr=abr_list[0]).first() # Download the audio filename = "audio.mp3" audio_stream.download(filename=filename) return ( gr.Audio(value=filename, label="Original audio", elem_id="original_audio", visible=True, interactive=False), gr.Textbox(value=filename, label="Stream page", elem_id="stream_page", visible=False) ) elif stream_page == TWITCH: # Get the video id video_id = re.search("\d{10}", url).group(0) # Download the video filename = "audio.mkv" subprocess.run(["twitch-dl", "download", "--overwrite", "-q", "audio_only", "--output", filename, video_id]) return ( gr.Audio(value=filename, label="Original audio", elem_id="original_audio", visible=True, interactive=False), gr.Textbox(value=filename, label="Stream page", elem_id="stream_page", visible=False) ) def trascribe_audio(audio_path): audio = whisper.load_audio(audio_path) audio = whisper.pad_or_trim(audio) mel = whisper.log_mel_spectrogram(audio).to(model.device) _, probs = model.detect_language(mel) options = whisper.DecodingOptions(fp16 = False) result = whisper.decode(model, mel, options) # Save the result to a file filename = "result.txt" with open(filename, "w") as f: f.write(result.text) # Remove audio file # subprocess.run(["rm", audio_path]) return ( result.text, gr.Textbox(value=filename, label="Original audio transcribed", elem_id="original_audio_transcribed", visible=False) ) def translate(original_audio_transcribed_path, source_languaje, target_languaje): # Translate with open(original_audio_transcribed_path, "r") as f: text = f.read() translated = text # Save the result to a file filename = "translated_text.txt" with open(filename, "w") as f: f.write(text) # Remove audio file # subprocess.run(["rm", original_audio_transcribed_path]) return ( translated, gr.Textbox(value=filename, label="Original audio translated", elem_id="original_audio_translated", visible=False) ) def tex2speech(original_audio_translated_path): pass with gr.Blocks() as demo: with gr.Row(variant="panel"): url_textbox = gr.Textbox(placeholder="Add video URL here", label="Video URL", elem_id="video_url", scale=1, interactive=True) copy_button = gr.Button(size="sm", icon="icons/copy.svg", value="", min_width="10px", scale=0) delete_button = gr.Button(size="sm", icon="icons/delete.svg", value="", min_width="10px", scale=0) copy_button.click(fn=copy_url_from_clipboard, outputs=url_textbox) stream_page = gr.Textbox(label="Stream page", elem_id="stream_page", visible=False) visible = False with gr.Row(equal_height=False): image = gr.Image(visible=visible, scale=1) with gr.Column(): with gr.Row(): source_languaje = gr.Dropdown(visible=visible, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True) target_languaje = gr.Dropdown(visible=visible, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True) with gr.Row(): get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=visible) transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible) original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible, interactive=False) original_audio_path = gr.Textbox(label="Stream page", elem_id="stream_page", visible=False) original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=visible) original_audio_transcribed_path = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", visible=False) original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=visible) original_audio_translated_path = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", visible=False) translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=visible) url_textbox.change( fn=is_valid_url, inputs=url_textbox, outputs=[ image, source_languaje, target_languaje, get_audio_button, stream_page, original_audio, original_audio_transcribed, translated_audio, transcribe_audio_button, original_audio_translated, ] ) delete_button.click( fn=clear_video_url, outputs=[ url_textbox, image, source_languaje, target_languaje, get_audio_button, transcribe_audio_button, original_audio, original_audio_transcribed, translated_audio, original_audio_translated, ] ) get_audio_button.click(fn=get_audio_from_video, inputs=[url_textbox, stream_page], outputs=[original_audio, original_audio_path]) original_audio.change(fn=trascribe_audio, inputs=original_audio_path, outputs=[original_audio_transcribed, original_audio_transcribed_path]) original_audio_transcribed.change(fn=translate, inputs=[original_audio_transcribed_path, source_languaje, target_languaje], outputs=[original_audio_translated, original_audio_translated_path]) original_audio_translated.change(fn=tex2speech, inputs=original_audio_translated_path, outputs=translated_audio) demo.launch()