Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import pyperclip | |
| import urllib.parse as urlparse | |
| from pytube import YouTube | |
| import re | |
| import subprocess | |
| import time | |
| from lang_list import ORIGINAL_LANGUAGE_NAME_TO_CODE, S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES | |
| import torch | |
| import whisper | |
| # get device | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| # device = torch.device("cpu") | |
| model = whisper.load_model("large-v2", device=device) | |
| YOUTUBE = "youtube" | |
| TWITCH = "twitch" | |
| ERROR = "error" | |
| def copy_url_from_clipboard(): | |
| return pyperclip.paste() | |
| def clear_video_url(): | |
| visible = False | |
| image = gr.Image(visible=visible, scale=1) | |
| source_languaje = gr.Dropdown(visible=visible, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True) | |
| target_languaje = gr.Dropdown(visible=visible, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True) | |
| get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=visible) | |
| transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible) | |
| original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible, interactive=False) | |
| original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=visible) | |
| original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=visible) | |
| translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=visible) | |
| transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible) | |
| return ( | |
| "", | |
| image, | |
| source_languaje, | |
| target_languaje, | |
| get_audio_button, | |
| transcribe_audio_button, | |
| original_audio, | |
| original_audio_transcribed, | |
| translated_audio, | |
| original_audio_translated, | |
| ) | |
| def get_youtube_thumbnail(video_id): | |
| thumbnail_url = f"https://img.youtube.com/vi/{video_id}/0.jpg" | |
| return thumbnail_url | |
| def get_youtube_video_id(url): | |
| if "youtu.be" in url.lower(): | |
| yt = YouTube(url) | |
| thumbnail_url = yt.thumbnail_url | |
| return thumbnail_url | |
| else: | |
| parsed_url = urlparse.urlparse(url) | |
| video_id = urlparse.parse_qs(parsed_url.query).get('v') | |
| if video_id: | |
| thumbnail_url = get_youtube_thumbnail(video_id[0]) | |
| return thumbnail_url | |
| else: | |
| return None | |
| def is_valid_url(url): | |
| source_languaje = gr.Dropdown(visible=True, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True) | |
| target_languaje = gr.Dropdown(visible=True, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True) | |
| get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=True) | |
| original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=True, interactive=False) | |
| original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=True) | |
| original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=True) | |
| translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=True) | |
| transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=True) | |
| if "youtube" in url.lower() or "youtu.be" in url.lower(): | |
| thumbnail = get_youtube_video_id(url) | |
| if thumbnail: | |
| return ( | |
| gr.Image(value=thumbnail, visible=True, show_download_button=False, container=False), | |
| source_languaje, | |
| target_languaje, | |
| get_audio_button, | |
| gr.Textbox(value=YOUTUBE, label="Stream page", elem_id="stream_page", visible=False), | |
| original_audio, | |
| original_audio_transcribed, | |
| translated_audio, | |
| transcribe_audio_button, | |
| original_audio_translated, | |
| ) | |
| else: | |
| return ( | |
| gr.Image(value="assets/youtube-no-thumbnails.webp", visible=True, show_download_button=False, container=False), | |
| source_languaje, | |
| target_languaje, | |
| get_audio_button, | |
| gr.Textbox(value=YOUTUBE, label="Stream page", elem_id="stream_page", visible=False), | |
| original_audio, | |
| original_audio_transcribed, | |
| translated_audio, | |
| transcribe_audio_button, | |
| original_audio_translated, | |
| ) | |
| elif "twitch" in url.lower() or "twitch.tv" in url.lower(): | |
| return ( | |
| gr.Image(value="assets/twitch.webp", visible=True, show_download_button=False, container=False), | |
| source_languaje, | |
| target_languaje, | |
| get_audio_button, | |
| gr.Textbox(value=TWITCH, label="Stream page", elem_id="stream_page", visible=False), | |
| original_audio, | |
| original_audio_transcribed, | |
| translated_audio, | |
| transcribe_audio_button, | |
| original_audio_translated, | |
| ) | |
| else: | |
| visible = False | |
| image = gr.Image(value="assets/youtube_error.webp", visible=visible, show_download_button=False, container=False) | |
| source_languaje = gr.Dropdown(visible=visible, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True) | |
| target_languaje = gr.Dropdown(visible=visible, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True) | |
| get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=visible) | |
| stream_page = gr.Textbox(value=ERROR, label="Stream page", elem_id="stream_page", visible=visible) | |
| original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible, interactive=False) | |
| original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=visible) | |
| original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=visible) | |
| translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=visible) | |
| transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible) | |
| return ( | |
| image, | |
| source_languaje, | |
| target_languaje, | |
| get_audio_button, | |
| stream_page, | |
| original_audio, | |
| original_audio_transcribed, | |
| translated_audio, | |
| transcribe_audio_button, | |
| original_audio_translated, | |
| ) | |
| def get_audio_from_video(url, stream_page): | |
| if stream_page == YOUTUBE: | |
| yt = YouTube(url) | |
| audio_streams = yt.streams.filter(mime_type="audio/mp4") | |
| # Get all available audio bitrates | |
| abr_list = [] | |
| for stream in audio_streams: | |
| abr_list.append(stream.abr) | |
| abr_list = sorted(set(abr_list)) | |
| # Get the highest audio bitrate | |
| audio_stream = audio_streams.filter(abr=abr_list[0]).first() | |
| # Download the audio | |
| filename = "audio.mp3" | |
| audio_stream.download(filename=filename) | |
| return ( | |
| gr.Audio(value=filename, label="Original audio", elem_id="original_audio", visible=True, interactive=False), | |
| gr.Textbox(value=filename, label="Stream page", elem_id="stream_page", visible=False) | |
| ) | |
| elif stream_page == TWITCH: | |
| # Get the video id | |
| video_id = re.search("\d{10}", url).group(0) | |
| # Download the video | |
| filename = "audio.mkv" | |
| subprocess.run(["twitch-dl", "download", "--overwrite", "-q", "audio_only", "--output", filename, video_id]) | |
| return ( | |
| gr.Audio(value=filename, label="Original audio", elem_id="original_audio", visible=True, interactive=False), | |
| gr.Textbox(value=filename, label="Stream page", elem_id="stream_page", visible=False) | |
| ) | |
| def trascribe_audio(audio_path): | |
| audio = whisper.load_audio(audio_path) | |
| audio = whisper.pad_or_trim(audio) | |
| mel = whisper.log_mel_spectrogram(audio).to(model.device) | |
| _, probs = model.detect_language(mel) | |
| options = whisper.DecodingOptions(fp16 = False) | |
| result = whisper.decode(model, mel, options) | |
| # Save the result to a file | |
| filename = "result.txt" | |
| with open(filename, "w") as f: | |
| f.write(result.text) | |
| # Remove audio file | |
| # subprocess.run(["rm", audio_path]) | |
| return ( | |
| result.text, | |
| gr.Textbox(value=filename, label="Original audio transcribed", elem_id="original_audio_transcribed", visible=False) | |
| ) | |
| def translate(original_audio_transcribed_path, source_languaje, target_languaje): | |
| # Translate | |
| with open(original_audio_transcribed_path, "r") as f: | |
| text = f.read() | |
| translated = text | |
| # Save the result to a file | |
| filename = "translated_text.txt" | |
| with open(filename, "w") as f: | |
| f.write(text) | |
| # Remove audio file | |
| # subprocess.run(["rm", original_audio_transcribed_path]) | |
| return ( | |
| translated, | |
| gr.Textbox(value=filename, label="Original audio translated", elem_id="original_audio_translated", visible=False) | |
| ) | |
| def tex2speech(original_audio_translated_path): | |
| pass | |
| with gr.Blocks() as demo: | |
| with gr.Row(variant="panel"): | |
| url_textbox = gr.Textbox(placeholder="Add video URL here", label="Video URL", elem_id="video_url", scale=1, interactive=True) | |
| copy_button = gr.Button(size="sm", icon="icons/copy.svg", value="", min_width="10px", scale=0) | |
| delete_button = gr.Button(size="sm", icon="icons/delete.svg", value="", min_width="10px", scale=0) | |
| copy_button.click(fn=copy_url_from_clipboard, outputs=url_textbox) | |
| stream_page = gr.Textbox(label="Stream page", elem_id="stream_page", visible=False) | |
| visible = False | |
| with gr.Row(equal_height=False): | |
| image = gr.Image(visible=visible, scale=1) | |
| with gr.Column(): | |
| with gr.Row(): | |
| source_languaje = gr.Dropdown(visible=visible, label="Source languaje", show_label=True, value="English", choices=ORIGINAL_LANGUAGE_NAME_TO_CODE, scale=1, interactive=True) | |
| target_languaje = gr.Dropdown(visible=visible, label="Target languaje", show_label=True, value="Español", choices=S2ST_TARGET_ORIGINAL_LANGUAGE_NAMES, scale=1, interactive=True) | |
| with gr.Row(): | |
| get_audio_button = gr.Button(size="lg", value="get audio", min_width="10px", scale=0, visible=visible) | |
| transcribe_audio_button = gr.Button(size="lg", value="transcribe audio", min_width="10px", scale=0, visible=visible) | |
| original_audio = gr.Audio(label="Original audio", elem_id="original_audio", visible=visible, interactive=False) | |
| original_audio_path = gr.Textbox(label="Stream page", elem_id="stream_page", visible=False) | |
| original_audio_transcribed = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", interactive=False, visible=visible) | |
| original_audio_transcribed_path = gr.Textbox(label="Original audio transcribed", elem_id="original_audio_transcribed", visible=False) | |
| original_audio_translated = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", interactive=False, visible=visible) | |
| original_audio_translated_path = gr.Textbox(label="Original audio translated", elem_id="original_audio_translated", visible=False) | |
| translated_audio = gr.Audio(label="Translated audio", elem_id="translated_audio", visible=visible) | |
| url_textbox.change( | |
| fn=is_valid_url, | |
| inputs=url_textbox, | |
| outputs=[ | |
| image, | |
| source_languaje, | |
| target_languaje, | |
| get_audio_button, | |
| stream_page, | |
| original_audio, | |
| original_audio_transcribed, | |
| translated_audio, | |
| transcribe_audio_button, | |
| original_audio_translated, | |
| ] | |
| ) | |
| delete_button.click( | |
| fn=clear_video_url, | |
| outputs=[ | |
| url_textbox, | |
| image, | |
| source_languaje, | |
| target_languaje, | |
| get_audio_button, | |
| transcribe_audio_button, | |
| original_audio, | |
| original_audio_transcribed, | |
| translated_audio, | |
| original_audio_translated, | |
| ] | |
| ) | |
| get_audio_button.click(fn=get_audio_from_video, inputs=[url_textbox, stream_page], outputs=[original_audio, original_audio_path]) | |
| original_audio.change(fn=trascribe_audio, inputs=original_audio_path, outputs=[original_audio_transcribed, original_audio_transcribed_path]) | |
| original_audio_transcribed.change(fn=translate, inputs=[original_audio_transcribed_path, source_languaje, target_languaje], outputs=[original_audio_translated, original_audio_translated_path]) | |
| original_audio_translated.change(fn=tex2speech, inputs=original_audio_translated_path, outputs=translated_audio) | |
| demo.launch() |