benjipeng commited on
Commit
aa22bc0
1 Parent(s): 5cdac43

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +0 -81
app.py CHANGED
@@ -22,87 +22,6 @@ model = VitsModel.from_pretrained("Matthijs/mms-tts-deu")
22
  tokenizer = VitsTokenizer.from_pretrained("Matthijs/mms-tts-deu")
23
 
24
 
25
- def translate(audio):
26
- outputs = asr_pipe(
27
- audio,
28
- max_new_tokens=256,
29
- generate_kwargs={"task": "transcribe", "language": "de"}
30
- )
31
- return outputs["text"]
32
-
33
-
34
- def synthesise(text):
35
- if len(text.strip()) == 0:
36
- return (16000, np.zeros(0).astype(np.int16))
37
-
38
- inputs = tokenizer(text, return_tensors="pt")
39
- input_ids = inputs["input_ids"]
40
-
41
- with torch.no_grad():
42
- outputs = model(input_ids)
43
-
44
- speech = outputs.audio[0]
45
- return speech.cpu()
46
-
47
-
48
- def speech_to_speech_translation(audio):
49
- translated_text = translate(audio)
50
- synthesised_speech = synthesise(translated_text)
51
- synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
52
- return 16000, synthesised_speech
53
-
54
- title = "Cascaded STST"
55
- description = """
56
- Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
57
- [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
58
- ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
59
- """
60
-
61
- demo = gr.Blocks()
62
-
63
- mic_translate = gr.Interface(
64
- fn=speech_to_speech_translation,
65
- inputs=gr.Audio(source="microphone", type="filepath"),
66
- outputs=gr.Audio(label="Generated Speech", type="numpy"),
67
- title=title,
68
- description=description,
69
- )
70
- file_translate = gr.Interface(
71
- fn=speech_to_speech_translation,
72
- inputs=gr.Audio(source="upload", type="filepath"),
73
- outputs=gr.Audio(label="Generated Speech", type="numpy"),
74
- examples=[["./example.wav"]],
75
- title=title,
76
- description=description,
77
- )
78
-
79
- with demo:
80
- gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
81
-
82
- demo.launch()import gradio as gr
83
- import numpy as np
84
- import torch
85
-
86
- from transformers import (
87
- VitsModel,
88
- VitsTokenizer,
89
- pipeline
90
- )
91
-
92
-
93
- device = "cuda:0" if torch.cuda.is_available() else "cpu"
94
-
95
- # load speech translation checkpoint
96
- asr_pipe = pipeline(
97
- "automatic-speech-recognition",
98
- model="openai/whisper-base",
99
- device=device
100
- )
101
-
102
- model = VitsModel.from_pretrained("Matthijs/mms-tts-deu")
103
- tokenizer = VitsTokenizer.from_pretrained("Matthijs/mms-tts-deu")
104
-
105
-
106
  def translate(audio):
107
  outputs = asr_pipe(
108
  audio,
 
22
  tokenizer = VitsTokenizer.from_pretrained("Matthijs/mms-tts-deu")
23
 
24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  def translate(audio):
26
  outputs = asr_pipe(
27
  audio,