[Experimental] Gruut support

app.py

@@ -16,13 +16,13 @@ voices = {}
 # else:
 for v in voicelist:
     voices[v] = styletts2importable.compute_style(f'voices/{v}.wav')
-def synthesize(text, voice):
+def synthesize(text, voice, use_gruut):
     if text.strip() == "":
         raise gr.Error("You must enter some text")
     if len(text) > 300:
         raise gr.Error("Text must be under 300 characters")
     v = voice.lower()
-    return (24000, styletts2importable.inference(text, voices[v], alpha=0.3, beta=0.7, diffusion_steps=7, embedding_scale=1))
+    return (24000, styletts2importable.inference(text, voices[v], alpha=0.3, beta=0.7, diffusion_steps=7, embedding_scale=1, use_gruut=use_gruut))
 def clsynthesize(text, voice):
     if text.strip() == "":
         raise gr.Error("You must enter some text")
@@ -43,10 +43,11 @@ with gr.Blocks() as vctk:
         with gr.Column(scale=1):
             inp = gr.Textbox(label="Text", info="What would you like StyleTTS 2 to read? It works better on full sentences.", interactive=True)
             voice = gr.Dropdown(voicelist, label="Voice", info="Select a default voice.", value='m-us-1', interactive=True)
+            use_gruut = gr.Checkbox(label="Use alternate phonemizer (Gruut) - Experimental")
         with gr.Column(scale=1):
             btn = gr.Button("Synthesize", variant="primary")
             audio = gr.Audio(interactive=False, label="Synthesized Audio")
-            btn.click(synthesize, inputs=[inp, voice], outputs=[audio], concurrency_limit=4)
+            btn.click(synthesize, inputs=[inp, voice, use_gruut], outputs=[audio], concurrency_limit=4)
 with gr.Blocks() as clone:
     with gr.Row():
         with gr.Column(scale=1):

gruut_phonemize.py

@@ -0,0 +1,10 @@
+from gruut import sentences
+
+
+def gphonemize(text):
+    phonemes = ''
+    for sent in sentences(text, lang="en-us"):
+        for word in sent:
+            if word.phonemes:
+                phonemes += ''.join(word.phonemes)
+    return phonemes

requirements.txt

@@ -18,4 +18,5 @@ git+https://github.com/resemble-ai/monotonic_align.git
 scipy
 phonemizer
 cached-path
-gradio
+gradio
+gruut

styletts2importable.py

@@ -1,4 +1,6 @@
 from cached_path import cached_path
+print("GRUUT")
+from gruut_phonemize import gphonemize
 
 # from dp.phonemizer import Phonemizer
 print("NLTK")
@@ -131,9 +133,12 @@ sampler = DiffusionSampler(
     clamp=False
 )
 
-def inference(text, ref_s, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1):
+def inference(text, ref_s, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1, use_gruut=False):
     text = text.strip()
-    ps = global_phonemizer.phonemize([text])
+    if use_gruut:
+        ps = gphonemize(text)
+    else:
+        ps = global_phonemizer.phonemize([text])
     ps = word_tokenize(ps[0])
     ps = ' '.join(ps)
     tokens = textclenaer(ps)
@@ -200,86 +205,92 @@ def inference(text, ref_s, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding
 
     return out.squeeze().cpu().numpy()[..., :-50] # weird pulse at the end of the model, need to be fixed later
 
-def LFinference(text, s_prev, ref_s, alpha = 0.3, beta = 0.7, t = 0.7, diffusion_steps=5, embedding_scale=1):
-  text = text.strip()
-  ps = global_phonemizer.phonemize([text])
-  ps = word_tokenize(ps[0])
-  ps = ' '.join(ps)
-  ps = ps.replace('``', '"')
-  ps = ps.replace("''", '"')
+def LFinference(text, s_prev, ref_s, alpha = 0.3, beta = 0.7, t = 0.7, diffusion_steps=5, embedding_scale=1, use_gruut=False):
+    text = text.strip()
+    if use_gruut:
+        ps = gphonemize(text)
+    else:
+        ps = global_phonemizer.phonemize([text])
+    ps = word_tokenize(ps[0])
+    ps = ' '.join(ps)
+    ps = ps.replace('``', '"')
+    ps = ps.replace("''", '"')
 
-  tokens = textclenaer(ps)
-  tokens.insert(0, 0)
-  tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)
+    tokens = textclenaer(ps)
+    tokens.insert(0, 0)
+    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)
 
-  with torch.no_grad():
-      input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)
-      text_mask = length_to_mask(input_lengths).to(device)
+    with torch.no_grad():
+        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)
+        text_mask = length_to_mask(input_lengths).to(device)
 
-      t_en = model.text_encoder(tokens, input_lengths, text_mask)
-      bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())
-      d_en = model.bert_encoder(bert_dur).transpose(-1, -2)
+        t_en = model.text_encoder(tokens, input_lengths, text_mask)
+        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())
+        d_en = model.bert_encoder(bert_dur).transpose(-1, -2)
 
-      s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device),
+        s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device),
                                         embedding=bert_dur,
                                         embedding_scale=embedding_scale,
-                                          features=ref_s, # reference from the same speaker as the embedding
+                                            features=ref_s, # reference from the same speaker as the embedding
                                             num_steps=diffusion_steps).squeeze(1)
 
-      if s_prev is not None:
-          # convex combination of previous and current style
-          s_pred = t * s_prev + (1 - t) * s_pred
+        if s_prev is not None:
+            # convex combination of previous and current style
+            s_pred = t * s_prev + (1 - t) * s_pred
 
-      s = s_pred[:, 128:]
-      ref = s_pred[:, :128]
+        s = s_pred[:, 128:]
+        ref = s_pred[:, :128]
 
-      ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]
-      s = beta * s + (1 - beta)  * ref_s[:, 128:]
+        ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]
+        s = beta * s + (1 - beta)  * ref_s[:, 128:]
 
-      s_pred = torch.cat([ref, s], dim=-1)
+        s_pred = torch.cat([ref, s], dim=-1)
 
-      d = model.predictor.text_encoder(d_en,
+        d = model.predictor.text_encoder(d_en,
                                         s, input_lengths, text_mask)
 
-      x, _ = model.predictor.lstm(d)
-      duration = model.predictor.duration_proj(x)
+        x, _ = model.predictor.lstm(d)
+        duration = model.predictor.duration_proj(x)
 
-      duration = torch.sigmoid(duration).sum(axis=-1)
-      pred_dur = torch.round(duration.squeeze()).clamp(min=1)
+        duration = torch.sigmoid(duration).sum(axis=-1)
+        pred_dur = torch.round(duration.squeeze()).clamp(min=1)
 
 
-      pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))
-      c_frame = 0
-      for i in range(pred_aln_trg.size(0)):
-          pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1
-          c_frame += int(pred_dur[i].data)
+        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))
+        c_frame = 0
+        for i in range(pred_aln_trg.size(0)):
+            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1
+            c_frame += int(pred_dur[i].data)
 
-      # encode prosody
-      en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))
-      if model_params.decoder.type == "hifigan":
-          asr_new = torch.zeros_like(en)
-          asr_new[:, :, 0] = en[:, :, 0]
-          asr_new[:, :, 1:] = en[:, :, 0:-1]
-          en = asr_new
+        # encode prosody
+        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))
+        if model_params.decoder.type == "hifigan":
+            asr_new = torch.zeros_like(en)
+            asr_new[:, :, 0] = en[:, :, 0]
+            asr_new[:, :, 1:] = en[:, :, 0:-1]
+            en = asr_new
 
-      F0_pred, N_pred = model.predictor.F0Ntrain(en, s)
+        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)
 
-      asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))
-      if model_params.decoder.type == "hifigan":
-          asr_new = torch.zeros_like(asr)
-          asr_new[:, :, 0] = asr[:, :, 0]
-          asr_new[:, :, 1:] = asr[:, :, 0:-1]
-          asr = asr_new
+        asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))
+        if model_params.decoder.type == "hifigan":
+            asr_new = torch.zeros_like(asr)
+            asr_new[:, :, 0] = asr[:, :, 0]
+            asr_new[:, :, 1:] = asr[:, :, 0:-1]
+            asr = asr_new
 
-      out = model.decoder(asr,
-                              F0_pred, N_pred, ref.squeeze().unsqueeze(0))
+        out = model.decoder(asr,
+                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))
 
 
-  return out.squeeze().cpu().numpy()[..., :-100], s_pred # weird pulse at the end of the model, need to be fixed later
+    return out.squeeze().cpu().numpy()[..., :-100], s_pred # weird pulse at the end of the model, need to be fixed later
 
-def STinference(text, ref_s, ref_text, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1):
+def STinference(text, ref_s, ref_text, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1, use_gruut=False):
     text = text.strip()
-    ps = global_phonemizer.phonemize([text])
+    if use_gruut:
+        ps = gphonemize(text)
+    else:
+        ps = global_phonemizer.phonemize([text])
     ps = word_tokenize(ps[0])
     ps = ' '.join(ps)
 
@@ -288,7 +299,10 @@ def STinference(text, ref_s, ref_text, alpha = 0.3, beta = 0.7, diffusion_steps=
     tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)
 
     ref_text = ref_text.strip()
-    ps = global_phonemizer.phonemize([ref_text])
+    if use_gruut:
+        ps = gphonemize(text)
+    else:
+        ps = global_phonemizer.phonemize([ref_text])
     ps = word_tokenize(ps[0])
     ps = ' '.join(ps)