import gradio as gr import librosa import numpy as np import torch import requests from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan checkpoint = "microsoft/speecht5_tts" processor = SpeechT5Processor.from_pretrained(checkpoint) model = SpeechT5ForTextToSpeech.from_pretrained(checkpoint) vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan") speaker_embeddings = { "BDL": "spkemb/cmu_us_bdl_arctic-wav-arctic_a0009.npy", "CLB": "spkemb/cmu_us_clb_arctic-wav-arctic_a0144.npy", "SLT": "spkemb/cmu_us_slt_arctic-wav-arctic_a0508.npy", } def getNews(search_key): return requests.get ("https://newsapi.org/v2/everything?q=" +search_key+ "&pageSize=3&apiKey=3bca07c913ec4703a23f6ba03e15b30b").content.decode("utf-8") # return requests.get ("https://newsapi.org/v2/top-headlines?country=us&apiKey=3bca07c913ec4703a23f6ba03e15b30b").content.decode("utf-8" ) def getHeadlines(): return requests.get ("https://newsapi.org/v2/top-headlines?country=us&apiKey=3bca07c913ec4703a23f6ba03e15b30b").content.decode("utf-8") def predict(text, speaker): if len(text.strip()) == 0: return (16000, np.zeros(0).astype(np.int16)) #if Interest == "": # inputs = processor(text=getNews(Interest), # return_tensors="pt") #else: inputs = processor(text=getNews(text), return_tensors="pt") # limit input length input_ids = inputs["input_ids"] input_ids = input_ids[..., :model.config.max_text_positions] speaker_embedding = np.load("spkemb/cmu_us_bdl_arctic-wav-arctic_a0009.npy") speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0) speech = model.generate_speech(input_ids, speaker_embedding, vocoder=vocoder) speech = (speech.numpy() * 32767).astype(np.int16) return (16000, speech) title = "SpeechT5: Speech Synthesis" description = """ The SpeechT5 model is pre-trained on text as well as speech inputs, with targets that are also a mix of text and speech. By pre-training on text and speech at the same time, it learns unified representations for both, resulting in improved modeling capabilities. """ article = """

References: SpeechT5 paper | original GitHub | original weights

Speaker embeddings were generated from CMU ARCTIC using this script.

""" examples = [ ["It is not in the stars to hold our destiny but in ourselves.", "BDL (male)"], ["The octopus and Oliver went to the opera in October.", "CLB (female)"], ["She sells seashells by the seashore. I saw a kitten eating chicken in the kitchen.", "RMS (male)"], ["How much wood would a woodchuck chuck if a woodchuck could chuck wood? He would chuck, he would, as much as he could, and chuck as much wood as a woodchuck would if a woodchuck could chuck wood.", "CLB (female)"], ] gr.Interface( fn=predict, inputs=[ gr.Text(label="keyword"), gr.Radio(label="Interest", choices=[ "Minecraft", "Technology", "Politics", "KPop", "Surprise Me!" ], value="KPop"), ], outputs=[ gr.Audio(label="Generated Speech", type="numpy"), ], title=title, description=description, article=article, examples=examples, ).launch()