diff --git "a/tts.ipynb" "b/tts.ipynb"
deleted file mode 100644--- "a/tts.ipynb"
+++ /dev/null
@@ -1,707 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Text to Speech Playground"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
- " from .autonotebook import tqdm as notebook_tqdm\n"
- ]
- }
- ],
- "source": [
- "import os\n",
- "\n",
- "import torch\n",
- "import gradio as gr\n",
- "from TTS.api import TTS\n",
- "os.environ[\"COQUI_TOS_AGREED\"] = \"1\"\n",
- "# os.environ[\"PYTORCH_ENABLE_MPS_FALLBACK\"] = \"1\""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [],
- "source": [
- "from collections import namedtuple\n",
- "\n",
- "Voice = namedtuple('voice', ['name', 'neutral','angry'])\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 84,
- "metadata": {},
- "outputs": [],
- "source": [
- "voices = [\n",
- " Voice('Attenborough', neutral='audio/attenborough/neutral.wav', angry=None),\n",
- " Voice('Rick', neutral='audio/rick/neutral.wav', angry=None),\n",
- " Voice('Freeman', neutral='audio/freeman/neutral.wav', angry='audio/freeman/angry.wav'),\n",
- " Voice('Walken', neutral='audio/walken/neutral.wav', angry=None),\n",
- " Voice('Darth Wader', neutral='audio/darth/neutral.wav', angry=None),\n",
- "]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "[voice(name='Attenborough', neutral='audio/attenborough/neutral.mp3', angry=None),\n",
- " voice(name='Rick', neutral='audio/rick/neutral.mp3', angry=None),\n",
- " voice(name='Freeman', neutral='audio/freeman/neutral.mp3', angry='audio/freeman/angry.mp3'),\n",
- " voice(name='Walken', neutral='audio/walken/neutral.mp3', angry=None),\n",
- " voice(name='Darth Wader', neutral='audio/darth/neutral.mp3', angry=None)]"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "voices"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- " > tts_models/multilingual/multi-dataset/xtts_v2 is already downloaded.\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
- " _torch_pytree._register_pytree_node(\n",
- "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
- " _torch_pytree._register_pytree_node(\n",
- "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
- " _torch_pytree._register_pytree_node(\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- " > Using model: xtts\n"
- ]
- }
- ],
- "source": [
- "#load model for text to speech\n",
- "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
- "# device = \"mps\"\n",
- "tts_pipelins = TTS(\"tts_models/multilingual/multi-dataset/xtts_v2\").to(device)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [],
- "source": [
- "import IPython\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 81,
- "metadata": {},
- "outputs": [],
- "source": [
- "speaker_embedding_cache = {}"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 82,
- "metadata": {},
- "outputs": [],
- "source": [
- "def compute_speaker_embedding(voice_path: str, config, pipeline, cache):\n",
- " if voice_path not in cache:\n",
- " cache[voice_path] = pipeline.synthesizer.tts_model.get_conditioning_latents(\n",
- " audio_path=voice_path,\n",
- " gpt_cond_len=config.gpt_cond_len,\n",
- " gpt_cond_chunk_len=config.gpt_cond_chunk_len,\n",
- " max_ref_length=config.max_ref_len,\n",
- " sound_norm_refs=config.sound_norm_refs,\n",
- " )\n",
- " return cache[voice_path]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 87,
- "metadata": {},
- "outputs": [],
- "source": [
- "out = compute_speaker_embedding(voices[0].neutral, tts_pipelins.synthesizer.tts_config, tts_pipelins, speaker_embedding_cache)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- " > Text splitted to sentences.\n",
- "['Hey Petra, so you are hungry?', 'and you like me to prepare some strawberries for you?', 'do you like strawberries?']\n",
- " > Processing time: 15.77448582649231\n",
- " > Real-time factor: 1.7459813091024587\n"
- ]
- }
- ],
- "source": [
- "out = tts_pipelins.tts(\n",
- " \"Hello, I am Rick, pickle rick, you took a wrong turn and now you're stuck in a parallel universe\",\n",
- " speaker_wav=\"audio/freeman/neutral.wav\",\n",
- " language=\"en\",\n",
- " # file_path=\"out.wav\",\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [],
- "source": [
- "from typing import List\n",
- "import time"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 19,
- "metadata": {},
- "outputs": [],
- "source": [
- "ref_audio_path = \"audio/freeman/neutral.wav\""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 53,
- "metadata": {},
- "outputs": [],
- "source": [
- "config.max_ref_len = 360"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 78,
- "metadata": {},
- "outputs": [],
- "source": [
- "config = tts_pipelins.synthesizer.tts_config\n",
- "(gpt_cond_latent, speaker_embedding) = tts_pipelins.synthesizer.tts_model.get_conditioning_latents(\n",
- " audio_path=ref_audio_path,\n",
- " gpt_cond_len=config.gpt_cond_len,\n",
- " gpt_cond_chunk_len=config.gpt_cond_chunk_len,\n",
- " max_ref_length=config.max_ref_len,\n",
- " sound_norm_refs=config.sound_norm_refs,\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 107,
- "metadata": {},
- "outputs": [],
- "source": [
- "(gpt_cond_latent, speaker_embedding) = compute_speaker_embedding(voices[0].neutral, tts_pipelins.synthesizer.tts_config, tts_pipelins, speaker_embedding_cache)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 114,
- "metadata": {},
- "outputs": [],
- "source": [
- "import numpy as np"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 116,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "(205872,)"
- ]
- },
- "execution_count": 116,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "np.array(out)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 110,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "205872"
- ]
- },
- "execution_count": 110,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "len(out)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 128,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- " > Text splitted to sentences.\n",
- "['Something is up!']\n",
- " > Processing time: 2.9515581130981445\n",
- " > Real-time factor: 1.588292083019672\n"
- ]
- }
- ],
- "source": [
- "out = tts(\n",
- " tts_pipelins.synthesizer,\n",
- " \"Something is up!\",\n",
- " # speaker_wav=ref_audio_path,\n",
- " language_name=\"en\",\n",
- " speaker=None,\n",
- " gpt_cond_latent=gpt_cond_latent,\n",
- " speaker_embedding=speaker_embedding,\n",
- " speed=1.1,\n",
- " # file_path=\"out.wav\",\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 129,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- " "
- ],
- "text/plain": [
- ""
- ]
- },
- "execution_count": 129,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "IPython.display.Audio(out, rate=22050)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 66,
- "metadata": {},
- "outputs": [],
- "source": [
- "from TTS.vocoder.utils.generic_utils import interpolate_vocoder_input\n",
-<<<<<<< Updated upstream
- "\n",
-=======
- "# use_gl = self.vocoder_model is None\n",
->>>>>>> Stashed changes
- "def tts(\n",
- " self,\n",
- " text: str = \"\",\n",
- " language_name: str = \"\",\n",
- " reference_wav=None,\n",
- " gpt_cond_latent=None,\n",
- " speaker_embedding=None,\n",
- " split_sentences: bool = True,\n",
- " **kwargs,\n",
- ") -> List[int]:\n",
- " \"\"\"🐸 TTS magic. Run all the models and generate speech.\n",
- "\n",
- " Args:\n",
- " text (str): input text.\n",
- " speaker_name (str, optional): speaker id for multi-speaker models. Defaults to \"\".\n",
- " language_name (str, optional): language id for multi-language models. Defaults to \"\".\n",
- " speaker_wav (Union[str, List[str]], optional): path to the speaker wav for voice cloning. Defaults to None.\n",
- " style_wav ([type], optional): style waveform for GST. Defaults to None.\n",
- " style_text ([type], optional): transcription of style_wav for Capacitron. Defaults to None.\n",
- " reference_wav ([type], optional): reference waveform for voice conversion. Defaults to None.\n",
- " reference_speaker_name ([type], optional): speaker id of reference waveform. Defaults to None.\n",
- " split_sentences (bool, optional): split the input text into sentences. Defaults to True.\n",
- " **kwargs: additional arguments to pass to the TTS model.\n",
- " Returns:\n",
- " List[int]: [description]\n",
- " \"\"\"\n",
- " start_time = time.time()\n",
- " wavs = []\n",
- "\n",
- " if not text and not reference_wav:\n",
- " raise ValueError(\n",
- " \"You need to define either `text` (for sythesis) or a `reference_wav` (for voice conversion) to use the Coqui TTS API.\"\n",
- " )\n",
- "\n",
- " if text:\n",
- " sens = [text]\n",
- " if split_sentences:\n",
- " print(\" > Text splitted to sentences.\")\n",
- " sens = self.split_into_sentences(text)\n",
- " print(sens)\n",
- "\n",
- " if not reference_wav: # not voice conversion\n",
- " for sen in sens:\n",
- " outputs = self.tts_model.inference(\n",
- " sen,\n",
- " language_name,\n",
- " gpt_cond_latent,\n",
- " speaker_embedding,\n",
- " # GPT inference\n",
- " temperature=0.75,\n",
- " length_penalty=1.0,\n",
- " repetition_penalty=10.0,\n",
- " top_k=50,\n",
- " top_p=0.85,\n",
- " do_sample=True,\n",
- " **kwargs,\n",
- " )\n",
- " waveform = outputs[\"wav\"]\n",
- " if torch.is_tensor(waveform) and waveform.device != torch.device(\"cpu\") and not use_gl:\n",
- " waveform = waveform.cpu()\n",
- " if not use_gl:\n",
- " waveform = waveform.numpy()\n",
- " waveform = waveform.squeeze()\n",
- "\n",
- " # trim silence\n",
- " if \"do_trim_silence\" in self.tts_config.audio and self.tts_config.audio[\"do_trim_silence\"]:\n",
- " waveform = trim_silence(waveform, self.tts_model.ap)\n",
- "\n",
- " wavs += list(waveform)\n",
- " wavs += [0] * 10000\n",
- "\n",
- "\n",
- " # compute stats\n",
- " process_time = time.time() - start_time\n",
- " audio_time = len(wavs) / self.tts_config.audio[\"sample_rate\"]\n",
- " print(f\" > Processing time: {process_time}\")\n",
- " print(f\" > Real-time factor: {process_time / audio_time}\")\n",
- " return wavs"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "type(tts_pipelins)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "IPython.display.Audio(out, rate=22050)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def text_to_speech(voice, tts):\n",
- " return voice.neutral"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- " tts.tts_to_file(text= str(quest_processing[0]),\n",
- " file_path=\"output.wav\",\n",
- " speaker_wav=f'Audio_Files/{voice}.wav',\n",
- " language=quest_processing[3],\n",
- " emotion = \"angry\")\n",
- "\n",
- " audio_path = \"output.wav\"\n",
- " return audio_path, state['context'], state"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 90,
- "metadata": {},
- "outputs": [],
- "source": [
- "voice_options = []\n",
- "for voice in voices:\n",
- " if voice.neutral:\n",
- " voice_options.append(f\"{voice.name} - Neutral\")\n",
- " if voice.angry:\n",
- " voice_options.append(f\"{voice.name} - Angry\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 101,
- "metadata": {},
- "outputs": [],
- "source": [
- "def voice_from_text(voice):\n",
- " for v in voices:\n",
- " if voice == f\"{v.name} - Neutral\":\n",
- " return v.neutral\n",
- " if voice == f\"{v.name} - Angry\":\n",
- " return v.angry"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 121,
- "metadata": {},
- "outputs": [],
- "source": [
- "def tts_gradio(text, voice, state):\n",
- " print(text, voice, state)\n",
- " voice_path = voice_from_text(voice)\n",
- " (gpt_cond_latent, speaker_embedding) = compute_speaker_embedding(voice_path, tts_pipelins.synthesizer.tts_config, tts_pipelins, speaker_embedding_cache)\n",
- " out = tts(\n",
- " tts_pipelins.synthesizer,\n",
- " text,\n",
- " language_name=\"en\",\n",
- " speaker=None,\n",
- " gpt_cond_latent=gpt_cond_latent,\n",
- " speaker_embedding=speaker_embedding,\n",
- " speed=1.1,\n",
- " # file_path=\"out.wav\",\n",
- " )\n",
- " return (22050, np.array(out)), dict(text=text, voice=voice)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 122,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "dict_keys(['audio/attenborough/neutral.wav'])"
- ]
- },
- "execution_count": 122,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "speaker_embedding_cache.keys()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 127,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "This is going to be fun, let's enjoy ourselves\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Closing server running on port: 7860\n",
- "Running on local URL: http://0.0.0.0:7860\n",
- "\n",
- "To create a public link, set `share=True` in `launch()`.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "This is going to be fun, let's enjoy ourselves Darth Wader - Neutral None\n",
- " > Text splitted to sentences.\n",
- "[\"This is going to be fun, let's enjoy ourselves\"]\n",
- " > Processing time: 9.152068138122559\n",
- " > Real-time factor: 1.8119083325456329\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/gradio/processing_utils.py:390: UserWarning: Trying to convert audio automatically from float64 to 16-bit int format.\n",
- " warnings.warn(warning.format(data.dtype))\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "This is going to be fun, let's enjoy ourselves Darth Wader - Neutral {'text': \"This is going to be fun, let's enjoy ourselves\", 'voice': 'Darth Wader - Neutral'}\n",
- " > Text splitted to sentences.\n",
- "[\"This is going to be fun, let's enjoy ourselves\"]\n",
- " > Processing time: 7.824646234512329\n",
- " > Real-time factor: 1.8261372721316347\n",
- "Keyboard interruption in main thread... closing server.\n"
- ]
- },
- {
- "data": {
- "text/plain": []
- },
- "execution_count": 127,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "#INTERFACE WITH AUDIO TO AUDIO\n",
- "\n",
- "#to be able to use the microphone on chrome, you will have to go to chrome://flags/#unsafely-treat-insecure-origin-as-secure and enter http://10.186.115.21:7860/ \n",
- "#in \"Insecure origins treated as secure\", enable it and relaunch chrome\n",
- "\n",
- "\n",
- "model_answer= ''\n",
- "general_context= \"This is going to be fun, let's enjoy ourselves\"\n",
- "# Define the initial state with some initial context.\n",
- "print(general_context)\n",
- "initial_state = {'context': general_context}\n",
- "initial_context= initial_state['context']\n",
- "# Create the Gradio interface.\n",
- "iface = gr.Interface(\n",
- " fn=tts_gradio,\n",
- " inputs=[\n",
- " gr.Textbox(value=initial_context, visible=True, label='Enter the text to be converted to speech', placeholder=\"This is going to be fun, let's enjoy ourselves\", lines=5),\n",
- " gr.Radio(choices=voice_options, label='Choose a voice', value=voice_options[0], show_label=True), # Radio button for voice selection\n",
- " gr.State() # This will keep track of the context state across interactions.\n",
- " ],\n",
- " outputs=[\n",
- " gr.Audio(label = 'output audio', autoplay=True),\n",
- " gr.State()\n",
- " ],\n",
- " flagging_options=['👎', '👍'],\n",
- ")\n",
- "#close all interfaces open to make the port available\n",
- "gr.close_all()\n",
- "# Launch the interface.\n",
- "iface.launch(debug=True, share=False, server_name=\"0.0.0.0\", server_port=7860, ssl_verify=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "base",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.11.8"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}