BadriNarayanan commited on
Commit
893081e
1 Parent(s): 17b9c60

deleted gradio file

Browse files
Files changed (1) hide show
  1. gradio_app.py +0 -1049
gradio_app.py DELETED
@@ -1,1049 +0,0 @@
1
- # import os
2
- # import re
3
- # import torch
4
- # import torchaudio
5
- # import gradio as gr
6
- # import numpy as np
7
- # import tempfile
8
- # from einops import rearrange
9
- # from vocos import Vocos
10
- # from pydub import AudioSegment
11
- # from model import CFM, UNetT, DiT, MMDiT
12
- # from cached_path import cached_path
13
- # from model.utils import (
14
- # load_checkpoint,
15
- # get_tokenizer,
16
- # convert_char_to_pinyin,
17
- # save_spectrogram,
18
- # )
19
- # from transformers import pipeline
20
- # import librosa
21
- # import click
22
-
23
- # device = (
24
- # "cuda"
25
- # if torch.cuda.is_available()
26
- # else "mps" if torch.backends.mps.is_available() else "cpu"
27
- # )
28
-
29
- # print(f"Using {device} device")
30
-
31
- # pipe = pipeline(
32
- # "automatic-speech-recognition",
33
- # model="openai/whisper-large-v3-turbo",
34
- # torch_dtype=torch.float16,
35
- # device=device,
36
- # )
37
-
38
- # # --------------------- Settings -------------------- #
39
-
40
- # target_sample_rate = 24000
41
- # n_mel_channels = 100
42
- # hop_length = 256
43
- # target_rms = 0.1
44
- # nfe_step = 32 # 16, 32
45
- # cfg_strength = 2.0
46
- # ode_method = "euler"
47
- # sway_sampling_coef = -1.0
48
- # speed = 1.0
49
- # # fix_duration = 27 # None or float (duration in seconds)
50
- # fix_duration = None
51
-
52
-
53
- # def load_model(exp_name, model_cls, model_cfg, ckpt_step):
54
- # ckpt_path = str(cached_path(f"hf://SWivid/F5-TTS/{exp_name}/model_{ckpt_step}.safetensors"))
55
- # # ckpt_path = f"ckpts/{exp_name}/model_{ckpt_step}.pt" # .pt | .safetensors
56
- # vocab_char_map, vocab_size = get_tokenizer("Emilia_ZH_EN", "pinyin")
57
- # model = CFM(
58
- # transformer=model_cls(
59
- # **model_cfg, text_num_embeds=vocab_size, mel_dim=n_mel_channels
60
- # ),
61
- # mel_spec_kwargs=dict(
62
- # target_sample_rate=target_sample_rate,
63
- # n_mel_channels=n_mel_channels,
64
- # hop_length=hop_length,
65
- # ),
66
- # odeint_kwargs=dict(
67
- # method=ode_method,
68
- # ),
69
- # vocab_char_map=vocab_char_map,
70
- # ).to(device)
71
-
72
- # model = load_checkpoint(model, ckpt_path, device, use_ema = True)
73
-
74
- # return model
75
-
76
-
77
- # # load models
78
- # F5TTS_model_cfg = dict(
79
- # dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4
80
- # )
81
- # E2TTS_model_cfg = dict(dim=1024, depth=24, heads=16, ff_mult=4)
82
-
83
- # F5TTS_ema_model = load_model(
84
- # "F5TTS_Base", DiT, F5TTS_model_cfg, 1200000
85
- # )
86
- # E2TTS_ema_model = load_model(
87
- # "E2TTS_Base", UNetT, E2TTS_model_cfg, 1200000
88
- # )
89
-
90
-
91
- # def infer(ref_audio_orig, ref_text, gen_text, exp_name, remove_silence):
92
- # print(gen_text)
93
- # if len(gen_text) > 200:
94
- # raise gr.Error("Please keep your text under 200 chars.")
95
- # gr.Info("Converting audio...")
96
- # with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f:
97
- # aseg = AudioSegment.from_file(ref_audio_orig)
98
- # audio_duration = len(aseg)
99
- # if audio_duration > 15000:
100
- # gr.Warning("Audio is over 15s, clipping to only first 15s.")
101
- # aseg = aseg[:15000]
102
- # aseg.export(f.name, format="wav")
103
- # ref_audio = f.name
104
- # if exp_name == "F5-TTS":
105
- # ema_model = F5TTS_ema_model
106
- # elif exp_name == "E2-TTS":
107
- # ema_model = E2TTS_ema_model
108
-
109
- # if not ref_text.strip():
110
- # gr.Info("No reference text provided, transcribing reference audio...")
111
- # ref_text = outputs = pipe(
112
- # ref_audio,
113
- # chunk_length_s=30,
114
- # batch_size=128,
115
- # generate_kwargs={"task": "transcribe"},
116
- # return_timestamps=False,
117
- # )["text"].strip()
118
- # gr.Info("Finished transcription")
119
- # else:
120
- # gr.Info("Using custom reference text...")
121
- # audio, sr = torchaudio.load(ref_audio)
122
- # if audio.shape[0] > 1:
123
- # audio = torch.mean(audio, dim=0, keepdim=True)
124
-
125
- # rms = torch.sqrt(torch.mean(torch.square(audio)))
126
- # if rms < target_rms:
127
- # audio = audio * target_rms / rms
128
- # if sr != target_sample_rate:
129
- # resampler = torchaudio.transforms.Resample(sr, target_sample_rate)
130
- # audio = resampler(audio)
131
- # audio = audio.to(device)
132
-
133
- # # Prepare the text
134
- # text_list = [ref_text + gen_text]
135
- # final_text_list = convert_char_to_pinyin(text_list)
136
-
137
- # # Calculate duration
138
- # ref_audio_len = audio.shape[-1] // hop_length
139
- # # if fix_duration is not None:
140
- # # duration = int(fix_duration * target_sample_rate / hop_length)
141
- # # else:
142
- # zh_pause_punc = r"。,、;:?!"
143
- # ref_text_len = len(ref_text) + len(re.findall(zh_pause_punc, ref_text))
144
- # gen_text_len = len(gen_text) + len(re.findall(zh_pause_punc, gen_text))
145
- # duration = ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / speed)
146
-
147
- # # inference
148
- # gr.Info(f"Generating audio using {exp_name}")
149
- # with torch.inference_mode():
150
- # generated, _ = ema_model.sample(
151
- # cond=audio,
152
- # text=final_text_list,
153
- # duration=duration,
154
- # steps=nfe_step,
155
- # cfg_strength=cfg_strength,
156
- # sway_sampling_coef=sway_sampling_coef,
157
- # )
158
-
159
- # generated = generated[:, ref_audio_len:, :]
160
- # generated_mel_spec = rearrange(generated, "1 n d -> 1 d n")
161
- # gr.Info("Running vocoder")
162
- # vocos = Vocos.from_pretrained("charactr/vocos-mel-24khz")
163
- # generated_wave = vocos.decode(generated_mel_spec.cpu())
164
- # if rms < target_rms:
165
- # generated_wave = generated_wave * rms / target_rms
166
-
167
- # # wav -> numpy
168
- # generated_wave = generated_wave.squeeze().cpu().numpy()
169
-
170
- # if remove_silence:
171
- # gr.Info("Removing audio silences... This may take a moment")
172
- # non_silent_intervals = librosa.effects.split(generated_wave, top_db=30)
173
- # non_silent_wave = np.array([])
174
- # for interval in non_silent_intervals:
175
- # start, end = interval
176
- # non_silent_wave = np.concatenate(
177
- # [non_silent_wave, generated_wave[start:end]]
178
- # )
179
- # generated_wave = non_silent_wave
180
-
181
- # # spectogram
182
- # with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_spectrogram:
183
- # spectrogram_path = tmp_spectrogram.name
184
- # save_spectrogram(generated_mel_spec[0].cpu().numpy(), spectrogram_path)
185
-
186
- # return (target_sample_rate, generated_wave), spectrogram_path
187
-
188
-
189
- # with gr.Blocks() as app:
190
- # gr.Markdown(
191
- # """
192
- # # Antriksh AI
193
-
194
- # """
195
- # )
196
-
197
- # # Image
198
- # gr.Image(value="C:\\Users\\USER\\OneDrive\\Documents\\logo.jpg", width=300, height= 150 )
199
-
200
- # ref_audio_input = gr.Audio(label="Reference Audio", type="filepath")
201
- # gen_text_input = gr.Textbox(label="Text to Generate (max 200 chars.)", lines=4)
202
- # model_choice = gr.Radio(
203
- # choices=["F5-TTS", "E2-TTS"], label="Choose TTS Model", value="F5-TTS"
204
- # )
205
- # generate_btn = gr.Button("Synthesize", variant="primary")
206
- # with gr.Accordion("Advanced Settings", open=False):
207
- # ref_text_input = gr.Textbox(
208
- # label="Reference Text",
209
- # info="Leave blank to automatically transcribe the reference audio. If you enter text it will override automatic transcription.",
210
- # lines=2,
211
- # )
212
- # remove_silence = gr.Checkbox(
213
- # label="Remove Silences",
214
- # info="The model tends to produce silences, especially on longer audio. We can manually remove silences if needed. Note that this is an experimental feature and may produce strange results. This will also increase generation time.",
215
- # value=True,
216
- # )
217
-
218
- # audio_output = gr.Audio(label="Synthesized Audio")
219
- # spectrogram_output = gr.Image(label="Spectrogram")
220
-
221
- # generate_btn.click(
222
- # infer,
223
- # inputs=[
224
- # ref_audio_input,
225
- # ref_text_input,
226
- # gen_text_input,
227
- # model_choice,
228
- # remove_silence,
229
- # ],
230
- # outputs=[audio_output, spectrogram_output],
231
- # )
232
-
233
-
234
- # @click.command()
235
- # @click.option("--port", "-p", default=None, type=int, help="Port to run the app on")
236
- # @click.option("--host", "-H", default=None, help="Host to run the app on")
237
- # @click.option(
238
- # "--share",
239
- # "-s",
240
- # default=True,
241
- # is_flag=True,
242
- # help="Share the app via Gradio share link",
243
- # )
244
- # @click.option("--api", "-a", default=True, is_flag=True, help="Allow API access")
245
- # def main(port, host, share, api):
246
- # global app
247
- # print(f"Starting app...")
248
- # app.queue(api_open=api).launch(
249
- # server_name=host, server_port=port, share=True, show_api=api
250
- # )
251
-
252
-
253
- # if __name__ == "__main__":
254
- # main()
255
-
256
- import re
257
- import torch
258
- import torchaudio
259
- import gradio as gr
260
- import numpy as np
261
- import tempfile
262
- from einops import rearrange
263
- from vocos import Vocos
264
- from pydub import AudioSegment, silence
265
- from model import CFM, UNetT, DiT, MMDiT
266
- from cached_path import cached_path
267
- from model.utils import (
268
- load_checkpoint,
269
- get_tokenizer,
270
- convert_char_to_pinyin,
271
- save_spectrogram,
272
- )
273
- from transformers import pipeline
274
- import click
275
- import soundfile as sf
276
-
277
- try:
278
- import spaces
279
- USING_SPACES = True
280
- except ImportError:
281
- USING_SPACES = False
282
-
283
- def gpu_decorator(func):
284
- if USING_SPACES:
285
- return spaces.GPU(func)
286
- else:
287
- return func
288
-
289
- device = (
290
- "cuda"
291
- if torch.cuda.is_available()
292
- else "mps" if torch.backends.mps.is_available() else "cpu"
293
- )
294
-
295
- print(f"Using {device} device")
296
-
297
- pipe = pipeline(
298
- "automatic-speech-recognition",
299
- model="openai/whisper-large-v3-turbo",
300
- torch_dtype=torch.float16,
301
- device=device,
302
- )
303
- vocos = Vocos.from_pretrained("charactr/vocos-mel-24khz")
304
-
305
- # --------------------- Settings -------------------- #
306
-
307
- target_sample_rate = 24000
308
- n_mel_channels = 100
309
- hop_length = 256
310
- target_rms = 0.1
311
- nfe_step = 32 # 16, 32
312
- cfg_strength = 2.0
313
- ode_method = "euler"
314
- sway_sampling_coef = -1.0
315
- speed = 1.0
316
- fix_duration = None
317
-
318
-
319
- def load_model(repo_name, exp_name, model_cls, model_cfg, ckpt_step):
320
- ckpt_path = str(cached_path(f"hf://SWivid/{repo_name}/{exp_name}/model_{ckpt_step}.safetensors"))
321
- # ckpt_path = f"ckpts/{exp_name}/model_{ckpt_step}.pt" # .pt | .safetensors
322
- vocab_char_map, vocab_size = get_tokenizer("Emilia_ZH_EN", "pinyin")
323
- model = CFM(
324
- transformer=model_cls(
325
- **model_cfg, text_num_embeds=vocab_size, mel_dim=n_mel_channels
326
- ),
327
- mel_spec_kwargs=dict(
328
- target_sample_rate=target_sample_rate,
329
- n_mel_channels=n_mel_channels,
330
- hop_length=hop_length,
331
- ),
332
- odeint_kwargs=dict(
333
- method=ode_method,
334
- ),
335
- vocab_char_map=vocab_char_map,
336
- ).to(device)
337
-
338
- model = load_checkpoint(model, ckpt_path, device, use_ema = True)
339
-
340
- return model
341
-
342
-
343
- # load models
344
- F5TTS_model_cfg = dict(
345
- dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4
346
- )
347
- E2TTS_model_cfg = dict(dim=1024, depth=24, heads=16, ff_mult=4)
348
-
349
- F5TTS_ema_model = load_model(
350
- "F5-TTS", "F5TTS_Base", DiT, F5TTS_model_cfg, 1200000
351
- )
352
- E2TTS_ema_model = load_model(
353
- "E2-TTS", "E2TTS_Base", UNetT, E2TTS_model_cfg, 1200000
354
- )
355
-
356
- def chunk_text(text, max_chars=135):
357
- """
358
- Splits the input text into chunks, each with a maximum number of characters.
359
-
360
- Args:
361
- text (str): The text to be split.
362
- max_chars (int): The maximum number of characters per chunk.
363
-
364
- Returns:
365
- List[str]: A list of text chunks.
366
- """
367
- chunks = []
368
- current_chunk = ""
369
- # Split the text into sentences based on punctuation followed by whitespace
370
- sentences = re.split(r'(?<=[;:,.!?])\s+|(?<=[;:,。!?])', text)
371
-
372
- for sentence in sentences:
373
- if len(current_chunk.encode('utf-8')) + len(sentence.encode('utf-8')) <= max_chars:
374
- current_chunk += sentence + " " if sentence and len(sentence[-1].encode('utf-8')) == 1 else sentence
375
- else:
376
- if current_chunk:
377
- chunks.append(current_chunk.strip())
378
- current_chunk = sentence + " " if sentence and len(sentence[-1].encode('utf-8')) == 1 else sentence
379
-
380
- if current_chunk:
381
- chunks.append(current_chunk.strip())
382
-
383
- return chunks
384
-
385
- @gpu_decorator
386
- def infer_batch(ref_audio, ref_text, gen_text_batches, exp_name, remove_silence, cross_fade_duration=0.15, progress=gr.Progress()):
387
- if exp_name == "F5-TTS":
388
- ema_model = F5TTS_ema_model
389
- elif exp_name == "E2-TTS":
390
- ema_model = E2TTS_ema_model
391
-
392
- audio, sr = ref_audio
393
- if audio.shape[0] > 1:
394
- audio = torch.mean(audio, dim=0, keepdim=True)
395
-
396
- rms = torch.sqrt(torch.mean(torch.square(audio)))
397
- if rms < target_rms:
398
- audio = audio * target_rms / rms
399
- if sr != target_sample_rate:
400
- resampler = torchaudio.transforms.Resample(sr, target_sample_rate)
401
- audio = resampler(audio)
402
- audio = audio.to(device)
403
-
404
- generated_waves = []
405
- spectrograms = []
406
-
407
- for i, gen_text in enumerate(progress.tqdm(gen_text_batches)):
408
- # Prepare the text
409
- if len(ref_text[-1].encode('utf-8')) == 1:
410
- ref_text = ref_text + " "
411
- text_list = [ref_text + gen_text]
412
- final_text_list = convert_char_to_pinyin(text_list)
413
-
414
- # Calculate duration
415
- ref_audio_len = audio.shape[-1] // hop_length
416
- zh_pause_punc = r"。,、;:?!"
417
- ref_text_len = len(ref_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, ref_text))
418
- gen_text_len = len(gen_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, gen_text))
419
- duration = ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / speed)
420
-
421
- # inference
422
- with torch.inference_mode():
423
- generated, _ = ema_model.sample(
424
- cond=audio,
425
- text=final_text_list,
426
- duration=duration,
427
- steps=nfe_step,
428
- cfg_strength=cfg_strength,
429
- sway_sampling_coef=sway_sampling_coef,
430
- )
431
-
432
- generated = generated[:, ref_audio_len:, :]
433
- generated_mel_spec = rearrange(generated, "1 n d -> 1 d n")
434
- generated_wave = vocos.decode(generated_mel_spec.cpu())
435
- if rms < target_rms:
436
- generated_wave = generated_wave * rms / target_rms
437
-
438
- # wav -> numpy
439
- generated_wave = generated_wave.squeeze().cpu().numpy()
440
-
441
- generated_waves.append(generated_wave)
442
- spectrograms.append(generated_mel_spec[0].cpu().numpy())
443
-
444
- # Combine all generated waves with cross-fading
445
- if cross_fade_duration <= 0:
446
- # Simply concatenate
447
- final_wave = np.concatenate(generated_waves)
448
- else:
449
- final_wave = generated_waves[0]
450
- for i in range(1, len(generated_waves)):
451
- prev_wave = final_wave
452
- next_wave = generated_waves[i]
453
-
454
- # Calculate cross-fade samples, ensuring it does not exceed wave lengths
455
- cross_fade_samples = int(cross_fade_duration * target_sample_rate)
456
- cross_fade_samples = min(cross_fade_samples, len(prev_wave), len(next_wave))
457
-
458
- if cross_fade_samples <= 0:
459
- # No overlap possible, concatenate
460
- final_wave = np.concatenate([prev_wave, next_wave])
461
- continue
462
-
463
- # Overlapping parts
464
- prev_overlap = prev_wave[-cross_fade_samples:]
465
- next_overlap = next_wave[:cross_fade_samples]
466
-
467
- # Fade out and fade in
468
- fade_out = np.linspace(1, 0, cross_fade_samples)
469
- fade_in = np.linspace(0, 1, cross_fade_samples)
470
-
471
- # Cross-faded overlap
472
- cross_faded_overlap = prev_overlap * fade_out + next_overlap * fade_in
473
-
474
- # Combine
475
- new_wave = np.concatenate([
476
- prev_wave[:-cross_fade_samples],
477
- cross_faded_overlap,
478
- next_wave[cross_fade_samples:]
479
- ])
480
-
481
- final_wave = new_wave
482
-
483
- # Remove silence
484
- if remove_silence:
485
- with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f:
486
- sf.write(f.name, final_wave, target_sample_rate)
487
- aseg = AudioSegment.from_file(f.name)
488
- non_silent_segs = silence.split_on_silence(aseg, min_silence_len=1000, silence_thresh=-50, keep_silence=500)
489
- non_silent_wave = AudioSegment.silent(duration=0)
490
- for non_silent_seg in non_silent_segs:
491
- non_silent_wave += non_silent_seg
492
- aseg = non_silent_wave
493
- aseg.export(f.name, format="wav")
494
- final_wave, _ = torchaudio.load(f.name)
495
- final_wave = final_wave.squeeze().cpu().numpy()
496
-
497
- # Create a combined spectrogram
498
- combined_spectrogram = np.concatenate(spectrograms, axis=1)
499
-
500
- with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_spectrogram:
501
- spectrogram_path = tmp_spectrogram.name
502
- save_spectrogram(combined_spectrogram, spectrogram_path)
503
-
504
- return (target_sample_rate, final_wave), spectrogram_path
505
-
506
- @gpu_decorator
507
- def infer(ref_audio_orig, ref_text, gen_text, exp_name, remove_silence, cross_fade_duration=0.15):
508
-
509
- print(gen_text)
510
-
511
- gr.Info("Converting audio...")
512
- with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f:
513
- aseg = AudioSegment.from_file(ref_audio_orig)
514
-
515
- non_silent_segs = silence.split_on_silence(
516
- aseg, min_silence_len=1000, silence_thresh=-50, keep_silence=1000
517
- )
518
- non_silent_wave = AudioSegment.silent(duration=0)
519
- for non_silent_seg in non_silent_segs:
520
- non_silent_wave += non_silent_seg
521
- aseg = non_silent_wave
522
-
523
- audio_duration = len(aseg)
524
- if audio_duration > 15000:
525
- gr.Warning("Audio is over 15s, clipping to only first 15s.")
526
- aseg = aseg[:15000]
527
- aseg.export(f.name, format="wav")
528
- ref_audio = f.name
529
-
530
- if not ref_text.strip():
531
- gr.Info("No reference text provided, transcribing reference audio...")
532
- ref_text = pipe(
533
- ref_audio,
534
- chunk_length_s=30,
535
- batch_size=128,
536
- generate_kwargs={"task": "transcribe"},
537
- return_timestamps=False,
538
- )["text"].strip()
539
- gr.Info("Finished transcription")
540
- else:
541
- gr.Info("Using custom reference text...")
542
-
543
- # Add the functionality to ensure it ends with ". "
544
- if not ref_text.endswith(". "):
545
- if ref_text.endswith("."):
546
- ref_text += " "
547
- else:
548
- ref_text += ". "
549
-
550
- audio, sr = torchaudio.load(ref_audio)
551
-
552
- # Use the new chunk_text function to split gen_text
553
- max_chars = int(len(ref_text.encode('utf-8')) / (audio.shape[-1] / sr) * (25 - audio.shape[-1] / sr))
554
- gen_text_batches = chunk_text(gen_text, max_chars=max_chars)
555
- print('ref_text', ref_text)
556
- for i, batch_text in enumerate(gen_text_batches):
557
- print(f'gen_text {i}', batch_text)
558
-
559
- gr.Info(f"Generating audio using {exp_name} in {len(gen_text_batches)} batches")
560
- return infer_batch((audio, sr), ref_text, gen_text_batches, exp_name, remove_silence, cross_fade_duration)
561
-
562
-
563
- @gpu_decorator
564
- def generate_podcast(script, speaker1_name, ref_audio1, ref_text1, speaker2_name, ref_audio2, ref_text2, exp_name, remove_silence):
565
- # Split the script into speaker blocks
566
- speaker_pattern = re.compile(f"^({re.escape(speaker1_name)}|{re.escape(speaker2_name)}):", re.MULTILINE)
567
- speaker_blocks = speaker_pattern.split(script)[1:] # Skip the first empty element
568
-
569
- generated_audio_segments = []
570
-
571
- for i in range(0, len(speaker_blocks), 2):
572
- speaker = speaker_blocks[i]
573
- text = speaker_blocks[i+1].strip()
574
-
575
- # Determine which speaker is talking
576
- if speaker == speaker1_name:
577
- ref_audio = ref_audio1
578
- ref_text = ref_text1
579
- elif speaker == speaker2_name:
580
- ref_audio = ref_audio2
581
- ref_text = ref_text2
582
- else:
583
- continue # Skip if the speaker is neither speaker1 nor speaker2
584
-
585
- # Generate audio for this block
586
- audio, _ = infer(ref_audio, ref_text, text, exp_name, remove_silence)
587
-
588
- # Convert the generated audio to a numpy array
589
- sr, audio_data = audio
590
-
591
- # Save the audio data as a WAV file
592
- with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
593
- sf.write(temp_file.name, audio_data, sr)
594
- audio_segment = AudioSegment.from_wav(temp_file.name)
595
-
596
- generated_audio_segments.append(audio_segment)
597
-
598
- # Add a short pause between speakers
599
- pause = AudioSegment.silent(duration=500) # 500ms pause
600
- generated_audio_segments.append(pause)
601
-
602
- # Concatenate all audio segments
603
- final_podcast = sum(generated_audio_segments)
604
-
605
- # Export the final podcast
606
- with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
607
- podcast_path = temp_file.name
608
- final_podcast.export(podcast_path, format="wav")
609
-
610
- return podcast_path
611
-
612
- def parse_speechtypes_text(gen_text):
613
- # Pattern to find (Emotion)
614
- pattern = r'\((.*?)\)'
615
-
616
- # Split the text by the pattern
617
- tokens = re.split(pattern, gen_text)
618
-
619
- segments = []
620
-
621
- current_emotion = 'Regular'
622
-
623
- for i in range(len(tokens)):
624
- if i % 2 == 0:
625
- # This is text
626
- text = tokens[i].strip()
627
- if text:
628
- segments.append({'emotion': current_emotion, 'text': text})
629
- else:
630
- # This is emotion
631
- emotion = tokens[i].strip()
632
- current_emotion = emotion
633
-
634
- return segments
635
-
636
- def update_speed(new_speed):
637
- global speed
638
- speed = new_speed
639
- return f"Speed set to: {speed}"
640
-
641
- with gr.Blocks() as app_credits:
642
- gr.Markdown("""
643
- # Credits
644
-
645
- * [mrfakename](https://github.com/fakerybakery) for the original [online demo](https://huggingface.co/spaces/mrfakename/E2-F5-TTS)
646
- * [RootingInLoad](https://github.com/RootingInLoad) for the podcast generation
647
- * [jpgallegoar](https://github.com/jpgallegoar) for multiple speech-type generation
648
- """)
649
- with gr.Blocks() as app_tts:
650
- gr.Markdown("# Batched TTS")
651
- ref_audio_input = gr.Audio(label="Reference Audio", type="filepath")
652
- gen_text_input = gr.Textbox(label="Text to Generate", lines=10)
653
- model_choice = gr.Radio(
654
- choices=["F5-TTS", "E2-TTS"], label="Choose TTS Model", value="F5-TTS"
655
- )
656
- generate_btn = gr.Button("Synthesize", variant="primary")
657
- with gr.Accordion("Advanced Settings", open=False):
658
- ref_text_input = gr.Textbox(
659
- label="Reference Text",
660
- info="Leave blank to automatically transcribe the reference audio. If you enter text it will override automatic transcription.",
661
- lines=2,
662
- )
663
- remove_silence = gr.Checkbox(
664
- label="Remove Silences",
665
- info="The model tends to produce silences, especially on longer audio. We can manually remove silences if needed. Note that this is an experimental feature and may produce strange results. This will also increase generation time.",
666
- value=False,
667
- )
668
- speed_slider = gr.Slider(
669
- label="Speed",
670
- minimum=0.3,
671
- maximum=2.0,
672
- value=speed,
673
- step=0.1,
674
- info="Adjust the speed of the audio.",
675
- )
676
- cross_fade_duration_slider = gr.Slider(
677
- label="Cross-Fade Duration (s)",
678
- minimum=0.0,
679
- maximum=1.0,
680
- value=0.15,
681
- step=0.01,
682
- info="Set the duration of the cross-fade between audio clips.",
683
- )
684
- speed_slider.change(update_speed, inputs=speed_slider)
685
-
686
- audio_output = gr.Audio(label="Synthesized Audio")
687
- spectrogram_output = gr.Image(label="Spectrogram")
688
-
689
- generate_btn.click(
690
- infer,
691
- inputs=[
692
- ref_audio_input,
693
- ref_text_input,
694
- gen_text_input,
695
- model_choice,
696
- remove_silence,
697
- cross_fade_duration_slider,
698
- ],
699
- outputs=[audio_output, spectrogram_output],
700
- )
701
-
702
- with gr.Blocks() as app_podcast:
703
- gr.Markdown("# Podcast Generation")
704
- speaker1_name = gr.Textbox(label="Speaker 1 Name")
705
- ref_audio_input1 = gr.Audio(label="Reference Audio (Speaker 1)", type="filepath")
706
- ref_text_input1 = gr.Textbox(label="Reference Text (Speaker 1)", lines=2)
707
-
708
- speaker2_name = gr.Textbox(label="Speaker 2 Name")
709
- ref_audio_input2 = gr.Audio(label="Reference Audio (Speaker 2)", type="filepath")
710
- ref_text_input2 = gr.Textbox(label="Reference Text (Speaker 2)", lines=2)
711
-
712
- script_input = gr.Textbox(label="Podcast Script", lines=10,
713
- placeholder="Enter the script with speaker names at the start of each block, e.g.:\nSean: How did you start studying...\n\nMeghan: I came to my interest in technology...\nIt was a long journey...\n\nSean: That's fascinating. Can you elaborate...")
714
-
715
- podcast_model_choice = gr.Radio(
716
- choices=["F5-TTS", "E2-TTS"], label="Choose TTS Model", value="F5-TTS"
717
- )
718
- podcast_remove_silence = gr.Checkbox(
719
- label="Remove Silences",
720
- value=True,
721
- )
722
- generate_podcast_btn = gr.Button("Generate Podcast", variant="primary")
723
- podcast_output = gr.Audio(label="Generated Podcast")
724
-
725
- def podcast_generation(script, speaker1, ref_audio1, ref_text1, speaker2, ref_audio2, ref_text2, model, remove_silence):
726
- return generate_podcast(script, speaker1, ref_audio1, ref_text1, speaker2, ref_audio2, ref_text2, model, remove_silence)
727
-
728
- generate_podcast_btn.click(
729
- podcast_generation,
730
- inputs=[
731
- script_input,
732
- speaker1_name,
733
- ref_audio_input1,
734
- ref_text_input1,
735
- speaker2_name,
736
- ref_audio_input2,
737
- ref_text_input2,
738
- podcast_model_choice,
739
- podcast_remove_silence,
740
- ],
741
- outputs=podcast_output,
742
- )
743
-
744
- def parse_emotional_text(gen_text):
745
- # Pattern to find (Emotion)
746
- pattern = r'\((.*?)\)'
747
-
748
- # Split the text by the pattern
749
- tokens = re.split(pattern, gen_text)
750
-
751
- segments = []
752
-
753
- current_emotion = 'Regular'
754
-
755
- for i in range(len(tokens)):
756
- if i % 2 == 0:
757
- # This is text
758
- text = tokens[i].strip()
759
- if text:
760
- segments.append({'emotion': current_emotion, 'text': text})
761
- else:
762
- # This is emotion
763
- emotion = tokens[i].strip()
764
- current_emotion = emotion
765
-
766
- return segments
767
-
768
- with gr.Blocks() as app_emotional:
769
- # New section for emotional generation
770
- gr.Markdown(
771
- """
772
- # Multiple Speech-Type Generation
773
-
774
- This section allows you to upload different audio clips for each speech type. 'Regular' emotion is mandatory. You can add additional speech types by clicking the "Add Speech Type" button. Enter your text in the format shown below, and the system will generate speech using the appropriate emotions. If unspecified, the model will use the regular speech type. The current speech type will be used until the next speech type is specified.
775
-
776
- **Example Input:**
777
-
778
- (Regular) Hello, I'd like to order a sandwich please. (Surprised) What do you mean you're out of bread? (Sad) I really wanted a sandwich though... (Angry) You know what, darn you and your little shop, you suck! (Whisper) I'll just go back home and cry now. (Shouting) Why me?!
779
- """
780
- )
781
-
782
- gr.Markdown("Upload different audio clips for each speech type. 'Regular' emotion is mandatory. You can add additional speech types by clicking the 'Add Speech Type' button.")
783
-
784
- # Regular speech type (mandatory)
785
- with gr.Row():
786
- regular_name = gr.Textbox(value='Regular', label='Speech Type Name', interactive=False)
787
- regular_audio = gr.Audio(label='Regular Reference Audio', type='filepath')
788
- regular_ref_text = gr.Textbox(label='Reference Text (Regular)', lines=2)
789
-
790
- # Additional speech types (up to 99 more)
791
- max_speech_types = 100
792
- speech_type_names = []
793
- speech_type_audios = []
794
- speech_type_ref_texts = []
795
- speech_type_delete_btns = []
796
-
797
- for i in range(max_speech_types - 1):
798
- with gr.Row():
799
- name_input = gr.Textbox(label='Speech Type Name', visible=False)
800
- audio_input = gr.Audio(label='Reference Audio', type='filepath', visible=False)
801
- ref_text_input = gr.Textbox(label='Reference Text', lines=2, visible=False)
802
- delete_btn = gr.Button("Delete", variant="secondary", visible=False)
803
- speech_type_names.append(name_input)
804
- speech_type_audios.append(audio_input)
805
- speech_type_ref_texts.append(ref_text_input)
806
- speech_type_delete_btns.append(delete_btn)
807
-
808
- # Button to add speech type
809
- add_speech_type_btn = gr.Button("Add Speech Type")
810
-
811
- # Keep track of current number of speech types
812
- speech_type_count = gr.State(value=0)
813
-
814
- # Function to add a speech type
815
- def add_speech_type_fn(speech_type_count):
816
- if speech_type_count < max_speech_types - 1:
817
- speech_type_count += 1
818
- # Prepare updates for the components
819
- name_updates = []
820
- audio_updates = []
821
- ref_text_updates = []
822
- delete_btn_updates = []
823
- for i in range(max_speech_types - 1):
824
- if i < speech_type_count:
825
- name_updates.append(gr.update(visible=True))
826
- audio_updates.append(gr.update(visible=True))
827
- ref_text_updates.append(gr.update(visible=True))
828
- delete_btn_updates.append(gr.update(visible=True))
829
- else:
830
- name_updates.append(gr.update())
831
- audio_updates.append(gr.update())
832
- ref_text_updates.append(gr.update())
833
- delete_btn_updates.append(gr.update())
834
- else:
835
- # Optionally, show a warning
836
- # gr.Warning("Maximum number of speech types reached.")
837
- name_updates = [gr.update() for _ in range(max_speech_types - 1)]
838
- audio_updates = [gr.update() for _ in range(max_speech_types - 1)]
839
- ref_text_updates = [gr.update() for _ in range(max_speech_types - 1)]
840
- delete_btn_updates = [gr.update() for _ in range(max_speech_types - 1)]
841
- return [speech_type_count] + name_updates + audio_updates + ref_text_updates + delete_btn_updates
842
-
843
- add_speech_type_btn.click(
844
- add_speech_type_fn,
845
- inputs=speech_type_count,
846
- outputs=[speech_type_count] + speech_type_names + speech_type_audios + speech_type_ref_texts + speech_type_delete_btns
847
- )
848
-
849
- # Function to delete a speech type
850
- def make_delete_speech_type_fn(index):
851
- def delete_speech_type_fn(speech_type_count):
852
- # Prepare updates
853
- name_updates = []
854
- audio_updates = []
855
- ref_text_updates = []
856
- delete_btn_updates = []
857
-
858
- for i in range(max_speech_types - 1):
859
- if i == index:
860
- name_updates.append(gr.update(visible=False, value=''))
861
- audio_updates.append(gr.update(visible=False, value=None))
862
- ref_text_updates.append(gr.update(visible=False, value=''))
863
- delete_btn_updates.append(gr.update(visible=False))
864
- else:
865
- name_updates.append(gr.update())
866
- audio_updates.append(gr.update())
867
- ref_text_updates.append(gr.update())
868
- delete_btn_updates.append(gr.update())
869
-
870
- speech_type_count = max(0, speech_type_count - 1)
871
-
872
- return [speech_type_count] + name_updates + audio_updates + ref_text_updates + delete_btn_updates
873
-
874
- return delete_speech_type_fn
875
-
876
- for i, delete_btn in enumerate(speech_type_delete_btns):
877
- delete_fn = make_delete_speech_type_fn(i)
878
- delete_btn.click(
879
- delete_fn,
880
- inputs=speech_type_count,
881
- outputs=[speech_type_count] + speech_type_names + speech_type_audios + speech_type_ref_texts + speech_type_delete_btns
882
- )
883
-
884
- # Text input for the prompt
885
- gen_text_input_emotional = gr.Textbox(label="Text to Generate", lines=10)
886
-
887
- # Model choice
888
- model_choice_emotional = gr.Radio(
889
- choices=["F5-TTS", "E2-TTS"], label="Choose TTS Model", value="F5-TTS"
890
- )
891
-
892
- with gr.Accordion("Advanced Settings", open=False):
893
- remove_silence_emotional = gr.Checkbox(
894
- label="Remove Silences",
895
- value=True,
896
- )
897
-
898
- # Generate button
899
- generate_emotional_btn = gr.Button("Generate Emotional Speech", variant="primary")
900
-
901
- # Output audio
902
- audio_output_emotional = gr.Audio(label="Synthesized Audio")
903
- @gpu_decorator
904
- def generate_emotional_speech(
905
- regular_audio,
906
- regular_ref_text,
907
- gen_text,
908
- *args,
909
- ):
910
- num_additional_speech_types = max_speech_types - 1
911
- speech_type_names_list = args[:num_additional_speech_types]
912
- speech_type_audios_list = args[num_additional_speech_types:2 * num_additional_speech_types]
913
- speech_type_ref_texts_list = args[2 * num_additional_speech_types:3 * num_additional_speech_types]
914
- model_choice = args[3 * num_additional_speech_types]
915
- remove_silence = args[3 * num_additional_speech_types + 1]
916
-
917
- # Collect the speech types and their audios into a dict
918
- speech_types = {'Regular': {'audio': regular_audio, 'ref_text': regular_ref_text}}
919
-
920
- for name_input, audio_input, ref_text_input in zip(speech_type_names_list, speech_type_audios_list, speech_type_ref_texts_list):
921
- if name_input and audio_input:
922
- speech_types[name_input] = {'audio': audio_input, 'ref_text': ref_text_input}
923
-
924
- # Parse the gen_text into segments
925
- segments = parse_speechtypes_text(gen_text)
926
-
927
- # For each segment, generate speech
928
- generated_audio_segments = []
929
- current_emotion = 'Regular'
930
-
931
- for segment in segments:
932
- emotion = segment['emotion']
933
- text = segment['text']
934
-
935
- if emotion in speech_types:
936
- current_emotion = emotion
937
- else:
938
- # If emotion not available, default to Regular
939
- current_emotion = 'Regular'
940
-
941
- ref_audio = speech_types[current_emotion]['audio']
942
- ref_text = speech_types[current_emotion].get('ref_text', '')
943
-
944
- # Generate speech for this segment
945
- audio, _ = infer(ref_audio, ref_text, text, model_choice, remove_silence, 0)
946
- sr, audio_data = audio
947
-
948
- generated_audio_segments.append(audio_data)
949
-
950
- # Concatenate all audio segments
951
- if generated_audio_segments:
952
- final_audio_data = np.concatenate(generated_audio_segments)
953
- return (sr, final_audio_data)
954
- else:
955
- gr.Warning("No audio generated.")
956
- return None
957
-
958
- generate_emotional_btn.click(
959
- generate_emotional_speech,
960
- inputs=[
961
- regular_audio,
962
- regular_ref_text,
963
- gen_text_input_emotional,
964
- ] + speech_type_names + speech_type_audios + speech_type_ref_texts + [
965
- model_choice_emotional,
966
- remove_silence_emotional,
967
- ],
968
- outputs=audio_output_emotional,
969
- )
970
-
971
- # Validation function to disable Generate button if speech types are missing
972
- def validate_speech_types(
973
- gen_text,
974
- regular_name,
975
- *args
976
- ):
977
- num_additional_speech_types = max_speech_types - 1
978
- speech_type_names_list = args[:num_additional_speech_types]
979
-
980
- # Collect the speech types names
981
- speech_types_available = set()
982
- if regular_name:
983
- speech_types_available.add(regular_name)
984
- for name_input in speech_type_names_list:
985
- if name_input:
986
- speech_types_available.add(name_input)
987
-
988
- # Parse the gen_text to get the speech types used
989
- segments = parse_emotional_text(gen_text)
990
- speech_types_in_text = set(segment['emotion'] for segment in segments)
991
-
992
- # Check if all speech types in text are available
993
- missing_speech_types = speech_types_in_text - speech_types_available
994
-
995
- if missing_speech_types:
996
- # Disable the generate button
997
- return gr.update(interactive=False)
998
- else:
999
- # Enable the generate button
1000
- return gr.update(interactive=True)
1001
-
1002
- gen_text_input_emotional.change(
1003
- validate_speech_types,
1004
- inputs=[gen_text_input_emotional, regular_name] + speech_type_names,
1005
- outputs=generate_emotional_btn
1006
- )
1007
- with gr.Blocks() as app:
1008
- gr.Markdown(
1009
- """
1010
- # Antriksh AI
1011
- """
1012
- )
1013
-
1014
- # Add the image here
1015
- gr.Image(
1016
- value="C:\\Users\\USER\\Downloads\\logo-removebg-preview.png",
1017
- label="AI System Logo",
1018
- show_label=False,
1019
- width=300,
1020
- height=150
1021
- )
1022
-
1023
- gr.TabbedInterface([app_tts, app_podcast, app_emotional, app_credits], ["TTS", "Podcast", "Multi-Style", "Credits"])
1024
-
1025
-
1026
- @click.command()
1027
- @click.option("--port", "-p", default=None, type=int, help="Port to run the app on")
1028
- @click.option("--host", "-H", default=None, help="Host to run the app on")
1029
- @click.option(
1030
- "--share",
1031
- "-s",
1032
- default=False,
1033
- is_flag=True,
1034
- help="Share the app via Gradio share link",
1035
- )
1036
- @click.option("--api", "-a", default=True, is_flag=True, help="Allow API access")
1037
- def main(port, host, share, api):
1038
- global app
1039
- print(f"Starting app...")
1040
- app.queue(api_open=api).launch(
1041
- server_name=host, server_port=port, share=share, show_api=api
1042
- )
1043
-
1044
-
1045
- if __name__ == "__main__":
1046
- if not USING_SPACES:
1047
- main()
1048
- else:
1049
- app.queue().launch()