First try

app.py

@@ -217,84 +217,32 @@ def generate_base(subject, setting):
     play_steps_in_s = 4.0
     play_steps = int(frame_rate * play_steps_in_s)
 
-    gr.Info("Generating Audio")
     description = "Jenny speaks at an average pace with a calm delivery in a very confined sounding environment with clear audio quality."
-    story_tokens = tokenizer(model_input_tokens, return_tensors="pt", padding=True).input_ids.to(device)
-    description_tokens = tokenizer([description for _ in range(len(model_input_tokens))], return_tensors="pt").input_ids.to(device)
-    speech_output = model.generate(input_ids=description_tokens, prompt_input_ids=story_tokens)
-    speech_output = [output.cpu().numpy() for output in speech_output]
-    gr.Info("Generated Audio")
-    return None, None, {"audio": speech_output, "text": model_input_tokens}
-    
-def stream_audio(state):
-    speech_output = state["audio"]
-    sentences = state["text"]
-
-    gr.Info("Reading Story")
-
-    story = ""
-    for sentence, new_audio in zip(sentences, speech_output):
-#         print(f"i, j, time: {i}, {j} {datetime.datetime.now()}")
-        print(f"Sample of length: {round(new_audio.shape[0] / sampling_rate, 2)} seconds")
-        story += f"{sentence}\n"
-        yield story, numpy_to_mp3(new_audio, sampling_rate=sampling_rate)
-
-    # BATCH_SIZE = 4
-    # for i in range(0, len(model_input), BATCH_SIZE):
-    #     inputs = model_input[i:min(i + BATCH_SIZE, len(model_input))]
-    #     story_tokens = tokenizer(inputs, return_tensors="pt", padding=True).input_ids.to(device)
-    #     description_tokens = tokenizer([description for _ in range(len(inputs))], return_tensors="pt").input_ids.to(device)
-    #     speech_output = model.generate(input_ids=description_tokens, prompt_input_ids=story_tokens)
-        
-    #     speech_output = [output.cpu().numpy() for output in speech_output]
-    #     for j, new_audio in enumerate(speech_output):
-    #         if i + j == 0:
-    #             gr.Info("Reading story", duration=3)
-    #         print(f"i, j, time: {i}, {j} {datetime.datetime.now()}")
-    #         print(f"Sample of length: {round(new_audio.shape[0] / sampling_rate, 2)} seconds")
-    #         yield story,  numpy_to_mp3(new_audio, sampling_rate=sampling_rate)
-        
-    #     if len(inputs) != 0:
-    # input_ids = tokenizer(description, return_tensors="pt").input_ids.to(device)
-    # story = tokenizer(model_input, return_tensors="pt", padding=True).input_ids.to(device)
-
-    # speech_output = model.generate(input_ids=description_tokens, prompt_input_ids=story)
-        
-    # speech_output = [output.cpu().numpy() for output in speech_output]
-
-    # for i, new_audio in enumerate(speech_output):
-    #     if i == 0:
-    #         gr.Info("Reading story", duration=3)
-    #     print(f"Sample of length: {round(new_audio.shape[0] / sampling_rate, 2)} seconds")
-    #     yield story, numpy_to_mp3(new_audio, sampling_rate=sampling_rate)
-    
-    # print(f"{i}-th part generated")
-    # pieces += [*speech_output, silence.copy()]
-
-    # for i, sentence in enumerate(model_input):
-    #     streamer = ParlerTTSStreamer(model, device=device, play_steps=play_steps)
-
-    #     prompt = tokenizer(sentence, return_tensors="pt").to(device)
-
-    #     generation_kwargs = dict(
-    #         input_ids=inputs.input_ids,
-    #         prompt_input_ids=prompt.input_ids,
-    #         streamer=streamer,
-    #         do_sample=True,
-    #         temperature=1.0,
-    #         min_new_tokens=10,
-    #     )
-
-    #     set_seed(SEED)
-    #     thread = Thread(target=model.generate, kwargs=generation_kwargs)
-    #     thread.start()
-
-        # for new_audio in streamer:
-        #     if i == 0:
-        #         gr.Info("Reading story", duration=3)
-        #     print(f"Sample of length: {round(new_audio.shape[0] / sampling_rate, 2)} seconds")
-        #     yield story, numpy_to_mp3(new_audio, sampling_rate=sampling_rate)
+    description_tokens = tokenizer(description, return_tensors="pt").to(device)
 
+    for i, sentence in enumerate(model_input):
+        streamer = ParlerTTSStreamer(model, device=device, play_steps=play_steps)
+
+        prompt = tokenizer(sentence, return_tensors="pt").to(device)
+
+        generation_kwargs = dict(
+            input_ids=description_tokens.input_ids,
+            prompt_input_ids=prompt.input_ids,
+            streamer=streamer,
+            do_sample=True,
+            temperature=1.0,
+            min_new_tokens=10,
+        )
+
+        set_seed(SEED)
+        thread = Thread(target=model.generate, kwargs=generation_kwargs)
+        thread.start()
+
+        for new_audio in streamer:
+            if i == 0:
+                gr.Info("Reading story", duration=3)
+            print(f"Sample of length: {round(new_audio.shape[0] / sampling_rate, 2)} seconds")
+            yield story, numpy_to_mp3(new_audio, sampling_rate=sampling_rate)
 
 with gr.Blocks() as block:
     gr.HTML(
@@ -319,5 +267,4 @@ with gr.Blocks() as block:
     state = gr.State()
     run_button.click(fn=generate_base, inputs=inputs, outputs=[story, audio_out, state]).success(stream_audio, inputs=state, outputs=outputs)
 
-block.queue()
-block.launch(share=True)
+block.launch()