import whisper model = whisper.load_model("base") model.device import gradio as gr from keybert import KeyBERT import random as r from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler import torch model_id = 'prompthero/midjourney-v4-diffusion' #"stabilityai/stable-diffusion-2" # Use the Euler scheduler here instead scheduler = EulerDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") pipe = StableDiffusionPipeline.from_pretrained(model_id , torch_dtype=torch.float16) #pipe = StableDiffusionPipeline.from_pretrained(model_id, scheduler=scheduler, revision="fp16", torch_dtype=torch.float16) pipe = pipe.to("cuda") # from IPython.display import Image from PIL import Image import time import matplotlib.pyplot as plt import numpy as np import PIL # import cv2 def transcribe(audio,prompt_num,user_keywords): # load audio and pad/trim it to fit 30 seconds audio1 = whisper.load_audio(audio) audio1 = whisper.pad_or_trim(audio1) # make log-Mel spectrogram and move to the same device as the model mel = whisper.log_mel_spectrogram(audio1).to(model.device) # detect the spoken language _, probs = model.detect_language(mel) print(f"Detected language: {max(probs, key=probs.get)}") # decode the audio options = whisper.DecodingOptions() result = whisper.decode(model, mel, options) print(result.text) # model = whisper.load_model("base") audio2 = whisper.load_audio(audio) final_result = model.transcribe(audio2) print(final_result["text"]) return final_result["text"],int(prompt_num),user_keywords def keywords(text,prompt_num,user_keywords): # ub = UrlBuilder("demo.imgix.net") kw_model = KeyBERT() a = kw_model.extract_keywords(text, keyphrase_ngram_range=(1, 3), stop_words=None) set_1 = [i[0] for i in a] b = kw_model.extract_keywords(text, keyphrase_ngram_range=(1, 3), stop_words='english', use_maxsum=True, nr_candidates=20, top_n=5) set_2 = [i[0] for i in b] c = kw_model.extract_keywords(text, keyphrase_ngram_range=(1, 3), stop_words='english', use_mmr=True, diversity=0.7) set_3 = [i[0] for i in c] d = kw_model.extract_keywords(text, keyphrase_ngram_range=(1, 3), stop_words='english', use_mmr=True, diversity=0.2) set_4 = [i[0] for i in d] keyword_pool = set_1 + set_2 + set_3 + set_4 print("keywords: ", keyword_pool, "length: ", len(keyword_pool)) generated_prompts = [] count = 0 while count != int(prompt_num): sentence = [] style_prompts = ["perfect shading, soft studio lighting, ultra-realistic, photorealistic, octane render, cinematic lighting, hdr, in-frame, 4k, 8k, edge lighting", "detailed, colourful, psychedelic, unreal engine, octane render, blender effect", "mechanical features, cybernetic eyes, baroque, rococo, anodized titanium highly detailed mechanisms, gears, fiber, cogs, bulbs, wires, cables, 70mm, Canon EOS 6D Mark II, 4k, 35mm (FX, Full-Frame), f/2.5, extremely detailed, very high details, photorealistic, hi res, hdr, UHD, hyper-detailed, ultra-realistic, vibrant, centered, vivid colors, Wide angle, zoom out", "detailed, soft ambiance, japanese influence, unreal engine 5, octane render", "perfect shading, soft studio lighting, ultra-realistic, photorealistic, octane render, cinematic lighting, hdr, in-frame, 4k, 8k, edge lighting --v 4"] my_list = user_keywords.split(',') print(my_list) # for i in range(len(my_list)): # sentence.append(my_list[i]) # numb = 5 for i in range(len(my_list)): # print("keyword_pool",keyword_pool, len(keyword_pool)) sentence.append("mdjrny-v4 style") for i in range (len(my_list)): sentence.append(my_list[i]) rand_1 = r.randint(1, 4) if rand_1 == 1: sentence.append(r.choice(set_1)) sentence.append(r.choice(set_1)) sentence.append(r.choice(set_2)) sentence.append(r.choice(set_3)) sentence.append(r.choice(set_4)) elif rand_1 == 2: sentence.append(r.choice(set_2)) sentence.append(r.choice(set_2)) sentence.append(r.choice(set_1)) sentence.append(r.choice(set_3)) sentence.append(r.choice(set_4)) elif rand_1 == 3: sentence.append(r.choice(set_3)) sentence.append(r.choice(set_3)) sentence.append(r.choice(set_1)) sentence.append(r.choice(set_2)) sentence.append(r.choice(set_4)) else: sentence.append(r.choice(set_4)) sentence.append(r.choice(set_4)) sentence.append(r.choice(set_1)) sentence.append(r.choice(set_2)) sentence.append(r.choice(set_3)) # Add Style Tail Prompt sentence.append(r.choice(style_prompts)) print("sentence: ", sentence) # Formatting Data as comma-delimited for Mid Journey myprompt = ', '.join(str(e) for e in sentence) sentence = [] print("prompt: ",myprompt) generated_prompts.append(myprompt) count += 1 print("no. of prompts: ", len(generated_prompts)) print("generated prompts: ", generated_prompts) count = 0 images = [] # np_images = [] while count != int(len(generated_prompts)): for i in generated_prompts: count += 1 print(i) image = pipe(i, height=768, width=768, guidance_scale = 10).images[0] # image.save("/content/drive/MyDrive/ColabNotebooks/GeneratedImages/" + "sd_image_" +str(count)+ ".png") images.append(image) # pick the image which is the smallest, and resize the others to match it (can be arbitrary image shape here) min_shape = sorted( [(np.sum(i.size), i.size ) for i in images])[0][1] imgs_comb = np.hstack([i.resize(min_shape) for i in images]) # save that beautiful picture imgs_comb = Image.fromarray( imgs_comb) # imgs_comb.save("/content/drive/MyDrive/ColabNotebooks/GeneratedImages/" + "Combined.png") # return imgs_comb #for combined image return images speech_text = gr.Interface(fn=transcribe, inputs=[gr.Audio(source="microphone", type="filepath"),gr.Number(placeholder = "Number of Images to be generated (int): "),gr.Textbox(placeholder = "Additional keywords (comma delimitied): ")], outputs=["text","number","text"], title = 'Speech to Image Generator', enable_queue=True) text_prompts = gr.Interface(fn=keywords, inputs=["text","number","text"], outputs=gr.Gallery(label="Generated images", show_label=False, elem_id="gallery").style(grid=[2], height="auto"), title = 'Speech to Image Generator', enable_queue=True) gr.Series(speech_text,text_prompts).launch(inline = False, share=True, enable_queue=True).queue()