File size: 5,914 Bytes
04646cc ddb246b 04646cc 00fcfb1 04646cc 00fcfb1 04646cc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 |
import gradio as gr
import torch
from spectro import wav_bytes_from_spectrogram_image
from diffusers import StableDiffusionPipeline
from transformers import BlipForConditionalGeneration, BlipProcessor
from share_btn import community_icon_html, loading_icon_html, share_js
model_id = "riffusion/riffusion-model-v1"
blip_model_id = "Salesforce/blip-image-captioning-base"
pipe = StableDiffusionPipeline.from_pretrained(model_id)
pipe = pipe.to("cpu") #cuda
blip_model = BlipForConditionalGeneration.from_pretrained(blip_model_id, torch_dtype=torch.float).to("CPU") #CUDA
processor = BlipProcessor.from_pretrained(blip_model_id)
def predict(image):
inputs = processor(image, return_tensors="pt").to("cpu", torch.float) #cuda
output_blip = blip_model.generate(**inputs)
prompt = processor.decode(output_blip[0], skip_special_tokens=True)
spec = pipe(prompt).images[0]
print(spec)
wav = wav_bytes_from_spectrogram_image(spec)
with open("output.wav", "wb") as f:
f.write(wav[0].getbuffer())
return spec, 'output.wav', gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
title = """
<div style="text-align: center; max-width: 500px; margin: 0 auto;">
<div
style="
display: inline-flex;
align-items: center;
gap: 0.8rem;
font-size: 1.75rem;
margin-bottom: 10px;
"
>
<h1 style="font-weight: 600; margin-bottom: 7px;">
Riffusion real-time prompt to image and to music generation system
</h1>
</div>
<p style="margin-bottom: 10px;font-size: 94%;font-weight: 100;line-height: 1.5em;">
Describe a musical prompt and generate a respective spectrogram image & musical sound associated with.
</div>
"""
article = """
<p style="font-size: 0.8em;line-height: 1.2em;border: 1px solid #374151;border-radius: 8px;padding: 20px;">
About the model: Riffusion is a latent text2img diffusion model capable of generating spectrogram images from a given text input prompts. These generated spectrograms are again then utilised to get converted into audio clips.
<br />—
<br />The Riffusion model was created by fine-tuning the Stable-Diffusion-v1-5 checkpoint.
<br />—
<br />The model is intended for research purposes only. Possible research areas and tasks include
generation of artworks, audio, and use in creative processes, applications in educational or creative tools, research on generative models.
</p>
<div class="footer">
<p>
<a href="https://huggingface.co/riffusion/riffusion-model-v1" target="_blank">Riffusion model</a> by Seth Forsgren and Hayk Martiros -
<a href="https://github.com/salesforce/BLIP" target="_blank"> BLIP Model </a> by Junnan Li et al. - Demo forked from 🤗 <a href="https://huggingface.co/nightfury" target="_blank">Nightfury</a>'s demo
</p>
</div>
"""
css = '''
#col-container, #col-container-2 {max-width: 510px; margin-left: auto; margin-right: auto;}
a {text-decoration-line: underline; font-weight: 600;}
div#record_btn > .mt-6 {
margin-top: 0!important;
}
div#record_btn > .mt-6 button {
width: 100%;
height: 40px;
}
.footer {
margin-bottom: 45px;
margin-top: 10px;
text-align: center;
border-bottom: 1px solid #e5e5e5;
}
.footer>p {
font-size: .8rem;
display: inline-block;
padding: 0 10px;
transform: translateY(10px);
background: white;
}
.dark .footer {
border-color: #303030;
}
.dark .footer>p {
background: #0b0f19;
}
.animate-spin {
animation: spin 1s linear infinite;
}
@keyframes spin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
#share-btn-container {
display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; width: 13rem;
}
#share-btn {
all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important;right:0;
}
#share-btn * {
all: unset;
}
#share-btn-container div:nth-child(-n+2){
width: auto !important;
min-height: 0px !important;
}
#share-btn-container .wrap {
display: none !important;
}
'''
with gr.Blocks(css=css) as demo:
with gr.Column(elem_id="col-container"):
gr.HTML(title)
# prompt_input = gr.Textbox(placeholder="a cat diva singing in a New York jazz club", label="Musical prompt", elem_id="prompt-in")
image_input = gr.Image()
send_btn = gr.Button(value="Get a new riffusion spectrogram ! ", elem_id="submit-btn")
with gr.Column(elem_id="col-container-2"):
spectrogram_output = gr.Image(label="riffusion spectrogram image result", elem_id="img-out")
sound_output = gr.Audio(type='filepath', label="riffusion spectrogram sound", elem_id="music-out")
with gr.Group(elem_id="share-btn-container"):
community_icon = gr.HTML(community_icon_html, visible=False)
loading_icon = gr.HTML(loading_icon_html, visible=False)
share_button = gr.Button("Share to community", elem_id="share-btn", visible=False)
gr.HTML(article)
send_btn.click(predict, inputs=[image_input], outputs=[spectrogram_output, sound_output, share_button, community_icon, loading_icon])
share_button.click(None, [], [], _js=share_js)
demo.queue(max_size=250).launch(debug=True)
|