Create app.py

app.py

@@ -0,0 +1,155 @@
+import gradio as gr
+
+import torch
+
+from spectro import wav_bytes_from_spectrogram_image
+from diffusers import StableDiffusionPipeline
+
+from transformers import BlipForConditionalGeneration, BlipProcessor
+
+from share_btn import community_icon_html, loading_icon_html, share_js
+
+model_id = "riffusion/riffusion-model-v1"
+blip_model_id = "Salesforce/blip-image-captioning-base"
+pipe = StableDiffusionPipeline.from_pretrained(model_id)
+pipe = pipe.to("cuda")
+
+blip_model = BlipForConditionalGeneration.from_pretrained(blip_model_id, torch_dtype=torch.float16).to("cuda")
+processor = BlipProcessor.from_pretrained(blip_model_id)
+
+def predict(image):
+    inputs = processor(image, return_tensors="pt").to("cuda", torch.float16)
+    output_blip = blip_model.generate(**inputs)
+    prompt = processor.decode(output_blip[0], skip_special_tokens=True)
+    
+    spec = pipe(prompt).images[0]
+    print(spec)
+    wav = wav_bytes_from_spectrogram_image(spec)
+    with open("output.wav", "wb") as f:
+        f.write(wav[0].getbuffer())
+    return spec, 'output.wav', gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
+
+title = """
+    <div style="text-align: center; max-width: 500px; margin: 0 auto;">
+        <div
+        style="
+            display: inline-flex;
+            align-items: center;
+            gap: 0.8rem;
+            font-size: 1.75rem;
+            margin-bottom: 10px;
+        "
+        >
+        <h1 style="font-weight: 600; margin-bottom: 7px;">
+            Riffusion real-time prompt to image and to music generation system
+        </h1>
+        </div>
+        <p style="margin-bottom: 10px;font-size: 94%;font-weight: 100;line-height: 1.5em;">
+        Describe a musical prompt and generate a respective spectrogram image & musical sound associated with.
+    </div>
+"""
+
+article = """
+    <p style="font-size: 0.8em;line-height: 1.2em;border: 1px solid #374151;border-radius: 8px;padding: 20px;">
+    About the model: Riffusion is a latent text2img diffusion model capable of generating spectrogram images from a given text input prompts. These generated spectrograms are again then utilised to get converted into audio clips.
+    <br />—
+    <br />The Riffusion model was created by fine-tuning the Stable-Diffusion-v1-5 checkpoint.
+    <br />—
+    <br />The model is intended for research purposes only. Possible research areas and tasks include 
+    generation of artworks, audio, and use in creative processes, applications in educational or creative tools, research on generative models.
+    </p>
+    <div class="footer">
+        <p>
+        <a href="https://huggingface.co/riffusion/riffusion-model-v1" target="_blank">Riffusion model</a> by Seth Forsgren and Hayk Martiros - 
+        <a href="https://github.com/salesforce/BLIP" target="_blank"> BLIP Model </a> by Junnan Li et al. - Demo forked from 🤗 <a href="https://huggingface.co/nightfury" target="_blank">Nightfury</a>'s demo
+        </p>
+    </div>
+"""
+
+css = '''
+    #col-container, #col-container-2 {max-width: 510px; margin-left: auto; margin-right: auto;}
+    a {text-decoration-line: underline; font-weight: 600;}
+    div#record_btn > .mt-6 {
+        margin-top: 0!important;
+    }
+    div#record_btn > .mt-6 button {
+        width: 100%;
+        height: 40px;
+    }
+    .footer {
+        margin-bottom: 45px;
+        margin-top: 10px;
+        text-align: center;
+        border-bottom: 1px solid #e5e5e5;
+    }
+    .footer>p {
+        font-size: .8rem;
+        display: inline-block;
+        padding: 0 10px;
+        transform: translateY(10px);
+        background: white;
+    }
+    .dark .footer {
+        border-color: #303030;
+    }
+    .dark .footer>p {
+        background: #0b0f19;
+    }
+    .animate-spin {
+        animation: spin 1s linear infinite;
+    }
+    @keyframes spin {
+        from {
+            transform: rotate(0deg);
+        }
+        to {
+            transform: rotate(360deg);
+        }
+    }
+    #share-btn-container {
+        display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; width: 13rem;
+    }
+    #share-btn {
+        all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important;right:0;
+    }
+    #share-btn * {
+        all: unset;
+    }
+    #share-btn-container div:nth-child(-n+2){
+        width: auto !important;
+        min-height: 0px !important;
+    }
+    #share-btn-container .wrap {
+        display: none !important;
+    }
+'''
+ 
+
+
+with gr.Blocks(css=css) as demo:
+    
+    with gr.Column(elem_id="col-container"):
+        
+        gr.HTML(title)
+        
+        # prompt_input = gr.Textbox(placeholder="a cat diva singing in a New York jazz club", label="Musical prompt", elem_id="prompt-in")
+        image_input = gr.Image()
+        send_btn = gr.Button(value="Get a new riffusion spectrogram ! ", elem_id="submit-btn")
+            
+    with gr.Column(elem_id="col-container-2"):
+        
+        spectrogram_output = gr.Image(label="riffusion spectrogram image result", elem_id="img-out")
+        sound_output = gr.Audio(type='filepath', label="riffusion spectrogram sound", elem_id="music-out")
+        
+        with gr.Group(elem_id="share-btn-container"):
+            community_icon = gr.HTML(community_icon_html, visible=False)
+            loading_icon = gr.HTML(loading_icon_html, visible=False)
+            share_button = gr.Button("Share to community", elem_id="share-btn", visible=False)
+        
+        gr.HTML(article)
+    
+    send_btn.click(predict, inputs=[image_input], outputs=[spectrogram_output, sound_output, share_button, community_icon, loading_icon])
+    share_button.click(None, [], [], _js=share_js)
+
+demo.queue(max_size=250).launch(debug=True)
+