app.py

import gradio as gr
import spaces
import torch
import torchaudio
import io
import base64
import uuid
import os
import time
import re
import threading
import gc
import random
import numpy as np
from einops import rearrange
from huggingface_hub import login
from stable_audio_tools import get_pretrained_model
from stable_audio_tools.inference.generation import generate_diffusion_cond
from gradio_client import Client, handle_file
from contextlib import contextmanager

# Global model storage
model_cache = {}
model_lock = threading.Lock()

@contextmanager
def resource_cleanup():
    """Lightweight context manager - let zerogpu handle memory management"""
    try:
        yield
    finally:
        # Minimal cleanup - let zerogpu handle the heavy lifting
        if torch.cuda.is_available():
            torch.cuda.synchronize()
        # Removed aggressive empty_cache() and gc.collect() calls

def load_stable_audio_model():
    """Load stable-audio-open-small model if not already loaded."""
    with model_lock:
        if 'stable_audio_model' not in model_cache:
            print("🔄 Loading stable-audio-open-small model...")
            load_start = time.time()
            
            # Authenticate with HF
            hf_token = os.getenv('HF_TOKEN')
            if hf_token:
                login(token=hf_token)
                print(f"✅ HF authenticated")
            
            # Load model
            model, config = get_pretrained_model("stabilityai/stable-audio-open-small")
            device = "cuda" if torch.cuda.is_available() else "cpu"
            model = model.to(device)
            if device == "cuda":
                model = model.half()
            
            load_time = time.time() - load_start
            print(f"✅ Model loaded on {device} in {load_time:.2f}s")
            
            # Aggressive model persistence - warm up with dummy generation
            print("🔥 Warming up model...")
            warmup_start = time.time()
            try:
                dummy_conditioning = [{"prompt": "test", "seconds_total": 12}]
                with torch.no_grad():
                    _ = generate_diffusion_cond(
                        model,
                        steps=1,  # Minimal steps for warmup
                        cfg_scale=1.0,
                        conditioning=dummy_conditioning,
                        sample_size=config["sample_size"],
                        sampler_type="pingpong",
                        device=device,
                        seed=42
                    )
                warmup_time = time.time() - warmup_start
                print(f"🔥 Model warmed up in {warmup_time:.2f}s")
            except Exception as e:
                print(f"⚠️ Warmup failed (but continuing): {e}")
            
            model_cache['stable_audio_model'] = model
            model_cache['stable_audio_config'] = config
            model_cache['stable_audio_device'] = device
            print(f"✅ Stable Audio model ready for fast generation!")
        else:
            print("♻️ Using cached model (should be fast!)")
        
        return (model_cache['stable_audio_model'], 
                model_cache['stable_audio_config'], 
                model_cache['stable_audio_device'])

@spaces.GPU(duration=12)
def generate_stable_audio_loop(prompt, loop_type, bpm, bars, seed=-1):
    """Generate a BPM-aware loop using stable-audio-open-small"""
    try:
        total_start = time.time()
        
        # Model loading timing
        load_start = time.time()
        model, config, device = load_stable_audio_model()
        load_time = time.time() - load_start
        
        # Calculate loop duration based on BPM and bars
        seconds_per_beat = 60.0 / bpm
        seconds_per_bar = seconds_per_beat * 4  # 4/4 time
        target_loop_duration = seconds_per_bar * bars
        
        # Enhance prompt based on loop type and BPM
        if loop_type == "drums":
            enhanced_prompt = f"{prompt} drum loop {bpm}bpm"
            negative_prompt = "melody, harmony, pitched instruments, vocals, singing"
        else:  # instruments
            enhanced_prompt = f"{prompt} instrumental loop {bpm}bpm"
            negative_prompt = "drums, percussion, kick, snare, hi-hat"
        
        # Set seed
        if seed == -1:
            seed = random.randint(0, 2**32 - 1)
        
        torch.manual_seed(seed)
        if device == "cuda":
            torch.cuda.manual_seed(seed)
        
        print(f"🎵 Generating {loop_type} loop:")
        print(f"   Enhanced prompt: {enhanced_prompt}")
        print(f"   Target duration: {target_loop_duration:.2f}s ({bars} bars at {bpm}bpm)")
        print(f"   Seed: {seed}")
        
        # Prepare conditioning
        conditioning_start = time.time()
        conditioning = [{
            "prompt": enhanced_prompt,
            "seconds_total": 12  # Model generates 12s max
        }]
        
        negative_conditioning = [{
            "prompt": negative_prompt,
            "seconds_total": 12
        }]
        conditioning_time = time.time() - conditioning_start
        
        # Generation timing
        generation_start = time.time()
        
        # Removed aggressive resource cleanup wrapper
        # Clear GPU cache once before generation (not after)
        if device == "cuda":
            torch.cuda.empty_cache()
        
        with torch.cuda.amp.autocast(enabled=(device == "cuda")):
            output = generate_diffusion_cond(
                model,
                steps=8,  # Fast generation
                cfg_scale=1.0,  # Good balance for loops
                conditioning=conditioning,
                negative_conditioning=negative_conditioning,
                sample_size=config["sample_size"],
                sampler_type="pingpong",
                device=device,
                seed=seed
            )
        
        generation_time = time.time() - generation_start
        
        # Post-processing timing
        postproc_start = time.time()
        
        # Post-process audio
        output = rearrange(output, "b d n -> d (b n)")  # (2, N) stereo
        output = output.to(torch.float32).div(torch.max(torch.abs(output))).clamp(-1, 1)
        
        # Extract the loop portion
        sample_rate = config["sample_rate"]
        loop_samples = int(target_loop_duration * sample_rate)
        available_samples = output.shape[1]
        
        if loop_samples > available_samples:
            loop_samples = available_samples
            actual_duration = available_samples / sample_rate
            print(f"⚠️ Requested {target_loop_duration:.2f}s, got {actual_duration:.2f}s")
        
        # Extract loop from beginning (cleanest beat alignment)
        loop_output = output[:, :loop_samples]
        loop_output_int16 = loop_output.mul(32767).to(torch.int16).cpu()
        
        # Save to temporary file
        loop_filename = f"loop_{loop_type}_{bpm}bpm_{bars}bars_{seed}.wav"
        torchaudio.save(loop_filename, loop_output_int16, sample_rate)
        
        postproc_time = time.time() - postproc_start
        total_time = time.time() - total_start
        actual_duration = loop_samples / sample_rate
        
        # Detailed timing breakdown
        print(f"⏱️ Timing breakdown:")
        print(f"   Model load: {load_time:.2f}s")
        print(f"   Conditioning: {conditioning_time:.3f}s") 
        print(f"   Generation: {generation_time:.2f}s")
        print(f"   Post-processing: {postproc_time:.3f}s")
        print(f"   Total: {total_time:.2f}s")
        print(f"✅ {loop_type.title()} loop: {actual_duration:.2f}s audio in {total_time:.2f}s")
        
        return loop_filename, f"Generated {actual_duration:.2f}s {loop_type} loop at {bpm}bpm ({bars} bars) in {total_time:.2f}s"
    
    except Exception as e:
        print(f"❌ Generation error: {str(e)}")
        return None, f"Error: {str(e)}"

def combine_loops(drums_audio, instruments_audio, bpm, bars, num_repeats):
    """Combine drum and instrument loops with specified repetitions"""
    try:
        if not drums_audio and not instruments_audio:
            return None, "No audio files to combine"
        
        # Calculate timing
        seconds_per_beat = 60.0 / bpm
        seconds_per_bar = seconds_per_beat * 4
        loop_duration = seconds_per_bar * bars
        total_duration = loop_duration * num_repeats
        
        print(f"🎛️ Combining loops:")
        print(f"   Loop duration: {loop_duration:.2f}s ({bars} bars)")
        print(f"   Repeats: {num_repeats}")
        print(f"   Total duration: {total_duration:.2f}s")
        
        combined_audio = None
        sample_rate = None
        
        # Process each audio file
        for audio_path, audio_type in [(drums_audio, "drums"), (instruments_audio, "instruments")]:
            if audio_path:
                # Load audio
                waveform, sr = torchaudio.load(audio_path)
                if sample_rate is None:
                    sample_rate = sr
                
                # Ensure we have the exact loop duration
                target_samples = int(loop_duration * sr)
                if waveform.shape[1] > target_samples:
                    waveform = waveform[:, :target_samples]
                elif waveform.shape[1] < target_samples:
                    # Pad if necessary
                    padding = target_samples - waveform.shape[1]
                    waveform = torch.cat([waveform, torch.zeros(waveform.shape[0], padding)], dim=1)
                
                # Repeat the loop
                repeated_waveform = waveform.repeat(1, num_repeats)
                
                print(f"   {audio_type}: {waveform.shape[1]/sr:.2f}s repeated {num_repeats}x = {repeated_waveform.shape[1]/sr:.2f}s")
                
                # Add to combined audio
                if combined_audio is None:
                    combined_audio = repeated_waveform
                else:
                    combined_audio = combined_audio + repeated_waveform
        
        if combined_audio is None:
            return None, "No valid audio to combine"
        
        # Normalize to prevent clipping
        combined_audio = combined_audio / torch.max(torch.abs(combined_audio))
        combined_audio = combined_audio.clamp(-1, 1)
        
        # Convert to int16 and save
        combined_audio_int16 = combined_audio.mul(32767).to(torch.int16)
        combined_filename = f"combined_{bpm}bpm_{bars}bars_{num_repeats}loops_{random.randint(1000, 9999)}.wav"
        torchaudio.save(combined_filename, combined_audio_int16, sample_rate)
        
        actual_duration = combined_audio.shape[1] / sample_rate
        status = f"Combined into {actual_duration:.2f}s audio ({num_repeats} × {bars} bars at {bpm}bpm)"
        
        print(f"✅ {status}")
        return combined_filename, status
    
    except Exception as e:
        print(f"❌ Combine error: {str(e)}")
        return None, f"Combine error: {str(e)}"

def transform_with_melodyflow_api(audio_path, prompt, solver="euler", flowstep=0.12):
    """Transform audio using Facebook/MelodyFlow space API"""
    if audio_path is None:
        return None, "❌ No audio file provided"
    
    try:
        # Initialize client for Facebook MelodyFlow space
        client = Client("facebook/MelodyFlow")
        
        # Set steps based on solver
        if solver == "midpoint":
            base_steps = 128
            effective_steps = base_steps // 2  # 64 effective steps
        else:  # euler
            base_steps = 125  
            effective_steps = base_steps // 5  # 25 effective steps
        
        print(f"🎛️ MelodyFlow transformation:")
        print(f"   Prompt: {prompt}")
        print(f"   Solver: {solver} ({effective_steps} effective steps)")
        print(f"   Flowstep: {flowstep}")
        
        # Call the MelodyFlow API
        result = client.predict(
            model="facebook/melodyflow-t24-30secs",
            text=prompt,
            solver=solver,
            steps=base_steps,
            target_flowstep=flowstep,
            regularize=solver == "euler",
            regularization_strength=0.2,
            duration=30,
            melody=handle_file(audio_path),
            api_name="/predict"
        )
        
        if result and len(result) > 0 and result[0]:
            # Save the result locally
            output_filename = f"melodyflow_transformed_{random.randint(1000, 9999)}.wav"
            import shutil
            shutil.copy2(result[0], output_filename)
            
            status_msg = f"✅ Transformed with prompt: '{prompt}' (flowstep: {flowstep}, {effective_steps} steps)"
            return output_filename, status_msg
        else:
            return None, "❌ MelodyFlow API returned no results"
            
    except Exception as e:
        return None, f"❌ MelodyFlow API error: {str(e)}"

def calculate_optimal_bars(bpm):
    """Calculate optimal bar count for given BPM to fit in ~10s"""
    seconds_per_beat = 60.0 / bpm
    seconds_per_bar = seconds_per_beat * 4
    max_duration = 10.0
    
    for bars in [8, 4, 2, 1]:
        if seconds_per_bar * bars <= max_duration:
            return bars
    return 1

# ========== GRADIO INTERFACE ==========

with gr.Blocks(title="stable-melodyflow") as iface:
    gr.Markdown("# stable-melodyflow (aka jerry and terry)")
    gr.Markdown("**generate synchronized drum and instrument loops with stable-audio-open-small (jerry), then transform with melodyflow (terry)!**")
    
    # ========== MODELS & PROJECT INFO ==========
    with gr.Accordion(" some info about these models", open=False):
        
        with gr.Accordion("🚀 stable-audio-open-small", open=False):
            gr.Markdown("""

            **stable-audio-open-small** is an incredibly fast model from the zachs and friends at Stability AI. It's capable of generating 12 seconds of audio in under a second, which gives rise to a lot of very interesting kinds of UX.

            

            **note about generation speed in this zerogpu space:** you'll notice generation times are a little slower here than if you were to use the model on a local gpu. that's just a result of the way zerogpu spaces work i think... let me know if there's a way to keep the model loaded in a zerogpu space!

            

            **links:**

            - 🤗 [model on HuggingFace](https://huggingface.co/stabilityai/stable-audio-open-small)

                        there's a docker container at this repo that can be spun up as a standalone api specifically for stable-audio-open-small:

            -  [stable-audio-api](https://github.com/betweentwomidnights/stable-audio-api)

            """)
        
        with gr.Accordion("🎛️ melodyflow", open=False):
            gr.Markdown("""

            **MelodyFlow** is a model by meta that can use regularized latent inversion to do transformations of input audio.

            

            It's not officially a part of the audiocraft repo yet, but we use it as a docker container in the backend for gary4live. i really enjoy turning my guitar riffs into orchestra.

            

            **links:**

            - 🤗 [Official MelodyFlow Space](https://huggingface.co/spaces/Facebook/MelodyFlow)  

                        

            -  [our melodyflow api](https://github.com/betweentwomidnights/melodyflow)

            """)
        
        with gr.Accordion("gary4live Project", open=False):
            gr.Markdown("""

            **gary4live** is a free/open source project that uses these models, along with musicGen, inside of ableton live to iterate on your projects with you. i run a backend myself so that we can all experiment with it, but you can also spin the backend up locally using docker-compose with our repo.

            

            **project Links:**

            -  [frontend repo](https://github.com/betweentwomidnights/gary4live)

            -  [backend repo](https://github.com/betweentwomidnights/gary-backend-combined)

            

            **installers:**

            -  [p.c. & mac installers on gumroad](https://thepatch.gumroad.com/l/gary4live)

            """)
    
    with gr.Accordion("how this works", open=False):
        gr.Markdown("""

        **workflow:**

        1. **set global bpm and bars** - affects both drum and instrument generation

        2. **generate drum loop** - creates BPM-aware percussion with negative prompting to attempt to get rid of instruments

        3. **generate instrument loop** - creates melodic/harmonic content with negative prompting to attempt to get rid of drums 

        4. **combine loops** - layer them together with repetitions (up to 30s)

        5. **transform** - use melodyflow to stylistically transform the combined result

        

        **features:**

        - bpm-aware generation ensures perfect sync between loops (most the time lol)

        - negative prompting separates drums from instruments (most the time)

        - smart bar calculation optimizes loop length for the BPM

        """)
    
    # ========== GLOBAL CONTROLS ==========
    gr.Markdown("## 🎛️ global settings")
    
    with gr.Row():
        global_bpm = gr.Dropdown(
            label="global bpm",
            choices=[90, 100, 110, 120, 130, 140, 150],
            value=120,
            info="bpm applied to both drum and instrument generation. keep this the same for the combine step to work correctly"
        )
        
        global_bars = gr.Dropdown(
            label="loop length (bars)",
            choices=[1, 2, 4],
            value=4,
            info="number of bars for each loop. keep this the same for both pieces of audio"
        )
        
        base_prompt = gr.Textbox(
            label="base prompt",
            value="lofi hiphop with pianos",
            placeholder="e.g., 'aggressive techno', 'lofi hiphop', 'chillwave', 'liquid drum and bass'",
            info="prompt applied to either loop. make it more drum/instrument specific for best results"
        )
    
    # Auto-suggest optimal bars based on BPM
    def update_suggested_bars(bpm):
        optimal = calculate_optimal_bars(bpm)
        return gr.update(info=f"Suggested: {optimal} bars for {bpm}bpm (≤10s)")
    
    global_bpm.change(update_suggested_bars, inputs=[global_bpm], outputs=[global_bars])
    
    # ========== LOOP GENERATION ==========
    gr.Markdown("## step one: generate individual loops")
    
    with gr.Row():
        with gr.Column():
            gr.Markdown("### drums")
            generate_drums_btn = gr.Button("generate drums", variant="primary", size="lg")
            drums_audio = gr.Audio(label="drum loop", type="filepath")
            drums_status = gr.Textbox(label="status", value="ready to generate")
            
        with gr.Column():
            gr.Markdown("### instruments")
            generate_instruments_btn = gr.Button("generate instruments", variant="secondary", size="lg")
            instruments_audio = gr.Audio(label="instrument loop", type="filepath")
            instruments_status = gr.Textbox(label="status", value="Ready to generate")
    
    # Seed controls
    with gr.Row():
        drums_seed = gr.Number(label="drums seed", value=-1, info="-1 for random")
        instruments_seed = gr.Number(label="instruments seed", value=-1, info="-1 for random")
    
    # ========== COMBINATION ==========
    gr.Markdown("## step two: combine loops")
    
    with gr.Row():
        num_repeats = gr.Slider(
            label="number of repetitions",
            minimum=1,
            maximum=5,
            step=1,
            value=2,
            info="how many times to repeat each loop (creates longer audio). aim for 30 seconds max"
        )
        combine_btn = gr.Button("combine", variant="primary", size="lg")
    
    combined_audio = gr.Audio(label="combined loops", type="filepath")
    combine_status = gr.Textbox(label="status", value="Generate loops first")
    
    # ========== MELODYFLOW TRANSFORMATION ==========
    gr.Markdown("## step three: transform with melodyflow")
    
    with gr.Row():
        with gr.Column():
            transform_prompt = gr.Textbox(
                label="transformation prompt",
                value="aggressive industrial techno with distorted sounds",
                placeholder="describe the style of transformation",
                lines=2
            )
            
        with gr.Column():
            transform_solver = gr.Dropdown(
                label="solver",
                choices=["euler", "midpoint"],
                value="euler",
                info="EULER: faster (25 steps), MIDPOINT: slower (64 steps)"
            )
            transform_flowstep = gr.Slider(
                label="transform intensity",
                minimum=0.0,
                maximum=0.15,
                step=0.01,
                value=0.12,
                info="Lower = more dramatic transformation"
            )
    
    transform_btn = gr.Button("transform audio", variant="secondary", size="lg")
    transformed_audio = gr.Audio(label="transformed audio", type="filepath")
    transform_status = gr.Textbox(label="status", value="Combine audio first")
    
    # ========== EVENT HANDLERS ==========
    
    # Generate drums
    generate_drums_btn.click(
        generate_stable_audio_loop,
        inputs=[base_prompt, gr.State("drums"), global_bpm, global_bars, drums_seed],
        outputs=[drums_audio, drums_status]
    )
    
    # Generate instruments  
    generate_instruments_btn.click(
        generate_stable_audio_loop,
        inputs=[base_prompt, gr.State("instruments"), global_bpm, global_bars, instruments_seed],
        outputs=[instruments_audio, instruments_status]
    )
    
    # Combine loops
    combine_btn.click(
        combine_loops,
        inputs=[drums_audio, instruments_audio, global_bpm, global_bars, num_repeats],
        outputs=[combined_audio, combine_status]
    )
    
    # Transform with MelodyFlow
    transform_btn.click(
        transform_with_melodyflow_api,
        inputs=[combined_audio, transform_prompt, transform_solver, transform_flowstep],
        outputs=[transformed_audio, transform_status]
    )
    
    # # ========== EXAMPLES ==========
    # gr.Markdown("## 🎯 Example Workflows")
    
    # examples = gr.Examples(
    #     examples=[
    #         ["techno", 128, 4, "aggressive industrial techno"],
    #         ["jazz", 110, 2, "smooth lo-fi jazz with vinyl crackle"],
    #         ["ambient", 90, 8, "ethereal ambient soundscape"],
    #         ["hip-hop", 100, 4, "classic boom bap hip-hop"],
    #         ["drum and bass", 140, 4, "liquid drum and bass"],
    #     ],
    #     inputs=[base_prompt, global_bpm, global_bars, transform_prompt],
    # )

if __name__ == "__main__":
    iface.launch()