Spaces:
Runtime error
Runtime error
| """ | |
| Copyright (c) Meta Platforms, Inc. and affiliates. | |
| All rights reserved. | |
| This source code is licensed under the license found in the | |
| LICENSE file in the root directory of this source tree. | |
| """ | |
| from tempfile import NamedTemporaryFile | |
| import torch | |
| import gradio as gr | |
| from audiocraft.models import MusicGen | |
| from audiocraft.data.audio import audio_write | |
| MODEL = None | |
| img_to_text = gr.load(name="spaces/fffiloni/CLIP-Interrogator-2") | |
| def load_model(version): | |
| print("Loading model", version) | |
| return MusicGen.get_pretrained(version) | |
| def predict(uploaded_image, melody, duration): | |
| text = img_to_text(uploaded_image, 'best', 4, fn_index=1)[0] | |
| global MODEL | |
| topk = int(250) | |
| if MODEL is None or MODEL.name != "melody": | |
| MODEL = load_model("melody") | |
| if duration > MODEL.lm.cfg.dataset.segment_duration: | |
| raise gr.Error("MusicGen currently supports durations of up to 30 seconds!") | |
| MODEL.set_generation_params( | |
| use_sampling=True, | |
| top_k=250, | |
| top_p=0, | |
| temperature=1.0, | |
| cfg_coef=3.0, | |
| duration=duration, | |
| ) | |
| if melody: | |
| sr, melody = melody[0], torch.from_numpy(melody[1]).to(MODEL.device).float().t().unsqueeze(0) | |
| print(melody.shape) | |
| if melody.dim() == 2: | |
| melody = melody[None] | |
| melody = melody[..., :int(sr * MODEL.lm.cfg.dataset.segment_duration)] | |
| output = MODEL.generate_with_chroma( | |
| descriptions=[text], | |
| melody_wavs=melody, | |
| melody_sample_rate=sr, | |
| progress=False | |
| ) | |
| else: | |
| output = MODEL.generate(descriptions=[text], progress=False) | |
| output = output.detach().cpu().float()[0] | |
| with NamedTemporaryFile("wb", suffix=".wav", delete=False) as file: | |
| audio_write(file.name, output, MODEL.sample_rate, strategy="loudness", add_suffix=False) | |
| #waveform_video = gr.make_waveform(file.name) | |
| return file.name | |
| with gr.Blocks() as demo: | |
| gr.Markdown( | |
| """ | |
| # Image to MusicGen | |
| This is the demo by @fffiloni for Image to [MusicGen](https://github.com/facebookresearch/audiocraft), a simple and controllable model for music generation | |
| presented at: ["Simple and Controllable Music Generation"](https://huggingface.co/papers/2306.05284), using Clip Interrogator to get an image description as init text. | |
| <br/> | |
| <a href="https://huggingface.co/spaces/musicgen/MusicGen?duplicate=true" style="display: inline-block;margin-top: .5em;margin-right: .25em;" target="_blank"> | |
| <img style="margin-bottom: 0em;display: inline;margin-top: -.25em;" src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a> | |
| for longer sequences, more control and no queue.</p> | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Column(): | |
| uploaded_image = gr.Image(label="Input Image", interactive=True, source="upload", type="filepath") | |
| melody = gr.Audio(source="upload", type="numpy", label="Melody Condition (optional)", interactive=True) | |
| with gr.Row(): | |
| submit = gr.Button("Submit") | |
| #with gr.Row(): | |
| # model = gr.Radio(["melody", "medium", "small", "large"], label="Model", value="melody", interactive=True) | |
| with gr.Row(): | |
| duration = gr.Slider(minimum=1, maximum=30, value=10, step=1, label="Duration", interactive=True) | |
| #with gr.Row(): | |
| # topk = gr.Number(label="Top-k", value=250, interactive=True) | |
| # topp = gr.Number(label="Top-p", value=0, interactive=True) | |
| # temperature = gr.Number(label="Temperature", value=1.0, interactive=True) | |
| # cfg_coef = gr.Number(label="Classifier Free Guidance", value=3.0, interactive=True) | |
| with gr.Column(): | |
| output = gr.Audio(label="Generated Music") | |
| submit.click(predict, inputs=[uploaded_image, melody, duration], outputs=[output]) | |
| gr.Markdown( | |
| """ | |
| ### More details | |
| The model will generate a short music extract based on the image you provided. | |
| You can generate up to 30 seconds of audio. | |
| This demo is set to use only the Melody model | |
| 1. Melody -- a music generation model capable of generating music condition on text and melody inputs. **Note**, you can also use text only. | |
| 2. Small -- a 300M transformer decoder conditioned on text only. | |
| 3. Medium -- a 1.5B transformer decoder conditioned on text only. | |
| 4. Large -- a 3.3B transformer decoder conditioned on text only (might OOM for the longest sequences.) | |
| When using `melody`, ou can optionaly provide a reference audio from | |
| which a broad melody will be extracted. The model will then try to follow both the description and melody provided. | |
| You can also use your own GPU or a Google Colab by following the instructions on our repo. | |
| See [github.com/facebookresearch/audiocraft](https://github.com/facebookresearch/audiocraft) | |
| for more details. | |
| """ | |
| ) | |
| demo.queue(max_size=32).launch() | |