app.py

import os
import subprocess
import sys


def install(package):
    if '=' in package:
        package_name, package_version = package.split('==')
    else:
        package_name = package
        package_version = None
    try:
        subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", package_name])
        print(f"Successfully uninstalled {package}")
    except subprocess.CalledProcessError:
        print(f"Package {package} was not installed, proceeding with installation")
    subprocess.check_call([sys.executable, "-m", "pip", "install", package])

# install('pydantic==2.0.0')
# install('gradio==4.44.0')
# install('spacy==3.7')

debug = False
is_prod = True
if os.environ.get('PROD_MODE') == 'local':
    is_prod = False
else:
    debug = False

import pickle

import gradio as gr
import os

if not is_prod:

    import os
    os.environ['HF_HOME'] = '/proj/afosr/metavoice/cache'
    os.environ['TRANSFORMERS_CACHE'] = '/proj/afosr/metavoice/cache'
    os.environ['HF_DATASETS_CACHE'] = '/proj/afosr/metavoice/cache'
    os.environ['HF_METRICS_CACHE'] = '/proj/afosr/metavoice/cache'
    os.environ['HF_MODULES_CACHE'] = '/proj/afosr/metavoice/cache'
    ffmpeg_path = '/home/hc3295/ffmpegg_build/bin'
    os.environ['PATH'] += os.pathsep + ffmpeg_path


import torch
if not debug:
    import shutil
    import tempfile
    import time
    from pathlib import Path

    import librosa
    
    from huggingface_hub import snapshot_download

    from fam.llm.adapters import FlattenedInterleavedEncodec2Codebook
    from fam.llm.decoders import EncodecDecoder
    from fam.llm.fast_inference_utils import build_model, main
    from fam.llm.inference import (
        EncodecDecoder,
        InferenceConfig,
        Model,
        TiltedEncodec,
        TrainedBPETokeniser,
        get_cached_embedding,
        get_cached_file,    
        get_enhancer,
    )
    from fam.llm.utils import (
        check_audio_file,
        get_default_dtype,
        get_device,
        normalize_text,
    )



DESCRIPTION = ""
if not torch.cuda.is_available():
    DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
if torch.cuda.is_available():
    if not debug:
        model_name = "metavoiceio/metavoice-1B-v0.1"
        seed = 1337
        output_dir = "outputs"
        _dtype = get_default_dtype()
        _device = 'cuda:0'

        _model_dir = snapshot_download(repo_id=model_name)
        first_stage_adapter = FlattenedInterleavedEncodec2Codebook(end_of_audio_token=1024)
        output_dir = output_dir
        os.makedirs(output_dir, exist_ok=True)

        second_stage_ckpt_path = f"{_model_dir}/second_stage.pt"
        config_second_stage = InferenceConfig(
            ckpt_path=second_stage_ckpt_path,
            num_samples=1,
            seed=seed,
            device=_device,
            dtype=_dtype,
            compile=False,
            init_from="resume",
            output_dir=output_dir,
        )
        data_adapter_second_stage = TiltedEncodec(end_of_audio_token=1024)
        llm_second_stage = Model(
            config_second_stage, TrainedBPETokeniser, EncodecDecoder, data_adapter_fn=data_adapter_second_stage.decode
        )
        enhancer = get_enhancer("df")

        precision = {"float16": torch.float16, "bfloat16": torch.bfloat16}[_dtype]
        model, tokenizer, smodel, model_size = build_model(
            precision=precision,
            checkpoint_path=Path(f"{_model_dir}/first_stage.pt"),
            spk_emb_ckpt_path=Path(f"{_model_dir}/speaker_encoder.pt"),
            device=_device,
            compile=True,
            compile_prefill=True,
        )

def generate_sample(text, emo_dir = None, source_path = None, emo_path = None, neutral_path = None, strength = 0.1, top_p = 0.95, guidance_scale = 3.0, preset_dropdown = None, toggle = None):

    print('text', text)
    print('emo_dir', emo_dir)
    print('source_path', source_path)
    print('emo_path', emo_path)
    print('neutral_path', neutral_path)
    print('strength', strength)
    print('top_p', top_p)
    print('guidance_scale', guidance_scale)

    if toggle == RADIO_CHOICES[0]:
        source_path = PRESET_VOICES[preset_dropdown]
    source_path = get_cached_file(source_path)
    check_audio_file(source_path)
    source_emb = get_cached_embedding(source_path, smodel).to(device=_device, dtype=precision)

    if emo_dir == EMO_NAMES[0]:
        emo_path = get_cached_file(emo_path)
        check_audio_file(emo_path)
        emo_emb = get_cached_embedding(emo_path, smodel).to(device=_device, dtype=precision)

        neutral_path = get_cached_file(neutral_path)
        check_audio_file(neutral_path)
        neutral_emb = get_cached_embedding(neutral_path, smodel).to(device=_device, dtype=precision)

        emo_dir = emo_emb - neutral_emb
        emo_dir = emo_dir / torch.norm(emo_dir, p=2)
    else:
        emo_dir = torch.tensor(ALL_EMO_DIRS[emo_dir], device=_device, dtype=precision)
    
    
    edited_emb = source_emb + strength * emo_dir
    edited_emb = edited_emb.to(device=_device, dtype=precision)

    temperature=1.0
    text = normalize_text(text)

    start = time.time()
    # first stage LLM
    tokens = main(
        model=model,
        tokenizer=tokenizer,
        model_size=model_size,
        prompt=text,
        spk_emb=edited_emb,
        top_p=torch.tensor(top_p, device=_device, dtype=precision),
        guidance_scale=torch.tensor(guidance_scale, device=_device, dtype=precision),
        temperature=torch.tensor(temperature, device=_device, dtype=precision),
    )
    text_ids, extracted_audio_ids = first_stage_adapter.decode([tokens])

    b_speaker_embs = edited_emb.unsqueeze(0)

    # second stage LLM + multi-band diffusion model
    wav_files = llm_second_stage(
        texts=[text],
        encodec_tokens=[torch.tensor(extracted_audio_ids, dtype=torch.int32, device=_device).unsqueeze(0)],
        speaker_embs=b_speaker_embs,
        batch_size=1,
        guidance_scale=None,
        top_p=None,
        top_k=200,
        temperature=1.0,
        max_new_tokens=None,
    )

    wav_file = wav_files[0]
    with tempfile.NamedTemporaryFile(suffix=".wav") as enhanced_tmp:
        enhancer(str(wav_file) + ".wav", enhanced_tmp.name)
        shutil.copy2(enhanced_tmp.name, str(wav_file) + ".wav")
        print(f"\nSaved audio to {wav_file}.wav")
    
    output_path = str(wav_file) + ".wav"
    return output_path


ALL_EMO_DIRS = pickle.load(open('all_emo_dirs.pkl', 'rb'))
EMO_NAMES = ['Upload your own sample'] + list(ALL_EMO_DIRS.keys())

RADIO_CHOICES = ["Preset voices", "Upload your voice"]
MAX_CHARS = 220
PRESET_VOICES = {
    # female
    "Bria": "https://cdn.themetavoice.xyz/speakers%2Fbria.mp3",
    # male
    "Alex": "https://cdn.themetavoice.xyz/speakers/alex.mp3",
    "Jacob": "https://cdn.themetavoice.xyz/speakers/jacob.wav",
}


def denormalise_top_p(top_p):
    # returns top_p in the range [0.9, 1.0]
    return round(0.9 + top_p / 100, 2)


def denormalise_guidance(guidance):
    # returns guidance in the range [1.0, 3.0]
    return 1 + ((guidance - 1) * (3 - 1)) / (5 - 1)


def _check_file_size(path):
    if not path:
        return
    filesize = os.path.getsize(path)
    filesize_mb = filesize / 1024 / 1024
    if filesize_mb >= 50:
        raise gr.Error(f"Please upload a sample less than 20MB for voice cloning. Provided: {round(filesize_mb)} MB")


def _handle_edge_cases(to_say, upload_target):
    if not to_say:
        raise gr.Error("Please provide text to synthesise")

    if len(to_say) > MAX_CHARS:
        gr.Warning(
            f"Max {MAX_CHARS} characters allowed. Provided: {len(to_say)} characters. Truncating and generating speech...Result at the end can be unstable as a result."
        )

    if not upload_target:
        return

    check_audio_file(upload_target)  # check file duration to be atleast 30s
    _check_file_size(upload_target)


def tts(to_say, top_p, guidance, toggle, preset_dropdown, upload_target):
    try:
        d_top_p = denormalise_top_p(top_p)
        d_guidance = denormalise_guidance(guidance)

        _handle_edge_cases(to_say, upload_target)

        to_say = to_say if len(to_say) < MAX_CHARS else to_say[:MAX_CHARS]

        return TTS_MODEL.synthesise(
            text=to_say,
            spk_ref_path=PRESET_VOICES[preset_dropdown] if toggle == RADIO_CHOICES[0] else upload_target,
            top_p=d_top_p,
            guidance_scale=d_guidance,
        )
    except Exception as e:
        raise gr.Error(f"Something went wrong. Reason: {str(e)}")


def change_voice_selection_layout(choice):
    if choice == RADIO_CHOICES[0]:
        return [gr.update(visible=True), gr.update(visible=False)]

    return [gr.update(visible=False), gr.update(visible=True)]

def change_emotion_selection_layout(choice):
    if choice == EMO_NAMES[0]:
        return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)]
    else:
        return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)]

title = """
<!-- Google Tag Manager -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','GTM-5N27BQH8');</script>
<!-- End Google Tag Manager -->

</style>
<h1 style="margin-top: 10px;" class="page-title">Demo for <span style="margin-left: 10px;background-color: #E0FEE4;padding: 15px;border-radius: 10px;">🎛️ EmoKnob</span></h1>

<!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-5N27BQH8"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) -->

"""

description = """

- EmoKnob applies control of emotion over arbitrary speaker.
- EmoKnob <b>extracts emotion from a pair of emotional and neutral audio from the same speaker.</b>
- In this demo, you can select from a few preset voices and upload your own emotional samples to clone.
- You can then apply control of a preset emotion or extract emotion from your own pair of emotional and neutral audio.
- You can adjust the strength of the emotion by using the slider.

Check out our [project page](https://emoknob.cs.columbia.edu/) for more details.

EmoKnob is uses [MetaVoice](https://github.com/metavoiceio/metavoice-src) as voice cloning backbone.
"""

with gr.Blocks(title="EmoKnob: EmoKnob: Enhance Voice Cloning with Fine-Grained Emotion Control") as demo:
    gr.Markdown(title)
    gr.Markdown(description)
    gr.Image("https://raw.githubusercontent.com/tonychenxyz/emoknob/main/docs/assets/emo-knob-teaser-1.svg", show_label=False, container=False)

    with gr.Row():
        with gr.Column():
            to_say = gr.TextArea(
                label=f"What should I say!? (max {MAX_CHARS} characters).",
                lines=4,
                value="To be or not to be, that is the question.",
            )



                # voice select
                
            with gr.Row(), gr.Column():
                toggle = gr.Radio(choices=RADIO_CHOICES, label="Choose voice", value=RADIO_CHOICES[0])
                

                with gr.Row() as row_1:
                    preset_dropdown = gr.Dropdown(
                        PRESET_VOICES.keys(), label="Preset voices", value=list(PRESET_VOICES.keys())[0]
                    )

                    with gr.Accordion("Preview: Preset voices", open=False):
                        for label, path in PRESET_VOICES.items():
                            gr.Audio(value=path, label=label)

                with gr.Row(visible=False) as row_2:
                    upload_target = gr.Audio(
                        sources=["upload"],
                        type="filepath",
                        label="Upload a clean sample to clone.",
                    )

                    
            with gr.Row(), gr.Column():
                strength = gr.Slider(
                        value=0.3,
                        minimum=0.0,
                        maximum=1.0,
                        step=0.01,
                        label="Strength - how strong the emotion is. Recommended value is between 0.0 and 0.6.",
                    )
                
                with gr.Row():
                    emotion_name = gr.Radio(choices=EMO_NAMES, label="Emotion", value=EMO_NAMES[1])  # Set default to second option



                with gr.Row(visible=False) as row_3:
                    upload_neutral = gr.Audio(
                        sources=["upload"],
                        type="filepath",
                        label="Neutral sample for emotion extraction.",
                    )

                    upload_emo = gr.Audio(
                        sources=["upload"],
                        type="filepath",
                        label="Emotional sample for emotion extraction.",
                    )

            with gr.Row(), gr.Column():
                # voice settings
                top_p = gr.Slider(
                    value=0.95,
                    minimum=0.0,
                    maximum=10.0,
                    step=1.0,
                    label="Speech Stability - improves text following for a challenging speaker",
                )
                guidance = gr.Slider(
                    value=3.0,
                    minimum=1.0,
                    maximum=5.0,
                    step=1.0,
                    label="Speaker similarity - How closely to match speaker identity and speech style.",
                )

            emotion_name.change(
                change_emotion_selection_layout,
                inputs=emotion_name,
                outputs=[row_3, upload_neutral, upload_emo],
            )

            toggle.change(
                change_voice_selection_layout,
                inputs=toggle,
                outputs=[row_1, row_2],
            )

        with gr.Column():
            speech = gr.Audio(
                type="filepath",
                label="Model says...",
            )

    submit = gr.Button("Generate Speech")
    submit.click(
        fn=generate_sample,
        inputs=[to_say, emotion_name, upload_target, upload_emo, upload_neutral, strength, top_p, guidance, preset_dropdown, toggle],
        outputs=speech,
    )


demo.launch()