spaces.py

import sys
import gradio as gr
import requests
import io
import base64
import json
import tempfile
import os
import numpy as np
import random
from PIL import Image as PILImage, ImageDraw, ImageFont
from moviepy.editor import *
import textwrap
from pydub import AudioSegment
import datetime
import cairosvg
import anthropic
import concurrent.futures

# Initialize Anthropic client
client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))

# ElevenLabs API key
elevenlabs_api_key = os.getenv("ELEVENLABS_API_KEY")

def get_voices():
    url = "https://api.elevenlabs.io/v1/voices"
    headers = {
        "xi-api-key": elevenlabs_api_key
    }
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        voices = response.json()["voices"]
        return [(voice["name"], voice["voice_id"]) for voice in voices]
    return []

def get_convo_list(description):
    prompt =f"Your task is to return a JSON object representing a complete conversation containing a key 'turns' with a value which is just a list of objects containing 'turn_number', an integer, and 'message', the message for that turn. Ensure you return as many turns as the user specifies, if they specify. Remember, each turn is a turn in a conversation between a phone agent (female) and a human (male) unless otherwise specified. The phone agent should speak first, unless otherwise specified. The conversation is described as:\n{description}.\nCritically, ensure that the human turns employ filler words (uh, uhhhh, ummmm, yeahhh, hm, hmm, etc with repeated letters to denote thinking...) and realistic language without using *sounds effects*. I repeat, do NOT use *sound effects*. Additionally, do not over-use filler words or start every human response with them. The goal is to sound realistic, not exagerrated. The AI should be conversational, employing transition phrases. The AI should always end their response with a question except when saying goodbye. Additionally, digits spaced out. For instance, the human might say: 'My phone number is 8 3 1... 5 4 8... 9 2 2 3...' instead of writing it out. They might also say 'My email is steve at gmail dot com.' where it is written out. Now provide the JSON."
    new_output = ""
    total_tokens = 350

    with client.messages.stream(
        max_tokens=8000,
        messages=[
            {"role": "user", "content": prompt}
        ],
        model="claude-3-5-sonnet-20241022",
        temperature=0.1,
    ) as stream:
        for text in stream.text_stream:
            new_output += text

    first_brace = new_output.find('{')
    last_brace = new_output.rfind('}')
    new_output = new_output[first_brace:last_brace+1]
    new_output = json.loads(new_output)
    output_list = []
    for i in new_output["turns"]:
        output_list.append(i['message'])
    return output_list

def download_and_convert_svg_to_png(svg_url):
    response = requests.get(svg_url)
    if response.status_code == 200:
        svg_data = response.content
        png_data = cairosvg.svg2png(bytestring=svg_data)
        image = PILImage.open(io.BytesIO(png_data))
        return image
    else:
        print(f"Failed to download SVG image from {svg_url}")
        return None

def generate_speech(text, voice_id, stability=0.8, style=0):
    model_id = "eleven_multilingual_v2"
    url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"
    payload = {
        "text": text,
        "model_id": model_id,
        "voice_settings": {
            "stability": stability,
            "similarity_boost": 0.5,
            "use_speaker_boost": True,
            "style": style,
        }
    }
    headers = {
        "xi-api-key": elevenlabs_api_key,
        "Accept": "audio/mpeg"
    }
    response = requests.request("POST", url, json=payload, headers=headers)
    if response.status_code == 200:
        return response.content
    else:
        print(f"Error generating speech: {response.status_code} - {response.text}")
        return None

def create_text_image(text, logo_image, text_color, image_size=(1920, 1080), bg_color="#0e2e28", font_size=70, logo_scale=0.05):
    bg_color_rgb = PILImage.new("RGB", (1, 1), color=bg_color).getpixel((0, 0))
    text_color_rgb = PILImage.new("RGB", (1, 1), color=text_color).getpixel((0, 0))

    img = PILImage.new('RGB', image_size, color=bg_color_rgb)
    draw = ImageDraw.Draw(img)

    logo_aspect_ratio = logo_image.width / logo_image.height
    logo_height = int(image_size[1] * logo_scale)
    logo_width = int(logo_height * logo_aspect_ratio)
    logo_image = logo_image.resize((logo_width, logo_height))
    logo_position = (int(image_size[0] * 0.05), int(image_size[1] / 2 - logo_height / 2))
    img.paste(logo_image, logo_position, logo_image.convert('RGBA'))

    text_area_x = logo_position[0] + logo_width + int(image_size[0] * 0.05)
    text_area_width = image_size[0] - text_area_x - int(image_size[0] * 0.05)

    try:
        import cv2
        font_path = os.path.join(cv2.__path__[0],'qt','fonts','DejaVuSans.ttf')
        font = ImageFont.truetype(font_path, size=font_size)
    except IOError:
        font = ImageFont.load_default()

    max_chars_per_line = int(text_area_width / (font_size * 0.6))
    wrapped_text = textwrap.fill(text, width=max_chars_per_line)

    draw_img = PILImage.new('RGB', (text_area_width, image_size[1]))
    draw_draw = ImageDraw.Draw(draw_img)
    try:
        bbox = draw_draw.multiline_textbbox((0, 0), wrapped_text, font=font, align='left')
    except AttributeError:
        bbox = draw_draw.textbox((0, 0), wrapped_text, font=font, align='left')
    text_height = bbox[3] - bbox[1]
    text_position = (text_area_x, int((image_size[1] - text_height) / 2))

    draw.multiline_text(text_position, wrapped_text, fill=text_color_rgb, font=font, align='left')

    return img

def trim_silence_from_end(audio_segment, silence_threshold=-50.0, chunk_size=10):
    duration_ms = len(audio_segment)
    trim_ms = 0

    while trim_ms < duration_ms:
        start_index = duration_ms - trim_ms - chunk_size
        if start_index < 0:
            start_index = 0
        chunk = audio_segment[start_index:duration_ms - trim_ms]
        if chunk.dBFS > silence_threshold:
            break
        trim_ms += chunk_size

    if trim_ms > 0:
        return audio_segment[:duration_ms - trim_ms]
    else:
        return audio_segment

def add_silence_to_audio(audio_content, silence_duration=0):
    silence = AudioSegment.silent(duration=silence_duration)
    original_audio = AudioSegment.from_file(io.BytesIO(audio_content), format="mp3")
    original_audio = trim_silence_from_end(original_audio)
    new_audio = silence + original_audio
    audio_io = io.BytesIO()
    new_audio.export(audio_io, format="wav", parameters=["-ar", "44100"])
    audio_io.seek(0)
    return audio_io.read()

def create_video_clip(image, duration, target_resolution=(1920, 1080)):
    image = image.convert('RGB')
    img_array = np.array(image)
    clip = ImageClip(img_array)
    clip = clip.resize(newsize=target_resolution)
    return clip.set_duration(duration)

def process_message(args):
    i, message, logo_image, voice_ids, male_stability, male_style = args
    voice_id = voice_ids[i % len(voice_ids)]

    if i % len(voice_ids) == 0:
        text_color = "#cdfa8a"
        stability = 0.8
        style = 0
    else:
        text_color = "#FFFFFF"
        stability = male_stability
        style = male_style

    try:
        audio_content = generate_speech(message, voice_id, stability=stability, style=style)
        if audio_content is None:
            return (None, None, None)

        audio_data = add_silence_to_audio(audio_content, silence_duration=0)

        temp_audio_file = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
        temp_audio_file.write(audio_data)
        temp_audio_file.close()
        temp_audio_path = temp_audio_file.name

        audio_clip = AudioFileClip(temp_audio_path)
        audio_duration = audio_clip.duration

        image = create_text_image(message, logo_image, text_color, font_size=30, logo_scale=0.07)
        video_clip = create_video_clip(image, duration=audio_duration)
        audio_clip = audio_clip.set_duration(video_clip.duration)
        audio_clip = audio_clip.audio_fadeout(0.2)
        video_clip = video_clip.set_audio(audio_clip)

        return (video_clip, audio_clip, temp_audio_path)
    except Exception as e:
        print(f"Error processing message {i+1}: {e}")
        return (None, None, None)

def generate_conversation_video(messages, voice_ids, logo_url, male_stability, male_style):
    logo_image = download_and_convert_svg_to_png(logo_url)
    if logo_image is None:
        return None

    video_clips = []
    audio_clips = []
    temp_audio_paths = []

    args = [(i, message, logo_image, voice_ids, male_stability, male_style) for i, message in enumerate(messages)]
    max_workers = 5

    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
        results = list(executor.map(process_message, args))

    for i, (video_clip, audio_clip, temp_audio_path) in enumerate(results):
        if video_clip and audio_clip:
            if i > 0:
                gap_duration = random.uniform(0.6, 1.3)
                silence = AudioClip(lambda t: 0, duration=gap_duration)
                previous_frame = video_clips[-1].get_frame(-1)
                gap_clip = ImageClip(previous_frame).set_duration(gap_duration)
                video_clips.append(gap_clip)
                audio_clips.append(silence)
            
            video_clips.append(video_clip)
            audio_clips.append(audio_clip)
            temp_audio_paths.append(temp_audio_path)
        else:
            if temp_audio_path:
                os.unlink(temp_audio_path)

    if not video_clips or not audio_clips:
        return None

    final_audio = concatenate_audioclips(audio_clips)
    video_clips_no_audio = [clip.without_audio() for clip in video_clips]
    final_video = concatenate_videoclips(video_clips_no_audio, method="chain")
    final_video = final_video.set_audio(final_audio)

    temp_video_path = tempfile.mktemp(suffix='.mp4')
    final_video.write_videofile(
        temp_video_path,
        fps=2,
        codec="libx264",
        audio_codec="aac",
        audio_bitrate="192k",
        temp_audiofile='temp-audio.m4a',
        remove_temp=True,
        verbose=False,
        logger=None
    )

    # Cleanup
    for clip in audio_clips:
        clip.close()
    for path in temp_audio_paths:
        if os.path.exists(path):
            os.unlink(path)

    return temp_video_path

def generate_video(description, female_voice, male_voice, male_stability=0.65, male_style=0.35):
    voice_ids = [
        female_voice,  # First speaker (female)
        male_voice     # Second speaker (male)
    ]
    logo_url = "https://opencall.ai/images/logo-symbol.svg"
    
    messages = get_convo_list(description)
    video_path = generate_conversation_video(messages, voice_ids, logo_url, male_stability, male_style)
    
    return video_path

# Get available voices
voices = get_voices()
default_female_id = "cgSgspJ2msm6clMCkdW9"  # Default female voice ID
default_male_id = "3Niy6MUaDzcs7Liw7dFs"    # Default male voice ID

# Create voice selection dropdowns
female_voice_names = [(voice[0], voice[1]) for voice in voices]
male_voice_names = [(voice[0], voice[1]) for voice in voices]

# Set default selections
default_female_idx = next((i for i, v in enumerate(female_voice_names) if v[1] == default_female_id), 0)
default_male_idx = next((i for i, v in enumerate(male_voice_names) if v[1] == default_male_id), 0)

# Create Gradio interface
iface = gr.Interface(
    fn=generate_video,
    inputs=[
        gr.Textbox(
            label="Enter conversation description",
            lines=5,
            placeholder="Describe the conversation you want to generate...",
            info="You can be specific about the number of turns, tone, and content of the conversation"
        ),
        gr.Dropdown(
            choices=female_voice_names,
            value=female_voice_names[default_female_idx][1],
            label="Female Voice",
            type="value",
            info="Select the voice for the phone agent"
        ),
        gr.Dropdown(
            choices=male_voice_names,
            value=male_voice_names[default_male_idx][1],
            label="Male Voice",
            type="value",
            info="Select the voice for the customer"
        ),
        gr.Slider(
            minimum=0.1,
            maximum=1.0,
            value=0.65,
            label="Male Voice Stability",
            info="Controls the consistency of the male voice (default: 0.65)"
        ),
        gr.Slider(
            minimum=0.1,
            maximum=1.0,
            value=0.35,
            label="Male Voice Style",
            info="Controls the expressiveness of the male voice (default: 0.35)"
        )
    ],
    outputs=gr.Video(label="Generated Video"),
    title="AI Conversation Video Generator",
    description="Generate a video conversation between two speakers based on your description."
)

iface.launch()