CingenAI / core /visual_engine.py
mgbam's picture
Update core/visual_engine.py
d4d0117 verified
raw
history blame
40.7 kB
# core/visual_engine.py
from PIL import Image, ImageDraw, ImageFont, ImageOps
import base64
import mimetypes
import numpy as np
import os
import openai
import requests
import io
import time
import random
import logging
from moviepy.editor import (ImageClip, VideoFileClip, concatenate_videoclips, TextClip,
CompositeVideoClip, AudioFileClip)
import moviepy.video.fx.all as vfx
try: # MONKEY PATCH
if hasattr(Image, 'Resampling') and hasattr(Image.Resampling, 'LANCZOS'):
if not hasattr(Image, 'ANTIALIAS'): Image.ANTIALIAS = Image.Resampling.LANCZOS
elif hasattr(Image, 'LANCZOS'):
if not hasattr(Image, 'ANTIALIAS'): Image.ANTIALIAS = Image.LANCZOS
elif not hasattr(Image, 'ANTIALIAS'): print("WARNING: Pillow ANTIALIAS/Resampling issue.")
except Exception as e_mp: print(f"WARNING: ANTIALIAS patch error: {e_mp}")
logger = logging.getLogger(__name__)
# logger.setLevel(logging.DEBUG)
ELEVENLABS_CLIENT_IMPORTED = False; ElevenLabsAPIClient = None; Voice = None; VoiceSettings = None
try:
from elevenlabs.client import ElevenLabs as ImportedElevenLabsClient
from elevenlabs import Voice as ImportedVoice, VoiceSettings as ImportedVoiceSettings
ElevenLabsAPIClient = ImportedElevenLabsClient; Voice = ImportedVoice; VoiceSettings = ImportedVoiceSettings
ELEVENLABS_CLIENT_IMPORTED = True; logger.info("ElevenLabs client components imported.")
except Exception as e_11l_imp: logger.warning(f"ElevenLabs client import failed: {e_11l_imp}. Audio disabled.")
RUNWAYML_SDK_IMPORTED = False; RunwayMLAPIClientClass = None
try:
from runwayml import RunwayML as ImportedRunwayMLAPIClientClass
RunwayMLAPIClientClass = ImportedRunwayMLAPIClientClass; RUNWAYML_SDK_IMPORTED = True
logger.info("RunwayML SDK imported.")
except Exception as e_rwy_imp: logger.warning(f"RunwayML SDK import failed: {e_rwy_imp}. RunwayML disabled.")
class VisualEngine:
DEFAULT_FONT_SIZE_PIL = 10; PREFERRED_FONT_SIZE_PIL = 20
VIDEO_OVERLAY_FONT_SIZE = 30; VIDEO_OVERLAY_FONT_COLOR = 'white'
DEFAULT_MOVIEPY_FONT = 'DejaVu-Sans-Bold'; PREFERRED_MOVIEPY_FONT = 'Liberation-Sans-Bold'
def __init__(self, output_dir="temp_cinegen_media", default_elevenlabs_voice_id="Rachel"):
self.output_dir = output_dir; os.makedirs(self.output_dir, exist_ok=True)
self.font_filename_pil_preference = "DejaVuSans-Bold.ttf"
font_paths = [ self.font_filename_pil_preference, f"/usr/share/fonts/truetype/dejavu/{self.font_filename_pil_preference}", f"/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf", f"/System/Library/Fonts/Supplemental/Arial.ttf", f"C:/Windows/Fonts/arial.ttf", f"/usr/local/share/fonts/truetype/mycustomfonts/arial.ttf"]
self.resolved_font_path_pil = next((p for p in font_paths if os.path.exists(p)), None)
self.active_font_pil = ImageFont.load_default(); self.active_font_size_pil = self.DEFAULT_FONT_SIZE_PIL; self.active_moviepy_font_name = self.DEFAULT_MOVIEPY_FONT
if self.resolved_font_path_pil:
try: self.active_font_pil = ImageFont.truetype(self.resolved_font_path_pil, self.PREFERRED_FONT_SIZE_PIL); self.active_font_size_pil = self.PREFERRED_FONT_SIZE_PIL; logger.info(f"Pillow font: {self.resolved_font_path_pil} sz {self.active_font_size_pil}."); self.active_moviepy_font_name = 'DejaVu-Sans-Bold' if "dejavu" in self.resolved_font_path_pil.lower() else ('Liberation-Sans-Bold' if "liberation" in self.resolved_font_path_pil.lower() else self.DEFAULT_MOVIEPY_FONT)
except IOError as e_font: logger.error(f"Pillow font IOError '{self.resolved_font_path_pil}': {e_font}. Default.")
else: logger.warning("Preferred Pillow font not found. Default.")
self.openai_api_key = None; self.USE_AI_IMAGE_GENERATION = False; self.dalle_model = "dall-e-3"; self.image_size_dalle3 = "1792x1024"
self.video_frame_size = (1280, 720)
self.elevenlabs_api_key = None; self.USE_ELEVENLABS = False; self.elevenlabs_client_instance = None; self.elevenlabs_voice_id = default_elevenlabs_voice_id
if VoiceSettings and ELEVENLABS_CLIENT_IMPORTED: self.elevenlabs_voice_settings_obj = VoiceSettings(stability=0.60, similarity_boost=0.80, style=0.15, use_speaker_boost=True)
else: self.elevenlabs_voice_settings_obj = None
self.pexels_api_key = None; self.USE_PEXELS = False
self.runway_api_key = None; self.USE_RUNWAYML = False; self.runway_ml_sdk_client_instance = None
if RUNWAYML_SDK_IMPORTED and RunwayMLAPIClientClass and os.getenv("RUNWAYML_API_SECRET"):
try: self.runway_ml_sdk_client_instance = RunwayMLAPIClientClass(); self.USE_RUNWAYML = True; logger.info("RunwayML Client init from env var at startup.")
except Exception as e_rwy_init: logger.error(f"Initial RunwayML client init failed: {e_rwy_init}"); self.USE_RUNWAYML = False
logger.info("VisualEngine initialized.")
def set_openai_api_key(self, k): self.openai_api_key=k; self.USE_AI_IMAGE_GENERATION=bool(k); logger.info(f"DALL-E: {'Ready' if self.USE_AI_IMAGE_GENERATION else 'Disabled'}")
def set_elevenlabs_api_key(self, k, vid=None):
self.elevenlabs_api_key=k;
if vid: self.elevenlabs_voice_id = vid; logger.info(f"11L Voice ID updated to: {vid}")
if k and ELEVENLABS_CLIENT_IMPORTED and ElevenLabsAPIClient:
try: self.elevenlabs_client_instance = ElevenLabsAPIClient(api_key=k); self.USE_ELEVENLABS=True; logger.info(f"11L Client: Ready (Voice:{self.elevenlabs_voice_id})")
except Exception as e: logger.error(f"11L client init err: {e}. Disabled.", exc_info=True); self.USE_ELEVENLABS=False; self.elevenlabs_client_instance=None
else: self.USE_ELEVENLABS = False; logger.info(f"11L Disabled (key/SDK).")
def set_pexels_api_key(self, k): self.pexels_api_key=k; self.USE_PEXELS=bool(k); logger.info(f"Pexels: {'Ready' if self.USE_PEXELS else 'Disabled'}")
def set_runway_api_key(self, k):
self.runway_api_key = k
if k:
if RUNWAYML_SDK_IMPORTED and RunwayMLAPIClientClass:
if not self.runway_ml_sdk_client_instance:
try:
orig_secret = os.getenv("RUNWAYML_API_SECRET")
if not orig_secret: os.environ["RUNWAYML_API_SECRET"]=k; logger.info("Temp set RUNWAYML_API_SECRET for SDK.")
self.runway_ml_sdk_client_instance=RunwayMLAPIClientClass(); self.USE_RUNWAYML=True; logger.info("RunwayML Client init via set_key.")
if not orig_secret: del os.environ["RUNWAYML_API_SECRET"]; logger.info("Cleared temp RUNWAYML_API_SECRET.")
except Exception as e: logger.error(f"RunwayML Client init in set_key fail: {e}", exc_info=True); self.USE_RUNWAYML=False;self.runway_ml_sdk_client_instance=None
else: self.USE_RUNWAYML=True; logger.info("RunwayML Client already init.")
else: logger.warning("RunwayML SDK not imported. Disabled."); self.USE_RUNWAYML=False
else: self.USE_RUNWAYML=False; self.runway_ml_sdk_client_instance=None; logger.info("RunwayML Disabled (no key).")
def _image_to_data_uri(self, img_path):
try:
mime, _ = mimetypes.guess_type(img_path)
if not mime: ext=os.path.splitext(img_path)[1].lower(); mime_map={".png":"image/png",".jpg":"image/jpeg",".jpeg":"image/jpeg",".webp":"image/webp"}; mime=mime_map.get(ext,"application/octet-stream");
if mime=="application/octet-stream": logger.warning(f"Unknown MIME for {img_path}, using {mime}.")
with open(img_path,"rb") as f_img: enc_str=base64.b64encode(f_img.read()).decode('utf-8')
uri=f"data:{mime};base64,{enc_str}"; logger.debug(f"Data URI for {os.path.basename(img_path)} (MIME:{mime}): {uri[:100]}..."); return uri
except FileNotFoundError: logger.error(f"Img not found {img_path} for data URI."); return None
except Exception as e: logger.error(f"Error converting {img_path} to data URI:{e}",exc_info=True); return None
def _map_resolution_to_runway_ratio(self, w, h):
r_str=f"{w}:{h}"; supp_r=["1280:720","720:1280","1104:832","832:1104","960:960","1584:672"];
if r_str in supp_r: return r_str
logger.warning(f"Res {r_str} not in Gen-4 list. Default 1280:720."); return "1280:720"
def _get_text_dimensions(self, txt, font):
dh=getattr(font,'size',self.active_font_size_pil);
if not txt: return 0,dh
try:
if hasattr(font,'getbbox'):b=font.getbbox(txt);w=b[2]-b[0];h=b[3]-b[1];return w,h if h>0 else dh
elif hasattr(font,'getsize'):w,h=font.getsize(txt);return w,h if h>0 else dh
else: return int(len(txt)*dh*0.6),int(dh*1.2)
except Exception as e:logger.warning(f"Err _get_text_dimensions:{e}");return int(len(txt)*self.active_font_size_pil*0.6),int(self.active_font_size_pil*1.2)
def _create_placeholder_image_content(self, desc, fname, sz=None):
# (Keep robust placeholder logic from before)
if sz is None: sz=self.video_frame_size; img=Image.new('RGB',sz,color=(20,20,40));drw=ImageDraw.Draw(img);pad=25;maxw=sz[0]-(2*pad);lns=[]
if not desc: desc="(Placeholder)"
wds=desc.split();curr_ln=""
for idx,w in enumerate(wds):
prosp_add=w+(" "if idx<len(wds)-1 else"");test_ln=curr_ln+prosp_add
curr_w,_=self._get_text_dimensions(test_ln,self.active_font_pil)
if curr_w==0 and test_ln.strip():curr_w=len(test_ln)*(self.active_font_size_pil*0.6)
if curr_w<=maxw:curr_ln=test_ln
else:
if curr_ln.strip():lns.append(curr_ln.strip())
curr_ln=prosp_add
if curr_ln.strip():lns.append(curr_ln.strip())
if not lns and desc:
avg_cw,_=self._get_text_dimensions("W",self.active_font_pil);avg_cw=avg_cw or(self.active_font_size_pil*0.6)
cpl=int(maxw/avg_cw)if avg_cw>0 else 20;lns.append(desc[:cpl]+("..."if len(desc)>cpl else""))
elif not lns:lns.append("(PH Error)")
_,slh=self._get_text_dimensions("Ay",self.active_font_pil);slh=slh if slh>0 else self.active_font_size_pil+2
maxl=min(len(lns),(sz[1]-(2*pad))//(slh+2))if slh>0 else 1;maxl=max(1,maxl)
yp=pad+(sz[1]-(2*pad)-maxl*(slh+2))/2.0
for i in range(maxl):
lt=lns[i];lw,_=self._get_text_dimensions(lt,self.active_font_pil)
if lw==0 and lt.strip():lw=len(lt)*(self.active_font_size_pil*0.6)
xp=(sz[0]-lw)/2.0
try:drw.text((xp,yp),lt,font=self.active_font_pil,fill=(200,200,180))
except Exception as e:logger.error(f"Pillow d.text err:{e} for '{lt}'")
yp+=slh+2
if i==6 and maxl>7:
try:drw.text((xp,yp),"...",font=self.active_font_pil,fill=(200,200,180))
except Exception as e:logger.error(f"Pillow ellipsis err:{e}");break
fpath=os.path.join(self.output_dir,fname)
try:img.save(fpath);return fpath
except Exception as e:logger.error(f"Save PH img '{fpath}' err:{e}",exc_info=True);return None
def _search_pexels_image(self, q_str, out_fn_base):
# (Keep robust Pexels logic from before)
if not self.USE_PEXELS or not self.pexels_api_key: return None
h={"Authorization":self.pexels_api_key};p={"query":q_str,"per_page":1,"orientation":"landscape","size":"large2x"}
base_n_px,_=os.path.splitext(out_fn_base);px_fn=base_n_px+f"_pexels_{random.randint(1000,9999)}.jpg";fp_px=os.path.join(self.output_dir,px_fn)
try:
logger.info(f"Pexels: Search '{q_str}'");eff_q=" ".join(q_str.split()[:5]);p["query"]=eff_q
resp_px=requests.get("https://api.pexels.com/v1/search",headers=h,params=p,timeout=20);resp_px.raise_for_status();data_px=resp_px.json()
if data_px.get("photos") and len(data_px["photos"]) > 0:
ph_det=data_px["photos"][0];ph_url=ph_det.get("src",{}).get("large2x")
if not ph_url:logger.warning(f"Pexels: 'large2x' URL missing for '{eff_q}'.");return None
img_resp=requests.get(ph_url,timeout=60);img_resp.raise_for_status();img_pil=Image.open(io.BytesIO(img_resp.content))
if img_pil.mode!='RGB':img_pil=img_pil.convert('RGB')
img_pil.save(fp_px);logger.info(f"Pexels: Saved to {fp_px}");return fp_px
else:logger.info(f"Pexels: No photos for '{eff_q}'.");return None
except requests.exceptions.RequestException as e:logger.error(f"Pexels ReqExc '{q_str}':{e}",exc_info=False);return None
except Exception as e:logger.error(f"Pexels GenErr '{q_str}':{e}",exc_info=True);return None
def _generate_video_clip_with_runwayml(self, motion_prompt, input_img_path, scene_id_base_fn, duration_s=5):
# (Keep robust RunwayML placeholder/integration logic from before)
if not self.USE_RUNWAYML or not self.runway_ml_sdk_client_instance: logger.warning("RunwayML skip: Not enabled/client not init."); return None
if not input_img_path or not os.path.exists(input_img_path): logger.error(f"Runway Gen-4 needs input img. Invalid: {input_img_path}"); return None
img_data_uri = self._image_to_data_uri(input_img_path);
if not img_data_uri: return None
rwy_dur = 10 if duration_s >= 8 else 5; rwy_ratio = self._map_resolution_to_runway_ratio(self.video_frame_size[0],self.video_frame_size[1])
rwy_base_name,_=os.path.splitext(scene_id_base_fn);rwy_out_fn=rwy_base_name+f"_runway_gen4_d{rwy_dur}s.mp4";rwy_out_fp=os.path.join(self.output_dir,rwy_out_fn)
logger.info(f"Runway Gen-4 task: motion='{motion_prompt[:70]}...', img='{os.path.basename(input_img_path)}', dur={rwy_dur}s, ratio='{rwy_ratio}'")
try:
rwy_task_sub = self.runway_ml_sdk_client_instance.image_to_video.create(model='gen4_turbo',prompt_image=img_data_uri,prompt_text=motion_prompt,duration=rwy_dur,ratio=rwy_ratio)
rwy_task_id = rwy_task_sub.id; logger.info(f"Runway task ID: {rwy_task_id}. Polling...")
poll_s=10;max_p_count=36;poll_t_start=time.time()
while time.time()-poll_t_start < max_p_count*poll_s:
time.sleep(poll_s);rwy_task_det=self.runway_ml_sdk_client_instance.tasks.retrieve(id=rwy_task_id)
logger.info(f"Runway task {rwy_task_id} status: {rwy_task_det.status}")
if rwy_task_det.status=='SUCCEEDED':
rwy_out_url=getattr(getattr(rwy_task_det,'output',None),'url',None) or (getattr(rwy_task_det,'artifacts',None)and rwy_task_det.artifacts and hasattr(rwy_task_det.artifacts[0],'url')and rwy_task_det.artifacts[0].url) or (getattr(rwy_task_det,'artifacts',None)and rwy_task_det.artifacts and hasattr(rwy_task_det.artifacts[0],'download_url')and rwy_task_det.artifacts[0].download_url)
if not rwy_out_url:logger.error(f"Runway task {rwy_task_id} SUCCEEDED, no output URL. Details:{vars(rwy_task_det)if hasattr(rwy_task_det,'__dict__')else rwy_task_det}");return None
logger.info(f"Runway task {rwy_task_id} SUCCEEDED. Downloading: {rwy_out_url}")
vid_resp=requests.get(rwy_out_url,stream=True,timeout=300);vid_resp.raise_for_status()
with open(rwy_out_fp,'wb')as f:
for chk in vid_resp.iter_content(chunk_size=8192):f.write(chk)
logger.info(f"Runway Gen-4 video saved: {rwy_out_fp}");return rwy_out_fp
elif rwy_task_det.status in['FAILED','ABORTED','ERROR']:
rwy_err_msg=getattr(rwy_task_det,'error_message',None)or getattr(getattr(rwy_task_det,'output',None),'error',"Unknown Runway error.")
logger.error(f"Runway task {rwy_task_id} status:{rwy_task_det.status}. Error:{rwy_err_msg}");return None
logger.warning(f"Runway task {rwy_task_id} timed out.");return None
except AttributeError as e:logger.error(f"RunwayML SDK AttrError:{e}. SDK methods changed?",exc_info=True);return None
except Exception as e:logger.error(f"Runway Gen-4 API error:{e}",exc_info=True);return None
def _create_placeholder_video_content(self, text_desc, fname, duration=4, size=None):
# (Keep robust placeholder video logic from before)
if size is None: size = self.video_frame_size; fp = os.path.join(self.output_dir, fname); tc = None
try: tc = TextClip(text_desc, fontsize=50, color='white', font=self.video_overlay_font, bg_color='black', size=size, method='caption').set_duration(duration); tc.write_videofile(fp, fps=24, codec='libx264', preset='ultrafast', logger=None, threads=2); logger.info(f"Generic placeholder video: {fp}"); return fp
except Exception as e: logger.error(f"Generic placeholder video error {fp}: {e}", exc_info=True); return None
finally:
if tc and hasattr(tc, 'close'):
try: tc.close()
except Exception as e_cl_phv: logger.warning(f"Ignoring error closing placeholder TextClip: {e_cl_phv}")
def generate_scene_asset(self, img_prompt, motion_prompt, scene_dict, scene_id_fn_base, gen_as_vid=False, rwy_dur=5):
# (Keep robust asset generation logic from before, ensuring parameters match)
asset_base_name,_=os.path.splitext(scene_id_fn_base); asset_info_obj={'path':None,'type':'none','error':True,'prompt_used':img_prompt,'error_message':'Asset gen init failed'}; base_img_path_for_rwy=None
base_img_fn = asset_base_name + ("_base_for_video.png" if gen_as_vid else ".png"); base_img_fp = os.path.join(self.output_dir, base_img_fn)
if self.USE_AI_IMAGE_GENERATION and self.openai_api_key:
max_r,att_c=2,0
for att_idx in range(max_r):
att_c=att_idx+1
try:
logger.info(f"Att {att_c} DALL-E (base img): {img_prompt[:70]}...");oai_client=openai.OpenAI(api_key=self.openai_api_key,timeout=90.0);oai_resp=oai_client.images.generate(model=self.dalle_model,prompt=img_prompt,n=1,size=self.image_size_dalle3,quality="hd",response_format="url",style="vivid");oai_url=oai_resp.data[0].url;oai_rev_p=getattr(oai_resp.data[0],'revised_prompt',None)
if oai_rev_p:logger.info(f"DALL-E revised: {oai_rev_p[:70]}...")
oai_img_get_resp=requests.get(oai_url,timeout=120);oai_img_get_resp.raise_for_status();oai_pil_img=Image.open(io.BytesIO(oai_img_get_resp.content))
if oai_pil_img.mode!='RGB':oai_pil_img=oai_pil_img.convert('RGB')
oai_pil_img.save(base_img_fp);logger.info(f"DALL-E base img saved: {base_img_fp}");base_img_path_for_rwy=base_img_fp;asset_info_obj={'path':base_img_fp,'type':'image','error':False,'prompt_used':img_prompt,'revised_prompt':oai_rev_p};break
except openai.RateLimitError as e:logger.warning(f"OpenAI RateLimit Att {att_c}:{e}.Retry...");time.sleep(5*att_c);asset_info_obj['error_message']=str(e)
except openai.APIError as e:logger.error(f"OpenAI APIError Att {att_c}:{e}");asset_info_obj['error_message']=str(e);break
except requests.exceptions.RequestException as e:logger.error(f"Requests Err DALL-E Att {att_c}:{e}");asset_info_obj['error_message']=str(e);break
except Exception as e:logger.error(f"General DALL-E Err Att {att_c}:{e}",exc_info=True);asset_info_obj['error_message']=str(e);break
if asset_info_obj['error']:logger.warning(f"DALL-E failed after {att_c} attempts for base img.")
if asset_info_obj['error'] and self.USE_PEXELS:
logger.info("Trying Pexels for base img.");px_q=scene_dict.get('pexels_search_query_감독',f"{scene_dict.get('emotional_beat','')} {scene_dict.get('setting_description','')}");px_p=self._search_pexels_image(px_q,base_img_fn)
if px_p:base_img_path_for_rwy=px_p;asset_info_obj={'path':px_p,'type':'image','error':False,'prompt_used':f"Pexels:{px_q}"}
else:curr_err=asset_info_obj.get('error_message',"");asset_info_obj['error_message']=(curr_err+" Pexels failed for base.").strip()
if asset_info_obj['error']:
logger.warning("Base img (DALL-E/Pexels) failed. Using placeholder.");ph_p_txt=asset_info_obj.get('prompt_used',img_prompt);ph_img_p=self._create_placeholder_image_content(f"[Base Placeholder]{ph_p_txt[:70]}...",base_img_fn)
if ph_img_p:base_img_path_for_rwy=ph_img_p;asset_info_obj={'path':ph_img_p,'type':'image','error':False,'prompt_used':ph_p_txt}
else:curr_err=asset_info_obj.get('error_message',"");asset_info_obj['error_message']=(curr_err+" Base placeholder failed.").strip()
if gen_as_vid:
if not base_img_path_for_rwy:logger.error("RunwayML video: base img failed.");asset_info_obj['error']=True;asset_info_obj['error_message']=(asset_info_obj.get('error_message',"")+" Base img miss, Runway abort.").strip();asset_info_obj['type']='none';return asset_info_obj
if self.USE_RUNWAYML:
rwy_vid_p=self._generate_video_clip_with_runwayml(motion_prompt,base_img_path_for_rwy,asset_base_name,rwy_dur)
if rwy_vid_p and os.path.exists(rwy_vid_p):asset_info_obj={'path':rwy_vid_p,'type':'video','error':False,'prompt_used':motion_prompt,'base_image_path':base_img_path_for_rwy}
else:logger.warning(f"RunwayML video failed for {asset_base_name}. Fallback to base img.");asset_info_obj['error']=True;asset_info_obj['error_message']=(asset_info_obj.get('error_message',"Base img ok.")+" RunwayML video fail; use base img.").strip();asset_info_obj['path']=base_img_path_for_rwy;asset_info_obj['type']='image';asset_info_obj['prompt_used']=img_prompt
else:logger.warning("RunwayML selected but disabled. Use base img.");asset_info_obj['error']=True;asset_info_obj['error_message']=(asset_info_obj.get('error_message',"Base img ok.")+" RunwayML disabled; use base img.").strip();asset_info_obj['path']=base_img_path_for_rwy;asset_info_obj['type']='image';asset_info_obj['prompt_used']=img_prompt
return asset_info_obj
def generate_narration_audio(self, narration_text, output_fn="narration_overall.mp3"):
# (Corrected version from previous response)
if not self.USE_ELEVENLABS or not self.elevenlabs_client_instance or not narration_text: logger.info("11L conditions not met. Skip audio."); return None
narration_fp = os.path.join(self.output_dir, output_fn)
try:
logger.info(f"11L audio (Voice:{self.elevenlabs_voice_id}): \"{narration_text[:70]}...\"")
stream_method = None
if hasattr(self.elevenlabs_client_instance,'text_to_speech') and hasattr(self.elevenlabs_client_instance.text_to_speech,'stream'): stream_method=self.elevenlabs_client_instance.text_to_speech.stream; logger.info("Using 11L .text_to_speech.stream()")
elif hasattr(self.elevenlabs_client_instance,'generate_stream'): stream_method=self.elevenlabs_client_instance.generate_stream; logger.info("Using 11L .generate_stream()")
elif hasattr(self.elevenlabs_client_instance,'generate'):
logger.info("Using 11L .generate() (non-streaming).")
voice_p = Voice(voice_id=str(self.elevenlabs_voice_id),settings=self.elevenlabs_voice_settings_obj) if Voice and self.elevenlabs_voice_settings_obj else str(self.elevenlabs_voice_id)
audio_b = self.elevenlabs_client_instance.generate(text=narration_text,voice=voice_p,model="eleven_multilingual_v2")
with open(narration_fp,"wb") as f_audio: f_audio.write(audio_b); logger.info(f"11L audio (non-stream): {narration_fp}"); return narration_fp
else: logger.error("No recognized 11L audio method."); return None
if stream_method:
voice_stream_params={"voice_id":str(self.elevenlabs_voice_id)}
if self.elevenlabs_voice_settings_obj:
if hasattr(self.elevenlabs_voice_settings_obj,'model_dump'): voice_stream_params["voice_settings"]=self.elevenlabs_voice_settings_obj.model_dump()
elif hasattr(self.elevenlabs_voice_settings_obj,'dict'): voice_stream_params["voice_settings"]=self.elevenlabs_voice_settings_obj.dict()
else: voice_stream_params["voice_settings"]=self.elevenlabs_voice_settings_obj
audio_iter = stream_method(text=narration_text,model_id="eleven_multilingual_v2",**voice_stream_params)
with open(narration_fp,"wb") as f_audio_stream:
for chunk_item in audio_iter:
if chunk_item: f_audio_stream.write(chunk_item)
logger.info(f"11L audio (stream): {narration_fp}"); return narration_fp
except AttributeError as e_11l_attr: logger.error(f"11L SDK AttrError: {e_11l_attr}. SDK/methods changed?", exc_info=True); return None
except Exception as e_11l_gen: logger.error(f"11L audio gen error: {e_11l_gen}", exc_info=True); return None
def assemble_animatic_from_assets(self, asset_data_list, overall_narration_path=None, output_filename="final_video.mp4", fps=24):
# (Keep the version with robust image processing, C-contiguous array, debug saves, and pix_fmt)
# This is the most critical part for your "blank/corrupted video" issue.
if not asset_data_list: logger.warning("No assets for animatic."); return None
processed_moviepy_clips_list = []; narration_audio_clip_mvpy = None; final_video_output_clip = None
logger.info(f"Assembling from {len(asset_data_list)} assets. Target Frame: {self.video_frame_size}.")
for i_asset, asset_info_item_loop in enumerate(asset_data_list):
path_of_asset, type_of_asset, duration_for_scene = asset_info_item_loop.get('path'), asset_info_item_loop.get('type'), asset_info_item_loop.get('duration', 4.5)
num_of_scene, action_in_key = asset_info_item_loop.get('scene_num', i_asset + 1), asset_info_item_loop.get('key_action', '')
logger.info(f"S{num_of_scene}: Path='{path_of_asset}', Type='{type_of_asset}', Dur='{duration_for_scene}'s")
if not (path_of_asset and os.path.exists(path_of_asset)): logger.warning(f"S{num_of_scene}: Not found '{path_of_asset}'. Skip."); continue
if duration_for_scene <= 0: logger.warning(f"S{num_of_scene}: Invalid duration ({duration_for_scene}s). Skip."); continue
active_scene_clip = None # Clip for this iteration
try:
if type_of_asset == 'image':
pil_img_original = Image.open(path_of_asset)
logger.debug(f"S{num_of_scene} (0-Load): Original loaded. Mode:{pil_img_original.mode}, Size:{pil_img_original.size}")
pil_img_original.save(os.path.join(self.output_dir,f"debug_0_ORIGINAL_S{num_of_scene}.png"))
img_rgba_intermediate = pil_img_original.convert('RGBA') if pil_img_original.mode != 'RGBA' else pil_img_original.copy().convert('RGBA')
logger.debug(f"S{num_of_scene} (1-ToRGBA): Converted to RGBA. Mode:{img_rgba_intermediate.mode}, Size:{img_rgba_intermediate.size}")
img_rgba_intermediate.save(os.path.join(self.output_dir,f"debug_1_AS_RGBA_S{num_of_scene}.png"))
thumbnailed_img_rgba = img_rgba_intermediate.copy()
resample_filter_pil = Image.Resampling.LANCZOS if hasattr(Image.Resampling,'LANCZOS') else Image.BILINEAR
thumbnailed_img_rgba.thumbnail(self.video_frame_size, resample_filter_pil)
logger.debug(f"S{num_of_scene} (2-Thumbnail): Thumbnailed RGBA. Mode:{thumbnailed_img_rgba.mode}, Size:{thumbnailed_img_rgba.size}")
thumbnailed_img_rgba.save(os.path.join(self.output_dir,f"debug_2_THUMBNAIL_RGBA_S{num_of_scene}.png"))
canvas_for_compositing_rgba = Image.new('RGBA', self.video_frame_size, (0,0,0,0))
pos_x_paste = (self.video_frame_size[0] - thumbnailed_img_rgba.width) // 2
pos_y_paste = (self.video_frame_size[1] - thumbnailed_img_rgba.height) // 2
canvas_for_compositing_rgba.paste(thumbnailed_img_rgba, (pos_x_paste, pos_y_paste), thumbnailed_img_rgba)
logger.debug(f"S{num_of_scene} (3-PasteOnRGBA): Image pasted onto transparent RGBA canvas. Mode:{canvas_for_compositing_rgba.mode}, Size:{canvas_for_compositing_rgba.size}")
canvas_for_compositing_rgba.save(os.path.join(self.output_dir,f"debug_3_COMPOSITED_RGBA_S{num_of_scene}.png"))
final_rgb_image_for_pil = Image.new("RGB", self.video_frame_size, (0, 0, 0)) # Opaque black background
if canvas_for_compositing_rgba.mode == 'RGBA':
final_rgb_image_for_pil.paste(canvas_for_compositing_rgba, mask=canvas_for_compositing_rgba.split()[3])
else: final_rgb_image_for_pil.paste(canvas_for_compositing_rgba)
logger.debug(f"S{num_of_scene} (4-ToRGB): Final RGB image created. Mode:{final_rgb_image_for_pil.mode}, Size:{final_rgb_image_for_pil.size}")
debug_path_img_pre_numpy = os.path.join(self.output_dir,f"debug_4_PRE_NUMPY_RGB_S{num_of_scene}.png");
final_rgb_image_for_pil.save(debug_path_img_pre_numpy);
logger.info(f"CRITICAL DEBUG: Saved PRE_NUMPY_RGB_S{num_of_scene} (image fed to NumPy) to {debug_path_img_pre_numpy}")
numpy_frame_arr = np.array(final_rgb_image_for_pil, dtype=np.uint8)
if not numpy_frame_arr.flags['C_CONTIGUOUS']: numpy_frame_arr = np.ascontiguousarray(numpy_frame_arr, dtype=np.uint8)
logger.debug(f"S{num_of_scene} (5-NumPy): Final NumPy array for MoviePy. Shape:{numpy_frame_arr.shape}, DType:{numpy_frame_arr.dtype}, Flags:{numpy_frame_arr.flags}")
if numpy_frame_arr.size == 0 or numpy_frame_arr.ndim != 3 or numpy_frame_arr.shape[2] != 3: logger.error(f"S{num_of_scene}: Invalid NumPy array shape/size ({numpy_frame_arr.shape}) for ImageClip. Skipping."); continue
base_image_clip_mvpy = ImageClip(numpy_frame_arr, transparent=False, ismask=False).set_duration(duration_for_scene)
logger.debug(f"S{num_of_scene} (6-ImageClip): Base ImageClip created. Duration: {base_image_clip_mvpy.duration}")
debug_path_moviepy_frame = os.path.join(self.output_dir,f"debug_7_MOVIEPY_FRAME_S{num_of_scene}.png")
try: base_image_clip_mvpy.save_frame(debug_path_moviepy_frame, t=min(0.1, base_image_clip_mvpy.duration / 2 if base_image_clip_mvpy.duration > 0 else 0.1))
logger.info(f"CRITICAL DEBUG: Saved frame FROM MOVIEPY ImageClip for S{num_of_scene} to {debug_path_moviepy_frame}")
except Exception as e_save_mvpy_frame: logger.error(f"DEBUG: Error saving frame FROM MOVIEPY ImageClip S{num_of_scene}: {e_save_mvpy_frame}", exc_info=True)
fx_image_clip_mvpy = base_image_clip_mvpy
try:
scale_end_kb_val = random.uniform(1.03, 1.08)
if duration_for_scene > 0: fx_image_clip_mvpy = base_image_clip_mvpy.fx(vfx.resize, lambda t_val: 1 + (scale_end_kb_val - 1) * (t_val / duration_for_scene)).set_position('center'); logger.debug(f"S{num_of_scene} (8-KenBurns): Ken Burns applied.")
else: logger.warning(f"S{num_of_scene}: Duration zero, skipping Ken Burns.")
except Exception as e_kb_fx_loop: logger.error(f"S{num_of_scene} Ken Burns error: {e_kb_fx_loop}", exc_info=False)
active_scene_clip = fx_image_clip_mvpy
elif type_of_asset == 'video':
# (Video processing logic as before)
source_video_clip_obj=None
try:
logger.debug(f"S{num_of_scene}: Loading VIDEO asset: {path_of_asset}")
source_video_clip_obj=VideoFileClip(path_of_asset,target_resolution=(self.video_frame_size[1],self.video_frame_size[0])if self.video_frame_size else None, audio=False)
temp_video_clip_obj_loop=source_video_clip_obj
if source_video_clip_obj.duration!=duration_for_scene:
if source_video_clip_obj.duration>duration_for_scene:temp_video_clip_obj_loop=source_video_clip_obj.subclip(0,duration_for_scene)
else:
if duration_for_scene/source_video_clip_obj.duration > 1.5 and source_video_clip_obj.duration>0.1:temp_video_clip_obj_loop=source_video_clip_obj.loop(duration=duration_for_scene)
else:temp_video_clip_obj_loop=source_video_clip_obj.set_duration(source_video_clip_obj.duration);logger.info(f"S{num_of_scene} Video clip ({source_video_clip_obj.duration:.2f}s) shorter than target ({duration_for_scene:.2f}s).")
active_scene_clip=temp_video_clip_obj_loop.set_duration(duration_for_scene)
if active_scene_clip.size!=list(self.video_frame_size):active_scene_clip=active_scene_clip.resize(self.video_frame_size)
logger.debug(f"S{num_of_scene}: Video asset processed. Final duration for scene: {active_scene_clip.duration:.2f}s")
except Exception as e_vid_load_loop:logger.error(f"S{num_of_scene} Video load error '{path_of_asset}':{e_vid_load_loop}",exc_info=True);continue
finally:
if source_video_clip_obj and source_video_clip_obj is not active_scene_clip and hasattr(source_video_clip_obj,'close'):
try: source_video_clip_obj.close()
except Exception as e_close_src_vid: logger.warning(f"S{num_of_scene}: Error closing source VideoFileClip: {e_close_src_vid}")
else: logger.warning(f"S{num_of_scene} Unknown asset type '{type_of_asset}'. Skipping."); continue
if active_scene_clip and action_in_key:
try:
dur_text_overlay_val=min(active_scene_clip.duration-0.5,active_scene_clip.duration*0.8)if active_scene_clip.duration>0.5 else active_scene_clip.duration; start_text_overlay_val=0.25
if dur_text_overlay_val > 0:
text_clip_for_overlay_obj=TextClip(f"Scene {num_of_scene}\n{action_in_key}",fontsize=self.VIDEO_OVERLAY_FONT_SIZE,color=self.VIDEO_OVERLAY_FONT_COLOR,font=self.active_moviepy_font_name,bg_color='rgba(10,10,20,0.7)',method='caption',align='West',size=(self.video_frame_size[0]*0.9,None),kerning=-1,stroke_color='black',stroke_width=1.5).set_duration(dur_text_overlay_val).set_start(start_text_overlay_val).set_position(('center',0.92),relative=True)
active_scene_clip=CompositeVideoClip([active_scene_clip,text_clip_for_overlay_obj],size=self.video_frame_size,use_bgclip=True)
logger.debug(f"S{num_of_scene}: Text overlay composited.")
else: logger.warning(f"S{num_of_scene}: Text overlay duration zero or negative ({dur_text_overlay_val}). Skipping text overlay.")
except Exception as e_txt_comp_loop:logger.error(f"S{num_of_scene} TextClip compositing error:{e_txt_comp_loop}. Proceeding without text for this scene.",exc_info=True)
if active_scene_clip: processed_moviepy_clips_list.append(active_scene_clip); logger.info(f"S{num_of_scene}: Asset successfully processed. Clip duration: {active_scene_clip.duration:.2f}s. Added to final list.")
except Exception as e_asset_loop_main_exc: logger.error(f"MAJOR UNHANDLED ERROR processing asset for S{num_of_scene} (Path: {path_of_asset}): {e_asset_loop_main_exc}", exc_info=True)
finally:
if active_scene_clip and hasattr(active_scene_clip,'close'):
try: active_scene_clip.close()
except Exception as e_close_active_err: logger.warning(f"S{num_of_scene}: Error closing active_scene_clip in error handler: {e_close_active_err}")
continue
if not processed_moviepy_clips_list: logger.warning("No MoviePy clips were successfully processed. Aborting animatic assembly before concatenation."); return None
transition_duration_val=0.75
try:
logger.info(f"Concatenating {len(processed_moviepy_clips_list)} processed clips for final animatic.");
if len(processed_moviepy_clips_list)>1: final_video_output_clip=concatenate_videoclips(processed_moviepy_clips_list, padding=-transition_duration_val if transition_duration_val > 0 else 0, method="compose")
elif processed_moviepy_clips_list: final_video_output_clip=processed_moviepy_clips_list[0]
if not final_video_output_clip: logger.error("Concatenation resulted in a None clip. Aborting."); return None
logger.info(f"Concatenated animatic base duration:{final_video_output_clip.duration:.2f}s")
if transition_duration_val > 0 and final_video_output_clip.duration > 0:
if final_video_output_clip.duration > transition_duration_val * 2: final_video_output_clip=final_video_output_clip.fx(vfx.fadein,transition_duration_val).fx(vfx.fadeout,transition_duration_val)
else: final_video_output_clip=final_video_output_clip.fx(vfx.fadein,min(transition_duration_val,final_video_output_clip.duration/2.0))
logger.debug("Applied fade in/out effects to final composite clip.")
if overall_narration_path and os.path.exists(overall_narration_path) and final_video_output_clip.duration > 0:
try: narration_audio_clip_mvpy=AudioFileClip(overall_narration_path); logger.info(f"Adding overall narration. Video duration: {final_video_output_clip.duration:.2f}s, Narration duration: {narration_audio_clip_mvpy.duration:.2f}s"); final_video_output_clip=final_video_output_clip.set_audio(narration_audio_clip_mvpy); logger.info("Overall narration successfully added to animatic.")
except Exception as e_narr_add_final:logger.error(f"Error adding overall narration to animatic:{e_narr_add_final}",exc_info=True)
elif final_video_output_clip.duration <= 0: logger.warning("Animatic has zero or negative duration before adding audio. Audio will not be added.")
if final_video_output_clip and final_video_output_clip.duration > 0:
final_output_path_str=os.path.join(self.output_dir,output_filename); logger.info(f"Writing final animatic video to: {final_output_path_str} (Target Duration: {final_video_output_clip.duration:.2f}s)")
num_threads = os.cpu_count(); num_threads = num_threads if isinstance(num_threads, int) and num_threads >= 1 else 2
final_video_output_clip.write_videofile(final_output_path_str, fps=fps, codec='libx264', preset='medium', audio_codec='aac', temp_audiofile=os.path.join(self.output_dir,f'temp-audio-{os.urandom(4).hex()}.m4a'), remove_temp=True, threads=num_threads, logger='bar', bitrate="5000k", ffmpeg_params=["-pix_fmt", "yuv420p"])
logger.info(f"Animatic video created successfully: {final_output_path_str}"); return final_output_path_str
else: logger.error("Final animatic clip is invalid or has zero duration. Cannot write video file."); return None
except Exception as e_vid_write_final_op: logger.error(f"Error during final animatic video file writing or composition stage: {e_vid_write_final_op}", exc_info=True); return None
finally:
logger.debug("Closing all MoviePy clips in `assemble_animatic_from_assets` main finally block.")
all_clips_for_closure = processed_moviepy_clips_list[:]
if narration_audio_clip_mvpy: all_clips_for_closure.append(narration_audio_clip_mvpy)
if final_video_output_clip: all_clips_for_closure.append(final_video_output_clip)
for clip_to_close_item_final in all_clips_for_closure:
if clip_to_close_item_final and hasattr(clip_to_close_item_final, 'close'):
try: clip_to_close_item_final.close()
except Exception as e_final_clip_close_op: logger.warning(f"Ignoring error while closing a MoviePy clip ({type(clip_to_close_item_final).__name__}): {e_final_clip_close_op}")