File size: 49,424 Bytes
e22eb13 e0b9b11 b1b7840 990e23e 92cb699 5089920 92cb699 200c5c4 b1b7840 e22eb13 b1b7840 e22eb13 b1b7840 f13d4b2 5089920 f13d4b2 5089920 b1b7840 5089920 b1b7840 4c2220b f13d4b2 287c9ca b1b7840 e0b9b11 b1b7840 55ef0ff b1b7840 55ef0ff b1b7840 55ef0ff b1b7840 55ef0ff b1b7840 d44d308 b1b7840 55ef0ff b1b7840 200c5c4 09d5c67 b1b7840 d44d308 b1b7840 55ef0ff b1b7840 55ef0ff b1b7840 55ef0ff b1b7840 55ef0ff b1b7840 55ef0ff 4da81e5 b1b7840 5089920 b1b7840 e22eb13 b1b7840 5089920 b1b7840 cb93f9c b1b7840 610a011 b1b7840 4da81e5 b1b7840 4da81e5 b1b7840 610a011 b1b7840 cb93f9c 610a011 b1b7840 610a011 b1b7840 610a011 8583908 b1b7840 610a011 b1b7840 610a011 b1b7840 3313da9 b1b7840 cb93f9c b1b7840 59af6e7 b1b7840 610a011 59af6e7 b1b7840 cb93f9c b1b7840 a219e07 b1b7840 b97795f b1b7840 610a011 b1b7840 754c854 3313da9 b1b7840 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 |
# core/visual_engine.py
from PIL import Image, ImageDraw, ImageFont, ImageOps
import base64
import mimetypes
import numpy as np
import os
import openai
import requests
import io
import time
import random
import logging
# --- MoviePy Imports ---
from moviepy.editor import (
ImageClip,
VideoFileClip,
concatenate_videoclips,
TextClip,
CompositeVideoClip,
AudioFileClip,
)
import moviepy.video.fx.all as vfx
# --- MONKEY PATCH for Pillow/MoviePy compatibility ---
try:
if hasattr(Image, "Resampling") and hasattr(Image.Resampling, "LANCZOS"): # Pillow 9+
if not hasattr(Image, "ANTIALIAS"):
Image.ANTIALIAS = Image.Resampling.LANCZOS
elif hasattr(Image, "LANCZOS"): # Pillow 8
if not hasattr(Image, "ANTIALIAS"):
Image.ANTIALIAS = Image.LANCZOS
elif not hasattr(Image, "ANTIALIAS"): # Fallback if no common resampling attributes found
print(
"WARNING: Pillow version lacks common Resampling attributes or ANTIALIAS. MoviePy effects might fail or look different."
)
except Exception as e_monkey_patch:
print(
f"WARNING: An unexpected error occurred during Pillow ANTIALIAS monkey-patch: {e_monkey_patch}"
)
logger = logging.getLogger(__name__)
# Consider setting level in main app if not already configured:
# logger.setLevel(logging.DEBUG) # For very verbose output during debugging
# --- External Service Client Imports ---
ELEVENLABS_CLIENT_IMPORTED = False
ElevenLabsAPIClient = None
Voice = None
VoiceSettings = None
try:
from elevenlabs.client import ElevenLabs as ImportedElevenLabsClient
from elevenlabs import Voice as ImportedVoice, VoiceSettings as ImportedVoiceSettings
ElevenLabsAPIClient = ImportedElevenLabsClient
Voice = ImportedVoice
VoiceSettings = ImportedVoiceSettings
ELEVENLABS_CLIENT_IMPORTED = True
logger.info("ElevenLabs client components imported successfully.")
except ImportError:
logger.warning(
"ElevenLabs SDK not found (pip install elevenlabs). Audio generation will be disabled."
)
except Exception as e_eleven_import:
logger.warning(
f"Error importing ElevenLabs client components: {e_eleven_import}. Audio generation disabled."
)
RUNWAYML_SDK_IMPORTED = False
RunwayMLAPIClient = None # Using a more specific name for the client class
try:
from runwayml import RunwayML as ImportedRunwayMLClient # Actual SDK import
RunwayMLAPIClient = ImportedRunwayMLClient
RUNWAYML_SDK_IMPORTED = True
logger.info("RunwayML SDK imported successfully.")
except ImportError:
logger.warning(
"RunwayML SDK not found (pip install runwayml). RunwayML video generation will be disabled."
)
except Exception as e_runway_sdk_import:
logger.warning(
f"Error importing RunwayML SDK: {e_runway_sdk_import}. RunwayML features disabled."
)
class VisualEngine:
DEFAULT_FONT_SIZE_PIL = 10 # For default Pillow font
PREFERRED_FONT_SIZE_PIL = 20 # For custom font
VIDEO_OVERLAY_FONT_SIZE = 30
VIDEO_OVERLAY_FONT_COLOR = "white"
# Standard font names ImageMagick (used by TextClip) is likely to find in Linux containers
DEFAULT_MOVIEPY_FONT = "DejaVu-Sans-Bold"
PREFERRED_MOVIEPY_FONT = "Liberation-Sans-Bold" # Often available
def __init__(
self, output_dir="temp_cinegen_media", default_elevenlabs_voice_id="Rachel"
):
self.output_dir = output_dir
os.makedirs(self.output_dir, exist_ok=True)
self.font_filename_pil = "DejaVuSans-Bold.ttf" # A more standard Linux font
font_paths_to_try = [
self.font_filename_pil, # If in working dir or PATH
f"/usr/share/fonts/truetype/dejavu/{self.font_filename_pil}",
f"/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf", # Alternative
f"/System/Library/Fonts/Supplemental/Arial.ttf", # macOS fallback
f"C:/Windows/Fonts/arial.ttf", # Windows fallback
f"/usr/local/share/fonts/truetype/mycustomfonts/arial.ttf", # User's previous custom path
]
self.font_path_pil_resolved = next(
(p for p in font_paths_to_try if os.path.exists(p)), None
)
self.font_pil = ImageFont.load_default() # Default
self.current_font_size_pil = self.DEFAULT_FONT_SIZE_PIL
if self.font_path_pil_resolved:
try:
self.font_pil = ImageFont.truetype(
self.font_path_pil_resolved, self.PREFERRED_FONT_SIZE_PIL
)
self.current_font_size_pil = self.PREFERRED_FONT_SIZE_PIL
logger.info(
f"Pillow font loaded: {self.font_path_pil_resolved} at size {self.current_font_size_pil}."
)
# Determine MoviePy font based on loaded PIL font
if "dejavu" in self.font_path_pil_resolved.lower():
self.video_overlay_font = "DejaVu-Sans-Bold"
elif "liberation" in self.font_path_pil_resolved.lower():
self.video_overlay_font = "Liberation-Sans-Bold"
else: # Fallback if custom font doesn't have an obvious ImageMagick name
self.video_overlay_font = self.DEFAULT_MOVIEPY_FONT
except IOError as e_font_load:
logger.error(
f"Pillow font loading IOError for '{self.font_path_pil_resolved}': {e_font_load}. Using default."
)
else:
logger.warning("Custom Pillow font not found. Using default.")
self.openai_api_key = None
self.USE_AI_IMAGE_GENERATION = False
self.dalle_model = "dall-e-3"
self.image_size_dalle3 = "1792x1024"
self.video_frame_size = (1280, 720)
self.elevenlabs_api_key = None
self.USE_ELEVENLABS = False
self.elevenlabs_client = None
self.elevenlabs_voice_id = default_elevenlabs_voice_id
if VoiceSettings and ELEVENLABS_CLIENT_IMPORTED:
self.elevenlabs_voice_settings = VoiceSettings(
stability=0.60,
similarity_boost=0.80,
style=0.15,
use_speaker_boost=True,
)
else:
self.elevenlabs_voice_settings = None
self.pexels_api_key = None
self.USE_PEXELS = False
self.runway_api_key = None
self.USE_RUNWAYML = False
self.runway_ml_client_instance = None # More specific name
# Attempt to initialize Runway client if SDK is present and env var might be set
if (
RUNWAYML_SDK_IMPORTED
and RunwayMLAPIClient
and os.getenv("RUNWAYML_API_SECRET")
):
try:
self.runway_ml_client_instance = RunwayMLAPIClient() # SDK uses env var
self.USE_RUNWAYML = True # Assume enabled if client initializes
logger.info(
"RunwayML Client initialized from RUNWAYML_API_SECRET env var at startup."
)
except Exception as e_runway_init_startup:
logger.error(
f"Initial RunwayML client init failed (env var RUNWAYML_API_SECRET might be invalid): {e_runway_init_startup}"
)
self.USE_RUNWAYML = False
logger.info("VisualEngine initialized.")
# --- API Key Setters ---
def set_openai_api_key(self, api_key):
self.openai_api_key = api_key
self.USE_AI_IMAGE_GENERATION = bool(api_key)
logger.info(
f"DALL-E ({self.dalle_model}) status: {'Ready' if self.USE_AI_IMAGE_GENERATION else 'Disabled'}"
)
def set_elevenlabs_api_key(self, api_key, voice_id_from_secret=None):
self.elevenlabs_api_key = api_key
if voice_id_from_secret:
self.elevenlabs_voice_id = voice_id_from_secret
if api_key and ELEVENLABS_CLIENT_IMPORTED and ElevenLabsAPIClient:
try:
self.elevenlabs_client = ElevenLabsAPIClient(api_key=api_key)
self.USE_ELEVENLABS = bool(self.elevenlabs_client)
logger.info(
f"ElevenLabs Client status: {'Ready' if self.USE_ELEVENLABS else 'Failed Initialization'} (Using Voice ID: {self.elevenlabs_voice_id})"
)
except Exception as e:
logger.error(
f"ElevenLabs client initialization error: {e}. Service Disabled.",
exc_info=True,
)
self.USE_ELEVENLABS = False
self.elevenlabs_client = None
else:
self.USE_ELEVENLABS = False
logger.info(
f"ElevenLabs Service Disabled (API key not provided or SDK import issue)."
)
def set_pexels_api_key(self, api_key):
self.pexels_api_key = api_key
self.USE_PEXELS = bool(api_key)
logger.info(
f"Pexels Search status: {'Ready' if self.USE_PEXELS else 'Disabled'}"
)
def set_runway_api_key(self, api_key):
self.runway_api_key = api_key # Store key regardless for potential direct HTTP use
if api_key:
if RUNWAYML_SDK_IMPORTED and RunwayMLAPIClient:
if not self.runway_ml_client_instance: # If not already initialized by env var
try:
# The RunwayML Python SDK expects the API key via the RUNWAYML_API_SECRET env var.
# If it's not set, we set it temporarily for client initialization.
original_env_secret = os.getenv("RUNWAYML_API_SECRET")
if not original_env_secret:
logger.info(
"Temporarily setting RUNWAYML_API_SECRET from provided key for SDK client init."
)
os.environ["RUNWAYML_API_SECRET"] = api_key
self.runway_ml_client_instance = RunwayMLAPIClient()
self.USE_RUNWAYML = True # SDK client successfully initialized
logger.info(
"RunwayML Client initialized successfully using provided API key."
)
if not original_env_secret: # Clean up if we set it
del os.environ["RUNWAYML_API_SECRET"]
logger.info(
"Cleared temporary RUNWAYML_API_SECRET env var."
)
except Exception as e_client_init:
logger.error(
f"RunwayML Client initialization via set_runway_api_key failed: {e_client_init}",
exc_info=True,
)
self.USE_RUNWAYML = False
self.runway_ml_client_instance = None
else: # Client was already initialized (likely via env var during __init__)
self.USE_RUNWAYML = True
logger.info(
"RunwayML Client was already initialized (likely from env var). API key stored."
)
else: # SDK not imported
logger.warning(
"RunwayML SDK not imported. API key stored, but integration requires SDK. Service effectively disabled."
)
self.USE_RUNWAYML = False
else: # No API key provided
self.USE_RUNWAYML = False
self.runway_ml_client_instance = None
logger.info("RunwayML Service Disabled (no API key provided).")
# --- Helper Methods ---
def _image_to_data_uri(self, image_path):
try:
mime_type, _ = mimetypes.guess_type(image_path)
if not mime_type:
ext = os.path.splitext(image_path)[1].lower()
mime_map = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"}
mime_type = mime_map.get(ext, "application/octet-stream")
if mime_type == "application/octet-stream":
logger.warning(
f"Could not determine MIME type for {image_path}, using default."
)
with open(image_path, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
data_uri = f"data:{mime_type};base64,{encoded_string}"
logger.debug(
f"Generated data URI for {os.path.basename(image_path)} (first 100 chars): {data_uri[:100]}..."
)
return data_uri
except FileNotFoundError:
logger.error(f"Image file not found at {image_path} for data URI conversion.")
return None
except Exception as e:
logger.error(
f"Error converting image {image_path} to data URI: {e}", exc_info=True
)
return None
def _map_resolution_to_runway_ratio(self, width, height):
ratio_str = f"{width}:{height}"
# Gen-4 supports: "1280:720", "720:1280", "1104:832", "832:1104", "960:960", "1584:672"
supported_ratios_gen4 = [
"1280:720",
"720:1280",
"1104:832",
"832:1104",
"960:960",
"1584:672",
]
if ratio_str in supported_ratios_gen4:
return ratio_str
# Fallback or find closest - for now, strict matching or default
logger.warning(
f"Resolution {ratio_str} not directly in Gen-4 supported list. Defaulting to 1280:720."
)
return "1280:720"
def _get_text_dimensions(self, text_content, font_object):
# (Robust version from before)
default_char_height = getattr(font_object, "size", self.current_font_size_pil)
if not text_content:
return 0, default_char_height
try:
if hasattr(font_object, "getbbox"):
bbox = font_object.getbbox(text_content)
w = bbox[2] - bbox[0]
h = bbox[3] - bbox[1]
return w, h if h > 0 else default_char_height
elif hasattr(font_object, "getsize"):
w, h = font_object.getsize(text_content)
return w, h if h > 0 else default_char_height
else:
return (
int(len(text_content) * default_char_height * 0.6),
int(default_char_height * 1.2),
)
except Exception as e:
logger.warning(f"Error in _get_text_dimensions: {e}")
return (
int(len(text_content) * self.current_font_size_pil * 0.6),
int(self.current_font_size_pil * 1.2),
)
def _create_placeholder_image_content(self, text_description, filename, size=None):
# (Corrected version from previous response)
if size is None:
size = self.video_frame_size
img = Image.new("RGB", size, color=(20, 20, 40))
d = ImageDraw.Draw(img)
padding = 25
max_w = size[0] - (2 * padding)
lines = []
if not text_description:
text_description = "(Placeholder Image)"
words = text_description.split()
current_line_text = ""
for word_idx, word in enumerate(words):
prospective_addition = word + (" " if word_idx < len(words) - 1 else "")
test_line_text = current_line_text + prospective_addition
current_w, _ = self._get_text_dimensions(test_line_text, self.font_pil)
if current_w == 0 and test_line_text.strip():
current_w = len(test_line_text) * (self.current_font_size_pil * 0.6) # Estimate
if current_w <= max_w:
current_line_text = test_line_text
else:
if current_line_text.strip():
lines.append(current_line_text.strip())
current_line_text = prospective_addition # Start new line
if current_line_text.strip():
lines.append(current_line_text.strip())
if not lines and text_description:
avg_char_w, _ = self._get_text_dimensions("W", self.font_pil)
avg_char_w = avg_char_w or (self.current_font_size_pil * 0.6)
chars_per_line = int(max_w / avg_char_w) if avg_char_w > 0 else 20
lines.append(
text_description[:chars_per_line]
+ ("..." if len(text_description) > chars_per_line else "")
)
elif not lines:
lines.append("(Placeholder Error)")
_, single_line_h = self._get_text_dimensions("Ay", self.font_pil)
single_line_h = single_line_h if single_line_h > 0 else self.current_font_size_pil + 2
max_lines = (
min(len(lines), (size[1] - (2 * padding)) // (single_line_h + 2))
if single_line_h > 0
else 1
)
max_lines = max(1, max_lines) # Ensure at least one line
y_pos = padding + (size[1] - (2 * padding) - max_lines * (single_line_h + 2)) / 2.0
for i in range(max_lines):
line_text = lines[i]
line_w, _ = self._get_text_dimensions(line_text, self.font_pil)
if line_w == 0 and line_text.strip():
line_w = len(line_text) * (self.current_font_size_pil * 0.6)
x_pos = (size[0] - line_w) / 2.0
try:
d.text((x_pos, y_pos), line_text, font=self.font_pil, fill=(200, 200, 180))
except Exception as e_draw:
logger.error(f"Pillow d.text error: {e_draw} for '{line_text}'")
y_pos += single_line_h + 2
if i == 6 and max_lines > 7:
try:
d.text((x_pos, y_pos), "...", font=self.font_pil, fill=(200, 200, 180))
except Exception as e_elip:
logger.error(f"Pillow d.text ellipsis error: {e_elip}")
break
filepath = os.path.join(self.output_dir, filename)
try:
img.save(filepath)
return filepath
except Exception as e_save:
logger.error(
f"Saving placeholder image '{filepath}' error: {e_save}", exc_info=True
)
return None
def _search_pexels_image(self, query, output_filename_base):
# <<< THIS IS THE CORRECTED METHOD >>>
if not self.USE_PEXELS or not self.pexels_api_key:
return None
headers = {"Authorization": self.pexels_api_key}
params = {"query": query, "per_page": 1, "orientation": "landscape", "size": "large2x"}
base_name_for_pexels, _ = os.path.splitext(output_filename_base)
pexels_filename = base_name_for_pexels + f"_pexels_{random.randint(1000,9999)}.jpg"
filepath = os.path.join(self.output_dir, pexels_filename)
try:
logger.info(f"Pexels: Searching for '{query}'")
effective_query = " ".join(query.split()[:5])
params["query"] = effective_query
response = requests.get(
"https://api.pexels.com/v1/search", headers=headers, params=params, timeout=20
)
response.raise_for_status()
data = response.json()
if data.get("photos") and len(data["photos"]) > 0:
photo_details = data["photos"][0]
photo_url = photo_details.get("src", {}).get("large2x")
if not photo_url:
logger.warning(
f"Pexels: 'large2x' URL missing for '{effective_query}'. Details: {photo_details}"
)
return None
image_response = requests.get(photo_url, timeout=60)
image_response.raise_for_status()
img_data_pil = Image.open(io.BytesIO(image_response.content))
if img_data_pil.mode != "RGB":
img_data_pil = img_data_pil.convert("RGB")
img_data_pil.save(filepath)
logger.info(f"Pexels: Image saved to {filepath}")
return filepath
else:
logger.info(f"Pexels: No photos for '{effective_query}'.")
return None
except requests.exceptions.RequestException as e_req:
logger.error(f"Pexels: RequestException for '{query}': {e_req}", exc_info=False)
return None # Less verbose for network
except Exception as e:
logger.error(f"Pexels: General error for '{query}': {e}", exc_info=True)
return None
# --- RunwayML Video Generation (Gen-4 Aligned with SDK) ---
def _generate_video_clip_with_runwayml(
self,
text_prompt_for_motion,
input_image_path,
scene_identifier_filename_base,
target_duration_seconds=5,
):
if not self.USE_RUNWAYML or not self.runway_ml_client_instance:
logger.warning("RunwayML not enabled or client not initialized. Cannot generate video clip.")
return None
if not input_image_path or not os.path.exists(input_image_path):
logger.error(
f"Runway Gen-4 requires an input image. Path not provided or invalid: {input_image_path}"
)
return None
image_data_uri = self._image_to_data_uri(input_image_path)
if not image_data_uri:
return None
runway_duration = 10 if target_duration_seconds >= 8 else 5 # Map to 5s or 10s for Gen-4
runway_ratio_str = self._map_resolution_to_runway_ratio(
self.video_frame_size[0], self.video_frame_size[1]
)
# Use a more descriptive output filename for Runway videos
base_name_for_runway, _ = os.path.splitext(scene_identifier_filename_base)
output_video_filename = base_name_for_runway + f"_runway_gen4_d{runway_duration}s.mp4"
output_video_filepath = os.path.join(self.output_dir, output_video_filename)
logger.info(
f"Initiating Runway Gen-4 task: motion='{text_prompt_for_motion[:100]}...', image='{os.path.basename(input_image_path)}', dur={runway_duration}s, ratio='{runway_ratio_str}'"
)
try:
# Using the RunwayML Python SDK structure
task_submission = self.runway_ml_client_instance.image_to_video.create(
model="gen4_turbo",
prompt_image=image_data_uri,
prompt_text=text_prompt_for_motion, # This is the motion prompt
duration=runway_duration,
ratio=runway_ratio_str,
# seed=random.randint(0, 4294967295), # Optional: for reproducibility
# Other Gen-4 params (motion_score, upscale, watermark etc. can be added here if available in SDK)
)
task_id = task_submission.id
logger.info(f"Runway Gen-4 task created with ID: {task_id}. Polling for completion...")
poll_interval_seconds = 10
max_polling_duration_seconds = 6 * 60 # 6 minutes
start_time = time.time()
while time.time() - start_time < max_polling_duration_seconds:
time.sleep(poll_interval_seconds)
task_details = self.runway_ml_client_instance.tasks.retrieve(id=task_id)
logger.info(f"Runway task {task_id} status: {task_details.status}")
if task_details.status == "SUCCEEDED":
# Determine output URL (this structure might vary based on SDK version)
output_url = None
if hasattr(task_details, "output") and task_details.output and hasattr(
task_details.output, "url"
):
output_url = task_details.output.url
elif (
hasattr(task_details, "artifacts")
and task_details.artifacts
and isinstance(task_details.artifacts, list)
and len(task_details.artifacts) > 0
):
first_artifact = task_details.artifacts[0]
if hasattr(first_artifact, "url"):
output_url = first_artifact.url
elif hasattr(first_artifact, "download_url"):
output_url = first_artifact.download_url
if not output_url:
logger.error(
f"Runway task {task_id} SUCCEEDED, but no output URL found. Details: {vars(task_details) if hasattr(task_details,'__dict__') else str(task_details)}"
)
return None
logger.info(f"Runway task {task_id} SUCCEEDED. Downloading video from: {output_url}")
video_response = requests.get(output_url, stream=True, timeout=300)
video_response.raise_for_status()
with open(output_video_filepath, "wb") as f:
for chunk in video_response.iter_content(chunk_size=8192):
f.write(chunk)
logger.info(
f"Runway Gen-4 video successfully downloaded to: {output_video_filepath}"
)
return output_video_filepath
elif task_details.status in ["FAILED", "ABORTED", "ERROR"]: # Added ERROR
error_msg = (
getattr(task_details, "error_message", None)
or getattr(getattr(task_details, "output", None), "error", "Unknown error from Runway task.")
)
logger.error(
f"Runway task {task_id} final status: {task_details.status}. Error: {error_msg}"
)
return None
logger.warning(
f"Runway task {task_id} timed out polling after {max_polling_duration_seconds} seconds."
)
return None
except AttributeError as ae: # If SDK methods are not as expected
logger.error(
f"AttributeError with RunwayML SDK: {ae}. Ensure SDK is up to date and methods/attributes match documentation.",
exc_info=True,
)
return None
except Exception as e_runway_call:
logger.error(
f"General error during Runway Gen-4 API call or processing: {e_runway_call}",
exc_info=True,
)
return None
def _create_placeholder_video_content(self, text_description, filename, duration=4, size=None):
# (Keeping as before)
if size is None:
size = self.video_frame_size
fp = os.path.join(self.output_dir, filename)
tc = None
try:
tc = TextClip(
text_description,
fontsize=50,
color="white",
font=self.video_overlay_font,
bg_color="black",
size=size,
method="caption",
).set_duration(duration)
tc.write_videofile(
fp, fps=24, codec="libx264", preset="ultrafast", logger=None, threads=2
)
logger.info(f"Generic placeholder video: {fp}")
return fp
except Exception as e:
logger.error(f"Generic placeholder video error {fp}: {e}", exc_info=True)
return None
finally:
if tc and hasattr(tc, "close"):
tc.close()
# --- generate_scene_asset (Main asset generation logic using Runway Gen-4 workflow) ---
def generate_scene_asset(
self,
image_generation_prompt_text,
motion_prompt_text_for_video,
scene_data,
scene_identifier_filename_base,
generate_as_video_clip=False,
runway_target_duration=5,
):
# (Logic updated for improved DALL路E and RunwayML fallback)
base_name, _ = os.path.splitext(scene_identifier_filename_base)
asset_info = {
"path": None,
"type": "none",
"error": True,
"prompt_used": image_generation_prompt_text,
"error_message": "Asset generation init failed",
}
input_image_for_runway_path = None
# Use a distinct name for the base image if it's only an intermediate step for video
base_image_filename = base_name + ("_base_for_video.png" if generate_as_video_clip else ".png")
base_image_filepath = os.path.join(self.output_dir, base_image_filename)
# STEP 1: Generate/acquire the base image via DALL路E
if self.USE_AI_IMAGE_GENERATION and self.openai_api_key:
try:
logger.info(f"Calling DALL路E with prompt: {image_generation_prompt_text[:70]}...")
response = openai.Image.create(
prompt=image_generation_prompt_text,
n=1,
size=self.image_size_dalle3,
model=self.dalle_model,
)
image_url = response["data"][0]["url"]
ir = requests.get(image_url, timeout=120)
ir.raise_for_status()
id_img = Image.open(io.BytesIO(ir.content))
if id_img.mode != "RGB":
id_img = id_img.convert("RGB")
id_img.save(base_image_filepath)
logger.info(f"DALL路E base image saved: {base_image_filepath}")
input_image_for_runway_path = base_image_filepath
asset_info = {
"path": base_image_filepath,
"type": "image",
"error": False,
"prompt_used": image_generation_prompt_text,
}
except openai.error.OpenAIError as e:
logger.warning(f"DALL路E error: {e}. Falling back to Pexels or placeholder.")
asset_info["error_message"] = str(e)
except Exception as e:
logger.error(f"Unexpected DALL路E error: {e}", exc_info=True)
asset_info["error_message"] = str(e)
# STEP 2: If DALL路E failed, try Pexels
if asset_info["error"] and self.USE_PEXELS:
logger.info("Attempting Pexels fallback for base image.")
pqt = scene_data.get(
"pexels_search_query_臧愲弲", f"{scene_data.get('emotional_beat','')} {scene_data.get('setting_description','')}"
)
pp = self._search_pexels_image(pqt, base_image_filename)
if pp:
input_image_for_runway_path = pp
asset_info = {
"path": pp,
"type": "image",
"error": False,
"prompt_used": f"Pexels:{pqt}",
}
else:
current_em = asset_info.get("error_message", "")
asset_info["error_message"] = (current_em + " Pexels fallback failed.").strip()
# STEP 3: If both DALL路E and Pexels failed, create placeholder
if asset_info["error"]:
logger.warning("Both DALL路E and Pexels failed. Creating placeholder image.")
ppt = asset_info.get("prompt_used", image_generation_prompt_text)
php = self._create_placeholder_image_content(
f"[Placeholder for] {ppt[:70]}...", base_image_filename
)
if php:
input_image_for_runway_path = php
asset_info = {
"path": php,
"type": "image",
"error": False,
"prompt_used": ppt,
}
else:
current_em = asset_info.get("error_message", "")
asset_info["error_message"] = (current_em + " Placeholder creation failed.").strip()
# STEP 4: If a video clip is requested, attempt RunwayML
if generate_as_video_clip:
if not input_image_for_runway_path or not os.path.exists(input_image_for_runway_path):
logger.error("No valid base image for RunwayML. Skipping video generation.")
asset_info["error"] = True
asset_info["error_message"] = (asset_info.get("error_message", "") + " No base image.").strip()
asset_info["type"] = "none"
return asset_info
if self.USE_RUNWAYML and self.runway_ml_client_instance:
video_path = self._generate_video_clip_with_runwayml(
motion_prompt_text_for_video,
input_image_for_runway_path,
base_name,
runway_target_duration,
)
if video_path and os.path.exists(video_path):
asset_info = {
"path": video_path,
"type": "video",
"error": False,
"prompt_used": motion_prompt_text_for_video,
"base_image_path": input_image_for_runway_path,
}
else:
logger.warning("RunwayML video generation failed. Returning base image instead.")
asset_info = {
"path": input_image_for_runway_path,
"type": "image",
"error": True,
"prompt_used": image_generation_prompt_text,
"error_message": (asset_info.get("error_message", "") + " RunwayML failed.").strip(),
}
else:
logger.warning("RunwayML not enabled or client not initialized. Skipping video generation.")
asset_info = {
"path": input_image_for_runway_path,
"type": "image",
"error": True,
"prompt_used": image_generation_prompt_text,
"error_message": (asset_info.get("error_message", "") + " RunwayML disabled.").strip(),
}
return asset_info
def generate_narration_audio(self, text_to_narrate, output_filename="narration_overall.mp3"):
# (Keep as before - robust enough)
if not self.USE_ELEVENLABS or not self.elevenlabs_client or not text_to_narrate:
logger.info("ElevenLabs audio skipped.")
return None
afp = os.path.join(self.output_dir, output_filename)
try:
logger.info(f"ElevenLabs audio (Voice:{self.elevenlabs_voice_id}): {text_to_narrate[:70]}...")
asm = None
if hasattr(self.elevenlabs_client, "text_to_speech") and hasattr(
self.elevenlabs_client.text_to_speech, "stream"
):
asm = self.elevenlabs_client.text_to_speech.stream
logger.info("Using ElevenLabs .text_to_speech.stream()")
elif hasattr(self.elevenlabs_client, "generate_stream"):
asm = self.elevenlabs_client.generate_stream
logger.info("Using ElevenLabs .generate_stream()")
elif hasattr(self.elevenlabs_client, "generate"):
logger.info("Using ElevenLabs .generate()")
vp = (
Voice(voice_id=str(self.elevenlabs_voice_id), settings=self.elevenlabs_voice_settings)
if Voice and self.elevenlabs_voice_settings
else str(self.elevenlabs_voice_id)
)
ab = self.elevenlabs_client.generate(
text=text_to_narrate, voice=vp, model="eleven_multilingual_v2"
)
with open(afp, "wb") as f:
f.write(ab)
logger.info(f"ElevenLabs audio (non-stream) saved: {afp}")
return afp
else:
logger.error("No ElevenLabs audio method available.")
return None
# If we have a streaming method (asm), use it
if asm:
vps = {"voice_id": str(self.elevenlabs_voice_id)}
if self.elevenlabs_voice_settings:
if hasattr(self.elevenlabs_voice_settings, "model_dump"):
vps["voice_settings"] = self.elevenlabs_voice_settings.model_dump()
elif hasattr(self.elevenlabs_voice_settings, "dict"):
vps["voice_settings"] = self.elevenlabs_voice_settings.dict()
else:
vps["voice_settings"] = self.elevenlabs_voice_settings
adi = asm(text=text_to_narrate, model_id="eleven_multilingual_v2", **vps)
with open(afp, "wb") as f:
for c in adi:
if c:
f.write(c)
logger.info(f"ElevenLabs audio (stream) saved: {afp}")
return afp
except Exception as e:
logger.error(f"ElevenLabs audio error: {e}", exc_info=True)
return None
# --- assemble_animatic_from_assets (Still contains crucial debug saves for blank video issue) ---
def assemble_animatic_from_assets(
self, asset_data_list, overall_narration_path=None, output_filename="final_video.mp4", fps=24
):
# (Keep the version with robust image processing, C-contiguous arrays, debug saves, and pix_fmt)
if not asset_data_list:
logger.warning("No assets for animatic.")
return None
processed_clips = []
narration_clip = None
final_clip = None
logger.info(f"Assembling from {len(asset_data_list)} assets. Frame: {self.video_frame_size}.")
for i, asset_info in enumerate(asset_data_list):
asset_path = asset_info.get("path")
asset_type = asset_info.get("type")
scene_dur = asset_info.get("duration", 4.5)
scene_num = asset_info.get("scene_num", i + 1)
key_action = asset_info.get("key_action", "")
logger.info(f"S{scene_num}: Path='{asset_path}', Type='{asset_type}', Dur='{scene_dur}'s")
if not (asset_path and os.path.exists(asset_path)):
logger.warning(f"S{scene_num}: Not found '{asset_path}'. Skip.")
continue
if scene_dur <= 0:
logger.warning(f"S{scene_num}: Invalid duration ({scene_dur}s). Skip.")
continue
current_scene_mvpy_clip = None
try:
if asset_type == "image":
pil_img = Image.open(asset_path)
logger.debug(f"S{scene_num}: Loaded img. Mode:{pil_img.mode}, Size:{pil_img.size}")
img_rgba = pil_img.convert("RGBA") if pil_img.mode != "RGBA" else pil_img.copy()
thumb = img_rgba.copy()
rf = Image.Resampling.LANCZOS if hasattr(Image.Resampling, "LANCZOS") else Image.BILINEAR
thumb.thumbnail(self.video_frame_size, rf)
cv_rgba = Image.new("RGBA", self.video_frame_size, (0, 0, 0, 0))
xo, yo = (
(self.video_frame_size[0] - thumb.width) // 2,
(self.video_frame_size[1] - thumb.height) // 2,
)
cv_rgba.paste(thumb, (xo, yo), thumb)
final_rgb_pil = Image.new("RGB", self.video_frame_size, (0, 0, 0))
final_rgb_pil.paste(cv_rgba, mask=cv_rgba.split()[3])
dbg_path = os.path.join(self.output_dir, f"debug_PRE_NUMPY_S{scene_num}.png")
final_rgb_pil.save(dbg_path)
logger.info(f"DEBUG: Saved PRE_NUMPY_S{scene_num} to {dbg_path}")
frame_np = np.array(final_rgb_pil, dtype=np.uint8)
if not frame_np.flags["C_CONTIGUOUS"]:
frame_np = np.ascontiguousarray(frame_np, dtype=np.uint8)
logger.debug(
f"S{scene_num}: NumPy for MoviePy. Shape:{frame_np.shape}, DType:{frame_np.dtype}, C-Contig:{frame_np.flags['C_CONTIGUOUS']}"
)
if frame_np.size == 0 or frame_np.ndim != 3 or frame_np.shape[2] != 3:
logger.error(f"S{scene_num}: Invalid NumPy. Skip.")
continue
clip_base = ImageClip(frame_np, transparent=False).set_duration(scene_dur)
mvpy_dbg_path = os.path.join(self.output_dir, f"debug_MOVIEPY_FRAME_S{scene_num}.png")
clip_base.save_frame(mvpy_dbg_path, t=0.1)
logger.info(f"DEBUG: Saved MOVIEPY_FRAME_S{scene_num} to {mvpy_dbg_path}")
clip_fx = clip_base
try:
es = random.uniform(1.03, 1.08)
clip_fx = clip_base.fx(
vfx.resize, lambda t: 1 + (es - 1) * (t / scene_dur) if scene_dur > 0 else 1
).set_position("center")
except Exception as e:
logger.error(f"S{scene_num} Ken Burns error: {e}", exc_info=False)
current_scene_mvpy_clip = clip_fx
elif asset_type == "video":
src_clip = None
try:
src_clip = VideoFileClip(
asset_path,
target_resolution=(
self.video_frame_size[1],
self.video_frame_size[0],
)
if self.video_frame_size
else None,
audio=False,
)
tmp_clip = src_clip
if src_clip.duration != scene_dur:
if src_clip.duration > scene_dur:
tmp_clip = src_clip.subclip(0, scene_dur)
else:
if scene_dur / src_clip.duration > 1.5 and src_clip.duration > 0.1:
tmp_clip = src_clip.loop(duration=scene_dur)
else:
tmp_clip = src_clip.set_duration(src_clip.duration)
logger.info(
f"S{scene_num} Video clip ({src_clip.duration:.2f}s) shorter than target ({scene_dur:.2f}s)."
)
current_scene_mvpy_clip = tmp_clip.set_duration(scene_dur)
if current_scene_mvpy_clip.size != list(self.video_frame_size):
current_scene_mvpy_clip = current_scene_mvpy_clip.resize(self.video_frame_size)
except Exception as e:
logger.error(f"S{scene_num} Video load error '{asset_path}':{e}", exc_info=True)
continue
finally:
if src_clip and src_clip is not current_scene_mvpy_clip and hasattr(src_clip, "close"):
src_clip.close()
else:
logger.warning(f"S{scene_num} Unknown asset type '{asset_type}'. Skip.")
continue
if current_scene_mvpy_clip and key_action:
try:
to_dur = (
min(current_scene_mvpy_clip.duration - 0.5, current_scene_mvpy_clip.duration * 0.8)
if current_scene_mvpy_clip.duration > 0.5
else current_scene_mvpy_clip.duration
)
to_start = 0.25
if to_dur > 0:
txt_c = TextClip(
f"Scene {scene_num}\n{key_action}",
fontsize=self.VIDEO_OVERLAY_FONT_SIZE,
color=self.VIDEO_OVERLAY_FONT_COLOR,
font=self.video_overlay_font,
bg_color="rgba(10,10,20,0.7)",
method="caption",
align="West",
size=(self.video_frame_size[0] * 0.9, None),
kerning=-1,
stroke_color="black",
stroke_width=1.5,
).set_duration(to_dur).set_start(to_start).set_position(
("center", 0.92), relative=True
)
current_scene_mvpy_clip = CompositeVideoClip(
[current_scene_mvpy_clip, txt_c], size=self.video_frame_size, use_bgclip=True
)
else:
logger.warning(f"S{scene_num}: Text overlay duration is zero. Skip text.")
except Exception as e:
logger.error(f"S{scene_num} TextClip error:{e}. No text.", exc_info=True)
if current_scene_mvpy_clip:
processed_clips.append(current_scene_mvpy_clip)
logger.info(f"S{scene_num} Processed. Dur:{current_scene_mvpy_clip.duration:.2f}s.")
except Exception as e:
logger.error(f"MAJOR Error S{scene_num} ({asset_path}):{e}", exc_info=True)
finally:
if current_scene_mvpy_clip and hasattr(current_scene_mvpy_clip, "close"):
try:
current_scene_mvpy_clip.close()
except:
pass
if not processed_clips:
logger.warning("No clips processed. Abort.")
return None
td = 0.75
try:
logger.info(f"Concatenating {len(processed_clips)} clips.")
if len(processed_clips) > 1:
final_clip = concatenate_videoclips(processed_clips, padding=-td if td > 0 else 0, method="compose")
elif processed_clips:
final_clip = processed_clips[0]
if not final_clip:
logger.error("Concatenation failed.")
return None
logger.info(f"Concatenated dur:{final_clip.duration:.2f}s")
if td > 0 and final_clip.duration > 0:
if final_clip.duration > td * 2:
final_clip = final_clip.fx(vfx.fadein, td).fx(vfx.fadeout, td)
else:
final_clip = final_clip.fx(vfx.fadein, min(td, final_clip.duration / 2.0))
if overall_narration_path and os.path.exists(overall_narration_path) and final_clip.duration > 0:
try:
narration_clip = AudioFileClip(overall_narration_path)
final_clip = final_clip.set_audio(narration_clip)
logger.info("Narration added.")
except Exception as e:
logger.error(f"Narration add error:{e}", exc_info=True)
elif final_clip.duration <= 0:
logger.warning("Video no duration. No audio.")
if final_clip and final_clip.duration > 0:
op = os.path.join(self.output_dir, output_filename)
logger.info(f"Writing video:{op} (Dur:{final_clip.duration:.2f}s)")
final_clip.write_videofile(
op,
fps=fps,
codec="libx264",
preset="medium",
audio_codec="aac",
temp_audiofile=os.path.join(self.output_dir, f"temp-audio-{os.urandom(4).hex()}.m4a"),
remove_temp=True,
threads=os.cpu_count() or 2,
logger="bar",
bitrate="5000k",
ffmpeg_params=["-pix_fmt", "yuv420p"],
)
logger.info(f"Video created:{op}")
return op
else:
logger.error("Final clip invalid. No write.")
return None
except Exception as e:
logger.error(f"Video write error:{e}", exc_info=True)
return None
finally:
logger.debug("Closing all MoviePy clips in `assemble_animatic_from_assets` finally block.")
all_clips_to_close = processed_clips + ([narration_clip] if narration_clip else []) + ([final_clip] if final_clip else [])
for clip_obj_to_close in all_clips_to_close:
if clip_obj_to_close and hasattr(clip_obj_to_close, "close"):
try:
clip_obj_to_close.close()
except Exception as e_close:
logger.warning(
f"Ignoring error while closing a clip: {type(clip_obj_to_close).__name__} - {e_close}"
)
|