Merge branch #Yakova/Rectifier' into 'azils3/Tester'

App/Generate/Schema.py

@@ -3,11 +3,23 @@ from pydantic import BaseModel, HttpUrl
 from pydantic import validator
 
 
+class Scene(BaseModel):
+    narration: str
+    image_prompts: List[str]
+    voice: str
+
+
+class Story(BaseModel):
+    scenes: List[Scene]
+    reference_image_url: Optional[str] = None
+    ip_adapter_weight: Optional[float] = 0.4  # Add this line
+
+
 class GeneratorRequest(BaseModel):
     prompt: str
-    grok: Optional[bool] = True
     batch_size: int = 4
-    model: str = "llama3-70b-8192"
+    grok: bool = False
+    model: str = "command"
 
 
 class GeneratorBulkRequest(BaseModel):

App/Generate/Story/Story.py

@@ -1,11 +1,12 @@
 import json
 from pydantic import BaseModel
-from typing import List
+from typing import List, Optional
 
 
 class Scene(BaseModel):
     narration: str
     image_prompts: List[str]
+    voice: Optional[str] = None
 
     def num_images(self):
         return len(self.image_prompts)
@@ -13,6 +14,9 @@ class Scene(BaseModel):
 
 class Story(BaseModel):
     scenes: List[Scene]
+    voice: Optional[str] = None
+    reference_image_url: Optional[str] = None
+    ip_adapter_weight: Optional[float] = 0.4
 
     @classmethod
     def from_dict(cls, data):

App/Generate/database/CharacterAPI.py

@@ -0,0 +1,123 @@
+import aiohttp
+import asyncio
+import os
+import uuid
+import tempfile
+from typing import List, Dict, Any
+
+from pydantic import BaseModel
+
+
+class AlignmentData(BaseModel):
+    word: str
+    start: float
+    end: float
+
+    def to_dict(self) -> dict:
+        return {
+            "word": self.word,
+            "alignedWord": self.word,
+            "startTime": self.start,
+            "endTime": self.end,
+            "hasFailedAlignment": False,
+        }
+
+
+class CharacterAITTS:
+    def __init__(self):
+        self.api_url = "https://yakova-embedding.hf.space"
+        self.dir = str(tempfile.mkdtemp())
+        self.descript = "https://yakova-embedding.hf.space"
+        self.headers = {"Connection": "keep-alive", "Content-Type": "application/json"}
+
+    async def _make_transcript(self, links, text):
+
+        data = {"audio_url": links, "text": text, "file_extenstion": ".mp3"}
+        response_data = await self._make_request(
+            "post", "descript_transcript", json=data, external=self.descript
+        )
+        if not response_data:
+            data["audio_url"] = data["audio_url"][0]
+            print(data)
+            response_data = await self.aligner(
+                "post",
+                "align/url",
+                json=data,
+            )
+            print(response_data)
+            response_data = self.process_alignments(
+                data=response_data["alignment"], offset=0
+            )
+        return response_data
+
+    def process_alignments(
+        self, data: List[Dict[str, Any]], offset: float = 0
+    ) -> List[Dict[str, Any]]:
+        alignments = [AlignmentData(**item) for item in data]
+        return [alignment.to_dict() for alignment in alignments]
+
+    async def aligner(
+        self,
+        method,
+        endpoint,
+        json=None,
+        external="https://yakova-aligner.hf.space/align/url",
+    ):
+        async with aiohttp.ClientSession() as session:
+            if external:
+                url = f"{external}"
+            else:
+                url = f"{self.api_url}/{endpoint}"
+            async with getattr(session, method)(url=url, json=json) as response:
+                return await response.json()
+
+    async def _make_request(self, method, endpoint, json=None, external=None):
+        async with aiohttp.ClientSession() as session:
+            if external:
+                url = f"{external}/{endpoint}"
+            else:
+                url = f"{self.api_url}/{endpoint}"
+            async with getattr(session, method)(url=url, json=json) as response:
+                return await response.json()
+
+    async def say(self, text, speaker=None):
+
+        data = {"text": text, "voice": speaker}
+
+        response_data = await self._make_request("post", "cai_tts", json=data)
+        # print(response_data)
+        audio_url = response_data["audio"]
+        temp = await self.download_file(audio_url)
+        return audio_url, temp
+
+    async def download_file(self, url):
+        filename = str(uuid.uuid4()) + ".mp3"
+        os.makedirs(self.dir, exist_ok=True)
+        save_path = os.path.join(self.dir, filename)
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                if response.status == 200:
+                    with open(save_path, "wb") as file:
+                        while True:
+                            chunk = await response.content.read(1024)
+                            if not chunk:
+                                break
+                            file.write(chunk)
+
+        return save_path
+
+
+# # Usage
+# async def main():
+#     tts = CharacterAITTS()
+#     url, temp = await tts.say(
+#         "Did you know that you don't have the balls to talk to me"
+#     )
+#     tranny = await tts._make_transcript(
+#         links=[url], text="Did you know that you don't have the balls to talk to me"
+#     )
+#     print(tranny)
+
+
+# # Run the main function
+# asyncio.run(main())

App/Generate/database/ElevenLab.py

@@ -3,6 +3,24 @@ import asyncio
 import os
 import uuid
 import tempfile
+from typing import List, Dict, Any
+
+from pydantic import BaseModel
+
+
+class AlignmentData(BaseModel):
+    word: str
+    start: float
+    end: float
+
+    def to_dict(self) -> dict:
+        return {
+            "word": self.word,
+            "alignedWord": self.word,
+            "startTime": self.start,
+            "endTime": self.end,
+            "hasFailedAlignment": False,
+        }
 
 
 class ElevenLab:
@@ -748,8 +766,41 @@ class ElevenLab:
         response_data = await self._make_request(
             "post", "descript_transcript", json=data, external=self.descript
         )
+        if not response_data:
+            data["audio_url"] = data["audio_url"][0]
+            print(data)
+            response_data = await self.aligner(
+                "post",
+                "align/url",
+                json=data,
+            )
+            print(response_data)
+            response_data = self.process_alignments(
+                data=response_data["alignment"], offset=0
+            )
         return response_data
 
+    def process_alignments(
+        self, data: List[Dict[str, Any]], offset: float = 0
+    ) -> List[Dict[str, Any]]:
+        alignments = [AlignmentData(**item) for item in data]
+        return [alignment.to_dict() for alignment in alignments]
+
+    async def aligner(
+        self,
+        method,
+        endpoint,
+        json=None,
+        external="https://yakova-aligner.hf.space/align/url",
+    ):
+        async with aiohttp.ClientSession() as session:
+            if external:
+                url = f"{external}"
+            else:
+                url = f"{self.api_url}/{endpoint}"
+            async with getattr(session, method)(url=url, json=json) as response:
+                return await response.json()
+
     async def _make_request(self, method, endpoint, json=None, external=None):
         async with aiohttp.ClientSession() as session:
             if external:
@@ -765,6 +816,7 @@ class ElevenLab:
             name = i["name"].split(" ")[0]
             if name.lower() == speaker.lower():
                 voiceId = i["voice_id"]
+                # voiceId = "yl2ZDV1MzN4HbQJbMihG"
         data = {"voiceId": voiceId, "text": text}
 
         response_data = await self._make_request("post", "convert", json=data)

App/Generate/database/Modal.py

@@ -0,0 +1,51 @@
+import aiohttp
+import asyncio
+import json
+from typing import Dict, Any
+
+
+class ModalImageGenerator:
+    def __init__(self, session: aiohttp.ClientSession):
+        self.session = session
+        self.base_url = (
+            "https://allanwatts705--kolors-app-model-web-inference.modal.run/"
+        )
+
+    async def generate_image(
+        self,
+        prompt: str,
+        reference_image_url: str = "https://image.lexica.art/full_webp/d6ddd5c5-060c-4aba-b9d0-cf0e02dc65bd",
+        ip_adapter_weight: float = 0.4,
+    ) -> Dict[str, Any]:
+        params = {
+            "ip_adapter_weight": ip_adapter_weight,
+            "reference_image_url": reference_image_url,
+            "prompt": prompt,
+        }
+
+        async with self.session.get(self.base_url, params=params) as response:
+            if response.status == 200:
+                return await response.json()
+            else:
+                raise Exception(f"Failed to generate image: {response.status}")
+
+    async def wait_for_image(
+        self, result: Dict[str, Any], max_attempts: int = 60, delay: int = 5
+    ) -> Dict[str, Any]:
+        attempts = 0
+        while attempts < max_attempts:
+            if result.get("image", {}).get("url"):
+                return result
+            await asyncio.sleep(delay)
+            attempts += 1
+
+        raise Exception("Image generation timed out")
+
+    async def generate_and_wait(
+        self, prompt: str, reference_image_url: str, ip_adapter_weight: float = 0.4
+    ) -> str:
+        result = await self.generate_image(
+            prompt, reference_image_url, ip_adapter_weight
+        )
+        final_result = await self.wait_for_image(result)
+        return final_result["image"]["url"]

App/Generate/database/Model.py

@@ -4,13 +4,17 @@ import asyncio, os
 import uuid, random
 from pydub import AudioSegment
 from .DescriptAPI import Speak
-from .ElevenLab import ElevenLab
+
+# from .ElevenLab import ElevenLab
+from .CharacterAPI import CharacterAITTS
 from .Vercel import AsyncImageGenerator
+from .Video3d import VideoGenerator
 import aiohttp
 from typing import List
 from pydantic import BaseModel
 import tempfile
 import json
+from .Modal import ModalImageGenerator
 
 SUPABASE = os.environ.get("SUPABASE", "RANDOM_STRING")
 database_url = SUPABASE
@@ -95,6 +99,7 @@ class Project(orm.Model):
 
             # generate transcripts
             temp = await scene.generate_scene_transcript(offset=self.start)
+            await asyncio.sleep(1)
             end_word = temp[-1]
 
             # narration of the story
@@ -113,34 +118,68 @@ class Project(orm.Model):
             )
             text_stream.extend(temp[:-1])
 
-            ## images and transitions
-            for image in scene.images:
-                file_name = str(uuid.uuid4()) + ".png"
-                self.links.append({"file_name": file_name, "link": image})
-                image_assets.append(
-                    {
-                        "type": "image",
-                        "name": file_name,
-                        "start": self.start,
-                        "end": self.start + scene.image_duration,
-                    }
-                )
-                self.start = self.start + scene.image_duration
-
-                # transitions between images
-                video_assets.append(
-                    {
-                        "type": "video",
-                        "name": "Effects/" + random.choice(transitions),
-                        "start": self.start - 1,
-                        "end": self.start + 2,
-                        "props": {
-                            "startFrom": 1 * 30,
-                            "endAt": 3 * 30,
-                            "volume": 0,
-                        },
-                    }
-                )
+            sample_image_extension = scene.images[0].split(".")[-1]
+
+            if sample_image_extension == "mp4":
+                ## moving images
+                for image in scene.images:
+                    file_name = str(uuid.uuid4()) + ".mp4"
+                    self.links.append({"file_name": file_name, "link": image})
+                    video_assets.append(
+                        {
+                            "type": "video",
+                            "name": file_name,
+                            "start": self.start,
+                            "loop": "true",
+                            "end": self.start + scene.image_duration,
+                            "props": {
+                                "volume": 0,
+                                "startFrom": 1 * 30,
+                                "endAt": 2.9 * 30,
+                                "playbackRate": 0.7,
+                                "style": {
+                                    "transform": "translate(-50%, -50%)",
+                                    "position": "absolute",
+                                    "top": "50%",
+                                    "left": "50%",
+                                    "width": 1920,
+                                    "height": 1080,
+                                    "objectFit": "cover",
+                                },
+                            },
+                        }
+                    )
+                    self.start = self.start + scene.image_duration
+
+            else:
+                ## images and transitions
+                for image in scene.images:
+                    file_name = str(uuid.uuid4()) + ".png"
+                    self.links.append({"file_name": file_name, "link": image})
+                    image_assets.append(
+                        {
+                            "type": "image",
+                            "name": file_name,
+                            "start": self.start,
+                            "end": self.start + scene.image_duration,
+                        }
+                    )
+                    self.start = self.start + scene.image_duration
+
+                ## transitions between images
+                # video_assets.append(
+                #     {
+                #         "type": "video",
+                #         "name": "Effects/" + random.choice(transitions),
+                #         "start": self.start - 1,
+                #         "end": self.start + 2,
+                #         "props": {
+                #             "startFrom": 1 * 30,
+                #             "endAt": 3 * 30,
+                #             "volume": 0,
+                #         },
+                #     }
+                # )
 
         self.assets.append({"type": "audio", "sequence": audio_assets})
         ## add the images to assets
@@ -160,12 +199,14 @@ class Project(orm.Model):
 
 
 class Scene(orm.Model):
-    tts = ElevenLab()
+    tts = CharacterAITTS()
+    voice = ""
     # eleven = ElevenLab()
     tablename = "scenes"
     registry = models
     fields = {
         "id": orm.Integer(primary_key=True),
+        "voice": orm.String(max_length=100, allow_null=True, default=""),
         "project": orm.ForeignKey(Project),
         "images": orm.JSON(default=None),
         "narration": orm.String(max_length=10_000, allow_null=True, default=""),
@@ -186,9 +227,13 @@ class Scene(orm.Model):
         transcript = await self.tts._make_transcript(links=links, text=text)
         return transform_alignment_data(data=transcript, offset=offset)
 
-    async def generate_scene_data(self):
+    async def generate_scene_data(
+        self, reference_image_url: str = None, ip_adapter_weight: float = 0.4
+    ):
         # Run narrate() and generate_images() concurrently
-        await asyncio.gather(self.narrate(), self.generate_images())
+        await asyncio.gather(
+            self.narrate(), self.generate_images(reference_image_url, ip_adapter_weight)
+        )
         self.calculate_durations()
 
     async def narrate(self):
@@ -197,12 +242,11 @@ class Scene(orm.Model):
         self.narration_link = link
 
     async def retry_narration_generation(self):
-        print(self.narration)
         retry_count = 0
         while retry_count < 3:
             try:
                 return await self.tts.say(
-                    text=self.narration + " master"
+                    text=self.narration + " master", speaker=self.voice
                 )  ### The blanks help to even stuff up.
             except Exception as e:
                 print(f"Failed to generate narration: {e}")
@@ -217,14 +261,23 @@ class Scene(orm.Model):
         self.narration_duration = int(len(audio_file) / 1000)
         self.image_duration = self.narration_duration / len(self.image_prompts)
 
-    async def generate_images(self):
+    async def generate_images(
+        self,
+        reference_image_url: str = "https://image.lexica.art/full_webp/d6ddd5c5-060c-4aba-b9d0-cf0e02dc65bd",
+        ip_adapter_weight: float = 0.4,
+    ):
         self.images = []
         async with aiohttp.ClientSession() as session:
-            image_generator = AsyncImageGenerator(session)
-            for payload in self.image_prompts:
-                result = await image_generator.generate_image(payload)
-                status = await image_generator.fetch_image_status(result["id"])
-                self.images.extend(status["output"])
+            image_generator = ModalImageGenerator(session)
+            for prompt in self.image_prompts:
+                try:
+                    image_url = await image_generator.generate_image(
+                        prompt, reference_image_url, ip_adapter_weight
+                    )
+                    self.images.append(image_url)
+                except Exception as e:
+                    print(f"Failed to generate image for prompt '{prompt}': {str(e)}")
+                await asyncio.sleep(1)  # Add a small delay between requests
 
 
 class Transition(orm.Model):
@@ -262,7 +315,7 @@ class BackgroundMusic(orm.Model):
 #         pass
 
 
-# # # Create the tables
+# # Create the tables
 # async def create_tables():
 #     datas = {
 #         "narration": "Welcome to a journey through some of history's strangest moments! Get ready to explore the bizarre, the unusual, and the downright weird.",

App/Generate/database/Vercel.py

@@ -1,5 +1,6 @@
 import aiohttp, asyncio, pprint
 from collections import deque
+import json
 
 
 class AsyncImageGenerator:
@@ -13,35 +14,56 @@ class AsyncImageGenerator:
             try:
                 url = f"{self.base}/predictions"
                 data = {
+                    #                     "input": {
+                    #                         "prompt": payload
+                    #                         + """
+                    # """,
+                    #                         # "aspect_ratio": "2:3",
+                    #                         "negative_prompt": "text, watermark, blurry, haze, low contrast, low quality, underexposed, ugly, deformed, boring, bad quality, cartoon, ((disfigured)), ((bad art)), ((deformed)), ((poorly drawn)), ((extra limbs)), ((close up)), ((b&w)), weird colors, blurry, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, extra limbs, disfigured, body out of frame, blurry, bad anatomy, blurred, watermark, grainy, signature, cut off, draft, low detail, low quality, double face, 2 faces, cropped, ugly, low-res, tiling, grainy, cropped, ostentatious, ugly, oversaturated, grain, low resolution, disfigured, blurry, bad anatomy, disfigured, poorly drawn face, mutant, mutated, extra limb, ugly, poorly drawn hands, missing limbs, blurred, floating limbs, disjointed limbs, deformed hands, blurred, out of focus, long neck, long body, ugly, disgusting, childish, cut off cropped, distorted, imperfect, surreal, bad hands, text, error, extra digit, fewer digits, cropped , worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, artist name, Lots of hands, extra limbs, extra fingers, conjoined fingers, deformed fingers, old, ugly eyes, imperfect eyes, skewed eyes , unnatural face, stiff face, stiff body, unbalanced body, unnatural body, lacking body, details are not clear, cluttered, details are sticky, details are low, distorted details, ugly hands, imperfect hands, (mutated hands and fingers:1.5), (long body :1.3), (mutation, poorly drawn :1.2) bad hands, fused ha nd, missing hand, disappearing arms, hands, disappearing thigh, disappearing calf, disappearing legs, ui, missing fingers",
+                    #                         "steps": 28,
+                    #                     },
                     # "input": {
-                    #     "prompt": payload,
-                    #     "negative_prompt": "CGI, Unreal, Airbrushed, Digital",
                     #     "width": 1024,
-                    #     "height": 1024,
+                    #     "height": 1536,
+                    #     "prompt": payload,
+                    #     "scheduler": "DPMSolver++",
                     #     "num_outputs": 1,
-                    #     "scheduler": "DPM++SDE",
-                    #     "num_inference_steps": 30,
-                    #     "guidance_scale": 2,
+                    #     "guidance_scale": 3,
+                    #     "negative_prompt": "doll opened mouth toy, text, sculpture pixar, cartoon, anime, low quality, plastic, 2 heads, 2 faces, poorly drawn, ugly, poorly drawn face, extra arms, extra legs, ugly, bad anatomy, bad fingers, ugly, deformed, noisy, blurry, distorted, grainy, nudity, nude",
+                    #     # "negative_prompt": "text, watermark, blurry, haze, low contrast, low quality, underexposed, ugly, deformed, boring, bad quality, cartoon, ((disfigured)), ((bad art)), ((deformed)), ((poorly drawn)), ((extra limbs)), ((close up)), ((b&w)), weird colors, blurry, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, extra limbs, disfigured, body out of frame, blurry, bad anatomy, blurred, watermark, grainy, signature, cut off, draft, low detail, low quality, double face, 2 faces, cropped, ugly, low-res, tiling, grainy, cropped, ostentatious, ugly, oversaturated, grain, low resolution, disfigured, blurry, bad anatomy, disfigured, poorly drawn face, mutant, mutated, extra limb, ugly, poorly drawn hands, missing limbs, blurred, floating limbs, disjointed limbs, deformed hands, blurred, out of focus, long neck, long body, ugly, disgusting, childish, cut off cropped, distorted, imperfect, surreal, bad hands, text, error, extra digit, fewer digits, cropped , worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, artist name, Lots of hands, extra limbs, extra fingers, conjoined fingers, deformed fingers, old, ugly eyes, imperfect eyes, skewed eyes , unnatural face, stiff face, stiff body, unbalanced body, unnatural body, lacking body, details are not clear, cluttered, details are sticky, details are low, distorted details, ugly hands, imperfect hands, (mutated hands and fingers:1.5), (long body :1.3), (mutation, poorly drawn :1.2) bad hands, fused ha nd, missing hand, disappearing arms, hands, disappearing thigh, disappearing calf, disappearing legs, ui, missing fingers",
+                    #     "num_inference_steps": 25,
                     # },
-                    "input": {
-                        "width": 1024,
-                        "height": 1536,
-                        "prompt": payload,
-                        "scheduler": "DPMSolver++",
-                        "num_outputs": 1,
-                        "guidance_scale": 3,
-                        "negative_prompt": "doll opened mouth toy, text, sculpture pixar, cartoon, anime, low quality, plastic, 2 heads, 2 faces, poorly drawn, ugly, poorly drawn face, extra arms, extra legs, ugly, bad anatomy, bad fingers, ugly, deformed, noisy, blurry, distorted, grainy, nudity, nude",
-                        # "negative_prompt": "text, watermark, blurry, haze, low contrast, low quality, underexposed, ugly, deformed, boring, bad quality, cartoon, ((disfigured)), ((bad art)), ((deformed)), ((poorly drawn)), ((extra limbs)), ((close up)), ((b&w)), weird colors, blurry, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, extra limbs, disfigured, body out of frame, blurry, bad anatomy, blurred, watermark, grainy, signature, cut off, draft, low detail, low quality, double face, 2 faces, cropped, ugly, low-res, tiling, grainy, cropped, ostentatious, ugly, oversaturated, grain, low resolution, disfigured, blurry, bad anatomy, disfigured, poorly drawn face, mutant, mutated, extra limb, ugly, poorly drawn hands, missing limbs, blurred, floating limbs, disjointed limbs, deformed hands, blurred, out of focus, long neck, long body, ugly, disgusting, childish, cut off cropped, distorted, imperfect, surreal, bad hands, text, error, extra digit, fewer digits, cropped , worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, artist name, Lots of hands, extra limbs, extra fingers, conjoined fingers, deformed fingers, old, ugly eyes, imperfect eyes, skewed eyes , unnatural face, stiff face, stiff body, unbalanced body, unnatural body, lacking body, details are not clear, cluttered, details are sticky, details are low, distorted details, ugly hands, imperfect hands, (mutated hands and fingers:1.5), (long body :1.3), (mutation, poorly drawn :1.2) bad hands, fused ha nd, missing hand, disappearing arms, hands, disappearing thigh, disappearing calf, disappearing legs, ui, missing fingers",
-                        "num_inference_steps": 25,
-                    },
+                    # "input": {"workflow_json": payload},
                     # "input": {
                     #     "prompt": payload,
                     #     "aspect_ratio": "9:16",
                     #     "output_quality": 79,
                     #     "negative_prompt": "doll opened mouth toy, text, sculpture pixar, cartoon, anime, low quality, plastic, 2 heads, 2 faces, poorly drawn, ugly, poorly drawn face, extra arms, extra legs, ugly, bad anatomy, bad fingers, ugly, deformed, noisy, blurry, distorted, grainy, nudity, nude",
                     # },
-                    "path": "models/playgroundai/playground-v2.5-1024px-aesthetic/versions/a45f82a1382bed5c7aeb861dac7c7d191b0fdf74d8d57c4a0e6ed7d4d0bf7d24",
-                    # "path": "models/stability-ai/stable-diffusion-3",
+                    "input": {
+                        "cfg": 2.5,
+                        # "seed": 42,
+                        # "image": "https://image.lexica.art/full_webp/e41b87fb-4cc3-43cd-a6e6-f3dbb08c2399",
+                        # "steps": 28,
+                        # "width": 1024,
+                        # "height": 1536,
+                        # "aspect_ratio": "16:9",
+                        "prompt": payload,
+                        # "sampler": "dpmpp_2m_sde_gpu",
+                        # "scheduler": "karras",
+                        # "output_format": "png",
+                        # "output_quality": 95,
+                        # "negative_prompt": "clouds, people, buildings",
+                        # "number_of_images": 1,
+                        # "ip_adapter_weight": 1,
+                        # "ip_adapter_weight_type": "style transfer precise",
+                    },
+                    # "path": "models/fofr/kolors-with-ipadapter/versions/5a1a92b2c0f81813225d48ed8e411813da41aa84e7582fb705d1af46eea36eed",
+                    # "path": "models/playgroundai/playground-v2.5-1024px-aesthetic/versions/a45f82a1382bed5c7aeb861dac7c7d191b0fdf74d8d57c4a0e6ed7d4d0bf7d24",
+                    # "path": "models/fofr/any-comfyui-workflow/versions/cd385285ba75685a040afbded7b79814a971f3febf46c5eab7c716e200c784e1",
+                    # "path": "models/fofr/sd3-explorer/versions/a9f4aebd943ad7db13de8e34debea359d5578d08f128e968f9a36c3e9b0148d4",
+                    "path": "models/bingbangboom-lab/flux-new-whimscape/versions/2e8de10f217bc56da163a0204cf09f89995eaf643459014803fae79753183682",
+                    # "path": "models/black-forest-labs/flux-schnell",
                     # "path": "models/lucataco/proteus-v0.4/versions/34a427535a3c45552b94369280b823fcd0e5c9710e97af020bf445c033d4569e",
                     # "path": "models/lucataco/juggernaut-xl-v9/versions/bea09cf018e513cef0841719559ea86d2299e05448633ac8fe270b5d5cd6777e",
                 }
@@ -73,6 +95,7 @@ class AsyncImageGenerator:
                 status["status"] = "404"
 
             while status["status"] != "succeeded":
+                print(status["status"])
                 try:
                     status = await self._fetch_image_status(image_id)
                 except Exception as e:
@@ -102,7 +125,9 @@ async def process_images(payloads):
                 result = await image_generator.generate_image(payload)
                 status = await image_generator.fetch_image_status(result["id"])
                 print(status["output"])
-                results.extend(status["output"])
+                if type(status["output"]) == list:
+                    results.extend(status["output"])
+                results.append(status["output"])
 
         for payload in payloads:
             tasks.append(payload)
@@ -118,11 +143,8 @@ async def process_images(payloads):
 # # # Example payloads
 # payloads = [
 #     """
-
-# [Underpainting sketch] [Abstract expressive ginger arcane girl fortune teller portrait] [Detailed room with candles, vines, stars] [Mystical scene with crystal orb, herbs, potions] [Burned paper, beautiful tender elements, mannerism, jewels, detailed eyes, celestial, shadow play, rough textures, light on perfect face, focus on detailed face, smoke, dynamic movement, rim lighting, mystical, magic, complex background, by Lockwood, Kuindzhi, Rockwell] high-quality portrait photography style 8K
-
-
-# """
+# comicbook illustration artistic, beautiful Awsome cat
+#      """
 # ]
 
 

App/Generate/database/Video3d.py

@@ -0,0 +1,120 @@
+import aiohttp
+import asyncio
+from itertools import chain
+
+
+class VideoGenerator:
+    def __init__(self):
+        self.base_urls = [f"https://yakova-depthflow-{i}.hf.space" for i in range(10)]
+        self.headers = {"accept": "application/json"}
+        self.default_params = {
+            "frame_rate": 30,
+            "duration": 3,
+            "quality": 1,
+            "ssaa": 0.8,
+            "raw": "false",
+        }
+
+    async def generate_video(self, base_url, params):
+        url = f"{base_url}/generate_video"
+
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                url, params=params, headers=self.headers
+            ) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    output_file = data.get("output_file")
+                    return output_file
+                else:
+                    print(f"Request to {url} failed with status: {response.status}")
+                    return None
+
+    async def check_video_ready(self, base_url, output_file):
+        url = f"{base_url}/download/{output_file}"
+
+        async with aiohttp.ClientSession() as session:
+            while True:
+                async with session.get(url, headers=self.headers) as response:
+                    if response.status == 200:
+                        video_content = await response.read()
+                        if len(video_content) > 0:
+                            return url
+                        else:
+                            print(
+                                f"Video {output_file} is ready but the file size is zero, retrying in 10 seconds..."
+                            )
+                            await asyncio.sleep(10)
+                    elif response.status == 404:
+                        data = await response.json()
+                        if data.get("detail") == "Video not found":
+                            print(
+                                f"Video {output_file} not ready yet, retrying in 10 seconds..."
+                            )
+                            await asyncio.sleep(180)
+                        else:
+                            print(f"Unexpected response for {output_file}: {data}")
+                            return None
+                    else:
+                        print(f"Request to {url} failed with status: {response.status}")
+                        return None
+
+    async def process_image(self, base_url, image_link):
+        params = self.default_params.copy()
+        params["image_link"] = image_link
+
+        output_file = await self.generate_video(base_url, params)
+        if output_file:
+            print(f"Generated video file id: {output_file} for image {image_link}")
+            video_url = await self.check_video_ready(base_url, output_file)
+            if video_url:
+                print(
+                    f"Video for {image_link} is ready and can be downloaded from: {video_url}"
+                )
+                return video_url
+            else:
+                print(f"Failed to get the video URL for {image_link}")
+                return None
+        else:
+            print(f"Failed to generate the video for {image_link}")
+            return None
+
+    def flatten(self, nested_list):
+        return list(chain.from_iterable(nested_list))
+
+    def nest(self, flat_list, nested_dims):
+        it = iter(flat_list)
+        return [[next(it) for _ in inner_list] for inner_list in nested_dims]
+
+    async def run(self, nested_image_links):
+        flat_image_links = self.flatten(nested_image_links)
+        tasks = []
+        base_index = 0
+
+        for image_link in flat_image_links:
+            base_url = self.base_urls[base_index % len(self.base_urls)]
+            tasks.append(self.process_image(base_url, image_link))
+            base_index += 1
+
+        flat_video_urls = await asyncio.gather(*tasks)
+        nested_video_urls = self.nest(flat_video_urls, nested_image_links)
+        return nested_video_urls
+
+
+# # Example usage
+# nested_image_links = [
+#     [
+#         "https://replicate.delivery/yhqm/mQId1rdf4Z3odCyB7cPsx1KwhHfdRc3w44eYAGNG9AQfV0dMB/out-0.png"
+#     ],
+#     [
+#         "https://replicate.delivery/yhqm/mQId1rdf4Z3odCyB7cPsx1KwhHfdRc3w44eYAGNG9AQfV0dMB/out-1.png",
+#         "https://replicate.delivery/yhqm/mQId1rdf4Z3odCyB7cPsx1KwhHfdRc3w44eYAGNG9AQfV0dMB/out-2.png",
+#     ],
+#     # Add more nested image links here
+# ]
+
+# loop = asyncio.get_event_loop()
+# video_generator = VideoGenerator()
+# nested_video_urls = loop.run_until_complete(video_generator.run(nested_image_links))
+
+# print("Generated video URLs:", nested_video_urls)

App/Generate/generatorRoutes.py

@@ -6,16 +6,29 @@ from .utils.HuggingChat import Hugging
 from .Story.Story import Story
 import asyncio, pprint, json
 from tqdm import tqdm
-from .database.Model import models, database_url, Scene, Project, database
+from .database.Model import (
+    models,
+    database_url,
+    Scene,
+    Project,
+    database,
+    VideoGenerator,
+)
 from .utils.RenderVideo import RenderVideo
 from .Prompts.StoryGen import Prompt
 from App.Editor.editorRoutes import celery_task, EditorRequest
 import uuid
 
 
-async def update_scene(model_scene):
-    await model_scene.generate_scene_data()
-    await model_scene.update(**model_scene.__dict__)
+async def update_scene(model_scene, reference_image_url=None, ip_adapter_weight=0.4):
+    await model_scene.generate_scene_data(reference_image_url, ip_adapter_weight)
+    await model_scene.update(
+        **{
+            k: v
+            for k, v in model_scene.__dict__.items()
+            if k not in ["reference_image_url", "ip_adapter_weight"]
+        }
+    )
 
 
 async def from_dict_generate(data: Story):
@@ -23,38 +36,70 @@ async def from_dict_generate(data: Story):
     await generate_assets(generated_story=generated_strory)
 
 
-async def generate_assets(generated_story: Story, batch_size=4):
+async def generate_assets(generated_story: Story, batch_size=4, threeD=True):
     x = await Project.objects.create(name=str(uuid.uuid4()))
 
     # Assuming generated_story.scenes is a list of scenes
-    scene_updates = []
     with tqdm(total=len(generated_story.scenes)) as pbar:
+
+        all_scenes: list[Scene] = []
+        # create the batches
         for i in range(0, len(generated_story.scenes), batch_size):
             batch = generated_story.scenes[
                 i : i + batch_size
             ]  # Get a batch of two story scenes
             batch_updates = []
 
+            # generate pictures or narration per batch
             for story_scene in batch:
                 model_scene = await Scene.objects.create(project=x)
                 model_scene.image_prompts = story_scene.image_prompts
                 model_scene.narration = story_scene.narration
-                await model_scene.update(**model_scene.__dict__)
+                model_scene.voice = story_scene.voice
+                await model_scene.update(
+                    **{
+                        k: v
+                        for k, v in model_scene.__dict__.items()
+                        if k not in ["reference_image_url", "ip_adapter_weight"]
+                    }
+                )
+                all_scenes.append(model_scene)
                 batch_updates.append(
-                    update_scene(model_scene)
+                    update_scene(
+                        model_scene,
+                        generated_story.reference_image_url,
+                        generated_story.ip_adapter_weight,
+                    )
                 )  # Append update coroutine to batch_updates
-            scene_updates.extend(batch_updates)  # Accumulate updates for later awaiting
+            # pause per batch
             await asyncio.gather(
                 *batch_updates
             )  # Await update coroutines for this batch
+
             pbar.update(len(batch))  # Increment progress bar by the size of the batch
 
+    ###### Here we generate the videos
+
+    if False:
+        vid_gen = VideoGenerator()
+        nested_images = []
+        for scene in all_scenes:
+            nested_images.append(scene.images)
+
+        results = await vid_gen.run(nested_image_links=nested_images)
+        print(results)
+        for result, _scene in zip(results, all_scenes):
+            _scene.images = result
+            await _scene.update(**_scene.__dict__)
+
     temp = await x.generate_json()
     # print(temp)
 
-    # await renderr.render_video(temp)
+    renderr = RenderVideo()
+    await renderr.render_video(temp)
     request = EditorRequest.model_validate(temp)
-    await celery_task(video_task=request)
+    #
+    # await celery_task(video_task=request)
 
 
 async def main(request: GeneratorRequest):
@@ -98,7 +143,7 @@ async def generate_video(
 
 @generator_router.post("/generate_video_from_json")
 async def generate_video_from_json(jsonReq: Story, background_task: BackgroundTasks):
-    background_task.add_task(from_dict_generate, jsonReq)
+    background_task.add_task(generate_assets, jsonReq)
     return {"task_id": "started"}
 
 

App/Generate/utils/Cohere.py

@@ -22,7 +22,11 @@ class VideoOutput(BaseModel):
 
 # Patching the Cohere client with the instructor for enhanced capabilities
 client = instructor.from_cohere(
-    cohere.Client(os.environ.get("COHERE_API", "RANDOM_STRING")),
+    cohere.Client(
+        os.environ.get(
+            "COHERE_API",
+        )
+    ),
     # max_tokens=5000,
     model="command-r-plus",
 )
@@ -36,7 +40,7 @@ def chatbot(prompt: str, model: str = "command-r-plus"):
 
     response: VideoOutput = client.chat.completions.create(
         model=model,
-        max_tokens=5000,
+        # max_tokens=5000,
         response_model=VideoOutput,
         messages=[
             {
@@ -46,3 +50,6 @@ def chatbot(prompt: str, model: str = "command-r-plus"):
         ],
     )
     return response.dict()
+
+
+# print(chatbot("A horror story"))

App/Generate/utils/RenderVideo.py

@@ -2,7 +2,9 @@ import aiohttp, json
 
 
 class RenderVideo:
-    def __init__(self, url="https://yakova-rectifier.hf.space/create-video"):
+    def __init__(
+        self, url="https://mbonea-mjema--rectifier-run-rectifier.modal.run/create-video"
+    ):
         self.url = url
 
     @staticmethod

App/Worker.py

@@ -284,7 +284,7 @@ async def cleanup_temp_directory(
 
 # @celery.task(name="All")
 async def celery_task(video_task: EditorRequest):
-    remotion_app_dir = os.path.join("/srv", "Remotion-app")
+    remotion_app_dir = os.path.join("./", "Remotion-app")
     project_id = str(uuid.uuid4())
     temp_dir = f"/tmp/{project_id}"
     output_dir = f"/tmp/{project_id}/out/video.mp4"

App/app.py

@@ -15,12 +15,14 @@ manager = WorkerClient()
 
 @app.on_event("startup")
 async def startup_event():
+    print("StartUp running")
     app.state.db = database
     app.state.models = models
-
+    
     try:
         # print(type(database.url), database_url)
         # await models.create_all()
+        print("Error registering worker")
         await models._create_all(str(database.url))
     except:
         print("failed to create")
@@ -30,7 +32,7 @@ async def startup_event():
             await database.connect()
         # await database.execute("pragma journal_mode=wal;")
 
-    await bot.start()
+    # await bot.start()
     # if SERVER_STATE.MASTER:
 
     # response = await manager.register_worker()

Dockerfile

@@ -71,7 +71,7 @@ COPY --chown=admin . /srv
 RUN chown -R admin:admin /srv
 RUN chmod 755 /srv
 USER admin
-CMD python -m uvicorn App.app:app --workers 1 --host 0.0.0.0 --port 7860 
+CMD python -m uvicorn App.app:app --workers 1 --host 0.0.0.0 --port 7860  --log-level debug
 
 # Expose port
 EXPOSE 7860

Remotion-app/package.json

@@ -17,8 +17,6 @@
 		"@remotion/transitions": "4.0.147",
 		"@remotion/zod-types": "4.0.147",
 		"@remotion/tailwind": "4.0.147",
-		"class-variance-authority": "^0.7.0",
-		"clsx": "^2.1.0",
 		"react": "^18.0.0",
 		"react-dom": "^18.0.0",
 		"remotion": "4.0.147",

Remotion-app/remotion.config.js

@@ -12,4 +12,4 @@ Config.overrideWebpackConfig((currentConfiguration) => {
 
 //Config.setBrowserExecutable("/usr/bin/chrome-headless-shell");
 Config.setVideoImageFormat('jpeg');
-// Config.setConcurrency(2);
+Config.setConcurrency(1);

Remotion-app/src/HelloWorld/TextStream.jsx

@@ -43,6 +43,7 @@ const TextStream = React.memo(() => {
 				backgroundColor: 'transparent',
 				justifyContent: 'center',
 				alignItems: 'center',
+				zIndex: 2,
 			}}
 		>
 			{memoizedTranscriptData.map((entry, index) => {

Remotion-app/src/HelloWorld/VideoStream.jsx

@@ -1,4 +1,4 @@
-import {Series} from 'remotion';
+import {Loop} from 'remotion';
 import React from 'react';
 import {Video, staticFile, useVideoConfig} from 'remotion';
 import videoSequences from './Assets/VideoSequences.json';
@@ -30,14 +30,20 @@ const VideoStream = React.memo(() => {
 
 const VideoX = React.memo(({entry}) => {
 	const {fps} = useVideoConfig();
+	const videoProps = {
+		pauseWhenBuffering: true,
+		startFrom: (fps * entry.props.startFrom) / 30,
+		endAt: (fps * entry.props.endAt) / 30,
+		volume: (fps * entry.props.volume) / 30,
+		src: staticFile(entry.name),
+		style: entry?.style ? entry.style : {},
+		playbackRate: entry.props.playbackRate ? entry.props.playbackRate : 1,
+	};
+
 	return (
-		<Video
-			pauseWhenBuffering
-			startFrom={(fps * entry.props.startFrom) / 30}
-			endAt={(fps * entry.props.endAt) / 30}
-			volume={(fps * entry.props.volume) / 30}
-			src={staticFile(entry.name)}
-		/>
+		<>
+			{entry?.loop ? <Video loop {...videoProps} /> : <Video {...videoProps} />}
+		</>
 	);
 });