fix

src/app/api/v1/edit/dialogues/processShot.ts

@@ -0,0 +1,49 @@
+
+import { ClapProject, ClapSegment, getClapAssetSourceType } from "@aitube/clap"
+import { getSpeechBackgroundAudioPrompt } from "@aitube/engine"
+
+import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
+import { generateSpeechWithParlerTTS } from "@/app/api/generators/speech/generateVoiceWithParlerTTS"
+
+export async function processShot({
+  shotSegment,
+  clap
+}: {
+  shotSegment: ClapSegment
+  clap: ClapProject
+}): Promise<void> {
+
+  const shotSegments: ClapSegment[] = clap.segments.filter(s =>
+    startOfSegment1IsWithinSegment2(s, shotSegment)
+  )
+
+  const shotDialogueSegments: ClapSegment[] = shotSegments.filter(s =>
+    s.category === "dialogue"
+  )
+
+  let shotDialogueSegment: ClapSegment | undefined = shotDialogueSegments.at(0)
+  
+  console.log(`[api/generate/dialogues] processShot: shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotDialogueSegments.length} dialogues)`)
+
+  if (shotDialogueSegment && !shotDialogueSegment.assetUrl) {
+    // console.log(`[api/generate/dialogues] generating audio..`)
+
+    try {
+      // this generates a mp3
+      shotDialogueSegment.assetUrl = await generateSpeechWithParlerTTS({
+        text: shotDialogueSegment.prompt,
+        audioId: getSpeechBackgroundAudioPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"]),
+        debug: true,
+      })
+      shotDialogueSegment.assetSourceType = getClapAssetSourceType(shotDialogueSegment.assetUrl)
+
+    } catch (err) {
+      console.log(`[api/generate/dialogues] processShot: failed to generate audio: ${err}`)
+      throw err
+    }
+
+    console.log(`[api/generate/dialogues] processShot: generated dialogue audio: ${shotDialogueSegment?.assetUrl?.slice?.(0, 50)}...`)
+  } else {
+    console.log(`[api/generate/dialogues] processShot: there is already a dialogue audio: ${shotDialogueSegment?.assetUrl?.slice?.(0, 50)}...`)
+  }
+}

src/app/api/v1/edit/dialogues/route.ts

@@ -1,11 +1,10 @@
 import { NextResponse, NextRequest } from "next/server"
 
-import { ClapProject, ClapSegment, getClapAssetSourceType, parseClap, serializeClap } from "@aitube/clap"
+import { ClapProject, ClapSegment, parseClap, serializeClap } from "@aitube/clap"
 
-import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
 import { getToken } from "@/app/api/auth/getToken"
-import { generateSpeechWithParlerTTS } from "@/app/api/generators/speech/generateVoiceWithParlerTTS"
-import { getSpeechBackgroundAudioPrompt } from "@aitube/engine"
+
+import { processShot } from "./processShot"
 
 // a helper to generate speech for a Clap
 export async function POST(req: NextRequest) {
@@ -27,43 +26,13 @@ export async function POST(req: NextRequest) {
     throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`)
   }
 
-  
-  for (const shotSegment of shotsSegments) {
-
-    const shotSegments: ClapSegment[] = clap.segments.filter(s =>
-      startOfSegment1IsWithinSegment2(s, shotSegment)
-    )
-
-    const shotDialogueSegments: ClapSegment[] = shotSegments.filter(s =>
-      s.category === "dialogue"
-    )
-
-    let shotDialogueSegment: ClapSegment | undefined = shotDialogueSegments.at(0)
-    
-    console.log(`[api/generate/dialogues] shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotDialogueSegments.length} dialogues)`)
-
-    if (shotDialogueSegment && !shotDialogueSegment.assetUrl) {
-      // console.log(`[api/generate/dialogues] generating audio..`)
-
-      try {
-        // this generates a mp3
-        shotDialogueSegment.assetUrl = await generateSpeechWithParlerTTS({
-          text: shotDialogueSegment.prompt,
-          audioId: getSpeechBackgroundAudioPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"]),
-          debug: true,
-        })
-        shotDialogueSegment.assetSourceType = getClapAssetSourceType(shotDialogueSegment.assetUrl)
-
-      } catch (err) {
-        console.log(`[api/generate/dialogues] failed to generate audio: ${err}`)
-        throw err
-      }
-   
-      console.log(`[api/generate/dialogues] generated dialogue audio: ${shotDialogueSegment?.assetUrl?.slice?.(0, 50)}...`)
-    } else {
-      console.log(`[api/generate/dialogues] there is already a dialogue audio: ${shotDialogueSegment?.assetUrl?.slice?.(0, 50)}...`)
-    }
-  }
+  // we process the shots in parallel (this will increase the queue size in the Gradio spaces)
+  await Promise.all(shotsSegments.map(shotSegment =>
+    processShot({
+      shotSegment,
+      clap
+    })
+  ))
 
   // console.log(`[api/generate/dialogues] returning the clap augmented with dialogues`)
 

src/app/api/v1/edit/storyboards/processShot.ts

@@ -0,0 +1,76 @@
+import { ClapProject, ClapSegment, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap"
+import { getVideoPrompt } from "@aitube/engine"
+
+import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
+
+import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
+import { generateStoryboard } from "./generateStoryboard"
+
+export async function processShot({
+  shotSegment,
+  clap
+}: {
+  shotSegment: ClapSegment
+  clap: ClapProject
+}): Promise<void> {
+
+  const shotSegments: ClapSegment[] = clap.segments.filter(s =>
+    startOfSegment1IsWithinSegment2(s, shotSegment)
+  )
+
+  const shotStoryboardSegments: ClapSegment[] = shotSegments.filter(s =>
+    s.category === "storyboard"
+  )
+
+  let shotStoryboardSegment: ClapSegment | undefined = shotStoryboardSegments.at(0)
+
+  // TASK 1: GENERATE MISSING STORYBOARD SEGMENT
+  if (!shotStoryboardSegment) {
+    shotStoryboardSegment = newSegment({
+      track: 1,
+      startTimeInMs: shotSegment.startTimeInMs,
+      endTimeInMs: shotSegment.endTimeInMs,
+      assetDurationInMs: shotSegment.assetDurationInMs,
+      category: "storyboard",
+      prompt: "",
+      assetUrl: "",
+      outputType: "image"
+    })
+
+    if (shotStoryboardSegment) {
+      clap.segments.push(shotStoryboardSegment)
+    }
+
+    console.log(`[api/generate/storyboards] processShot: generated storyboard segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
+  }
+  if (!shotStoryboardSegment) { throw new Error(`failed to generate a newSegment`) }
+
+  // TASK 2: GENERATE MISSING STORYBOARD PROMPT
+  if (!shotStoryboardSegment?.prompt) {
+    // storyboard is missing, let's generate it
+    shotStoryboardSegment.prompt = getVideoPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"])
+    console.log(`[api/generate/storyboards] processShot: generating storyboard prompt: ${shotStoryboardSegment.prompt}`)
+  }
+
+  // TASK 3: GENERATE MISSING STORYBOARD BITMAP
+  if (!shotStoryboardSegment.assetUrl) {
+    // console.log(`[api/generate/storyboards] generating image..`)
+
+    try {
+      shotStoryboardSegment.assetUrl = await generateStoryboard({
+        prompt: getPositivePrompt(shotStoryboardSegment.prompt),
+        width: clap.meta.width,
+        height: clap.meta.height,
+      })
+      shotStoryboardSegment.assetSourceType = getClapAssetSourceType(shotStoryboardSegment.assetUrl)
+    } catch (err) {
+      console.log(`[api/generate/storyboards] processShot: failed to generate an image: ${err}`)
+      throw err
+    }
+  
+    console.log(`[api/generate/storyboards] processShot: generated storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
+  } else {
+    console.log(`[api/generate/storyboards] processShot: there is already a storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
+  }
+
+}

src/app/api/v1/edit/storyboards/route.ts

@@ -1,13 +1,10 @@
 import { NextResponse, NextRequest } from "next/server"
 
-import { ClapProject, ClapSegment, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap"
-import { getVideoPrompt } from "@aitube/engine"
+import { ClapProject, ClapSegment, parseClap, serializeClap } from "@aitube/clap"
 
-import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
 import { getToken } from "@/app/api/auth/getToken"
 
-import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
-import { generateStoryboard } from "./generateStoryboard"
+import { processShot } from "./processShot"
 
 // a helper to generate storyboards for a Clap
 // this is mostly used by external apps such as the Stories Factory
@@ -35,63 +32,13 @@ export async function POST(req: NextRequest) {
     throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`)
   }
 
-  for (const shotSegment of shotsSegments) {
-
-    const shotSegments: ClapSegment[] = clap.segments.filter(s =>
-      startOfSegment1IsWithinSegment2(s, shotSegment)
-    )
-
-    const shotStoryboardSegments: ClapSegment[] = shotSegments.filter(s =>
-      s.category === "storyboard"
-    )
-
-    let shotStoryboardSegment: ClapSegment | undefined = shotStoryboardSegments.at(0)
-    
-    console.log(`[api/generate/storyboards] shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotStoryboardSegments.length} storyboards)`)
-  
-    // TASK 1: GENERATE MISSING STORYBOARD SEGMENT
-    if (!shotStoryboardSegment) {
-      shotStoryboardSegment = newSegment({
-        track: 1,
-        startTimeInMs: shotSegment.startTimeInMs,
-        endTimeInMs: shotSegment.endTimeInMs,
-        assetDurationInMs: shotSegment.assetDurationInMs,
-        category: "storyboard",
-        prompt: "",
-        assetUrl: "",
-        outputType: "image"
-      })
-      console.log(`[api/generate/storyboards] generated storyboard segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
-    }
-
-    // TASK 2: GENERATE MISSING STORYBOARD PROMPT
-    if (shotStoryboardSegment && !shotStoryboardSegment?.prompt) {
-      // storyboard is missing, let's generate it
-      shotStoryboardSegment.prompt = getVideoPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"])
-      console.log(`[api/generate/storyboards] generating storyboard prompt: ${shotStoryboardSegment.prompt}`)
-    }
-
-    // TASK 3: GENERATE MISSING STORYBOARD BITMAP
-    if (shotStoryboardSegment && !shotStoryboardSegment.assetUrl) {
-      // console.log(`[api/generate/storyboards] generating image..`)
-
-      try {
-        shotStoryboardSegment.assetUrl = await generateStoryboard({
-          prompt: getPositivePrompt(shotStoryboardSegment.prompt),
-          width: clap.meta.width,
-          height: clap.meta.height,
-        })
-        shotStoryboardSegment.assetSourceType = getClapAssetSourceType(shotStoryboardSegment.assetUrl)
-      } catch (err) {
-        console.log(`[api/generate/storyboards] failed to generate an image: ${err}`)
-        throw err
-      }
-   
-      console.log(`[api/generate/storyboards] generated storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
-    } else {
-      console.log(`[api/generate/storyboards] there is already a storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
-    }
-  }
+  // we process the shots in parallel (this will increase the queue size in the Gradio spaces)
+  await Promise.all(shotsSegments.map(shotSegment =>
+    processShot({
+      shotSegment,
+      clap
+    })
+  ))
 
   // console.log(`[api/generate/storyboards] returning the clap augmented with storyboards`)
 

src/app/api/v1/edit/videos/processShot.ts

@@ -0,0 +1,82 @@
+import { NextResponse, NextRequest } from "next/server"
+
+import { ClapProject, ClapSegment, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap"
+import { getVideoPrompt } from "@aitube/engine"
+
+import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
+import { getToken } from "@/app/api/auth/getToken"
+import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
+
+import { generateVideo } from "./generateVideo"
+
+export async function processShot({
+  shotSegment,
+  clap
+}: {
+  shotSegment: ClapSegment
+  clap: ClapProject
+}): Promise<void> {
+  const shotSegments: ClapSegment[] = clap.segments.filter(s =>
+    startOfSegment1IsWithinSegment2(s, shotSegment)
+  )
+
+  const shotVideoSegments: ClapSegment[] = shotSegments.filter(s =>
+    s.category === "video"
+  )
+
+  let shotVideoSegment: ClapSegment | undefined = shotVideoSegments.at(0)
+  
+  console.log(`[api/generate/videos] processShot: shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotVideoSegments.length} videos)`)
+
+  // TASK 1: GENERATE MISSING VIDEO SEGMENT
+  if (!shotVideoSegment) {
+    shotVideoSegment = newSegment({
+      track: 1,
+      startTimeInMs: shotSegment.startTimeInMs,
+      endTimeInMs: shotSegment.endTimeInMs,
+      assetDurationInMs: shotSegment.assetDurationInMs,
+      category: "video",
+      prompt: "",
+      assetUrl: "",
+      outputType: "video"
+    })
+
+    if (shotVideoSegment) {
+      clap.segments.push(shotVideoSegment)
+    }
+
+    console.log(`[api/generate/videos] processShot: generated video segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
+  }
+
+  if (!shotVideoSegment) {
+    throw new Error(`failed to generate a new segment`)
+  }
+
+  // TASK 2: GENERATE MISSING VIDEO PROMPT
+  if (!shotVideoSegment?.prompt) {
+    // video is missing, let's generate it
+    shotVideoSegment.prompt = getVideoPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"])
+    console.log(`[api/generate/videos] processShot: generating video prompt: ${shotVideoSegment.prompt}`)
+  }
+
+  // TASK 3: GENERATE MISSING VIDEO FILE
+  if (!shotVideoSegment.assetUrl) {
+    console.log(`[api/generate/videos] processShot: generating video file..`)
+
+    try {
+      shotVideoSegment.assetUrl = await generateVideo({
+        prompt: getPositivePrompt(shotVideoSegment.prompt),
+        width: clap.meta.width,
+        height: clap.meta.height,
+      })
+      shotVideoSegment.assetSourceType = getClapAssetSourceType(shotVideoSegment.assetUrl)
+    } catch (err) {
+      console.log(`[api/generate/videos] processShot: failed to generate a video file: ${err}`)
+      throw err
+    }
+  
+    console.log(`[api/generate/videos] processShot: generated video files: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
+  } else {
+    console.log(`[api/generate/videos] processShot: there is already a video file: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
+  }
+}

src/app/api/v1/edit/videos/route.ts

@@ -1,13 +1,10 @@
 import { NextResponse, NextRequest } from "next/server"
 
-import { ClapProject, ClapSegment, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap"
-import { getVideoPrompt } from "@aitube/engine"
+import { ClapProject, ClapSegment, parseClap, serializeClap } from "@aitube/clap"
 
-import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
 import { getToken } from "@/app/api/auth/getToken"
-import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
 
-import { generateVideo } from "./generateVideo"
+import { processShot } from "./processShot"
 
 
 // a helper to generate videos for a Clap
@@ -36,63 +33,13 @@ export async function POST(req: NextRequest) {
     throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`)
   }
 
-  for (const shotSegment of shotsSegments) {
-
-    const shotSegments: ClapSegment[] = clap.segments.filter(s =>
-      startOfSegment1IsWithinSegment2(s, shotSegment)
-    )
-
-    const shotVideoSegments: ClapSegment[] = shotSegments.filter(s =>
-      s.category === "video"
-    )
-
-    let shotVideoSegment: ClapSegment | undefined = shotVideoSegments.at(0)
-    
-    console.log(`[api/generate/videos] shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotVideoSegments.length} videos)`)
-  
-    // TASK 1: GENERATE MISSING VIDEO SEGMENT
-    if (!shotVideoSegment) {
-      shotVideoSegment = newSegment({
-        track: 1,
-        startTimeInMs: shotSegment.startTimeInMs,
-        endTimeInMs: shotSegment.endTimeInMs,
-        assetDurationInMs: shotSegment.assetDurationInMs,
-        category: "video",
-        prompt: "",
-        assetUrl: "",
-        outputType: "video"
-      })
-      console.log(`[api/generate/videos] generated video segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
-    }
-
-    // TASK 2: GENERATE MISSING VIDEO PROMPT
-    if (shotVideoSegment && !shotVideoSegment?.prompt) {
-      // video is missing, let's generate it
-      shotVideoSegment.prompt = getVideoPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"])
-      console.log(`[api/generate/videos] generating video prompt: ${shotVideoSegment.prompt}`)
-    }
-
-    // TASK 3: GENERATE MISSING VIDEO FILE
-    if (shotVideoSegment && !shotVideoSegment.assetUrl) {
-      console.log(`[api/generate/videos] generating video file..`)
-
-      try {
-        shotVideoSegment.assetUrl = await generateVideo({
-          prompt: getPositivePrompt(shotVideoSegment.prompt),
-          width: clap.meta.width,
-          height: clap.meta.height,
-        })
-        shotVideoSegment.assetSourceType = getClapAssetSourceType(shotVideoSegment.assetUrl)
-      } catch (err) {
-        console.log(`[api/generate/videos] failed to generate a video file: ${err}`)
-        throw err
-      }
-   
-      console.log(`[api/generate/videos] generated video files: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
-    } else {
-      console.log(`[api/generate/videos] there is already a video file: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
-    }
-  }
+  // we process the shots in parallel (this will increase the queue size in the Gradio spaces)
+  await Promise.all(shotsSegments.map(shotSegment =>
+    processShot({
+      shotSegment,
+      clap
+    })
+  ))
 
   console.log(`[api/generate/videos] returning the clap augmented with videos`)