up

src/app/api/utils/extractFirstFrame.ts

@@ -0,0 +1,81 @@
+import { promises as fs, existsSync } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+import { v4 as uuidv4 } from 'uuid';
+import ffmpeg from 'fluent-ffmpeg';
+
+const validFormats = ['jpeg', 'png', 'webp'];
+
+/**
+ * Extract the first frame from a video
+ * 
+ * @param param0
+ * @returns 
+ */
+export async function extractFirstFrame({
+  inputVideo,
+  outputFormat = 'jpeg'
+}: {
+  inputVideo?: string
+  outputFormat?: "jpeg" | "png" | "webp"
+
+}): Promise<string> {
+
+  if (!inputVideo) {
+    throw new Error(`inputVideo must be a file path or a base64 data-uri`);
+  }
+
+  if (!validFormats.includes(outputFormat)) {
+    throw new Error(`Invalid output format. Choose one of: ${validFormats.join(', ')}`);
+  }
+
+  // Handle base64 input
+  let videoFilePath = inputVideo;
+  if (inputVideo.startsWith('data:')) {
+    const matches = inputVideo.match(/^data:video\/(\w+);base64,(.*)$/);
+    if (!matches) {
+      throw new Error('Invalid base64 input provided.');
+    }
+    const extension = matches[1];
+    const base64Content = matches[2];
+    
+    videoFilePath = path.join(os.tmpdir(), `${uuidv4()}_inputVideo.${extension}`);
+    await fs.writeFile(videoFilePath, base64Content, 'base64');
+  } else if (!existsSync(videoFilePath)) {
+    throw new Error('Video file does not exist.');
+  }
+
+  // Create a temporary output file
+  const outputImagePath = path.join(os.tmpdir(), `${uuidv4()}.${outputFormat}`);
+
+  return new Promise((resolve, reject) => {
+    ffmpeg()
+      .input(videoFilePath)
+      .outputOptions([
+        '-vframes', '1',  // Extract only one frame
+        '-f', 'image2',   // Output format for the frame as image
+        '-an'             // Disable audio
+      ])
+      .output(outputImagePath)
+      .on('error', (err) => {
+        reject(new Error(`FFmpeg error: ${err.message}`));
+      })
+      .on('end', async () => {
+        try {
+          const imageBuffer = await fs.readFile(outputImagePath);
+          const imageBase64 = `data:image/${outputFormat};base64,${imageBuffer.toString('base64')}`;
+          resolve(imageBase64);
+        } catch (error) {
+          reject(new Error(`Error reading the image file: ${error}`));
+        } finally {
+          // Clean up temporary files
+          if (inputVideo.startsWith('data:')) {
+            await fs.unlink(videoFilePath);
+          }
+          await fs.unlink(outputImagePath);
+        }
+      })
+      .run();
+  });
+}

src/app/api/v1/edit/videos/processShot.ts

@@ -16,6 +16,7 @@ import { getVideoPrompt } from "@aitube/engine"
 import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
 
 import { render } from "@/app/api/v1/render"
+import { extractFirstFrame } from "@/app/api/utils/extractFirstFrame"
 
 export async function processShot({
   shotSegment,
@@ -42,7 +43,11 @@ export async function processShot({
 
   let shotVideoSegment: ClapSegment | undefined = shotVideoSegments.at(0)
   
-  // console.log("bug here?", turbo)
+  const shotStoryboardSegments: ClapSegment[] = shotSegments.filter(s =>
+    s.category === ClapSegmentCategory.STORYBOARD
+  )
+
+  let shotStoryboardSegment: ClapSegment | undefined = shotStoryboardSegments.at(0)
 
   console.log(`[api/edit/videos] processShot: shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotVideoSegments.length} videos)`)
 
@@ -85,7 +90,7 @@ export async function processShot({
 
   // TASK 3: GENERATE MISSING VIDEO FILE
   if (!shotVideoSegment.assetUrl) {
-    // console.log(`[api/edit/videos] processShot: generating video file..`)
+    console.log(`[api/edit/videos] processShot: generating video file..`)
 
     const debug = false
 
@@ -135,4 +140,56 @@ export async function processShot({
   } else {
     console.log(`[api/edit/videos] processShot: there is already a video file: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
   }
+
+  if (!shotVideoSegment.assetUrl) {
+    return
+  }
+
+  if (!shotStoryboardSegment) {
+    console.log(`[api/edit/videos] processShot: adding the missing storyboard segment`)
+
+    shotStoryboardSegment = newSegment({
+      track: 1,
+      startTimeInMs: shotSegment.startTimeInMs,
+      endTimeInMs: shotSegment.endTimeInMs,
+      assetDurationInMs: shotSegment.assetDurationInMs,
+      category: ClapSegmentCategory.STORYBOARD,
+      prompt: shotVideoSegment.prompt,
+      outputType: ClapOutputType.IMAGE,
+      status: "to_generate",
+    })
+
+    if (shotStoryboardSegment) {
+      existingClap.segments.push(shotStoryboardSegment)
+    }
+  }
+
+
+  //----------
+  if (
+    shotStoryboardSegment && 
+    (!shotStoryboardSegment.assetUrl || shotStoryboardSegment.status === "to_generate")
+  ) {
+    console.log(`[api/edit/videos] processShot: generating a missing storyboard asset`)
+
+    try {
+      shotStoryboardSegment.assetUrl = await extractFirstFrame({
+        inputVideo: shotVideoSegment.assetUrl,
+        outputFormat: "jpeg"
+      })
+      if (!shotStoryboardSegment.assetUrl) { throw new Error(`failed to extract the first frame`) }
+      console.warn(`[api/edit/videos] processShot: successfully fixed the missing storyboard`)
+      
+      shotStoryboardSegment.status = "completed"
+    } catch (err) {
+      console.warn(`[api/edit/videos] processShot: couldn't generate the missing storyboard (probably an error with the ffmpeg config). Message:`, err)
+      shotStoryboardSegment.status = "to_generate"
+    }
+
+
+    if (shotStoryboardSegment && mode !== ClapCompletionMode.FULL) {
+      newerClap.segments.push(shotStoryboardSegment)
+    }
+  }
+
 }