update

package-lock.json

@@ -8,9 +8,9 @@
       "name": "@aitube/website",
       "version": "0.0.0",
       "dependencies": {
-        "@aitube/clap": "0.0.20",
-        "@aitube/client": "0.0.29",
-        "@aitube/engine": "0.0.11",
+        "@aitube/clap": "0.0.22",
+        "@aitube/client": "0.0.31",
+        "@aitube/engine": "0.0.12",
         "@huggingface/hub": "0.12.3-oauth",
         "@huggingface/inference": "^2.7.0",
         "@jcoreio/async-throttle": "^1.6.0",
@@ -119,9 +119,9 @@
       }
     },
     "node_modules/@aitube/clap": {
-      "version": "0.0.20",
-      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.20.tgz",
-      "integrity": "sha512-QOEl3GTHRpaBtKGYoH+iabY9TfLYogOoIfc/YuwC01paO/woMASccjDloPGiPe9C0/P88AqB6s7bN3Dkz1v0TQ==",
+      "version": "0.0.22",
+      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.22.tgz",
+      "integrity": "sha512-oN+tfoy0fp95AiwNMVy6pbhkK9k/h/gZ3FTZPpF06bVLig4rZ/Bthyot+wy4E/hr9ERLeyGpfZVW8blRqHCAaw==",
       "dependencies": {
         "pure-uuid": "^1.8.1",
         "yaml": "^2.4.2"
@@ -131,22 +131,22 @@
       }
     },
     "node_modules/@aitube/client": {
-      "version": "0.0.29",
-      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.29.tgz",
-      "integrity": "sha512-7ASbsoKuk10RbF/d9uuGU2OBnMxrLd/O01rSH5GUYMTeWnJNjnEBy/klgCQxoA7qrO6VX7kJGI5MB2pfK9XJFA==",
+      "version": "0.0.31",
+      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.31.tgz",
+      "integrity": "sha512-xjdPYRruuEWe8KdmWpMYB1ELylBRF2M6+BYr0K3VAf/fv4qcRqda+F9amitJ27h0t5rvagA7+MLLKFqlF7BT/A==",
       "dependencies": {
         "query-string": "^9.0.0"
       },
       "peerDependencies": {
-        "@aitube/clap": "0.0.20"
+        "@aitube/clap": "0.0.22"
       }
     },
     "node_modules/@aitube/engine": {
-      "version": "0.0.11",
-      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.11.tgz",
-      "integrity": "sha512-XMjhyakLvYvEmJFlJVWIdxymDhpAcW9RnZfGXNCsCPrjkrb6nhuVm3RDSRO2JwNPe/i7T4DaEDVeWavDiyQNMA==",
+      "version": "0.0.12",
+      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.12.tgz",
+      "integrity": "sha512-MkV9FFk2JOjm6+ir5T4p65AzO95n1zjmXZevswer3ApXpMuMY0MuqC3krYhxj9cBF54S2C/mltUCk90cOHWP2Q==",
       "peerDependencies": {
-        "@aitube/clap": "0.0.20"
+        "@aitube/clap": "0.0.22"
       }
     },
     "node_modules/@alloc/quick-lru": {
@@ -3322,17 +3322,6 @@
         "node": ">= 8"
       }
     },
-    "node_modules/anymatch/node_modules/picomatch": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
-      "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
-      "engines": {
-        "node": ">=8.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/jonschlinkert"
-      }
-    },
     "node_modules/arg": {
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
@@ -3764,9 +3753,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001620",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001620.tgz",
-      "integrity": "sha512-WJvYsOjd1/BYUY6SNGUosK9DUidBPDTnOARHp3fSmFO1ekdxaY6nKRttEVrfMmYi80ctS0kz1wiWmm14fVc3ew==",
+      "version": "1.0.30001621",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001621.tgz",
+      "integrity": "sha512-+NLXZiviFFKX0fk8Piwv3PfLPGtRqJeq2TiNoUff/qB5KJgwecJTvCXDpmlyP/eCI/GUEmp/h/y5j0yckiiZrA==",
       "funding": [
         {
           "type": "opencollective",
@@ -6325,12 +6314,12 @@
       }
     },
     "node_modules/micromatch": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.6.tgz",
-      "integrity": "sha512-Y4Ypn3oujJYxJcMacVgcs92wofTHxp9FzfDpQON4msDefoC0lb3ETvQLOdLcbhSwU1bz8HrL/1sygfBIHudrkQ==",
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
+      "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
       "dependencies": {
         "braces": "^3.0.3",
-        "picomatch": "^4.0.2"
+        "picomatch": "^2.3.1"
       },
       "engines": {
         "node": ">=8.6"
@@ -6876,11 +6865,11 @@
       "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew=="
     },
     "node_modules/picomatch": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz",
-      "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
+      "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
       "engines": {
-        "node": ">=12"
+        "node": ">=8.6"
       },
       "funding": {
         "url": "https://github.com/sponsors/jonschlinkert"
@@ -7035,9 +7024,9 @@
       }
     },
     "node_modules/postcss-selector-parser": {
-      "version": "6.0.16",
-      "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.16.tgz",
-      "integrity": "sha512-A0RVJrX+IUkVZbW3ClroRWurercFhieevHB38sr2+l9eUClMqome3LmEmnhlNy+5Mr2EYN6B2Kaw9wYdd+VHiw==",
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.0.tgz",
+      "integrity": "sha512-UMz42UD0UY0EApS0ZL9o1XnLhSTtvvvLe5Dc2H2O56fvRZi+KulDyf5ctDhhtYJBGKStV2FL1fy6253cmLgqVQ==",
       "dependencies": {
         "cssesc": "^3.0.0",
         "util-deprecate": "^1.0.2"
@@ -7380,17 +7369,6 @@
         "node": ">=8.10.0"
       }
     },
-    "node_modules/readdirp/node_modules/picomatch": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
-      "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
-      "engines": {
-        "node": ">=8.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/jonschlinkert"
-      }
-    },
     "node_modules/reflect.getprototypeof": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.6.tgz",

package.json

@@ -10,9 +10,9 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@aitube/clap": "0.0.20",
-    "@aitube/client": "0.0.29",
-    "@aitube/engine": "0.0.11",
+    "@aitube/clap": "0.0.22",
+    "@aitube/client": "0.0.31",
+    "@aitube/engine": "0.0.12",
     "@huggingface/hub": "0.12.3-oauth",
     "@huggingface/inference": "^2.7.0",
     "@jcoreio/async-throttle": "^1.6.0",

src/app/api/generators/clap/constants.ts

@@ -1 +1,4 @@
 export const defaultSegmentDurationInMs = 2000
+// const defaultSegmentDurationInMs = 2916
+
+export const MAX_PROMPT_LENGTH_IN_CHARS = 1024

src/app/api/v1/create/index.ts

@@ -13,6 +13,7 @@ import { clapToLatentStory } from "../edit/entities/clapToLatentStory"
 import { generateRandomStory } from "./generateRandomStory"
 import { generateSoundPrompts } from "../edit/sounds/generateSoundPrompt"
 import { checkCaptions } from "./checkCaptions"
+import { MAX_PROMPT_LENGTH_IN_CHARS } from "../../generators/clap/constants"
 
 // a helper to generate Clap stories from a few sentences
 // this is mostly used by external apps such as the Stories Factory
@@ -29,7 +30,8 @@ export async function create(request: {
 }): Promise<ClapProject> {
 
   // we limit to 512 characters
-  let { prompt, hasCaptions } = checkCaptions(`${request?.prompt || ""}`.trim().slice(0, 512))
+  let { prompt, hasCaptions } = checkCaptions(`${request?.prompt || ""}`.trim().slice(0, MAX_PROMPT_LENGTH_IN_CHARS))
+
 
   // console.log("api/v1/create(): request:", request)
 
@@ -159,19 +161,19 @@ Output: `
     }))
 
     if (hasCaptions) {
-    clap.segments.push(newSegment({
-      track: 2,
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentElapsedTimeInMs + defaultSegmentDurationInMs,
-      assetDurationInMs: defaultSegmentDurationInMs,
-      category: ClapSegmentCategory.INTERFACE,
-      prompt: comment,
-      // assetUrl: `data:text/plain;base64,${btoa(comment)}`,
-      assetUrl: comment,
-      outputType: ClapOutputType.TEXT,
-      status: ClapSegmentStatus.TO_GENERATE,
-    }))
-  }
+      clap.segments.push(newSegment({
+        track: 2,
+        startTimeInMs: currentElapsedTimeInMs,
+        endTimeInMs: currentElapsedTimeInMs + defaultSegmentDurationInMs,
+        assetDurationInMs: defaultSegmentDurationInMs,
+        category: ClapSegmentCategory.INTERFACE,
+        prompt: comment,
+        // assetUrl: `data:text/plain;base64,${btoa(comment)}`,
+        assetUrl: comment,
+        outputType: ClapOutputType.TEXT,
+        status: ClapSegmentStatus.TO_GENERATE,
+      }))
+    }
 
     clap.segments.push(newSegment({
       track: 3,

src/app/api/v1/edit/story/extendClapStory.ts

@@ -11,6 +11,8 @@ import { ClapCompletionMode } from "@aitube/client"
 import { clapToLatentStory } from "../entities/clapToLatentStory"
 import { LatentStory } from "../../types"
 import { extendLatentStoryWithMoreShots } from "./extendLatentStoryWithMoreShots"
+import { MAX_PROMPT_LENGTH_IN_CHARS } from "@/app/api/generators/clap/constants"
+import { checkCaptions } from "../../create/checkCaptions"
 
 export async function extendClapStory({
   prompt: maybePrompt,
@@ -30,9 +32,26 @@ export async function extendClapStory({
   turbo: boolean
 }): Promise<void> {
 
-  const prompt = typeof maybePrompt === "string" && maybePrompt.length > 0
-    ? maybePrompt
-    : existingClap.meta.description
+  let hasCaptions = false
+
+  let currentElapsedTimeInMs = 0
+  for (const s of existingClap.segments) {
+    if (s.category === ClapSegmentCategory.INTERFACE) { hasCaptions = true }
+    if (s.endTimeInMs > currentElapsedTimeInMs) { currentElapsedTimeInMs = s.endTimeInMs }
+  }
+
+  const { prompt, hasCaptions: hasCaptionInNewPrompt } = checkCaptions(
+    (
+      typeof maybePrompt === "string" && maybePrompt.length > 0
+      ? maybePrompt
+      : existingClap.meta.description
+    ).trim().slice(0, MAX_PROMPT_LENGTH_IN_CHARS)
+  )
+
+  // if the video already contains captions but the user asks for none for the subsequent scenes
+  if (!hasCaptionInNewPrompt && hasCaptions) {
+    hasCaptions = false
+  }
 
   const latentStory: LatentStory[] = await clapToLatentStory(existingClap)
   
@@ -43,16 +62,12 @@ export async function extendClapStory({
     turbo,
   })
 
-  let hasCaptions = false
-
-  let currentElapsedTimeInMs = 0
-  for (const s of existingClap.segments) {
-    if (s.category === ClapSegmentCategory.INTERFACE) { hasCaptions = true }
-    if (s.endTimeInMs > currentElapsedTimeInMs) { currentElapsedTimeInMs = s.endTimeInMs }
-  }
+  // console.log(`extendClapStory: going to extend, starting at ${currentElapsedTimeInMs}`)
 
   // this is approximate - TTS generation will determine the final duration of each shot
+
   const defaultSegmentDurationInMs = 3000
+  // const defaultSegmentDurationInMs = 2916
 
   for (const { comment, image, voice } of shots) {
 

src/app/api/v1/edit/story/route.ts

@@ -21,7 +21,7 @@ export async function POST(req: NextRequest) {
   const qs = queryString.parseUrl(req.url || "")
   const query = (qs || {}).query
 
-  const prompt = parsePrompt(query?.p, false)
+  const prompt = parsePrompt(query?.p, false) // <- false, to make it non-mandatory
   const mode = parseCompletionMode(query?.c)
   const turbo = parseTurbo(query?.t)
   
@@ -38,10 +38,10 @@ export async function POST(req: NextRequest) {
     meta: existingClap.meta
   })
 
-    // console.log(`api/v1/edit/storyboards(): detected ${existingClap.segments.length} segments`)
+    // console.log(`api/v1/edit/story(): detected ${existingClap.segments.length} segments`)
   
     const allShotsSegments: ClapSegment[] = existingClap.segments.filter(s => s.category === ClapSegmentCategory.CAMERA)
-    console.log(`api/v1/edit/storyboards(): detected ${allShotsSegments.length} shots`)
+    console.log(`api/v1/edit/story(): detected ${allShotsSegments.length} shots`)
   
     await extendClapStory({
       prompt,