investigating..

src/app/api/resolvers/video/route.ts

@@ -39,6 +39,7 @@ export async function GET(req: NextRequest) {
     return NextResponse.json({ error: `access denied ${err}` }, { status: 400 });
   }
   */
+  console.log("[API] /api/resolvers/video")
 
   let prompt = ""
   try {
@@ -71,8 +72,13 @@ export async function GET(req: NextRequest) {
   prompt = getPositivePrompt(prompt)
   const negativePrompt = getNegativePrompt()
 
-  // console.log("calling await newRender")
 
+  console.log("calling await newRender with", {
+    prompt,
+    negativePrompt,
+  })
+
+  throw new Error("no! use render()!")
   let render = await newRender({
     prompt,
     negativePrompt,

src/app/api/v1/edit/videos/processShot.ts

@@ -17,7 +17,6 @@ import { getVideoPrompt } from "@aitube/engine"
 import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
 
 import { render } from "@/app/api/v1/render/animatediff-lcm-svd"
-// import { render } from "@/app/api/v1/render/animatediff-lightning"
 import { extractFirstFrame } from "@/app/api/utils/extractFirstFrame"
 
 export async function processShot({
@@ -33,6 +32,7 @@ export async function processShot({
   mode: ClapCompletionMode
   turbo: boolean
 }): Promise<void> {
+  // console.log(`processShot()`)
   const shotSegments: ClapSegment[] = filterSegments(
     ClapSegmentFilteringMode.BOTH,
     shotSegment,

src/app/api/v1/render/animatediff-lightning/cluster.ts

@@ -3,7 +3,7 @@ import { ClusterMachine } from "../../types"
 
 
 // video generation requires A100s so we need to be parcimonous here,
-// we shouldn't burn too many GPUs
+// we shouldn't burn too many GPUs - 3 seems like a good number
 export const nbClusterMachines = 2
 // make sure the machines are running!!
 

src/app/api/v1/render/route.ts

@@ -5,7 +5,6 @@ import { getValidNumber } from "@aitube/clap"
 import { throwIfInvalidToken } from "@/app/api/v1/auth/throwIfInvalidToken"
 import { getContentType } from "@/lib/data/getContentType"
 
-// import { render } from "./animatediff-lcm-svd"
 import { render } from "./animatediff-lightning"
 
 export async function POST(req: NextRequest, res: NextResponse) {

src/app/latent/search/page.tsx

@@ -1,11 +1,12 @@
 import { encode, decode } from 'js-base64'
-import { clapToDataUri, generateClapFromSimpleStory } from "@aitube/clap"
+import { clapToDataUri } from "@aitube/clap"
 
 import { LatentQueryProps } from "@/types/general"
 
 import { Main } from "../../main"
 import { getNewMediaInfo } from "../../api/generators/search/getNewMediaInfo"
 import { getToken } from "../../api/v1/auth/getToken"
+import { generateClapFromPrompt } from '@/components/interface/latent-engine/core/generateClapFromPrompt'
 
 // https://jmswrnr.com/blog/protecting-next-js-api-routes-query-parameters
 
@@ -28,7 +29,7 @@ export default async function LatentSearchPage({
   const latentMedia = getNewMediaInfo()
 
   latentMedia.clapUrl = await clapToDataUri(
-    generateClapFromSimpleStory({
+    generateClapFromPrompt({
       showIntroPoweredByEngine: false,
       showIntroDisclaimerAboutAI: false
     })

src/app/page.tsx

@@ -83,9 +83,13 @@ export default async function Page({ searchParams: { v: videoId } }: AppQueryPro
   })
   return (
     <div className="flex flex-col items-center justify-center h-screen v-screen bg-stone-900">
-      <div className="flex flex-col items-center justify-center text-center w-2/3 h-2/3">
-      <h1 className="text-5xl">AiTube is being re-imagined.</h1>
-      <p className="text-2xl mt-8">We will be back once better AI tech is available.</p>
+      <div className="
+      flex flex-col items-center justify-center text-center
+      w-3/4 h-full
+   
+      ">
+      <h1 className="text-stone-200 text-6xl font-thin">Say goodbye to static videos.</h1>
+      <p className="mt-12 text-stone-400 text-xl font-thin">Beta planned for Winter 2024. Follow <a href="x.com/@flngr" className="font-normal font-mono text-stone-400 hover:text-stone-300 hover:underline hover:underline-offset-2" target="_blank">@flngr</a> for updates.</p>
       </div>
     </div>
   )

src/components/interface/latent-engine/core/engine.tsx

@@ -2,6 +2,7 @@
 
 import React, { MouseEventHandler, useEffect, useRef, useState } from "react"
 import { useLocalStorage } from "usehooks-ts"
+import { ClapProject } from "@aitube/clap"
 
 import { cn } from "@/lib/utils/cn"
 import { MediaInfo } from "@/types/general"
@@ -13,8 +14,8 @@ import { ContentLayer } from "../components/content-layer"
 import { localStorageKeys } from "@/app/state/localStorageKeys"
 import { defaultSettings } from "@/app/state/defaultSettings"
 import { useStore } from "@/app/state/useStore"
-import { ClapProject, generateClapFromSimpleStory, serializeClap } from "@aitube/clap"
 import { theSimps } from "@/app/latent/samples"
+import { generateClapFromPrompt } from "./generateClapFromPrompt"
 
 function LatentEngine({
   media,
@@ -86,7 +87,7 @@ function LatentEngine({
 
         // TODO Julian work on the chunk mechanism
         
-        const mockClap: ClapProject = generateClapFromSimpleStory({
+        const mockClap: ClapProject = generateClapFromPrompt({
           story: theSimps,
           showIntroPoweredByEngine: false,
           showIntroDisclaimerAboutAI: false

src/components/interface/latent-engine/core/generateClapFromPrompt.ts

@@ -0,0 +1,124 @@
+import { ClapOutputType, ClapProject, ClapSegmentCategory, newClap, newSegment } from "@aitube/clap"
+
+import { defaultSegmentDurationInMs, demoStory } from "./samples"
+
+export function generateClapFromPrompt({
+  story = demoStory,
+  showIntroPoweredByEngine = false,
+  showIntroDisclaimerAboutAI = false,
+}: {
+  story?: string[]
+  showIntroPoweredByEngine?: boolean
+  showIntroDisclaimerAboutAI?: boolean
+} = {
+  story: demoStory,
+  showIntroPoweredByEngine: false,
+  showIntroDisclaimerAboutAI: false,
+}): ClapProject {
+
+  const clap = newClap({
+    meta: {
+      title: "Interactive Demo",
+      isInteractive: true,
+      isLoop: true,
+      description: story,
+      synopsis: story,
+    }
+  })
+
+  let startTimeInMs = 0
+  let endTimeInMs = defaultSegmentDurationInMs
+
+  if (showIntroPoweredByEngine) {
+    clap.segments.push(newSegment({
+      startTimeInMs,
+      endTimeInMs,
+      category: ClapSegmentCategory.INTERFACE,
+      prompt: "<BUILTIN:POWERED_BY_ENGINE>",
+      label: "disclaimer",
+      outputType: ClapOutputType.INTERFACE,
+    }))
+    startTimeInMs += defaultSegmentDurationInMs
+    endTimeInMs += defaultSegmentDurationInMs
+  }
+
+  if (showIntroDisclaimerAboutAI) {
+    clap.segments.push(newSegment({
+      startTimeInMs,
+      endTimeInMs,
+      category:ClapSegmentCategory.INTERFACE,
+      prompt: "<BUILTIN:DISCLAIMER_ABOUT_AI>",
+      label: "disclaimer",
+      outputType: ClapOutputType.INTERFACE,
+    }))
+    startTimeInMs += defaultSegmentDurationInMs
+    endTimeInMs += defaultSegmentDurationInMs
+  }
+
+  /*
+  clap.segments.push(
+    newSegment({
+      // id: string
+      // track: number
+      startTimeInMs,
+      endTimeInMs,
+      category: ClapSegmentCategory.INTERFACE,
+      // entityId: string
+      // sceneId: string
+      prompt: "a hello world",
+      label: "hello world",
+      outputType: ClapOutputType.INTERFACE,
+      // renderId: string
+      // status: ClapSegmentStatus
+      // assetUrl: string
+      // assetDurationInMs: number
+      // createdBy: ClapAuthor
+      // editedBy: ClapAuthor
+      // outputGain: number
+      // seed: number
+    })
+  )
+    startTimeInMs += defaultSegmentDurationInMs
+    endTimeInMs += defaultSegmentDurationInMs
+  */
+  
+ 
+
+  for (let prompt of story) {
+
+    clap.segments.push(newSegment({
+      track: 0,
+      startTimeInMs,
+      endTimeInMs,
+      category: ClapSegmentCategory.VIDEO,
+      prompt: "",
+      label: "video",
+      outputType: ClapOutputType.VIDEO,
+    }))
+    clap.segments.push(newSegment({
+      track: 1,
+      startTimeInMs,
+      endTimeInMs,
+      category: ClapSegmentCategory.GENERIC,
+      prompt,
+      label: prompt,
+      outputType: ClapOutputType.TEXT,
+    }))
+    clap.segments.push(newSegment({
+      track: 2,
+      startTimeInMs,
+      endTimeInMs,
+      category: ClapSegmentCategory.CAMERA,
+      prompt: "medium-shot",
+      label: "medium-shot",
+      outputType: ClapOutputType.TEXT,
+    }))
+
+    startTimeInMs += defaultSegmentDurationInMs
+    endTimeInMs += defaultSegmentDurationInMs
+  }
+
+  clap.meta.durationInMs = endTimeInMs
+
+  return clap
+}

src/components/interface/latent-engine/core/generators/fetchLatentClap.ts

@@ -4,7 +4,7 @@ export async function fetchLatentClap(prompt: string): Promise<ClapProject> {
 
   const requestUri = `/api/resolvers/clap?p=${encodeURIComponent(prompt)}`
 
-  console.log(`fetchLatentClap: calling ${requestUri}`)
+  // console.log(`fetchLatentClap: calling ${requestUri}`)
 
   const res = await fetch(requestUri)
 
@@ -12,7 +12,7 @@ export async function fetchLatentClap(prompt: string): Promise<ClapProject> {
 
   const clap = await parseClap(blob)
 
-  console.log(`fetchLatentClap: received = `, clap)
+  // console.log(`fetchLatentClap: received = `, clap)
 
   return clap
 }

src/components/interface/latent-engine/core/samples.ts

@@ -0,0 +1,28 @@
+export const defaultSegmentDurationInMs = 2500 // 2584
+
+export const fishDemoStory = [
+  "Siamese fighting fish, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
+  
+  // this one is magnificient!
+  "princess parrot fish, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
+  
+  "pacific ocean perch, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
+  
+  "Queen angelfish, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
+  
+  "sea turtle, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
+
+  "hippocampus, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
+]
+
+export const demoStory = [
+  ...fishDemoStory,
+
+  // "portrait of one man news anchor, 60yo, thin, fit, american, mustache, beard, wearing a suit, medium-shot, central park, outside, serious, bokeh, perfect",
+  
+  // "screenshot from Call of Duty, FPS game, nextgen, videogame screenshot, unreal engine, raytracing, perfect",
+
+  // "screenshot from a flight simulator, nextgen, videogame screenshot, unreal engine, raytracing, perfect",
+  // "screenshot from fallout3, fallout4, western, wasteland, 3rd person RPG, nextgen, videogame screenshot, unreal engine, raytracing, perfect",
+  // "portrait of single influencer woman, 30yo, thin, fit, american, wearing a red tshirt, medium-shot, central park, outside, serious, bokeh, perfect",
+]

src/components/interface/latent-engine/core/useLatentEngine.ts

@@ -356,21 +356,36 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
     let toPlay: HTMLVideoElement[] = []
     let toPreload: HTMLVideoElement[] = []
     
+    console.log("clap: ", clap)
+
+    ;(window as any).debugJuju = clap;
+
     for (let i = 0; i < videosSortedByStartAt.length; i++) {
       const video = videosSortedByStartAt[i]
       
       const segmentStartAt = getSegmentStartAt(video)
       const segmentEndAt = getSegmentEndAt(video)
 
+      console.log(`segs: `, {
+        segmentStartAt,
+        segmentEndAt
+      })
       // this segment has been spent, it should be discared
       if (segmentEndAt < positionInMs) {
+        console.log("A")
         toRecycle.push(video)
       } else if (segmentStartAt < positionInMs) {
+        console.log("B")
         toPlay.push(video)
         video.play()
         setZIndexDepthId(video, 10)
       } else {
+        console.log("C")
         toPreload.push(video)
+
+        // Julian: I'm adding that to see if it's the problem
+        //toRecycle.push(video)
+
         video.pause()
         setZIndexDepthId(video, 0)
       }
@@ -382,22 +397,29 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
     // to grab the max number of segments
     const maxBufferDurationInMs = positionInMs + (videoDurationInMs * 4)
     
+
     
-    console.log(`DEBUG: `, {
+    const prefilterSegmentsForPerformanceReasons: ClapSegment[] = clap.segments.filter(s =>
+      s.startTimeInMs >= positionInMs &&
+      s.startTimeInMs < maxBufferDurationInMs
+    )
+
+ 
+    console.log(`runVideoSimulationLoop: `, {
       positionInMs,
       videoModelDurationInSec,
       videoDurationInMs,
       "(videoDurationInMs * 4)":  (videoDurationInMs * 4),
       maxBufferDurationInMs,
-      segments: clap.segments
+      segments: clap.segments,
+      prefilterSegmentsForPerformanceReasons,
+      videosSortedByStartAt,
+      toPlay,
+      toPreload,
+      toRecycle
     })
-    
-    const prefilterSegmentsForPerformanceReasons: ClapSegment[] = clap.segments.filter(s =>
-      s.startTimeInMs >= positionInMs &&
-      s.startTimeInMs < maxBufferDurationInMs
-    )
 
-    console.log(`prefilterSegmentsForPerformanceReasons: `, prefilterSegmentsForPerformanceReasons)
+    // console.log(`prefilterSegmentsForPerformanceReasons: `, prefilterSegmentsForPerformanceReasons)
 
     // this tells us how much time is left
     let remainingTimeInMs = Math.max(0, clap.meta.durationInMs - positionInMs)
@@ -414,7 +436,7 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
     let bufferAheadOfCurrentPositionInMs = positionInMs
 
     for (let i = 0; i < toRecycle.length; i++) {
-      console.log(`got a spent video to recycle`)
+      // console.log(`got a spent video to recycle`)
       
       // we select the segments in the current shot
 
@@ -515,7 +537,7 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
  
     try {
       if (get().isPlaying) {
-        // console.log(`runSimulationLoop: rendering UI layer..`)
+        console.log(`runSimulationLoop: rendering UI layer..`)
 
         // note: for now we only display one panel at a time,
         // later we can try to see if we should handle more
@@ -577,8 +599,12 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
   // list to put into the buffer the one that should be displayed
   runRenderingLoop: () => {
     const {
+      clap,
       isLoaded,
       isPlaying,
+      isInteractive,
+      isLive,
+      isLoop,
       renderingIntervalId,
       renderingIntervalDelayInMs,
       renderingLastRenderAt,
@@ -597,12 +623,30 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
       document.querySelectorAll('.video-buffer')
     ) as HTMLVideoElement[]
  
+    // ------------ TIMELINE CURSOR PROGRESSION CYCLE -------------
+    // the following implements the mechanism of moving the cursor
+    // within the timeline
+
     const newRenderingLastRenderAt = performance.now()
     const elapsedInMs = newRenderingLastRenderAt - renderingLastRenderAt
 
-    // let's move inside the Clap file timeline
-    const newPositionInMs = positionInMs + elapsedInMs
+    let newPositionInMs = positionInMs
+    // this is were we decide what to do based on the current mode
+    if (isInteractive) {
+      console.log("interactive mode: nothing to do")
+      
+    } else {
+     
+      newPositionInMs += elapsedInMs
 
+      if (isLoop) {
+        if (newPositionInMs > 5000) {
+          console.log("end of loop detected! going back")
+          newPositionInMs = 0
+        }
+      }
+    }
+ 
     clearInterval(renderingIntervalId)
 
     set({