jbilcke-hf's picture
jbilcke-hf HF staff
use @aitube/engine
2eea766
raw
history blame
3.94 kB
import { NextResponse, NextRequest } from "next/server"
import { ClapProject, ClapSegment, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap"
import { getVideoPrompt } from "@aitube/engine"
import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
import { getToken } from "@/app/api/auth/getToken"
import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
import { generateVideo } from "./generateVideo"
// a helper to generate videos for a Clap
// this is mostly used by external apps such as the Stories Factory
// this function will:
//
// - add missing videos to the shots
// - add missing video prompts
// - add missing video files
export async function POST(req: NextRequest) {
const jwtToken = await getToken({ user: "anonymous" })
const blob = await req.blob()
const clap: ClapProject = await parseClap(blob)
if (!clap?.segments) { throw new Error(`no segment found in the provided clap!`) }
console.log(`[api/generate/videos] detected ${clap.segments.length} segments`)
const shotsSegments: ClapSegment[] = clap.segments.filter(s => s.category === "camera")
console.log(`[api/generate/videos] detected ${shotsSegments.length} shots`)
if (shotsSegments.length > 32) {
throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`)
}
for (const shotSegment of shotsSegments) {
const shotSegments: ClapSegment[] = clap.segments.filter(s =>
startOfSegment1IsWithinSegment2(s, shotSegment)
)
const shotVideoSegments: ClapSegment[] = shotSegments.filter(s =>
s.category === "video"
)
let shotVideoSegment: ClapSegment | undefined = shotVideoSegments.at(0)
console.log(`[api/generate/videos] shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotVideoSegments.length} videos)`)
// TASK 1: GENERATE MISSING VIDEO SEGMENT
if (!shotVideoSegment) {
shotVideoSegment = newSegment({
track: 1,
startTimeInMs: shotSegment.startTimeInMs,
endTimeInMs: shotSegment.endTimeInMs,
assetDurationInMs: shotSegment.assetDurationInMs,
category: "video",
prompt: "",
assetUrl: "",
outputType: "video"
})
console.log(`[api/generate/videos] generated video segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
}
// TASK 2: GENERATE MISSING VIDEO PROMPT
if (shotVideoSegment && !shotVideoSegment?.prompt) {
// video is missing, let's generate it
shotVideoSegment.prompt = getVideoPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"])
console.log(`[api/generate/videos] generating video prompt: ${shotVideoSegment.prompt}`)
}
// TASK 3: GENERATE MISSING VIDEO FILE
if (shotVideoSegment && !shotVideoSegment.assetUrl) {
console.log(`[api/generate/videos] generating video file..`)
try {
shotVideoSegment.assetUrl = await generateVideo({
prompt: getPositivePrompt(shotVideoSegment.prompt),
width: clap.meta.width,
height: clap.meta.height,
})
shotVideoSegment.assetSourceType = getClapAssetSourceType(shotVideoSegment.assetUrl)
} catch (err) {
console.log(`[api/generate/videos] failed to generate a video file: ${err}`)
throw err
}
console.log(`[api/generate/videos] generated video files: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
} else {
console.log(`[api/generate/videos] there is already a video file: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
}
}
console.log(`[api/generate/videos] returning the clap augmented with videos`)
return new NextResponse(await serializeClap(clap), {
status: 200,
headers: new Headers({ "content-type": "application/x-gzip" }),
})
}