Spaces:
Running
Running
import { NextResponse, NextRequest } from "next/server" | |
import { ClapProject, ClapSegment, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap" | |
import { getVideoPrompt } from "@aitube/engine" | |
import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2" | |
import { getToken } from "@/app/api/auth/getToken" | |
import { getPositivePrompt } from "@/app/api/utils/imagePrompts" | |
import { generateVideo } from "./generateVideo" | |
// a helper to generate videos for a Clap | |
// this is mostly used by external apps such as the Stories Factory | |
// this function will: | |
// | |
// - add missing videos to the shots | |
// - add missing video prompts | |
// - add missing video files | |
export async function POST(req: NextRequest) { | |
const jwtToken = await getToken({ user: "anonymous" }) | |
const blob = await req.blob() | |
const clap: ClapProject = await parseClap(blob) | |
if (!clap?.segments) { throw new Error(`no segment found in the provided clap!`) } | |
console.log(`[api/generate/videos] detected ${clap.segments.length} segments`) | |
const shotsSegments: ClapSegment[] = clap.segments.filter(s => s.category === "camera") | |
console.log(`[api/generate/videos] detected ${shotsSegments.length} shots`) | |
if (shotsSegments.length > 32) { | |
throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`) | |
} | |
for (const shotSegment of shotsSegments) { | |
const shotSegments: ClapSegment[] = clap.segments.filter(s => | |
startOfSegment1IsWithinSegment2(s, shotSegment) | |
) | |
const shotVideoSegments: ClapSegment[] = shotSegments.filter(s => | |
s.category === "video" | |
) | |
let shotVideoSegment: ClapSegment | undefined = shotVideoSegments.at(0) | |
console.log(`[api/generate/videos] shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotVideoSegments.length} videos)`) | |
// TASK 1: GENERATE MISSING VIDEO SEGMENT | |
if (!shotVideoSegment) { | |
shotVideoSegment = newSegment({ | |
track: 1, | |
startTimeInMs: shotSegment.startTimeInMs, | |
endTimeInMs: shotSegment.endTimeInMs, | |
assetDurationInMs: shotSegment.assetDurationInMs, | |
category: "video", | |
prompt: "", | |
assetUrl: "", | |
outputType: "video" | |
}) | |
console.log(`[api/generate/videos] generated video segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`) | |
} | |
// TASK 2: GENERATE MISSING VIDEO PROMPT | |
if (shotVideoSegment && !shotVideoSegment?.prompt) { | |
// video is missing, let's generate it | |
shotVideoSegment.prompt = getVideoPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"]) | |
console.log(`[api/generate/videos] generating video prompt: ${shotVideoSegment.prompt}`) | |
} | |
// TASK 3: GENERATE MISSING VIDEO FILE | |
if (shotVideoSegment && !shotVideoSegment.assetUrl) { | |
console.log(`[api/generate/videos] generating video file..`) | |
try { | |
shotVideoSegment.assetUrl = await generateVideo({ | |
prompt: getPositivePrompt(shotVideoSegment.prompt), | |
width: clap.meta.width, | |
height: clap.meta.height, | |
}) | |
shotVideoSegment.assetSourceType = getClapAssetSourceType(shotVideoSegment.assetUrl) | |
} catch (err) { | |
console.log(`[api/generate/videos] failed to generate a video file: ${err}`) | |
throw err | |
} | |
console.log(`[api/generate/videos] generated video files: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`) | |
} else { | |
console.log(`[api/generate/videos] there is already a video file: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`) | |
} | |
} | |
console.log(`[api/generate/videos] returning the clap augmented with videos`) | |
return new NextResponse(await serializeClap(clap), { | |
status: 200, | |
headers: new Headers({ "content-type": "application/x-gzip" }), | |
}) | |
} | |