File size: 3,516 Bytes
53aa97a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 |
import { v4 as uuidv4, validate as uuidValidate } from "uuid"
import { HfInference } from "@huggingface/inference"
// convert a request (which might be invalid)
import { VideoAPIRequest, Video } from "../types.mts"
import { getValidNumber } from "./getValidNumber.mts"
import { getValidResolution } from "./getValidResolution.mts"
import { parseShotRequest } from "./parseShotRequest.mts"
import { generateSeed } from "./generateSeed.mts"
import { sequenceFormatVersion } from "../config.mts"
// const hfi = new HfInference(process.env._VC_HF_API_TOKEN)
// const hf = hfi.endpoint(process.env.VC_INFERENCE_ENDPOINT_URL)
export const parseVideoRequest = async (ownerId: string, request: VideoAPIRequest): Promise<Video> => {
// we don't want people to input their own ID or we might have trouble,
// such as people attempting to use a non-UUID, a file path (to hack us), etc
const id = uuidv4()
if (typeof request.prompt === "string" && request.prompt.length > 0) {
console.log("we have a valid prompt:", request.prompt)
// TODO: use llama2 to populate this!
request.sequence = {
videoPrompt: request.prompt,
}
request.shots = [{
shotPrompt: request.prompt,
environmentPrompt: "",
photographyPrompt: "",
actionPrompt: "",
}]
}
// console.log("continuing..")
const video: Video = {
// ------------ VideoSequenceMeta -------------
id,
ownerId,
// describe the whole movie
videoPrompt: `${request.sequence.videoPrompt || ''}`,
// describe the background audio (crowd, birds, wind, sea etc..)
backgroundAudioPrompt: `${request.sequence.backgroundAudioPrompt || ''}`,
// describe the foreground audio (cars revving, footsteps, objects breaking, explosion etc)
foregroundAudioPrompt: `${request.sequence.foregroundAudioPrompt || ''}`,
// describe the main actor visible in the shot (optional)
actorPrompt: `${request.sequence.actorPrompt || ''}`,
// describe the main actor voice (man, woman, old, young, amused, annoyed.. etc)
actorVoicePrompt: `${request.sequence.actorVoicePrompt || ''}`,
// describe the main actor dialogue line
actorDialoguePrompt: `${request.sequence.actorDialoguePrompt || ''}`,
seed: getValidNumber(request.sequence.seed, 0, 2147483647, generateSeed()),
noise: request.sequence.noise === true,
noiseAmount: request.sequence.noise === true ? 2 : 0,
steps: getValidNumber(request.sequence.steps, 10, 50, 45),
fps: getValidNumber(request.sequence.fps, 8, 60, 30),
resolution: getValidResolution(request.sequence.resolution),
outroTransition: 'staticfade',
outroDurationMs: 3000,
// ---------- VideoSequenceData ---------
version: sequenceFormatVersion,
fileName: `${ownerId}_${id}.mp4`,
status: "pending",
hasAssembledVideo: false,
hasGeneratedSpecs: false,
nbCompletedShots: 0,
progressPercent: 0,
createdAt: new Date().toISOString(),
completedAt: null,
completed: false,
error: '',
// ------- the VideoShot -----
shots: [],
}
// console.log("we are still good..")
const maybeShots = Array.isArray(request.shots) ? request.shots : []
// console.log("let's try..")
for (const maybeShot of maybeShots) {
// console.log("trying shot", maybeShot)
try {
const shot = await parseShotRequest(video, maybeShot)
video.shots.push(shot)
} catch (err) {
console.log(`error parsing shot: `, maybeShot)
}
}
return video
} |