src/index.mts

import { createReadStream, existsSync } from "node:fs"
import path from "node:path"

import { validate as uuidValidate } from "uuid"
import express from "express"
import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene, ImageAnalysisRequest, ImageAnalysisResponse, SoundAnalysisResponse, SoundAnalysisRequest } from "./types.mts"

import { parseVideoRequest } from "./utils/requests/parseVideoRequest.mts"
import { savePendingVideo } from "./scheduler/savePendingVideo.mts"
import { getVideo } from "./scheduler/getVideo.mts"
import { main } from "./main.mts"
import { completedFilesDirFilePath } from "./config.mts"
import { markVideoAsToDelete } from "./scheduler/markVideoAsToDelete.mts"
import { markVideoAsToAbort } from "./scheduler/markVideoAsToAbort.mts"
import { markVideoAsToPause } from "./scheduler/markVideoAsToPause.mts"
import { markVideoAsPending } from "./scheduler/markVideoAsPending.mts"
import { getPendingVideos } from "./scheduler/getPendingVideos.mts"
import { hasValidAuthorization } from "./utils/requests/hasValidAuthorization.mts"
import { getAllVideosForOwner } from "./scheduler/getAllVideosForOwner.mts"
import { initFolders } from "./utils/filesystem/initFolders.mts"
import { sortVideosByYoungestFirst } from "./scheduler/sortVideosByYoungestFirst.mts"
import { getRenderedScene, renderScene } from "./production/renderScene.mts"
import { parseRenderRequest } from "./utils/requests/parseRenderRequest.mts"
import { loadRenderedSceneFromCache } from "./utils/requests/loadRenderedSceneFromCache.mts"

import { upscaleImage } from "./providers/image-upscaling/upscaleImage.mts"
import { analyzeImage } from "./providers/image-caption/analyzeImageWithIDEFICSAndNastyHack.mts"
// import { speechToText } from "./speechToText/speechToTextWithWhisperLib.mts"

initFolders()
// to disable all processing (eg. to debug)
// then comment the following line:
main()

const app = express()
const port = 7860

let isRendering = false

// fix this error: "PayloadTooLargeError: request entity too large"
// there are multiple version because.. yeah well, it's Express!
// app.use(bodyParser.json({limit: '50mb'}));
//app.use(bodyParser.urlencoded({limit: '50mb', extended: true}));
app.use(express.json({limit: '50mb'}));
app.use(express.urlencoded({limit: '50mb', extended: true}));

// an image analyzing pipeline
app.post("/analyze", async (req, res) => {

  const request = req.body as ImageAnalysisRequest

  if (!request.prompt) {
    console.log("Invalid prompt")
    res.status(400)
    res.write(JSON.stringify({ result: "", error: "invalid prompt" }))
    res.end()
    return
  }

  if (!request.image) {
    console.log("Invalid image")
    res.status(400)
    res.write(JSON.stringify({ result: "", error: "invalid image" }))
    res.end()
    return
  }

  /*
  console.log("/analyze called with: ", {
    prompt: request.prompt,
    image: request.image.slice(0, 50)
  })
  */

  const response: ImageAnalysisResponse = {
    result: "",
    error: ""
  }

  try {
    response.result = await analyzeImage(request.image, request.prompt)
  } catch (err) {
    console.log("failed to analyze the image!")
    console.log(err)
    response.error = `failed to analyze the image: ${err}`
  }

  if (response.error.length > 0) {
    // console.log("server error")
    res.status(500)
    res.write(JSON.stringify(response))
    res.end()
    return
  } else {
    // console.log("all good")
    res.status(200)
    res.write(JSON.stringify(response))
    res.end()
    return
  }
})


// a sound recognition pipeline
// it is currently broken
/*
app.post("/listen", async (req, res) => {

  const request = req.body as SoundAnalysisRequest

  if (!request.sound) {
    console.log("Invalid sound")
    res.status(400)
    res.write(JSON.stringify({ result: "", error: "invalid sound" }))
    res.end()
    return
  }

  console.log("/listen called with: ", {
    sound: request.sound.slice(0, 50)
  })

  const response: SoundAnalysisResponse = {
    result: "",
    error: ""
  }

  try {
    response.result = await speechToText(request.sound)
  } catch (err) {
    console.log("failed to listen to the sound!")
    console.log(err)
    response.error = `failed to listen to the sound: ${err}`
  }

  if (response.error.length > 0) {
    // console.log("server error")
    res.status(500)
    res.write(JSON.stringify(response))
    res.end()
    return
  } else {
    // console.log("all good")
    res.status(200)
    res.write(JSON.stringify(response))
    res.end()
    return
  }
})
*/

// a "fast track" pipeline
app.post("/render", async (req, res) => {

  if (!hasValidAuthorization(req.headers)) {
    console.log("Invalid authorization")
    res.status(401)
    res.write(JSON.stringify({ error: "invalid token" }))
    res.end()
    return
  }
  
  console.log(req.body)

  const request = parseRenderRequest(req.body as RenderRequest)

  if (!request.prompt) {
    console.log("Invalid prompt")
    res.status(400)
    res.write(JSON.stringify({ url: "", error: "invalid prompt" }))
    res.end()
    return
  }

  if (request.cache === "use") {
    // console.log("client requested to use the cache")
    try {
      const cached = await loadRenderedSceneFromCache(request)
      const cachedJson = JSON.stringify(cached)
      // console.log(`request ${request} is in cache!`)
      res.status(200)
      res.write(cachedJson)
      res.end()
      return
    } catch (err) {
      // console.log("request not found in cache: "+ err)
      // move along
    }
  } else if (request.cache === "renew") {
    // console.log("client requested to renew the cache")
  }

  let response: RenderedScene = {
    renderId: "",
    status: "pending",
    assetUrl: "",
    alt: request.prompt || "",
    maskUrl: "",
    error: "",
    segments: []
  }

  try {
    response = await renderScene(request)
  } catch (err) {
    // console.log("failed to render scene!")
    response.error = `failed to render scene: ${err}`
  }

  if (response.error === "already rendering") {
    // console.log("server busy")
    res.status(200)
    res.write(JSON.stringify(response))
    res.end()
    return
  } else if (response.error.length > 0) {
    // console.log("server error")
    res.status(500)
    res.write(JSON.stringify(response))
    res.end()
    return
  } else {
    // console.log("all good")
    res.status(200)
    res.write(JSON.stringify(response))
    res.end()
    return
  }
})

// upscale an arbitrary image
app.post("/upscale", async (req, res) => {

  if (!hasValidAuthorization(req.headers)) {
    console.log("Invalid authorization")
    res.status(401)
    res.write(JSON.stringify({ error: "invalid token" }))
    res.end()
    return
  }

  const image = `${req.body.image}`

  if (!image) {
    console.error("invalid input image")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid input image` }))
    res.end()
    return
  }

  let response = {
    assetUrl: "",
    error: "",
  }

  let assetUrl = ""
  try {
    try {
      assetUrl = await upscaleImage(image, req.body.factor)
    } catch (err) {
      // hmm.. let's try again?
      try {
        assetUrl = await upscaleImage(image, req.body.factor)
      } catch (err) {
        throw new Error(`second attempt to upscale the image failed (${err})`)
      }
    }
    if (!assetUrl) {
      throw new Error(`no image to return`)
    }

    response.assetUrl = assetUrl
    // console.log(`request ${renderId} is already in cache, so we return that`)
    res.status(200)
    res.write(JSON.stringify(response))
    res.end()
    return
  } catch (err) {
    response.error = `${err}`
    console.error(`${err}`)
    res.status(500)
    res.write(JSON.stringify(response))
    res.end()
  }
})


// used to upscale an existing image
// the image has to be completed for this to work
// ie it has to be in cache
// it's a bit of a hack endpoint I've added to reduce the laod
// on the AI Comic Factory, but we don't really need it for other applications
app.post("/upscale/:renderId", async (req, res) => {

  const renderId = `${req.params.renderId}`

  if (!uuidValidate(renderId)) {
    console.error("invalid render id")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid render id` }))
    res.end()
    return
  }

  let response = {
    assetUrl: "",
    error: "",
  }

  let assetUrl = ""
  try {
    // we still try to search for it in the cache
    const cached = await loadRenderedSceneFromCache(undefined, renderId)
    try {
      assetUrl = await upscaleImage(cached.assetUrl, 2)
    } catch (err) {
      // hmm.. let's try again?
      try {
        assetUrl = await upscaleImage(cached.assetUrl, 2)
      } catch (err) {
        throw new Error(`second attempt to upscale the image failed (${err})`)
      }
    }
    if (!assetUrl) {
      throw new Error(`no image to return`)
    }

    response.assetUrl = assetUrl
    // console.log(`request ${renderId} is already in cache, so we return that`)
    res.status(200)
    res.write(response)
    res.end()
    return
  } catch (err) {
    // console.log("renderId not found in cache: "+ err)
    // move along
    response.error = `${err}`
    console.error(`${err}`)
    res.status(500)
    res.write(JSON.stringify(response))
    res.end()
  }
})

// a "fast track" pipeline
app.get("/render/:renderId", async (req, res) => {

  const renderId = `${req.params.renderId}`

  if (!uuidValidate(renderId)) {
    console.error("invalid render id")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid render id` }))
    res.end()
    return
  }

  try {
    // we still try to search for it in the cache
    const cached = await loadRenderedSceneFromCache(undefined, renderId)
    const cachedJson = JSON.stringify(cached)
    // console.log(`request ${renderId} is already in cache, so we return that`)
    res.status(200)
    res.write(cachedJson)
    res.end()
    return
  } catch (err) {
    // console.log("renderId not found in cache: "+ err)
    // move along
  }

  let response: RenderedScene = {
    renderId: "",
    status: "pending",
    assetUrl: "",
    alt: "",
    error: "",
    maskUrl: "",
    segments: []
  }

  // console.log("going to render the scene!")
  try {
    response = await getRenderedScene(renderId)
  } catch (err) {
    // console.log("failed to render scene!")
    response.error = `failed to render scene: ${err}`
  }

  if (response.error === "already rendering") {
    console.log("server busy")
    res.status(200)
    res.write(JSON.stringify(response))
    res.end()
    return
  } else if (response.error.length > 0) {
    // console.log("server error")
    res.status(500)
    res.write(JSON.stringify(response))
    res.end()
    return
  } else {
    // console.log("all good")
    res.status(200)
    res.write(JSON.stringify(response))
    res.end()
    return
  }
})

app.post("/:ownerId", async (req, res) => {
  const request = req.body as VideoAPIRequest

  if (!hasValidAuthorization(req.headers)) {
    console.log("Invalid authorization")
    res.status(401)
    res.write(JSON.stringify({ error: "invalid token" }))
    res.end()
    return
  }
  
  const ownerId = req.params.ownerId

  if (!uuidValidate(ownerId)) {
    console.error("invalid owner id")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid owner id` }))
    res.end()
    return
  }

  let video: Video = null

  console.log(`creating video from request..`)
  // console.log(`request: `, JSON.stringify(request))
  if (!request?.prompt?.length) {
    console.error(`failed to create video (prompt is empty})`)
    res.status(400)
    res.write(JSON.stringify({ error: "prompt is empty" }))
    res.end()
    return
  }
  try {
    video = await parseVideoRequest(ownerId, request)
  } catch (err) {
    console.error(`failed to create video: ${video} (${err})`)
    res.status(400)
    res.write(JSON.stringify({ error: "query seems to be malformed" }))
    res.end()
    return
  }

  console.log(`saving video ${video.id}`)
  try {
    await savePendingVideo(video)
    res.status(200)
    res.write(JSON.stringify(video))
    res.end()
  } catch (err) {
    console.error(err)
    res.status(500)
    res.write(JSON.stringify({ error: "couldn't save the video" }))
    res.end()
  }
})

app.get("/:ownerId/:videoId\.mp4", async (req, res) => {
    
  /*
  for simplicity, let's skip auth when fetching videos
  the UUIDs cannot easily be guessed anyway

  if (!hasValidAuthorization(req.headers)) {
    console.log("Invalid authorization")
    res.status(401)
    res.write(JSON.stringify({ error: "invalid token" }))
    res.end()
    return
  }
  */

  const ownerId = req.params.ownerId
  console.log("downloading..")

  if (!uuidValidate(ownerId)) {
    console.error("invalid owner id")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid owner id` }))
    res.end()
    return
  }

  const videoId = req.params.videoId

  if (!uuidValidate(videoId)) {
    console.error("invalid video id")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid video id` }))
    res.end()
    return
  }

  let video: Video = null
  try {
    video = await getVideo(ownerId, videoId)
    console.log(`returning video ${videoId} to owner ${ownerId}`)
  } catch (err) {
    res.status(404)
    res.write(JSON.stringify({ error: "this video doesn't exist" }))
    res.end()
    return
  }

  const completedFilePath = path.join(completedFilesDirFilePath, video.fileName)

  // note: we DON'T want to use the pending file path, as there may be operations on it
  // (ie. a process might be busy writing stuff to it)
  const filePath = existsSync(completedFilePath) ? completedFilePath : ""
  if (!filePath) {
    res.status(400)
    res.write(JSON.stringify({ error: "video exists, but cannot be previewed yet" }))
    res.end()
    return
  }

  // file path exists, let's try to read it
  try {
    // do we need this?
    // res.status(200)
    // res.setHeader("Content-Type", "media/mp4")
    console.log(`creating a video read stream from ${filePath}`)
    const stream = createReadStream(filePath)
  
    stream.on('close', () => {
      console.log(`finished streaming the video`)
      res.end()
    })
    
    stream.pipe(res)
  } catch (err) {
    console.error(`failed to read the video file at ${filePath}: ${err}`)
    res.status(500)
    res.write(JSON.stringify({ error: "failed to read the video file" }))
    res.end()
  }
})

// get metadata (json)
app.get("/:ownerId/:videoId", async (req, res) => {
    
  if (!hasValidAuthorization(req.headers)) {
    console.log("Invalid authorization")
    res.status(401)
    res.write(JSON.stringify({ error: "invalid token" }))
    res.end()
    return
  }

  const ownerId = req.params.ownerId

  if (!uuidValidate(ownerId)) {
    console.error("invalid owner id")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid owner id` }))
    res.end()
    return
  }

  const videoId = req.params.videoId

  if (!uuidValidate(videoId)) {
    console.error("invalid video id")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid video id` }))
    res.end()
    return
  }

  try {
    const video = await getVideo(ownerId, videoId)
    res.status(200)
    res.write(JSON.stringify(video))
    res.end()
  } catch (err) {
    console.error(err)
    res.status(404)
    res.write(JSON.stringify({ error: "couldn't find this video" }))
    res.end()
  }
})

// only get the videos for a specific owner
app.get("/:ownerId", async (req, res) => {
  if (!hasValidAuthorization(req.headers)) {
    console.log("Invalid authorization")
    res.status(401)
    res.write(JSON.stringify({ error: "invalid token" }))
    res.end()
    return
  }

  const ownerId = req.params.ownerId

  if (!uuidValidate(ownerId)) {
    console.error(`invalid owner d ${ownerId}`)
    res.status(400)
    res.write(JSON.stringify({ error: `invalid owner id ${ownerId}` }))
    res.end()
    return
  }

  try {
    const videos = await getAllVideosForOwner(ownerId)
    sortVideosByYoungestFirst(videos)

    res.status(200)
    res.write(JSON.stringify(videos.filter(video => video.status !== "delete"), null, 2))
    res.end()
  } catch (err) {
    console.error(err)
    res.status(500)
    res.write(JSON.stringify({ error: `couldn't get the videos for owner ${ownerId}` }))
    res.end()
  }
})

// caption an audio sample
/*
app.post("/audio/caption", async (req, res) => {

  if (!hasValidAuthorization(req.headers)) {
    console.log("Invalid authorization")
    res.status(401)
    res.write(JSON.stringify({ error: "invalid token" }))
    res.end()
    return
  }

  const image = `${req.body.image}`

  if (!image) {
    console.error("invalid input image")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid input image` }))
    res.end()
    return
  }

  let response = {
    caption: "",
    error: "",
  }

  let caption = ""
  try {
    try {
      caption = await audioToCaption(audio, req.body.factor)
    } catch (err) {
      // hmm.. let's try again?
      try {
        caption = await audioToCaption(audio, req.body.factor)
      } catch (err) {
        throw new Error(`second attempt to caption the audio failed (${err})`)
      }
    }
    if (!caption) {
      throw new Error(`no caption to return`)
    }

    response.caption = caption
    res.status(200)
    res.write(JSON.stringify(response))
    res.end()
    return
  } catch (err) {
    response.error = `${err}`
    console.error(`${err}`)
    res.status(500)
    res.write(JSON.stringify(response))
    res.end()
  }
})

*/

// get all pending videos - this is for admin usage only
app.get("/", async (req, res) => {
  if (!hasValidAuthorization(req.headers)) {
    // this is what users will see in the space - but no need to show something scary
    console.log("Invalid authorization")
    res.status(200)
    res.write(`<html><head></head><body>
VideoChain is a toolchain to render scenes using AI. It is used by <a href="https://jbilcke-hf-videoquest.hf.space" target="_blank">VideoQuest</a>, a generative AI game.
    </body></html>`)
    res.end()
    // res.status(401)
    // res.write(JSON.stringify({ error: "invalid token" }))
    // res.end()
    return
  }

  try {
    const videos = await getPendingVideos()
    res.status(200)
    res.write(JSON.stringify(videos, null, 2))
    res.end()
  } catch (err) {
    console.error(err)
    res.status(500)
    res.write(JSON.stringify({ error: "couldn't get the videos" }))
    res.end()
  }
})


// edit a video
app.patch("/:ownerId/:videoId", async (req, res) => {
    
  if (!hasValidAuthorization(req.headers)) {
    console.log("Invalid authorization")
    res.status(401)
    res.write(JSON.stringify({ error: "invalid token" }))
    res.end()
    return
  }

  const ownerId = req.params.ownerId

  if (!uuidValidate(ownerId)) {
    console.error(`invalid owner id ${ownerId}`)
    res.status(400)
    res.write(JSON.stringify({ error: `invalid owner id ${ownerId}` }))
    res.end()
    return
  }

  const videoId = req.params.videoId

  if (!uuidValidate(videoId)) {
    console.error(`invalid video id ${videoId}`)
    res.status(400)
    res.write(JSON.stringify({ error: `invalid video id ${videoId}` }))
    res.end()
    return
  }

  let status: VideoStatus = "unknown"
  try {
    const request = req.body as { status: VideoStatus }
    if (['pending', 'abort', 'delete', 'pause'].includes(request.status)) {
      status = request.status
    } else {
      throw new Error(`invalid video status "${request.status}"`)
    }
  } catch (err) {
    console.error(`invalid parameter (${err})`)
    res.status(401)
    res.write(JSON.stringify({ error: `invalid parameter (${err})` }))
    res.end()
    return
  }

  switch (status) {
    case 'delete': 
      try {
        await markVideoAsToDelete(ownerId, videoId)
        console.log(`deleting video ${videoId}`)
        res.status(200)
        res.write(JSON.stringify({ success: true }))
        res.end()
      } catch (err) {
        console.error(`failed to delete video ${videoId} (${err})`)
        res.status(500)
        res.write(JSON.stringify({ error: `failed to delete video ${videoId}` }))
        res.end()
      }
      break

    case 'abort': 
      try {
        await markVideoAsToAbort(ownerId, videoId)
        console.log(`aborted video ${videoId}`)
        res.status(200)
        res.write(JSON.stringify({ success: true }))
        res.end()
      } catch (err) {
        console.error(`failed to abort video ${videoId} (${err})`)
        res.status(500)
        res.write(JSON.stringify({ error: `failed to abort video ${videoId}` }))
        res.end()
      }
      break

    case 'pause': 
      try {
        await markVideoAsToPause(ownerId, videoId)
        console.log(`paused video ${videoId}`)
        res.status(200)
        res.write(JSON.stringify({ success: true }))
        res.end()
      } catch (err) {
        console.error(`failed to pause video ${videoId} (${err})`)
        res.status(500)
        res.write(JSON.stringify({ error: `failed to pause video ${videoId}` }))
        res.end()
      }
      break
    
    case 'pending': 
      try {
        await markVideoAsPending(ownerId, videoId)
        console.log(`unpausing video ${videoId}`)
        res.status(200)
        res.write(JSON.stringify({ success: true }))
        res.end()
      } catch (err) {
        console.error(`failed to unpause video ${videoId} (${err})`)
        res.status(500)
        res.write(JSON.stringify({ error: `failed to unpause video ${videoId}` }))
        res.end()
      }
      break

    default:
      console.log(`unsupported status ${status}`)
      res.status(401)
      res.write(JSON.stringify({ error: `unsupported status ${status}` }))
      res.end()
  }
})

// delete a video - this is legacy, we should use other functions instead
/*
app.delete("/:id", async (req, res) => {
    
  if (!hasValidAuthorization(req.headers)) {
    console.log("Invalid authorization")
    res.status(401)
    res.write(JSON.stringify({ error: "invalid token" }))
    res.end()
    return
  }

  const [ownerId, videoId] = `${req.params.id}`.split("_")

  if (!uuidValidate(ownerId)) {
    console.error("invalid owner id")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid owner id` }))
    res.end()
    return
  }

  if (!uuidValidate(videoId)) {
    console.error("invalid video id")
    res.status(400)
    res.write(JSON.stringify({ error: `invalid video id` }))
    res.end()
    return
  }

  // ecurity note: we always check the existence if the video first
  // that's because we are going to delete all the associated files with a glob,
  // so we must be sure the id is not a system path or something ^^
  let video: Video = null
  try {
    video = await getVideo(ownerId, videoId)
  } catch (err) {
    console.error(err)
    res.status(404)
    res.write(JSON.stringify({ error: "couldn't find this video" }))
    res.end()
    return
  }

  try {
    await markVideoAsToDelete(ownerId, videoId)
    res.status(200)
    res.write(JSON.stringify({ success: true }))
    res.end()
  } catch (err) {
    console.error(err)
    res.status(500)
    res.write(JSON.stringify({ success: false, error: "failed to delete the video" }))
    res.end()
  }
})
*/

app.listen(port, () => { console.log(`Open http://localhost:${port}`) })