Duplicate from jbilcke-hf/VideoChain-API

Co-authored-by: Julian Bilcke <[email protected]>

.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+npm-debug.log
+models
+sandbox
+audio.pipe
+video.pipe

.gitignore

@@ -0,0 +1,8 @@
+node_modules
+*.log
+*.bin
+.DS_Store
+.venv
+*.mp4
+sandbox
+scripts

.nvmrc

@@ -0,0 +1 @@
+v18.16.0

Dockerfile

@@ -0,0 +1,41 @@
+FROM node:18
+# try this maybe
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt update
+
+# For FFMPEG and gl concat
+RUN apt --yes install ffmpeg curl build-essential python3 python3-dev libx11-dev libxext-dev libxext6 libglu1-mesa-dev xvfb libxi-dev libglew-dev pkg-config
+
+# For Puppeteer
+RUN apt --yes install libnss3 libatk1.0-0 libatk-bridge2.0-0 libcups2 libgbm1 libasound2 libpangocairo-1.0-0 libxss1 libgtk-3-0
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -o -u 1000 user
+
+# Switch to the "user" user
+USER user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Install app dependencies
+# A wildcard is used to ensure both package.json AND package-lock.json are copied
+# where available (npm@5+)
+COPY --chown=user package*.json $HOME/app
+
+RUN npm install
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+EXPOSE 7860
+
+# we can't use this (it time out)
+# CMD [ "xvfb-run", "-s", "-ac -screen 0 1920x1080x24", "npm", "run", "start" ]
+CMD [ "npm", "run", "start" ]

LICENSE.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,12 @@
+---
+title: VideoChain API
+emoji: 🎥 🔗
+colorFrom: black
+colorTo: white
+sdk: docker
+pinned: false
+app_port: 7860
+duplicated_from: jbilcke-hf/VideoChain-API
+---
+
+A micro service to generate videos

TODO.md

@@ -0,0 +1,6 @@
+
+
+to  allow multiple videos to be processed a the same time:
+
+[ ] yield from the loop at each step
+[ ] random processing of videos

package.json

@@ -0,0 +1,41 @@
+{
+  "name": "videochain-api",
+  "version": "1.0.0",
+  "description": "A service which wraps and chains video and audio spaces together",
+  "main": "src/index.mts",
+  "scripts": {
+    "start": "node --loader ts-node/esm src/index.mts",
+    "test:submitVideo": "node --loader ts-node/esm src/tests/submitVideo.mts",
+    "test:checkStatus": "node --loader ts-node/esm src/tests/checkStatus.mts",
+    "test:downloadFileToTmp": "node --loader ts-node/esm src/tests/downloadFileToTmp.mts",
+    "test:stuff": "node --loader ts-node/esm src/utils/segmentImage.mts",
+    "docker": "npm run docker:build && npm run docker:run",
+    "docker:build": "docker build -t videochain-api .",
+    "docker:run": "docker run -it -p 7860:7860 videochain-api"
+  },
+  "author": "Julian Bilcke <[email protected]>",
+  "license": "Apache License",
+  "dependencies": {
+    "@gorgonjs/gorgon": "^1.4.1",
+    "@gradio/client": "^0.1.4",
+    "@huggingface/inference": "^2.6.1",
+    "@types/express": "^4.17.17",
+    "@types/ffmpeg-concat": "^1.1.2",
+    "@types/uuid": "^9.0.2",
+    "eventsource-parser": "^1.0.0",
+    "express": "^4.18.2",
+    "ffmpeg-concat": "^1.3.0",
+    "fluent-ffmpeg": "^2.1.2",
+    "fs-extra": "^11.1.1",
+    "gpt-tokens": "^1.1.1",
+    "node-fetch": "^3.3.1",
+    "openai": "^3.3.0",
+    "puppeteer": "^20.8.0",
+    "resize-base64": "^1.0.12",
+    "sharp": "^0.32.4",
+    "temp-dir": "^3.0.0",
+    "ts-node": "^10.9.1",
+    "uuid": "^9.0.0",
+    "yaml": "^2.3.1"
+  }
+}

src/config.mts

@@ -0,0 +1,14 @@
+import path from "node:path"
+
+export const storagePath = `${process.env.VC_STORAGE_PATH || './sandbox'}`
+
+export const metadataDirPath = path.join(storagePath, "metadata")
+export const pendingMetadataDirFilePath = path.join(metadataDirPath, "pending")
+export const completedMetadataDirFilePath =  path.join(metadataDirPath, "completed")
+
+export const filesDirPath = path.join(storagePath, "files")
+export const pendingFilesDirFilePath = path.join(filesDirPath, "pending")
+export const completedFilesDirFilePath =  path.join(filesDirPath, "completed")
+
+export const shotFormatVersion = 1
+export const sequenceFormatVersion = 1

src/index.mts

@@ -0,0 +1,605 @@
+import { createReadStream, existsSync } from "node:fs"
+import path from "node:path"
+
+import { v4 as uuidv4, validate as uuidValidate } from "uuid"
+import express from "express"
+
+import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene } from "./types.mts"
+import { parseVideoRequest } from "./utils/parseVideoRequest.mts"
+import { savePendingVideo } from "./scheduler/savePendingVideo.mts"
+import { getVideo } from "./scheduler/getVideo.mts"
+import { main } from "./main.mts"
+import { completedFilesDirFilePath } from "./config.mts"
+import { markVideoAsToDelete } from "./scheduler/markVideoAsToDelete.mts"
+import { markVideoAsToAbort } from "./scheduler/markVideoAsToAbort.mts"
+import { markVideoAsToPause } from "./scheduler/markVideoAsToPause.mts"
+import { markVideoAsPending } from "./scheduler/markVideoAsPending.mts"
+import { getPendingVideos } from "./scheduler/getPendingVideos.mts"
+import { hasValidAuthorization } from "./utils/hasValidAuthorization.mts"
+import { getAllVideosForOwner } from "./scheduler/getAllVideosForOwner.mts"
+import { initFolders } from "./initFolders.mts"
+import { sortVideosByYoungestFirst } from "./utils/sortVideosByYoungestFirst.mts"
+import { generateVideo } from "./production/generateVideo.mts"
+import { generateSeed } from "./utils/generateSeed.mts"
+import { getRenderedScene, renderScene } from "./production/renderScene.mts"
+
+initFolders()
+// to disable all processing (eg. to debug)
+// then comment the following line:
+main()
+
+const app = express()
+const port = 7860
+
+app.use(express.json())
+
+let isRendering = false
+
+// a "fast track" pipeline
+app.post("/render", async (req, res) => {
+
+  const request = req.body as RenderRequest
+  console.log(req.body)
+  if (!request.prompt) {
+    console.log("Invalid prompt")
+    res.status(400)
+    res.write(JSON.stringify({ url: "", error: "invalid prompt" }))
+    res.end()
+    return
+  }
+
+  let response: RenderedScene = {
+    renderId: "",
+    status: "pending",
+    assetUrl: "",
+    maskBase64: "",
+    error: "",
+    segments: []
+  }
+  
+  try {
+    response = await renderScene(request)
+  } catch (err) {
+    // console.log("failed to render scene!")
+    response.error = `failed to render scene: ${err}`
+  }
+
+  if (response.error === "already rendering") {
+    console.log("server busy")
+    res.status(200)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  } else if (response.error.length > 0) {
+    // console.log("server error")
+    res.status(500)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  } else {
+    // console.log("all good")
+    res.status(200)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  }
+})
+
+// a "fast track" pipeline
+app.get("/render/:renderId", async (req, res) => {
+
+  const renderId = `${req.params.renderId}`
+
+  if (!uuidValidate(renderId)) {
+    console.error("invalid render id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid render id` }))
+    res.end()
+    return
+  }
+
+  let response: RenderedScene = {
+    renderId: "",
+    status: "pending",
+    assetUrl: "",
+    error: "",
+    maskBase64: "",
+    segments: []
+  }
+
+  try {
+    response = await getRenderedScene(renderId)
+  } catch (err) {
+    // console.log("failed to render scene!")
+    response.error = `failed to render scene: ${err}`
+  }
+
+  if (response.error === "already rendering") {
+    console.log("server busy")
+    res.status(200)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  } else if (response.error.length > 0) {
+    // console.log("server error")
+    res.status(500)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  } else {
+    // console.log("all good")
+    res.status(200)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  }
+})
+
+
+// a "fast track" pipeline
+/*
+app.post("/segment", async (req, res) => {
+
+  const request = req.body as RenderRequest
+  console.log(req.body)
+
+  let result: RenderedScene = {
+    assetUrl: "",
+    maskBase64: "",
+    error: "",
+    segments: []
+  }
+  
+  try {
+    result = await renderScene(request)
+  } catch (err) {
+    // console.log("failed to render scene!")
+    result.error = `failed to render scene: ${err}`
+  }
+
+  if (result.error === "already rendering") {
+    console.log("server busy")
+    res.status(200)
+    res.write(JSON.stringify({ url: "", error: result.error }))
+    res.end()
+    return
+  } else if (result.error.length > 0) {
+    // console.log("server error")
+    res.status(500)
+    res.write(JSON.stringify({ url: "", error: result.error }))
+    res.end()
+    return
+  } else {
+    // console.log("all good")
+    res.status(200)
+    res.write(JSON.stringify(result))
+    res.end()
+    return
+  }
+})
+*/
+
+
+
+app.post("/:ownerId", async (req, res) => {
+  const request = req.body as VideoAPIRequest
+
+  if (!hasValidAuthorization(req.headers)) {
+    console.log("Invalid authorization")
+    res.status(401)
+    res.write(JSON.stringify({ error: "invalid token" }))
+    res.end()
+    return
+  }
+  
+  const ownerId = req.params.ownerId
+
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.end()
+    return
+  }
+
+  let video: Video = null
+
+  console.log(`creating video from request..`)
+  console.log(`request: `, JSON.stringify(request))
+  if (!request?.prompt?.length) {
+    console.error(`failed to create video (prompt is empty})`)
+    res.status(400)
+    res.write(JSON.stringify({ error: "prompt is empty" }))
+    res.end()
+    return
+  }
+  try {
+    video = await parseVideoRequest(ownerId, request)
+  } catch (err) {
+    console.error(`failed to create video: ${video} (${err})`)
+    res.status(400)
+    res.write(JSON.stringify({ error: "query seems to be malformed" }))
+    res.end()
+    return
+  }
+
+  console.log(`saving video ${video.id}`)
+  try {
+    await savePendingVideo(video)
+    res.status(200)
+    res.write(JSON.stringify(video))
+    res.end()
+  } catch (err) {
+    console.error(err)
+    res.status(500)
+    res.write(JSON.stringify({ error: "couldn't save the video" }))
+    res.end()
+  }
+})
+
+
+app.get("/:ownerId/:videoId\.mp4", async (req, res) => {
+    
+  /*
+  for simplicity, let's skip auth when fetching videos
+  the UUIDs cannot easily be guessed anyway
+
+  if (!hasValidAuthorization(req.headers)) {
+    console.log("Invalid authorization")
+    res.status(401)
+    res.write(JSON.stringify({ error: "invalid token" }))
+    res.end()
+    return
+  }
+  */
+
+  const ownerId = req.params.ownerId
+  console.log("downloading..")
+
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.end()
+    return
+  }
+
+  const videoId = req.params.videoId
+
+  if (!uuidValidate(videoId)) {
+    console.error("invalid video id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid video id` }))
+    res.end()
+    return
+  }
+
+  let video: Video = null
+  try {
+    video = await getVideo(ownerId, videoId)
+    console.log(`returning video ${videoId} to owner ${ownerId}`)
+  } catch (err) {
+    res.status(404)
+    res.write(JSON.stringify({ error: "this video doesn't exist" }))
+    res.end()
+    return
+  }
+
+  const completedFilePath = path.join(completedFilesDirFilePath, video.fileName)
+
+  // note: we DON'T want to use the pending file path, as there may be operations on it
+  // (ie. a process might be busy writing stuff to it)
+  const filePath = existsSync(completedFilePath) ? completedFilePath : ""
+  if (!filePath) {
+    res.status(400)
+    res.write(JSON.stringify({ error: "video exists, but cannot be previewed yet" }))
+    res.end()
+    return
+  }
+
+  // file path exists, let's try to read it
+  try {
+    // do we need this?
+    // res.status(200)
+    // res.setHeader("Content-Type", "media/mp4")
+    console.log(`creating a video read stream from ${filePath}`)
+    const stream = createReadStream(filePath)
+  
+    stream.on('close', () => {
+      console.log(`finished streaming the video`)
+      res.end()
+    })
+    
+    stream.pipe(res)
+  } catch (err) {
+    console.error(`failed to read the video file at ${filePath}: ${err}`)
+    res.status(500)
+    res.write(JSON.stringify({ error: "failed to read the video file" }))
+    res.end()
+  }
+})
+
+// get metadata (json)
+app.get("/:ownerId/:videoId", async (req, res) => {
+    
+  if (!hasValidAuthorization(req.headers)) {
+    console.log("Invalid authorization")
+    res.status(401)
+    res.write(JSON.stringify({ error: "invalid token" }))
+    res.end()
+    return
+  }
+
+  const ownerId = req.params.ownerId
+
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.end()
+    return
+  }
+
+  const videoId = req.params.videoId
+
+  if (!uuidValidate(videoId)) {
+    console.error("invalid video id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid video id` }))
+    res.end()
+    return
+  }
+
+  try {
+    const video = await getVideo(ownerId, videoId)
+    res.status(200)
+    res.write(JSON.stringify(video))
+    res.end()
+  } catch (err) {
+    console.error(err)
+    res.status(404)
+    res.write(JSON.stringify({ error: "couldn't find this video" }))
+    res.end()
+  }
+})
+
+// only get the videos for a specific owner
+app.get("/:ownerId", async (req, res) => {
+  if (!hasValidAuthorization(req.headers)) {
+    console.log("Invalid authorization")
+    res.status(401)
+    res.write(JSON.stringify({ error: "invalid token" }))
+    res.end()
+    return
+  }
+
+  const ownerId = req.params.ownerId
+
+  if (!uuidValidate(ownerId)) {
+    console.error(`invalid owner d ${ownerId}`)
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id ${ownerId}` }))
+    res.end()
+    return
+  }
+
+  try {
+    const videos = await getAllVideosForOwner(ownerId)
+    sortVideosByYoungestFirst(videos)
+
+    res.status(200)
+    res.write(JSON.stringify(videos.filter(video => video.status !== "delete"), null, 2))
+    res.end()
+  } catch (err) {
+    console.error(err)
+    res.status(500)
+    res.write(JSON.stringify({ error: `couldn't get the videos for owner ${ownerId}` }))
+    res.end()
+  }
+})
+
+// get all pending videos - this is for admin usage only
+app.get("/", async (req, res) => {
+  if (!hasValidAuthorization(req.headers)) {
+    // this is what users will see in the space - but no need to show something scary
+    console.log("Invalid authorization")
+    res.status(200)
+    res.write(`<html><head></head><body>
+This space is the rendering engine used by various demos spaces, such as <a href="https://jbilcke-hf-fishtank.hf.space" target="_blank">FishTank</a> and <a href="https://jbilcke-hf-videochain-ui.hf.space" target="_blank">VideoChain UI</a>
+    </body></html>`)
+    res.end()
+    // res.status(401)
+    // res.write(JSON.stringify({ error: "invalid token" }))
+    // res.end()
+    return
+  }
+
+  try {
+    const videos = await getPendingVideos()
+    res.status(200)
+    res.write(JSON.stringify(videos, null, 2))
+    res.end()
+  } catch (err) {
+    console.error(err)
+    res.status(500)
+    res.write(JSON.stringify({ error: "couldn't get the videos" }))
+    res.end()
+  }
+})
+
+
+// edit a video
+app.patch("/:ownerId/:videoId", async (req, res) => {
+    
+  if (!hasValidAuthorization(req.headers)) {
+    console.log("Invalid authorization")
+    res.status(401)
+    res.write(JSON.stringify({ error: "invalid token" }))
+    res.end()
+    return
+  }
+
+  const ownerId = req.params.ownerId
+
+  if (!uuidValidate(ownerId)) {
+    console.error(`invalid owner id ${ownerId}`)
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id ${ownerId}` }))
+    res.end()
+    return
+  }
+
+  const videoId = req.params.videoId
+
+  if (!uuidValidate(videoId)) {
+    console.error(`invalid video id ${videoId}`)
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid video id ${videoId}` }))
+    res.end()
+    return
+  }
+
+  let status: VideoStatus = "unknown"
+  try {
+    const request = req.body as { status: VideoStatus }
+    if (['pending', 'abort', 'delete', 'pause'].includes(request.status)) {
+      status = request.status
+    } else {
+      throw new Error(`invalid video status "${request.status}"`)
+    }
+  } catch (err) {
+    console.error(`invalid parameter (${err})`)
+    res.status(401)
+    res.write(JSON.stringify({ error: `invalid parameter (${err})` }))
+    res.end()
+    return
+  }
+
+  switch (status) {
+    case 'delete': 
+      try {
+        await markVideoAsToDelete(ownerId, videoId)
+        console.log(`deleting video ${videoId}`)
+        res.status(200)
+        res.write(JSON.stringify({ success: true }))
+        res.end()
+      } catch (err) {
+        console.error(`failed to delete video ${videoId} (${err})`)
+        res.status(500)
+        res.write(JSON.stringify({ error: `failed to delete video ${videoId}` }))
+        res.end()
+      }
+      break
+
+    case 'abort': 
+      try {
+        await markVideoAsToAbort(ownerId, videoId)
+        console.log(`aborted video ${videoId}`)
+        res.status(200)
+        res.write(JSON.stringify({ success: true }))
+        res.end()
+      } catch (err) {
+        console.error(`failed to abort video ${videoId} (${err})`)
+        res.status(500)
+        res.write(JSON.stringify({ error: `failed to abort video ${videoId}` }))
+        res.end()
+      }
+      break
+
+    case 'pause': 
+      try {
+        await markVideoAsToPause(ownerId, videoId)
+        console.log(`paused video ${videoId}`)
+        res.status(200)
+        res.write(JSON.stringify({ success: true }))
+        res.end()
+      } catch (err) {
+        console.error(`failed to pause video ${videoId} (${err})`)
+        res.status(500)
+        res.write(JSON.stringify({ error: `failed to pause video ${videoId}` }))
+        res.end()
+      }
+      break
+    
+    case 'pending': 
+      try {
+        await markVideoAsPending(ownerId, videoId)
+        console.log(`unpausing video ${videoId}`)
+        res.status(200)
+        res.write(JSON.stringify({ success: true }))
+        res.end()
+      } catch (err) {
+        console.error(`failed to unpause video ${videoId} (${err})`)
+        res.status(500)
+        res.write(JSON.stringify({ error: `failed to unpause video ${videoId}` }))
+        res.end()
+      }
+      break
+
+    default:
+      console.log(`unsupported status ${status}`)
+      res.status(401)
+      res.write(JSON.stringify({ error: `unsupported status ${status}` }))
+      res.end()
+  }
+})
+
+// delete a video - this is legacy, we should use other functions instead
+/*
+app.delete("/:id", async (req, res) => {
+    
+  if (!hasValidAuthorization(req.headers)) {
+    console.log("Invalid authorization")
+    res.status(401)
+    res.write(JSON.stringify({ error: "invalid token" }))
+    res.end()
+    return
+  }
+
+  const [ownerId, videoId] = `${req.params.id}`.split("_")
+
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.end()
+    return
+  }
+
+  if (!uuidValidate(videoId)) {
+    console.error("invalid video id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid video id` }))
+    res.end()
+    return
+  }
+
+  // ecurity note: we always check the existence if the video first
+  // that's because we are going to delete all the associated files with a glob,
+  // so we must be sure the id is not a system path or something ^^
+  let video: Video = null
+  try {
+    video = await getVideo(ownerId, videoId)
+  } catch (err) {
+    console.error(err)
+    res.status(404)
+    res.write(JSON.stringify({ error: "couldn't find this video" }))
+    res.end()
+    return
+  }
+
+  try {
+    await markVideoAsToDelete(ownerId, videoId)
+    res.status(200)
+    res.write(JSON.stringify({ success: true }))
+    res.end()
+  } catch (err) {
+    console.error(err)
+    res.status(500)
+    res.write(JSON.stringify({ success: false, error: "failed to delete the video" }))
+    res.end()
+  }
+})
+*/
+
+app.listen(port, () => { console.log(`Open http://localhost:${port}`) })

src/initFolders.mts

@@ -0,0 +1,19 @@
+import {
+  metadataDirPath,
+  pendingMetadataDirFilePath,
+  completedMetadataDirFilePath,
+  filesDirPath,
+  pendingFilesDirFilePath,
+  completedFilesDirFilePath
+} from "./config.mts"
+import { createDirIfNeeded } from "./utils/createDirIfNeeded.mts"
+
+export const initFolders = () => {
+  console.log(`initializing folders..`)
+  createDirIfNeeded(metadataDirPath)
+  createDirIfNeeded(pendingMetadataDirFilePath)
+  createDirIfNeeded(completedMetadataDirFilePath)
+  createDirIfNeeded(filesDirPath)
+  createDirIfNeeded(pendingFilesDirFilePath)
+  createDirIfNeeded(completedFilesDirFilePath)
+}

src/llm/enrichVideoSpecsUsingLLM.mts

@@ -0,0 +1,75 @@
+import { ChatCompletionRequestMessage } from "openai"
+
+import { Video, VideoAPIRequest } from "../types.mts"
+import { generateYAML } from "./openai/generateYAML.mts"
+import { HallucinatedVideoRequest, OpenAIErrorResponse } from "./types.mts"
+import { getQueryChatMessages } from "../preproduction/prompts.mts"
+import { getValidNumber } from "../utils/getValidNumber.mts"
+import { parseShotRequest } from "../utils/parseShotRequest.mts"
+
+
+export const enrichVideoSpecsUsingLLM = async (video: Video): Promise<Video> => {
+
+  const messages: ChatCompletionRequestMessage[] = getQueryChatMessages(video.videoPrompt)
+  
+  const defaultValue = {} as unknown as HallucinatedVideoRequest
+
+  let hallucinatedVideo: HallucinatedVideoRequest
+  video.shots = []
+
+  try {
+    hallucinatedVideo = await generateYAML<HallucinatedVideoRequest>(
+      messages,
+      defaultValue
+    )
+    console.log("enrichVideoSpecsUsingLLM: hallucinatedVideo = ", hallucinatedVideo)
+  } catch (err) {
+
+    let error: OpenAIErrorResponse = err?.response?.data?.error as unknown as OpenAIErrorResponse
+    if (!error) {
+      error = { message: `${err || ""}` } as unknown as OpenAIErrorResponse
+    }
+    
+    console.error(JSON.stringify(error, null, 2))
+    throw new Error(`failed to call the LLM: ${error.message}`)
+  }
+
+  // const video = JSON.parse(JSON.stringify(referenceVideo)) as Video
+
+  // TODO here we should make some verifications and perhaps even some conversions
+  // betwen the LLM response and the actual format used in a videoRequest
+  video.backgroundAudioPrompt = hallucinatedVideo.backgroundAudioPrompt || video.backgroundAudioPrompt
+  video.foregroundAudioPrompt = hallucinatedVideo.foregroundAudioPrompt || video.foregroundAudioPrompt
+  video.actorPrompt = hallucinatedVideo.actorPrompt || video.actorPrompt
+  video.actorVoicePrompt = hallucinatedVideo.actorVoicePrompt || video.actorVoicePrompt
+
+  video.noise = typeof hallucinatedVideo.noise !== "undefined"
+    ? (`${hallucinatedVideo.noise || ""}`.toLowerCase() === "true")
+    : video.noise
+  
+  video.noiseAmount = typeof hallucinatedVideo.noiseAmount !== "undefined"
+    ? getValidNumber(hallucinatedVideo.noiseAmount, 0, 10, 2)
+    : video.noiseAmount
+
+  video.outroDurationMs = typeof hallucinatedVideo.outroDurationMs !== "undefined"
+    ? getValidNumber(hallucinatedVideo.outroDurationMs, 0, 3000, 500)
+    : video.outroDurationMs
+
+  const hallucinatedShots = Array.isArray(hallucinatedVideo.shots) ? hallucinatedVideo.shots : []
+
+
+  for (const hallucinatedShot of hallucinatedShots) {
+    const shot = await parseShotRequest(video, {
+      shotPrompt: hallucinatedShot.shotPrompt,
+      environmentPrompt: hallucinatedShot.environmentPrompt,
+      photographyPrompt: hallucinatedShot.photographyPrompt,
+      actionPrompt: hallucinatedShot.actionPrompt,
+      foregroundAudioPrompt: hallucinatedShot.foregroundAudioPrompt
+    })
+    video.shots.push(shot)
+  }
+
+  console.log("enrichVideoSpecsUsingLLM: video = ", video)
+
+  return video
+}

src/llm/openai/createChatCompletion.mts

@@ -0,0 +1,105 @@
+import { ChatCompletionRequestMessage } from "openai"
+import { GPTTokens } from "gpt-tokens"
+
+import { openai } from "./openai.mts"
+import { runModerationCheck } from "./runModerationCheck.mts"
+import { getUserContent } from "./getUserContent.mts"
+import { getTextPrompt } from "./getTextPrompt.mts"
+
+export const createChatCompletion = async (
+  messages: ChatCompletionRequestMessage[],
+  model = "gpt-4"
+): Promise<string> => {
+  // this is the part added by the user, and the one we need to check against the moderation API
+  const userContent = getUserContent(messages)
+
+  const check = await runModerationCheck(userContent)
+
+  if (check.flagged) {
+    console.error("Thoughtcrime: content flagged by the AI police", {
+      userContent,
+      moderationResult: check,
+    })
+    return "Thoughtcrime: content flagged by the AI police"
+  }
+
+  const rawPrompt = getTextPrompt(messages)
+
+
+  // for doc: https://www.npmjs.com/package/gpt-tokens
+  const usageInfo = new GPTTokens({
+    // Plus enjoy a 25% cost reduction for input tokens on GPT-3.5 Turbo (0.0015 per 1K input tokens)
+    plus    : false,
+    model   : "gpt-4",
+    messages: messages as any,
+  })
+
+  console.table({
+    "Tokens prompt": usageInfo.promptUsedTokens,
+    "Tokens completion": usageInfo.completionUsedTokens,
+    "Tokens total": usageInfo.usedTokens,
+  })
+
+  // Price USD:  0.000298
+  console.log("Price USD: ", usageInfo.usedUSD)
+
+  // const tokenLimit = 4000
+
+  const maxTokens = 4000 - usageInfo.promptUsedTokens
+
+  console.log("maxTokens:", maxTokens)
+  /*
+  console.log("settings:", {
+    tokenLimit,
+    promptLength: rawPrompt.length,
+    promptTokenLengh: rawPrompt.length / 1.9,
+    maxTokens
+  })
+
+  console.log("createChatCompletion(): raw prompt length:", rawPrompt.length)
+  console.log(
+    `createChatCompletion(): requesting ${maxTokens} of the ${tokenLimit} tokens availables`
+  )
+  */
+
+  console.log("query:", {
+    model,
+    // messages,
+    user: "Anonymous User",
+    temperature: 0.7,
+    max_tokens: maxTokens,
+    // stop: preset.stop?.length ? preset.stop : undefined,
+  })
+
+  const response = await openai.createChatCompletion({
+    model,
+    messages,
+    // TODO use the Hugging Face Login username here
+    user: "Anonymous User",
+    temperature: 0.7,
+
+    // 30 tokens is about 120 characters
+    // we don't want more, as it will take longer to respond
+    max_tokens: maxTokens,
+    // stop: preset.stop?.length ? preset.stop : undefined,
+  })
+
+  const { choices } = response.data
+
+  if (!choices.length) {
+    console.log("createChatCompletion(): no choice found in the LLM response..")
+    return ""
+  }
+  const firstChoice = choices[0]
+
+  if (firstChoice?.message?.role !== "assistant") {
+    console.log(
+      "createChatCompletion(): something went wrong, the model imagined the user response?!"
+    )
+    return ""
+  }
+
+  console.log("createChatCompletion(): response", firstChoice.message.content)
+
+  return firstChoice.message.content || ""
+}

src/llm/openai/createChatCompletionStream.mts

@@ -0,0 +1,66 @@
+import { ChatCompletionRequestMessage } from "openai"
+
+import { openai } from "./openai.mts"
+import { streamCompletion } from "./stream.mts"
+import { getTextPrompt } from "./getTextPrompt.mts"
+
+export const createChatCompletionStream = async (
+  messages: ChatCompletionRequestMessage[],
+  model: string,
+  onMessage: (message: string) => Promise<void>,
+  onEnd = () => Promise<void>
+) => {
+  try {
+    const rawPrompt = getTextPrompt(messages)
+
+    const tokenLimit = 4096 // 8000
+
+    const maxTokens = Math.round(tokenLimit - rawPrompt.length / 1.9)
+
+    const completion = await openai.createCompletion({
+      model,
+      prompt: messages,
+      temperature: 0.7,
+      max_tokens: Math.min(30, maxTokens),
+      stream: true,
+    })
+
+    for await (const message of streamCompletion(completion as any)) {
+      try {
+        const parsed = JSON.parse(message)
+        const { text } = parsed.choices[0]
+
+        try {
+          await onMessage(text)
+        } catch (err) {
+          console.error(
+            'Could not process stream message (callback failed)',
+            message,
+            err
+          )
+        }
+      } catch (error) {
+        console.error('Could not JSON parse stream message', message, error)
+      }
+    }
+    try {
+      await onEnd()
+    } catch (err) {
+      console.error('onEnd callback failed', err)
+    }
+  } catch (error: any) {
+    if (error.code) {
+      try {
+        const parsed = JSON.parse(error.body)
+        console.error('An error occurred during OpenAI request: ', parsed)
+      } catch (error) {
+        console.error(
+          'An error occurred during OpenAI request (invalid json): ',
+          error
+        )
+      }
+    } else {
+      console.error('An error occurred during OpenAI request', error)
+    }
+  }
+}

src/llm/openai/generateYAML.mts

@@ -0,0 +1,42 @@
+import { ChatCompletionRequestMessage } from "openai"
+import { parse } from "yaml"
+
+import { createChatCompletion } from "./createChatCompletion.mts"
+
+export const generateYAML = async <T,>(messages: ChatCompletionRequestMessage[] = [], defaultValue?: T): Promise<T> => {
+
+  const defaultResult = defaultValue || ({} as T)
+
+  if (!messages.length) {
+    return defaultResult
+  }
+
+  const output = await createChatCompletion(messages)
+
+  let raw = ""
+
+  // cleanup any remains of the markdown response
+  raw = output.split("```")[0]
+
+  // remove any remaining `
+  const input = raw.replaceAll("`", "")
+
+  try {
+    const obj = parse(input) as T
+
+    if (obj === null || typeof obj === undefined) {
+      throw new Error("couldn't parse YAML")
+    }
+
+    return obj
+  } catch (err) {
+    // just in case, we also try JSON!
+    const obj = JSON.parse(input) as T
+
+    if (obj === null || typeof obj === undefined) {
+      throw new Error("couldn't parse JSON")
+    }
+
+    return obj
+  }
+}

src/llm/openai/getTextPrompt.mts

@@ -0,0 +1,4 @@
+import { ChatCompletionRequestMessage } from "openai"
+
+export const getTextPrompt = (prompt: ChatCompletionRequestMessage[]) =>
+  prompt.reduce((acc, item) => acc.concat(item.content), "") || ""

src/llm/openai/getUserContent.mts

@@ -0,0 +1,7 @@
+import { ChatCompletionRequestMessage } from "openai"
+
+export const getUserContent = (prompt: ChatCompletionRequestMessage[]) =>
+  prompt
+    .filter((message) => message.role === "user")
+    .map((message) => message.content)
+    .join("\n")

src/llm/openai/openai.mts

@@ -0,0 +1,7 @@
+import { Configuration, OpenAIApi } from "openai"
+
+export const openai = new OpenAIApi(
+  new Configuration({
+    apiKey: process.env.VC_OPENAI_API_KEY
+  })
+)

src/llm/openai/runModerationCheck.mts

@@ -0,0 +1,30 @@
+import {
+  CreateModerationResponseResultsInnerCategories,
+  CreateModerationResponseResultsInnerCategoryScores,
+} from "openai"
+
+import { openai } from "./openai.mts"
+
+export const runModerationCheck = async (
+  input = ''
+): Promise<{
+  categories?: CreateModerationResponseResultsInnerCategories
+  category_scores?: CreateModerationResponseResultsInnerCategoryScores
+  flagged: boolean
+}> => {
+  if (!input || !input.length) {
+    console.log(`skipping moderation check as input length is too shot`)
+    return {
+      flagged: false,
+    }
+  }
+
+  const response = await openai.createModeration({ input })
+  const { results } = response.data
+
+  if (!results.length) {
+    throw new Error(`failed to call the moderation endpoint`)
+  }
+
+  return results[0]
+}

src/llm/openai/stream.mts

@@ -0,0 +1,35 @@
+import { Readable } from "node:stream"
+
+async function* chunksToLines(
+  chunksAsync: AsyncIterable<Buffer>
+): AsyncIterable<string> {
+  let previous = ""
+  for await (const chunk of chunksAsync) {
+    const bufferChunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+    previous += bufferChunk
+    let eolIndex
+    while ((eolIndex = previous.indexOf("\n")) >= 0) {
+      // line includes the EOL
+      const line = previous.slice(0, eolIndex + 1).trimEnd()
+      if (line === "data: [DONE]") break
+      if (line.startsWith("data: ")) yield line
+      previous = previous.slice(eolIndex + 1)
+    }
+  }
+}
+
+async function* linesToMessages(
+  linesAsync: AsyncIterable<string>
+): AsyncIterable<string> {
+  for await (const line of linesAsync) {
+    const message = line.substring("data :".length)
+
+    yield message
+  }
+}
+
+export async function* streamCompletion(
+  stream: Readable
+): AsyncGenerator<string, void, undefined> {
+  yield* linesToMessages(chunksToLines(stream))
+}

src/llm/types.mts

@@ -0,0 +1,25 @@
+// note: this has to exactly match what is in the prompt, in ../preproduction/prompts.mts
+export interface HallucinatedVideoRequest {
+  backgroundAudioPrompt: string; // describe the background audio (crowd, birds, wind, sea etc..)
+  foregroundAudioPrompt: string; // describe the foreground audio (cars revving, footsteps, objects breaking, explosion etc)
+  actorPrompt: string; // describe the physical look of the main actor visible in the shot (man, woman, old, young, hair, glasses, clothes etc)
+  actorVoicePrompt: string; // describe the main actor voice (man, woman, old, young, amused, annoyed.. etc)
+  noise: boolean; // whether to apply movie noise or not
+  noiseAmount: number; // (integer) the amount of ffmpeg noise (film grain) to apply. 0 is none, 10 is a lot
+  outroDurationMs: number; // in milliseconds. An outro generally only lasts between 0 and 3000 (3s)
+  
+  shots: Array<{
+    shotPrompt: string; // describe the main elements of a shot, in excruciating details. You must include ALL those parameters: characters, shot story, what is happening. How they look, the textures, the expressions, their clothes. The color, materials and style of clothes. 
+    environmentPrompt: string; // describe the environment, in excruciating details. You must include ALL those parameters: Lights, atmosphere and weather (misty, dust, clear, rain, snow..). Time of the day and hour of the day. Furnitures, their shape, style, era. The materials used for each object. The global time period, time of the day, era. Explain if anything is moving in the backgroung.
+    photographyPrompt: string; // describe the photography, in excruciating details. You must include ALL those parameters: Camera angle, position and movement. Type of shot and angle. Lighting. Mood. Settings. Tint of the lights. Position of the sun or moon. Shadows and their direction. Camera shutter speed, blur, bokeh, aperture.
+    actionPrompt: string; // describe the dynamics of a shot, in excruciating details. You must include ALL those parameters: What is happening, who and what is moving. Which entity are in movements. What are the directions, starting and ending position. At which speed entities or objects are moving. Is there motion blur, slow motion, timelapse etc.
+    foregroundAudioPrompt: string; // describe the sounds in a concise way (eg. ringing bells, underwater sound and whistling dolphin, cat mewong etc),
+  }>
+}
+
+export interface OpenAIErrorResponse {
+  message: string
+  type: string
+  param: any
+  code: any
+}

src/main.mts

@@ -0,0 +1,41 @@
+import { initFolders } from "./initFolders.mts"
+import { getPendingVideos } from "./scheduler/getPendingVideos.mts"
+import { processVideo } from "./scheduler/processVideo.mts"
+import { sortPendingVideosByLeastCompletedFirst } from "./utils/sortPendingVideosByLeastCompletedFirst.mts"
+
+export const main = async () => {
+
+  const videos = await getPendingVideos()
+  if (!videos.length) {
+    // console.log(`no job to process.. going to try in 200 ms`)
+    setTimeout(() => {
+      main()
+    }, 200)
+    return
+  }
+
+  console.log(`there are ${videos.length} pending videos`)
+
+  sortPendingVideosByLeastCompletedFirst(videos)
+
+  let somethingFailed = ""
+  await Promise.all(videos.map(async video => {
+    try {
+      const result = await processVideo(video)
+      return result
+    } catch (err) {
+      somethingFailed = `${err}`
+      // a video failed.. no big deal
+      return Promise.resolve(somethingFailed)
+    }
+  }))
+
+  if (somethingFailed) {
+    console.error(`one of the jobs failed: ${somethingFailed}, let's wait 5 seconds`)
+    setTimeout(() => { main() }, 5000)
+  } else {
+    console.log(`successfully worked on the jobs, let's immediately loop`)
+    setTimeout(() => { main() }, 50)
+  }
+
+}

src/preproduction/mocks.mts

@@ -0,0 +1,36 @@
+import { Video, VideoShot } from "../types.mts"
+
+export const mockShots: VideoShot[] = [
+  {
+    "shotPrompt": "In the extreme wide shot, a flock of ducks is converging on the Central Park, coming from multiple directions. Their feathers are glossy and clean, casting off varying degrees of green, brown and white",
+    "environmentPrompt": "Central Park at sunrise, the park looks slightly misty, the sky is tinged with shades of pink and orange as the day breaks. There's dew on the grass, and the leaves on trees are rustling in the light breeze",
+    "photographyPrompt": "Eye-level shot with a slight tilt in the camera, capturing the panorama of the park. There's natural lighting, sun just rising. The camera zooms out to capture the ducks entering the park. Shutter speed is slow to capture the movement of ducks",
+    "actionPrompt": "Large groups of ducks waddle into the park from various directions, some fly in groups, landing on the pond with small splashes. Movement is slow, slightly sped up to depict the invasion",
+    "foregroundAudioPrompt": "A symphony of soft quacking and rustling feathers",
+  },
+  {
+    "shotPrompt": "In the medium shot, a group of ducks are by the pond, pecking at the ground and frolicking in the water. One male mallard is particularly captivating with its emerald green head and healthy body",
+    "environmentPrompt": "It's a sunny spring day in Central Park. The pond is surrounded by lush, green vegetation and dappled with sunlight filtering through the leaves",
+    "photographyPrompt": "Low angle shot near the water level, the camera moves in a crane shot to capture ducks in action, and the camera's aperture is partially open. Natural sunlight creates playful shadows",
+    "actionPrompt": "Ducks are pecking at the ground, dabbling at the water's edge and frolicking in the pond. The camera tracks a particularly majestic mallard navigating through the pond",
+    "foregroundAudioPrompt": "Sounds of ducks quacking and splashing in the water"
+  },
+  {
+    "shotPrompt": "Close-up shot of a mother duck with ducklings following her in a line on the grass and into the water",
+    "environmentPrompt": "Central Park, by one of the smaller ponds, surrounded by green trees. Sun is high up giving off warm, radiant light",
+    "photographyPrompt": "High angle shot, focusing on the line of ducklings following their mother. The camera follows the ducklings. The setting is bright and clear with sun illuminating the ducklings",
+    "actionPrompt": "Mother duck is leading her ducklings from the grass into the water, the ducklings obediently follow, creating a neat line. The whole scene feels peaceful",
+    "foregroundAudioPrompt": "Ducklings' high pitched chirping, soft lapping of water at the edge of the pond"
+  }
+] as any
+
+export const mock: Video = {
+  "backgroundAudioPrompt": "City ambience mixed with the rustling leaves and the chirping birds in the park",
+  "foregroundAudioPrompt": "Rustling feathers, soft quacking, flapping wings, occasional splash in the pond",
+  "actorPrompt": "Main actors are ducks - a variety of breeds, mostly mallards: males with glossy green heads and females in mottled brown; all plump, medium-sized waterfowl",
+  "actorVoicePrompt": "Soft, low pitched quacking of adult ducks and higher pitched chirping of ducklings",
+  "noise": true,
+  "noiseAmount": 2,
+  "outroDurationMs": 1500,
+  "shots": mockShots
+} as any

src/preproduction/prompts.mts

@@ -0,0 +1,123 @@
+
+
+// types of movie shots: https://www.nfi.edu/types-of-film-shots/
+
+import { ChatCompletionRequestMessage } from "openai"
+
+export const getQueryChatMessages = (sceneDescription: string): ChatCompletionRequestMessage[] => {
+  return [
+  {
+    role: "system",
+    name: "moviemaking_rules",
+    content: `# Context
+You are an AI Movie Director Assistant, and you need to help generating input requests (movie "specs") for an automated movie generation API.
+The format expected by the API must be in YAML. The TypeScript schema for this YAML file is:
+\`\`\`typescript
+{
+  backgroundAudioPrompt: string; // describe the background audio (crowd, birds, wind, sea etc..)
+  foregroundAudioPrompt: string; // describe the foreground audio (cars revving, footsteps, objects breaking, explosion etc)
+  actorPrompt: string; // describe the physical look of the main actor visible in the shot (man, woman, old, young, hair, glasses, clothes etc)
+  actorVoicePrompt: string; // describe the main actor voice (man, woman, old, young, amused, annoyed.. etc)
+  noise: boolean; // whether to apply movie noise or not
+  noiseAmount: number; // (integer) the amount of noise (film grain) to apply. This is mapped from the FFmpeg filter (0 is none, 10 is already a lot)
+  outroDurationMs: number; // in milliseconds. An outro generally only lasts between 0 and 3000 (3s)
+  shots: Array<{
+    shotPrompt: string; // describe the main elements of a shot, in excruciating details. You must include ALL those parameters: characters, shot story, what is happening. How they look, the textures, the expressions, their clothes. The color, materials and style of clothes. 
+    environmentPrompt: string; // describe the environment, in excruciating details. You must include ALL those parameters: Lights, atmosphere and weather (misty, dust, clear, rain, snow..). Time of the day and hour of the day. Furnitures, their shape, style, era. The materials used for each object. The global time period, time of the day, era. Explain if anything is moving in the backgroung.
+    photographyPrompt: string; // describe the photography, in excruciating details. You must include ALL those parameters: Camera angle, position and movement. Type of shot and angle. Lighting. Mood. Settings. Tint of the lights. Position of the sun or moon. Shadows and their direction. Camera shutter speed, blur, bokeh, aperture.
+    actionPrompt: string; // describe the dynamics of a shot, in excruciating details. You must include ALL those parameters: What is happening, who and what is moving. Which entity are in movements. What are the directions, starting and ending position. At which speed entities or objects are moving. Is there motion blur, slow motion, timelapse etc.
+    foregroundAudioPrompt: string; // describe the sounds in a concise way (eg. ringing bells, underwater sound and whistling dolphin, cat mewong etc),
+  }>
+}
+\`\`\`
+# Guidelines for number of shots and their duration
+You will generate 1 shot by default, unless more or less are specified.
+A shot can only last 3 seconds max.
+So if you are asked to generate a 6 seconds videos, you need 2 shots, for a 9 seconds video, 3 shots, and so on.
+If you are asked to generate for instance a 11 seconds videos, you need three 3sec shots plus one 2sec shot.
+# Guidelines for writing descriptions
+Is it crucial to repeat the elements constituting a sequence of multiple shots verbatim from one shot to another.
+For instance, you will have to repeat exactly what a character or background look like, how they are dressed etc.
+This will ensure consistency from one scene to another.
+## Choosing the right words
+Here are some guidelines regarding film-making:
+- The distance your subject is to the camera impacts how the audience feels about them.
+- Subject will appear largest in a close-up or choker shot and smallest in a wide or long shot.
+- Camera movement is a technique for changing the relationship between the subject and the camera frame, controlling the delivery of the narrative. It helps to give additional meaning to what’s happening on the screen.
+- Do not hesitate to combine types of shots with camera movement shots and camera position (angle) shots.
+## Shots
+Single shot: where the shot only captures one subject.
+Two shot: which has only two characters.
+Three shot: when three characters are in the frame.
+Point-of-view shot (POV): shows the scene from the point of view of one of the characters, makes the audience feel that they are there seeing what the character is seeing.
+Over-the-shoulder shot (OTS): shows the subject from behind the shoulder of another character.
+Over-the-hip (OTH) shot, in which the camera is placed on the hip of one character and the focus is on the subject.
+Reverse angle shot: which is approximately 180 degrees opposite the previous shot.
+Reaction shot: which shows the character’s reaction to the previous shot.
+Weather shot: where the subject of the filming is the weather.
+Extreme wide shot/extreme long shot: used to show the subject and the entire area of the environment they are in.
+Wide shot/long shot: used to focus on the subject while still showing the scene the subject is in.
+Medium shot: shows the subject from the knees up, and is often referred to as the 3/4 shot.
+Medium close-up shot: The subject fills the frame. It is somewhere between a medium close-up and a close-up.
+Close-up shot: shows emotions and detailed reactions, with the subject filling the entire frame.
+Choker shot: shows the subject’s face from just above the eyebrows to just below the mouth and is between a close-up and an extreme close-up.
+Extreme close-up shot: shows the detail of an object, such as one a character is handling, or a person, such as just their eyes or moving lips.
+Full shot: similar to a wide shot except that it focuses on the character in the frame, showing them from head to toe.
+Cowboy shot: similar to the medium shot except that the character is shown from the hips or waist up.
+Establishing shot: a long shot at the beginning of a scene that shows objects, buildings, and other elements of a setting from a distance to establish where the next sequence of events takes place.
+## Camera angles
+Eye-level shot: This is when the camera is placed at the same height as the eyes of the characters.
+Low angle shot: This shot frames the subject from a low height, often used to emphasize differences in power between characters.
+Aerial shot/helicopter shot: Taken from way up high, this shot is usually from a drone or helicopter to establish the expanse of the surrounding landscape.
+High angle shot: This is when the subject is framed with the camera looking down at them.
+Birds-eye-view shot/overhead shot: This is a shot taken from way above the subject, usually including a significant amount of the surrounding environment to create a sense of scale or movement.
+Shoulder-level shot: This is where the camera is approximately the same height as the character’s shoulders.
+Hip-level shot: The camera is approximately at the height of the character’s hips.
+Knee-level shot: The camera is approximately at the same level as the character’s knees.
+Ground-level shot: When the height of the camera is at ground level with the character, this shot captures what’s happening on the ground the character is standing on.
+Dutch-angle/tilt shot: This is where the camera is tilted to the side.
+Cut-in shot: This type of shot cuts into the action on the screen to offer a different view of something happening in this main scene.
+Cutaway shot: As a shot that cuts away from the main action on the screen, it’s used to focus on secondary action and add more information for greater understanding for the audience.
+Master shot: A long shot that captures most or all of the action happening in a scene.
+Deep focus: A shot that keeps everything on the screen in sharp focus, including the foreground, background, and middle ground.
+Locked-down shot: With this shot, the camera is fixed in one position and the action continues off-screen.
+## Camera movements
+Zoom Shot: involves changing the focal length of the lens to zoom in or out during filming.
+Pan shot:  involves moving the camera from side to side to show something to the audience or help them better follow the sequence of events.
+Tilt shot: similar to a pan shot, except moving the camera up and down.
+Dolly shot: the camera is attached to a dolly that moves on tracks and can possibly move up and down.
+Truck shot: you move the entire camera on a fixed point and the motion goes from side to side.
+Pedestal shot: the entire camera is moved vertically, not just the angle of view, and is often combined with panning and/or tilting.
+Static/fixed shot: where there is no camera movement, and the shot emphasizes the movement of the subject in the environment.
+Arc shot: where the camera moves in an arc pattern around the subject to give the audience a better perspective of their surroundings.
+Crab shot: a less-common version of tracking a subject where the dolly the camera is on goes sideways.
+Dolly zoom shot: the position of the camera and focal length are changed simultaneously.
+Whip pan shot/swish pan shot: used to create a blur as you pan from one shot to the next.
+Tracking shot: the camera follows the subject, either from behind or at their side, moving with them.
+Whip tilt shot: used to create a blur panning from one shot to the next vertically.
+Bridging shot: denotes a shift in place or time.
+## Focus
+Focus pull: focus the lens to keep the subject within an acceptable focus range.
+Rack focus: focus is more aggressively shifted from subject A to subject B.
+Tilt-shift: parts of the image are in focus while other parts are out of focus.
+Deep focus: both the subject and the environment are in focus.
+Shallow focus: subject is crisp and in focus while the background is out of focus.
+## Camera angles
+High-angle
+Low-angle
+Over-the-shoulder
+Bird’s eye
+Dutch angle/tilt`
+  },
+  {
+    role: "user",
+    name: "movie_director",
+    content: `# Task
+Please generate the movie spec YAML based on the following description:
+${sceneDescription}.
+# YAML
+\`\`\`
+`
+  },
+]
+}

src/production/addAudioToVideo.mts

@@ -0,0 +1,43 @@
+import path from "node:path"
+
+import tmpDir from "temp-dir"
+import { v4 as uuidv4 } from "uuid"
+import ffmpeg from "fluent-ffmpeg"
+
+import { pendingFilesDirFilePath } from "../config.mts"
+import { moveFileFromTmpToPending } from "../utils/moveFileFromTmpToPending.mts"
+
+export const addAudioToVideo = async (
+  videoFileName: string,
+  audioFileName: string,
+
+  /*
+  * 0.0: mute the audio completely
+  * 0.5: set the audio to 50% of original volume (half volume)
+  * 1.0: maintain the audio at original volume (100% of original volume)
+  * 2.0: amplify the audio to 200% of original volume (double volume - might cause clipping)
+  */
+  volume: number = 1.0
+) => {
+  const inputFilePath = path.join(pendingFilesDirFilePath, videoFileName)
+  const audioFilePath = path.resolve(pendingFilesDirFilePath, audioFileName)
+
+  const tmpFileName = `${uuidv4()}.mp4`
+  const tempOutputFilePath = path.join(tmpDir, tmpFileName)
+
+  await new Promise((resolve, reject) => {
+    ffmpeg(inputFilePath)
+      .input(audioFilePath)
+      .audioFilters({ filter: 'volume', options: volume }) // add audio filter for volume
+      .outputOptions("-c:v copy")  // use video copy codec
+      .outputOptions("-c:a aac")   // use audio codec
+      .outputOptions("-map 0:v:0") // map video from 0th to 0th
+      .outputOptions("-map 1:a:0") // map audio from 1st to 0th
+      .outputOptions("-shortest") // finish encoding when shortest input stream ends
+      .output(tempOutputFilePath)
+      .on("end", resolve)
+      .on("error", reject)
+      .run()
+  })
+  await moveFileFromTmpToPending(tmpFileName, videoFileName)
+};

src/production/assembleShots.mts

@@ -0,0 +1,59 @@
+import path from "node:path"
+
+// due to Docker issues, we disable OpenGL transitions for now
+// import concat from 'ffmpeg-concat'
+import concat from './concatNoGL.mts'
+
+import { VideoShot } from '../types.mts'
+import { pendingFilesDirFilePath } from "../config.mts"
+import { normalizePendingVideoToTmpFilePath } from "./normalizePendingVideoToTmpFilePath.mts"
+
+export const assembleShots = async (shots: VideoShot[], fileName: string) => {
+
+  if (!Array.isArray(shots) || shots.length < 2) {
+    throw new Error(`need at least 2 shots`)
+  }
+
+  const transitions = [
+    {
+      name: 'circleOpen',
+      duration: 1000,
+    },
+    {
+      name: 'crossWarp',
+      duration: 800,
+    },
+    {
+      name: 'directionalWarp',
+      duration: 800,
+      // pass custom params to a transition
+      params: { direction: [1, -1] },
+    },
+    
+    /*
+    {
+      name: 'squaresWire',
+      duration: 2000,
+    },
+    */
+  ]
+
+  const videoFilePath = path.join(pendingFilesDirFilePath, fileName)
+
+  // before performing assembly, we must normalize images
+  const shotFilesPaths: string[] = []
+  for (let shot of shots) {
+    const normalizedShotFilePath = await normalizePendingVideoToTmpFilePath(shot.fileName)
+    shotFilesPaths.push(normalizedShotFilePath)
+  }
+
+  await concat({
+    output: videoFilePath,
+    videos: shotFilesPaths,
+    transitions: shotFilesPaths
+      .slice(0, shotFilesPaths.length - 1)
+      .map(
+        (vid) => transitions[Math.floor(Math.random() * transitions.length)]
+      ),
+  })
+}

src/production/concatNoGL.mts

@@ -0,0 +1,33 @@
+import ffmpeg from "fluent-ffmpeg";
+import fs from "fs";
+
+interface IConcatParams {
+    output: string;
+    videos: string[];
+    transitions: any;
+}
+
+const concat = async ({ output, videos }: IConcatParams): Promise<void> => {
+    if(!output || !Array.isArray(videos)) {
+        throw new Error("An output file and videos must be provided");
+    }
+
+    if(!videos.every(video => fs.existsSync(video))) {
+        throw new Error("All videos must exist");
+    }
+
+    const ffmpegCommand = ffmpeg();
+ 
+    videos.forEach((video) =>
+        ffmpegCommand.addInput(video)
+    );
+
+    return new Promise<void>((resolve, reject) => {
+        ffmpegCommand
+            .on('error', reject)
+            .on('end', resolve)
+            .mergeToFile(output);
+    });
+};
+  
+export default concat;

src/production/generateActor.mts

@@ -0,0 +1,50 @@
+import { promises as fs } from "node:fs"
+import path from "node:path"
+import tmpDir from "temp-dir"
+
+import { HfInference } from "@huggingface/inference"
+
+const hf = new HfInference(process.env.VC_HF_API_TOKEN)
+
+export const generateActor = async (prompt: string, fileName: string, seed: number) => {
+  const positivePrompt = [
+    `profile photo of ${prompt || ""}`,
+    "id picture",
+    "photoshoot",
+    "portrait photography",
+    "neutral expression",
+    "neutral background",
+    "studio photo",
+    "award winning",
+    "high resolution",
+    "photo realistic",
+    "intricate details",
+    "beautiful",
+  ]
+  const negativePrompt = [
+    "anime",
+    "drawing",
+    "painting",
+    "lowres",
+    "blurry",
+    "artificial"
+  ]
+
+  console.log(`generating actor: ${positivePrompt.join(", ")}`)
+
+  const blob = await hf.textToImage({
+    inputs: positivePrompt.join(", "),
+    model: "stabilityai/stable-diffusion-2-1",
+    parameters: {
+      negative_prompt: negativePrompt.join(", "),
+      // seed, no seed?
+    }
+  })
+
+  const filePath = path.resolve(tmpDir, fileName)
+
+  const buffer = Buffer.from(await blob.arrayBuffer())
+  await fs.writeFile(filePath, buffer, "utf8")
+
+  return filePath
+}

src/production/generateAudio.mts

@@ -0,0 +1,76 @@
+import { v4 as uuidv4 } from "uuid"
+import puppeteer from "puppeteer"
+
+import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
+import { moveFileFromTmpToPending } from "../utils/moveFileFromTmpToPending.mts"
+
+export const state = {
+  load: 0,
+}
+
+const instances: string[] = [
+  process.env.VC_AUDIO_GENERATION_SPACE_API_URL
+]
+
+// TODO we should use an inference endpoint instead
+export async function generateAudio(prompt: string, audioFileName: string) {
+
+  if (state.load === instances.length) {
+    throw new Error(`all audio generation servers are busy, try again later..`)
+  }
+
+  state.load += 1
+
+  try {
+    const instance = instances.shift()
+    instances.push(instance)
+
+    const browser = await puppeteer.launch({
+      headless: true,
+      protocolTimeout: 120000,
+    })
+
+    try {
+      const page = await browser.newPage()
+
+      await page.goto(instance, {
+        waitUntil: "networkidle2",
+      })
+
+      await new Promise(r => setTimeout(r, 3000))
+
+      const firstTextboxInput = await page.$('input[data-testid="textbox"]')
+
+      await firstTextboxInput.type(prompt)
+
+      // console.log("looking for the button to submit")
+      const submitButton = await page.$("button.lg")
+
+      // console.log("clicking on the button")
+      await submitButton.click()
+
+      await page.waitForSelector("a[download]", {
+        timeout: 120000, // no need to wait for too long, generation is quick
+      })
+
+      const audioRemoteUrl = await page.$$eval("a[download]", el => el.map(x => x.getAttribute("href"))[0])
+
+
+      // it is always a good idea to download to a tmp dir before saving to the pending dir
+      // because there is always a risk that the download will fail
+      
+      const tmpFileName = `${uuidv4()}.mp4`
+
+      await downloadFileToTmp(audioRemoteUrl, tmpFileName)
+      await moveFileFromTmpToPending(tmpFileName, audioFileName)
+    } catch (err) {
+      throw err
+    } finally {
+      await browser.close()
+    }
+  } catch (err) {
+    throw err
+  } finally {
+    state.load -= 1
+  }
+}

src/production/generateAudioLegacy.mts

@@ -0,0 +1,52 @@
+import { client } from '@gradio/client'
+
+import { generateSeed } from "../utils/generateSeed.mts"
+
+export const state = {
+  load: 0
+}
+
+const instances: string[] = [
+  process.env.VC_AUDIO_GENERATION_SPACE_API_URL
+]
+
+export const generateAudio = async (prompt: string, options?: {
+  seed: number;
+  nbFrames: number;
+  nbSteps: number;
+}) => {
+
+  if (state.load === instances.length) {
+    throw new Error(`all audio generation servers are busy, try again later..`)
+  }
+
+  state.load += 1
+
+  try {
+    const seed = options?.seed || generateSeed()
+    const nbFrames = options?.nbFrames || 24 // we can go up to 48 frames, but then upscaling quill require too much memory!
+    const nbSteps = options?.nbSteps || 35
+
+    const instance = instances.shift()
+    instances.push(instance)
+
+    const api = await client(instance, {
+          hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+    })
+
+    const rawResponse = await api.predict('/run', [		
+      prompt, // string  in 'Prompt' Textbox component		
+      seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component		
+      nbFrames, // 24 // it is the nb of frames per seconds I think?
+      nbSteps, // 10, (numeric value between 10 and 50) in 'Number of inference steps' Slider component
+    ]) as any
+    
+    const { name } = rawResponse?.data?.[0]?.[0] as { name: string, orig_name: string }
+
+    return `${instance}/file=${name}`
+  } catch (err) {
+    throw err
+  } finally {
+    state.load -= 1
+  }
+}

src/production/generateVideo.mts

@@ -0,0 +1,57 @@
+import { client } from "@gradio/client"
+
+import { generateSeed } from "../utils/generateSeed.mts"
+
+export const state = {
+  load: 0,
+}
+
+// we don't use replicas yet, because it ain't easy to get their hostname
+const instances: string[] = [
+  `${process.env.VC_ZEROSCOPE_SPACE_API_URL_1 || ""}`,
+  `${process.env.VC_ZEROSCOPE_SPACE_API_URL_2 || ""}`,
+  // `${process.env.VC_ZEROSCOPE_SPACE_API_URL_3 || ""}`,
+].filter(instance => instance?.length > 0)
+
+export const generateVideo = async (prompt: string, options?: {
+  seed: number;
+  nbFrames: number;
+  nbSteps: number;
+}) => {
+
+  if (state.load === instances.length) {
+    throw new Error(`all video generation servers are busy, try again later..`)
+  }
+
+  state.load += 1
+
+  try {
+    const seed = options?.seed || generateSeed()
+    const nbFrames = options?.nbFrames || 24 // we can go up to 48 frames, but then upscaling quill require too much memory!
+    const nbSteps = options?.nbSteps || 35
+
+    const instance = instances.shift()
+    instances.push(instance)
+
+    const api = await client(instance, {
+      hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+    })
+
+    const rawResponse = await api.predict('/run', [		
+      prompt, // string  in 'Prompt' Textbox component		
+      seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component		
+      nbFrames, // 24 // it is the nb of frames per seconds I think?
+      nbSteps, // 10, (numeric value between 10 and 50) in 'Number of inference steps' Slider component
+    ]) as any
+    
+    // console.log("rawResponse:", rawResponse)
+
+    const { name } = rawResponse?.data?.[0]?.[0] as { name: string, orig_name: string }
+
+    return `${instance}/file=${name}`
+  } catch (err) {
+    throw err
+  } finally {
+    state.load -= 1
+  }
+}

src/production/generateVoice.mts

@@ -0,0 +1,78 @@
+import puppeteer from "puppeteer"
+
+import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
+
+export const state = {
+  load: 0
+}
+
+const instances: string[] = [
+  process.env.VC_VOICE_GENERATION_SPACE_API_URL
+]
+
+// TODO we should use an inference endpoint instead
+export async function generateVoice(prompt: string, voiceFileName: string) {
+  if (state.load === instances.length) {
+    throw new Error(`all voice generation servers are busy, try again later..`)
+  }
+
+  state.load += 1
+
+  try {
+    const instance = instances.shift()
+    instances.push(instance)
+
+    console.log("instance:", instance)
+    
+    const browser = await puppeteer.launch({
+      headless: true,
+      protocolTimeout: 800000,
+    })
+
+    try {
+      const page = await browser.newPage()
+
+      await page.goto(instance, {
+        waitUntil: "networkidle2",
+      })
+
+      await new Promise(r => setTimeout(r, 3000))
+
+      const firstTextarea = await page.$('textarea[data-testid="textbox"]')
+
+      await firstTextarea.type(prompt)
+
+      // console.log("looking for the button to submit")
+      const submitButton = await page.$("button.lg")
+
+      // console.log("clicking on the button")
+      await submitButton.click()
+
+      await page.waitForSelector("audio", {
+        timeout: 800000, // need to be large enough in case someone else attemps to use our space
+      })
+
+      const voiceRemoteUrl = await page.$$eval("audio", el => el.map(x => x.getAttribute("src"))[0])
+
+
+      console.log({
+        voiceRemoteUrl,
+      })
+
+
+      console.log(`- downloading ${voiceFileName} from ${voiceRemoteUrl}`)
+
+      await downloadFileToTmp(voiceRemoteUrl, voiceFileName)
+
+      return voiceFileName
+    } catch (err) {
+      throw err
+    } finally {
+      await browser.close()
+    }
+  } catch (err) {
+    throw err
+  } finally {
+    state.load -= 1
+  }
+}

src/production/interpolateVideo.mts

@@ -0,0 +1,81 @@
+import path from "node:path"
+
+import { v4 as uuidv4 } from "uuid"
+import puppeteer from "puppeteer"
+
+import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
+import { pendingFilesDirFilePath } from "../config.mts"
+import { moveFileFromTmpToPending } from "../utils/moveFileFromTmpToPending.mts"
+
+export const state = {
+  load: 0
+}
+
+const instances: string[] = [
+  process.env.VC_VIDEO_INTERPOLATION_SPACE_API_URL
+]
+
+// TODO we should use an inference endpoint instead
+export async function interpolateVideo(fileName: string, steps: number, fps: number) {
+  if (state.load === instances.length) {
+    throw new Error(`all video interpolation servers are busy, try again later..`)
+  }
+
+  state.load += 1
+
+  try {
+    const inputFilePath = path.join(pendingFilesDirFilePath, fileName)
+
+    console.log(`interpolating ${fileName}`)
+    console.log(`warning: interpolateVideo parameter "${steps}" is ignored!`)
+    console.log(`warning: interpolateVideo parameter "${fps}" is ignored!`)
+
+    const instance = instances.shift()
+    instances.push(instance)
+
+    const browser = await puppeteer.launch({
+      headless: true,
+      protocolTimeout: 400000,
+    })
+
+    try {
+      const page = await browser.newPage()
+      await page.goto(instance, { waitUntil: 'networkidle2' })
+      
+      await new Promise(r => setTimeout(r, 3000))
+
+      const fileField = await page.$('input[type=file]')
+
+      // console.log(`uploading file..`)
+      await fileField.uploadFile(inputFilePath)
+
+      // console.log('looking for the button to submit')
+      const submitButton = await page.$('button.lg')
+
+      // console.log('clicking on the button')
+      await submitButton.click()
+
+      await page.waitForSelector('a[download="interpolated_result.mp4"]', {
+        timeout: 400000, // need to be large enough in case someone else attemps to use our space
+      })
+
+      const interpolatedFileUrl = await page.$$eval('a[download="interpolated_result.mp4"]', el => el.map(x => x.getAttribute("href"))[0])
+
+      // it is always a good idea to download to a tmp dir before saving to the pending dir
+      // because there is always a risk that the download will fail
+
+      const tmpFileName = `${uuidv4()}.mp4`
+
+      await downloadFileToTmp(interpolatedFileUrl, tmpFileName)
+      await moveFileFromTmpToPending(tmpFileName, fileName)
+    } catch (err) {
+      throw err
+    } finally {
+      await browser.close()
+    }
+  } catch (err) {
+    throw err
+  } finally {
+    state.load -= 1
+  }
+}

src/production/interpolateVideoLegacy.mts

@@ -0,0 +1,56 @@
+import { promises as fs } from "node:fs"
+import path from "node:path"
+import { Blob } from "buffer"
+
+import { client } from "@gradio/client"
+import tmpDir from "temp-dir"
+
+import { downloadFileToTmp } from '../utils/downloadFileToTmp.mts'
+
+export const state = {
+  load: 0
+}
+
+const instances: string[] = [
+  process.env.VC_VIDEO_INTERPOLATION_SPACE_API_URL
+]
+
+export const interpolateVideo = async (fileName: string, steps: number, fps: number) => {
+  if (state.load === instances.length) {
+    throw new Error(`all video interpolation servers are busy, try again later..`)
+  }
+
+  state.load += 1
+
+  try {
+    const inputFilePath = path.join(tmpDir, fileName)
+
+    const instance = instances.shift()
+    instances.push(instance)
+
+    const api = await client(instance, {
+      hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+    })
+
+    const video = await fs.readFile(inputFilePath)
+
+    const blob = new Blob([video], { type: 'video/mp4' })
+    // const blob = blobFrom(filePath)
+    const result = await api.predict(1, [
+      blob, 	// blob in 'parameter_5' Video component		
+      steps, // number (numeric value between 1 and 4) in 'Interpolation Steps' Slider component		
+      fps, // string (FALSE! it's a number)  in 'FPS output' Radio component
+    ])
+
+    const data = (result as any).data[0]
+    console.log('raw data:', data)
+    const { orig_name, data: remoteFilePath } = data
+    const remoteUrl = `${instance}/file=${remoteFilePath}`
+    console.log("remoteUrl:", remoteUrl)
+    await downloadFileToTmp(remoteUrl, fileName)
+  } catch (err) {
+    throw err
+  } finally {
+    state.load -= 1
+  }
+}

src/production/mergeAudio.mts

@@ -0,0 +1,49 @@
+import path from "node:path"
+
+import tmpDir from "temp-dir"
+import { v4 as uuidv4 } from "uuid"
+import ffmpeg from "fluent-ffmpeg"
+
+export const mergeAudio = async ({
+  input1FileName, 
+  input1Volume,
+  input2FileName,
+  input2Volume,
+  outputFileName = ''
+}: {
+  input1FileName: string, 
+  input1Volume: number,
+  input2FileName: string,
+  input2Volume: number,
+  outputFileName?: string 
+}): Promise<string> => {
+  outputFileName = `${uuidv4()}.m4a`
+
+  const input1FilePath = path.resolve(tmpDir, input1FileName)
+  const input2FilePath = path.resolve(tmpDir, input2FileName)
+  const outputFilePath = path.resolve(tmpDir, outputFileName)
+
+  const input1Ffmpeg = ffmpeg(input1FilePath)
+    .outputOptions("-map 0:a:0")
+    .audioFilters([{ filter: 'volume', options: input1Volume }]); // set volume for main audio
+  
+  const input2Ffmpeg = ffmpeg(input2FilePath)
+    .outputOptions("-map 1:a:0")
+    .audioFilters([{ filter: 'volume', options: input2Volume }]); // set volume for additional audio
+
+  await new Promise((resolve, reject) => {
+    ffmpeg()
+      .input(input1Ffmpeg)
+      .input(input2Ffmpeg)
+      .outputOptions("-c:a aac")   // use audio codec
+      .outputOptions("-shortest")  // finish encoding when shortest input stream ends
+      .output(outputFilePath)
+      .on("end", resolve)
+      .on("error", reject)
+      .run()
+  })
+
+  console.log(`merged audio from ${input1FileName} and ${input2FileName} into ${outputFileName}`)
+
+  return outputFileName
+}

src/production/normalizePendingVideoToTmpFilePath.mts

@@ -0,0 +1,33 @@
+import path from "node:path"
+
+import { v4 as uuidv4 } from "uuid"
+import tmpDir from "temp-dir"
+import ffmpeg from "fluent-ffmpeg"
+
+import { pendingFilesDirFilePath } from "../config.mts"
+
+export const normalizePendingVideoToTmpFilePath = async (fileName: string): Promise<string> => {
+  return new Promise((resolve,reject) => {
+
+    const tmpFileName = `${uuidv4()}.mp4`
+
+    const filePath = path.join(pendingFilesDirFilePath, fileName)
+    const tmpFilePath = path.join(tmpDir, tmpFileName)
+
+    ffmpeg.ffprobe(filePath, function(err,) {
+      if (err) { reject(err); return; }
+
+    ffmpeg(filePath)
+
+      .size("1280x720")
+
+      .save(tmpFilePath)
+      .on("end", async () => {
+        resolve(tmpFilePath)
+      })
+      .on("error", (err) => {
+        reject(err)
+      })
+    })
+  })
+}

src/production/postInterpolation.mts

@@ -0,0 +1,58 @@
+import path from "node:path"
+
+import { v4 as uuidv4 } from "uuid"
+import tmpDir from "temp-dir"
+import ffmpeg from "fluent-ffmpeg"
+import { moveFileFromTmpToPending } from "../utils/moveFileFromTmpToPending.mts"
+import { pendingFilesDirFilePath } from "../config.mts"
+
+export const postInterpolation = async (fileName: string, durationMs: number, nbFrames: number, noiseAmount: number): Promise<string> => {
+  return new Promise((resolve,reject) => {
+
+    const tmpFileName = `${uuidv4()}.mp4`
+
+    const filePath = path.join(pendingFilesDirFilePath, fileName)
+    const tmpFilePath = path.join(tmpDir, tmpFileName)
+
+    ffmpeg.ffprobe(filePath, function(err, metadata) {
+      if (err) { reject(err); return; }
+      
+      const durationInSec = durationMs / 1000
+
+      const currentVideoDurationInSec = metadata.format.duration
+      
+      console.log(`currentVideoDurationInSec in sec: ${currentVideoDurationInSec}s`)
+    
+      console.log(`target duration in sec: ${durationInSec}s (${durationMs}ms)`)
+    
+      // compute a ratio ex. 0.3 = 30% of the total length
+      const durationRatio = currentVideoDurationInSec / durationInSec
+      console.log(`durationRatio: ${durationRatio}`)
+
+    ffmpeg(filePath)
+
+      // convert to HD
+      .size("1280x720")
+
+      .videoFilters([
+        `setpts=0.5*PTS`, // we make the video faster
+        //'scale=-1:576:lanczos',
+        // 'unsharp=5:5:0.2:5:5:0.2', // not recommended, this make the video more "pixely"
+        `noise=c0s=${noiseAmount}:c0f=t+u` // add a movie grain noise
+      ])
+      .outputOptions([
+        `-r ${nbFrames}`,
+      ])
+
+      .save(tmpFilePath)
+      .on("end", async () => {
+        await moveFileFromTmpToPending(tmpFileName, fileName)
+
+        resolve(fileName)
+      })
+      .on("error", (err) => {
+        reject(err)
+      })
+    })
+  })
+}

src/production/renderImage.mts

@@ -0,0 +1,54 @@
+import { RenderedScene, RenderRequest } from "../types.mts"
+import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
+import { generateImageSDXL360AsBase64 } from "../utils/generateImageSDXL360.mts"
+import { generateSeed } from "../utils/generateSeed.mts"
+
+export async function renderImage(
+  request: RenderRequest,
+  response: RenderedScene,
+): Promise<RenderedScene> {
+
+  const isSpherical = request.projection === 'spherical'
+
+  const generateImageAsBase64 = isSpherical
+    ? generateImageSDXL360AsBase64
+    : generateImageSDXLAsBase64
+
+  console.log(`going to generate an image using ${request.projection || "default (cartesian)"} projection`)
+  
+  const params = {
+    positivePrompt: request.prompt,
+    seed: request.seed,
+    nbSteps: request.nbSteps,
+    width: request.width,
+    height: request.height
+  }
+
+  console.log(`calling generateImageAsBase64 with: `, JSON.stringify(params, null, 2))
+
+
+  // first we generate a quick low quality version
+  try {
+    response.assetUrl = await generateImageAsBase64(params)
+    console.log("successful generation!", response.assetUrl.slice(0, 30))
+    if (!response.assetUrl?.length) {
+      throw new Error(`the generated image is empty`)
+    }
+  } catch (err) {
+    console.error(`failed to render.. but let's try again!`)
+    try {
+      response.assetUrl = await generateImageAsBase64(params)
+      console.log("successful generation!", response.assetUrl.slice(0, 30))
+      if (!response.assetUrl?.length) {
+        throw new Error(`the generated image is empty`)
+      }
+    } catch (err) {
+      console.error(`failed to generate the image, although ${err}`)
+      response.error = `failed to render scene: ${err}`
+      response.status = "error"
+      response.assetUrl = ""
+    }
+  }
+
+  return response
+}

src/production/renderImageSegmentation.mts

@@ -0,0 +1,69 @@
+import path from "node:path"
+
+import { v4 as uuidv4 } from "uuid"
+import tmpDir from "temp-dir"
+
+import { RenderedScene, RenderRequest } from "../types.mts"
+import { segmentImage } from "../utils/segmentImage.mts"
+import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
+
+
+export async function renderImageSegmentation(
+  request: RenderRequest,
+  response: RenderedScene,
+): Promise<RenderedScene> {
+
+  const actionnables = Array.isArray(request.actionnables) ? request.actionnables : []
+
+  if (actionnables.length > 0) {
+    console.log("we have some actionnables:", actionnables)
+    console.log("going to grab the first frame")
+
+    const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
+
+    // console.log("beginning:", imageBase64.slice(0, 100))
+    await writeBase64ToFile(response.assetUrl, tmpImageFilePath)
+    console.log("wrote the image to ", tmpImageFilePath)
+  
+    if (!tmpImageFilePath) {
+      console.error("failed to get the image")
+      response.error = "failed to segment the image"
+      response.status = "error"
+    } else {
+      console.log("got the first frame! segmenting..")
+      try {
+        const result = await segmentImage(tmpImageFilePath, actionnables, request.width, request.height)
+        response.maskBase64 = result.pngInBase64
+        response.segments = result.segments
+  
+        console.log(`it worked the first time! got ${response.segments.length} segments`)
+      } catch (err) {
+        console.log("this takes too long :/ trying another server..")
+        try {
+          const result = await segmentImage(tmpImageFilePath, actionnables, request.width, request.height)
+          response.maskBase64 = result.pngInBase64
+          response.segments = result.segments
+         
+          console.log(`it worked the second time! got ${response.segments.length} segments`)
+        } catch (err) {
+          console.log("trying one last time, on a 3rd server..")
+          try {
+            const result = await segmentImage(tmpImageFilePath, actionnables, request.width, request.height)
+            response.maskBase64 = result.pngInBase64
+            response.segments = result.segments
+        
+            console.log(`it worked the third time! got ${response.segments.length} segments`)
+          } catch (err) {
+            console.log("yeah, all servers are busy it seems.. aborting")
+            response.error = "all servers are busy"
+            response.status = "error"
+          }
+        }
+      }
+    }
+  } else {
+    console.log("no actionnables: just returning the image, then")
+  }
+
+  return response
+}

src/production/renderPipeline.mts

@@ -0,0 +1,46 @@
+
+import { RenderedScene, RenderRequest } from "../types.mts"
+
+import { renderImage } from "./renderImage.mts"
+import { renderVideo } from "./renderVideo.mts"
+import { renderImageSegmentation } from "./renderImageSegmentation.mts"
+import { renderVideoSegmentation } from "./renderVideoSegmentation.mts"
+
+export async function renderPipeline(request: RenderRequest, response: RenderedScene) {
+  const isVideo = request?.nbFrames > 1
+
+  const renderContent = isVideo ? renderVideo : renderImage
+  const renderSegmentation  = isVideo ? renderVideoSegmentation : renderImageSegmentation 
+
+  if (isVideo) {
+    console.log(`rendering a video..`)
+  } else {
+    console.log(`rendering an image..`)
+  }
+  await renderContent(request, response)
+  await renderSegmentation(request, response)
+
+  /*
+  this is the optimized pipeline
+  However, right now it doesn't work because for some reason,
+  asking to generate the same seed + prompt on different nb of steps
+  doesn't generate the same image!
+
+  // first we need to wait for the low quality pre-render
+  await renderContent({
+    ...request,
+
+    // we are a bit more aggressive with the quality of the video preview
+    nbSteps: isVideo ? 8 : 16
+  }, response)
+
+  // then we can run both the segmentation and the high-res render at the same time
+  await Promise.all([
+    renderSegmentation(request, response),
+    renderContent(request, response)
+  ])
+  */
+
+  response.status = "completed"
+  response.error = ""
+}

src/production/renderScene.mts

@@ -0,0 +1,62 @@
+import { v4 as uuidv4 } from "uuid"
+
+import { RenderedScene, RenderRequest } from "../types.mts"
+import { generateSeed } from "../utils/generateSeed.mts"
+import { getValidNumber } from "../utils/getValidNumber.mts"
+import { renderPipeline } from "./renderPipeline.mts"
+
+const cache: Record<string, RenderedScene> = {}
+const cacheQueue: string[] = []
+const maxCacheSize = 1000
+
+export async function renderScene(request: RenderRequest): Promise<RenderedScene> {
+  // const key = getCacheKey(scene)
+  const renderId = uuidv4()
+
+  request.nbFrames = getValidNumber(request.nbFrames, 1, 24, 16)
+
+  const isVideo = request?.nbFrames === 1
+
+  // important: we need a consistent seed for our multiple rendering passes
+  request.seed = getValidNumber(request.seed, 0, 2147483647, generateSeed())
+  request.nbSteps = getValidNumber(request.nbSteps, 5, 50, 10)
+
+  if (isVideo) {
+    request.width = getValidNumber(request.width, 256, 1024, 1024)
+    request.height = getValidNumber(request.width, 256, 1024, 512)
+  } else {
+    request.width = getValidNumber(request.width, 256, 1280, 576)
+    request.height = getValidNumber(request.width, 256, 720, 320)
+  }
+
+  const response: RenderedScene = {
+    renderId,
+    status: "pending",
+    assetUrl: "",
+    error: "",
+    maskBase64: "",
+    segments: []
+  }
+
+  cache[renderId] = response
+  cacheQueue.push(renderId)
+  if (cacheQueue.length > maxCacheSize) {
+    const toRemove = cacheQueue.shift()
+    delete cache[toRemove]
+  }
+
+  // this is a fire-and-forget asynchronous pipeline:
+  // we start it, but we do not await for the response
+  renderPipeline(request, response)
+
+  console.log("renderScene: yielding the scene", response)
+  return response
+}
+
+export async function getRenderedScene(renderId: string): Promise<RenderedScene> {
+  const rendered = cache[renderId]
+  if (!rendered) {
+    throw new Error(`couldn't find any rendered scene with renderId ${renderId}`)
+  }
+  return cache[renderId]
+}

src/production/renderVideo.mts

@@ -0,0 +1,41 @@
+import { RenderedScene, RenderRequest } from "../types.mts"
+import { generateVideo } from "./generateVideo.mts"
+
+export async function renderVideo(
+  request: RenderRequest,
+  response: RenderedScene
+): Promise<RenderedScene> {
+
+  const params = {
+    seed: request.seed,
+    nbFrames: request.nbFrames,
+    nbSteps: request.nbSteps,
+  }
+
+  try {
+    response.assetUrl = await generateVideo(request.prompt, params)
+    // console.log("successfull generation")
+
+    if (!response.assetUrl?.length) {
+      throw new Error(`url for the generated video is empty`)
+    }
+  } catch (err) {
+    console.error(`failed to render the video scene.. but let's try again!`)
+
+    try {
+      response.assetUrl = await generateVideo(request.prompt, params)
+      // console.log("successfull generation")
+
+      if (!response.assetUrl?.length) {
+        throw new Error(`url for the generated video is empty`)
+      }
+      
+    } catch (err) {
+      console.error(`it failed the video for second time ${err}`)
+      response.error = `failed to render video scene: ${err}`
+      response.status = "error"
+    }
+  }
+
+  return response
+}

src/production/renderVideoSegmentation.mts

@@ -0,0 +1,52 @@
+import { v4 as uuidv4 } from "uuid"
+
+import { RenderedScene, RenderRequest } from "../types.mts"
+import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
+import { getFirstVideoFrame } from "../utils/getFirstVideoFrame.mts"
+import { segmentImage } from "../utils/segmentImage.mts"
+
+export async function renderVideoSegmentation(
+  request: RenderRequest,
+  response: RenderedScene
+): Promise<RenderedScene> {
+  
+  const actionnables = Array.isArray(request.actionnables) ? request.actionnables : []
+
+  if (actionnables.length > 0) {
+    console.log("we have some actionnables:", actionnables)
+    if (request.segmentation === "firstframe") {
+      console.log("going to grab the first frame")
+      const tmpVideoFilePath = await downloadFileToTmp(response.assetUrl, `${uuidv4()}`)
+      console.log("downloaded the first frame to ", tmpVideoFilePath)
+      const firstFrameFilePath = await getFirstVideoFrame(tmpVideoFilePath)
+      console.log("downloaded the first frame to ", firstFrameFilePath)
+      
+      if (!firstFrameFilePath) {
+        console.error("failed to get the image")
+        response.error = "failed to segment the image"
+        response.status = "error"
+      } else {
+        console.log("got the first frame! segmenting..")
+        const result = await segmentImage(firstFrameFilePath, actionnables, request.width, request.height)
+        response.maskBase64 = result.pngInBase64
+        response.segments = result.segments
+
+        // console.log("success!", {  segments })
+      }
+      /*
+      const jpgBase64 = await getFirstVideoFrame(tmpVideoFileName)
+      if (!jpgBase64) {
+        console.error("failed to get the image")
+        error = "failed to segment the image"
+      } else {
+        console.log(`got the first frame (${jpgBase64.length})`)
+
+        console.log("TODO: call segmentImage with the base64 image")
+        await segmentImage()
+      }
+      */
+    }
+  }
+
+  return response
+}

src/production/upscaleVideo.mts

@@ -0,0 +1,78 @@
+import path from "node:path"
+
+import { v4 as uuidv4 } from "uuid"
+import tmpDir from "temp-dir"
+import puppeteer from "puppeteer"
+
+import { downloadFileToTmp } from '../utils/downloadFileToTmp.mts'
+import { pendingFilesDirFilePath } from '../config.mts'
+import { moveFileFromTmpToPending } from "../utils/moveFileFromTmpToPending.mts"
+
+const instances: string[] = [
+  process.env.VC_VIDEO_UPSCALE_SPACE_API_URL
+]
+
+// TODO we should use an inference endpoint instead (or a space which bakes generation + upscale at the same time)
+export async function upscaleVideo(fileName: string, prompt: string) {
+  const instance = instances.shift()
+  instances.push(instance)
+
+  const browser = await puppeteer.launch({
+    // headless: true,
+    protocolTimeout: 800000,
+  })
+
+  try {
+    const page = await browser.newPage()
+
+    await page.goto(instance, {
+      waitUntil: 'networkidle2',
+    })
+
+    const promptField = await page.$('textarea')
+    await promptField.type(prompt)
+
+    const inputFilePath = path.join(pendingFilesDirFilePath, fileName)
+    // console.log(`local file to upscale: ${inputFilePath}`)
+    
+    await new Promise(r => setTimeout(r, 3000))
+
+    const fileField = await page.$('input[type=file]')
+
+    // console.log(`uploading file..`)
+    await fileField.uploadFile(inputFilePath)
+
+    // console.log('looking for the button to submit')
+    const submitButton = await page.$('button.lg')
+
+    // console.log('clicking on the button')
+    await submitButton.click()
+
+    /*
+    const client = await page.target().createCDPSession()
+
+    await client.send('Page.setDownloadBehavior', {
+      behavior: 'allow',
+      downloadPath: tmpDir,
+    })
+    */
+
+    await page.waitForSelector('a[download="xl_result.mp4"]', {
+      timeout: 800000, // need to be large enough in case someone else attemps to use our space
+    })
+
+    const upscaledFileUrl = await page.$$eval('a[download="xl_result.mp4"]', el => el.map(x => x.getAttribute("href"))[0])
+
+    // it is always a good idea to download to a tmp dir before saving to the pending dir
+    // because there is always a risk that the download will fail
+    
+    const tmpFileName = `${uuidv4()}.mp4`
+
+    await downloadFileToTmp(upscaledFileUrl, tmpFileName)
+    await moveFileFromTmpToPending(tmpFileName, fileName)
+  } catch (err) {
+    throw err
+  } finally {
+    await browser.close()
+  }
+}

src/scheduler/deleteVideo.mts

@@ -0,0 +1,28 @@
+import tmpDir from "temp-dir"
+import { validate as uuidValidate } from "uuid"
+
+import { completedMetadataDirFilePath, completedFilesDirFilePath, pendingMetadataDirFilePath, pendingFilesDirFilePath } from "../config.mts"
+import { deleteFilesWithName } from "../utils/deleteAllFilesWith.mts"
+
+
+// note: we make sure ownerId and videoId are *VALID*
+// otherwise an attacker could try to delete important files!
+export const deleteVideo = async (ownerId: string, videoId?: string) => {
+  if (!uuidValidate(ownerId)) {
+    throw new Error(`fatal error: ownerId ${ownerId} is invalid!`)
+  }
+
+  if (videoId && !uuidValidate(videoId)) {
+    throw new Error(`fatal error: videoId ${videoId} is invalid!`)
+  }
+  const id = videoId ? `${ownerId}_${videoId}` : ownerId
+
+  // this should delete everything, including audio files
+  // however we still have some temporary files with a name that is unique:
+  // we should probably rename those
+  await deleteFilesWithName(tmpDir, id)
+  await deleteFilesWithName(completedMetadataDirFilePath, id)
+  await deleteFilesWithName(completedFilesDirFilePath, id)
+  await deleteFilesWithName(pendingMetadataDirFilePath, id)
+  await deleteFilesWithName(pendingFilesDirFilePath, id)
+}

src/scheduler/getAllVideosForOwner.mts

@@ -0,0 +1,9 @@
+import { Video } from "../types.mts"
+import { getCompletedVideos } from "./getCompletedVideos.mts"
+import { getPendingVideos } from "./getPendingVideos.mts"
+
+export const getAllVideosForOwner = async (ownerId: string): Promise<Video[]> => {
+  const pendingVideos = await getPendingVideos(ownerId)
+  const completedVideos = await getCompletedVideos(ownerId)
+  return [...pendingVideos, ...completedVideos]
+}

src/scheduler/getCompletedVideos.mts

@@ -0,0 +1,9 @@
+import { Video } from "../types.mts"
+import { completedMetadataDirFilePath } from "../config.mts"
+import { readVideoMetadataFiles } from "./readVideoMetadataFiles.mts"
+
+export const getCompletedVideos = async (ownerId?: string): Promise<Video[]> => {
+  const completedVideos = await readVideoMetadataFiles(completedMetadataDirFilePath, ownerId)
+
+  return completedVideos
+}