jbilcke-hf HF staff commited on
Commit
096584a
Β·
1 Parent(s): f7ac73c

added code for async processing

Browse files
src/index.mts CHANGED
@@ -1,7 +1,7 @@
1
  import { createReadStream, existsSync } from "node:fs"
2
  import path from "node:path"
3
 
4
- import { validate as uuidValidate } from "uuid"
5
  import express from "express"
6
 
7
  import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene } from "./types.mts"
@@ -21,7 +21,7 @@ import { initFolders } from "./initFolders.mts"
21
  import { sortVideosByYoungestFirst } from "./utils/sortVideosByYoungestFirst.mts"
22
  import { generateVideo } from "./production/generateVideo.mts"
23
  import { generateSeed } from "./utils/generateSeed.mts"
24
- import { renderScene } from "./production/renderScene.mts"
25
 
26
  initFolders()
27
  // to disable all processing (eg. to debug)
@@ -48,7 +48,9 @@ app.post("/render", async (req, res) => {
48
  return
49
  }
50
 
51
- let result: RenderedScene = {
 
 
52
  assetUrl: "",
53
  maskBase64: "",
54
  error: "",
@@ -56,33 +58,84 @@ app.post("/render", async (req, res) => {
56
  }
57
 
58
  try {
59
- result = await renderScene(request)
60
  } catch (err) {
61
  // console.log("failed to render scene!")
62
- result.error = `failed to render scene: ${err}`
63
  }
64
 
65
- if (result.error === "already rendering") {
66
  console.log("server busy")
67
  res.status(200)
68
- res.write(JSON.stringify({ url: "", error: result.error }))
69
  res.end()
70
  return
71
- } else if (result.error.length > 0) {
72
  // console.log("server error")
73
  res.status(500)
74
- res.write(JSON.stringify({ url: "", error: result.error }))
75
  res.end()
76
  return
77
  } else {
78
  // console.log("all good")
79
  res.status(200)
80
- res.write(JSON.stringify(result))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81
  res.end()
82
  return
83
  }
84
  })
85
 
 
86
  // a "fast track" pipeline
87
  /*
88
  app.post("/segment", async (req, res) => {
 
1
  import { createReadStream, existsSync } from "node:fs"
2
  import path from "node:path"
3
 
4
+ import { v4 as uuidv4, validate as uuidValidate } from "uuid"
5
  import express from "express"
6
 
7
  import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene } from "./types.mts"
 
21
  import { sortVideosByYoungestFirst } from "./utils/sortVideosByYoungestFirst.mts"
22
  import { generateVideo } from "./production/generateVideo.mts"
23
  import { generateSeed } from "./utils/generateSeed.mts"
24
+ import { getRenderedScene, renderScene } from "./production/renderScene.mts"
25
 
26
  initFolders()
27
  // to disable all processing (eg. to debug)
 
48
  return
49
  }
50
 
51
+ let response: RenderedScene = {
52
+ renderId: "",
53
+ status: "pending",
54
  assetUrl: "",
55
  maskBase64: "",
56
  error: "",
 
58
  }
59
 
60
  try {
61
+ response = await renderScene(request)
62
  } catch (err) {
63
  // console.log("failed to render scene!")
64
+ response.error = `failed to render scene: ${err}`
65
  }
66
 
67
+ if (response.error === "already rendering") {
68
  console.log("server busy")
69
  res.status(200)
70
+ res.write(JSON.stringify(response))
71
  res.end()
72
  return
73
+ } else if (response.error.length > 0) {
74
  // console.log("server error")
75
  res.status(500)
76
+ res.write(JSON.stringify(response))
77
  res.end()
78
  return
79
  } else {
80
  // console.log("all good")
81
  res.status(200)
82
+ res.write(JSON.stringify(response))
83
+ res.end()
84
+ return
85
+ }
86
+ })
87
+
88
+ // a "fast track" pipeline
89
+ app.get("/render/:renderId", async (req, res) => {
90
+
91
+ const renderId = `${req.params.renderId}`
92
+
93
+ if (!uuidValidate(renderId)) {
94
+ console.error("invalid render id")
95
+ res.status(400)
96
+ res.write(JSON.stringify({ error: `invalid render id` }))
97
+ res.end()
98
+ return
99
+ }
100
+
101
+ let response: RenderedScene = {
102
+ renderId: "",
103
+ status: "pending",
104
+ assetUrl: "",
105
+ error: "",
106
+ maskBase64: "",
107
+ segments: []
108
+ }
109
+
110
+ try {
111
+ response = await getRenderedScene(renderId)
112
+ } catch (err) {
113
+ // console.log("failed to render scene!")
114
+ response.error = `failed to render scene: ${err}`
115
+ }
116
+
117
+ if (response.error === "already rendering") {
118
+ console.log("server busy")
119
+ res.status(200)
120
+ res.write(JSON.stringify(response))
121
+ res.end()
122
+ return
123
+ } else if (response.error.length > 0) {
124
+ // console.log("server error")
125
+ res.status(500)
126
+ res.write(JSON.stringify(response))
127
+ res.end()
128
+ return
129
+ } else {
130
+ // console.log("all good")
131
+ res.status(200)
132
+ res.write(JSON.stringify(response))
133
  res.end()
134
  return
135
  }
136
  })
137
 
138
+
139
  // a "fast track" pipeline
140
  /*
141
  app.post("/segment", async (req, res) => {
src/production/renderImage.mts ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { RenderedScene, RenderRequest } from "../types.mts"
2
+ import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
3
+ import { generateImageSDXL360AsBase64 } from "../utils/generateImageSDXL360.mts"
4
+ import { generateSeed } from "../utils/generateSeed.mts"
5
+
6
+ export async function renderImage(
7
+ request: RenderRequest,
8
+ response: RenderedScene,
9
+ ): Promise<RenderedScene> {
10
+
11
+ const isSpherical = request.projection === 'spherical'
12
+
13
+ const generateImageAsBase64 = isSpherical
14
+ ? generateImageSDXL360AsBase64
15
+ : generateImageSDXLAsBase64
16
+
17
+ console.log(`going to generate an image using ${request.projection || "default (cartesian)"} projection`)
18
+
19
+ const params = {
20
+ positivePrompt: request.prompt,
21
+ seed: request.seed,
22
+ nbSteps: request.nbSteps,
23
+ width: request.width,
24
+ request: request.height
25
+ }
26
+
27
+ console.log(`calling generateImageAsBase64 with: `, JSON.stringify(params, null, 2))
28
+
29
+
30
+ // first we generate a quick low quality version
31
+ try {
32
+ response.assetUrl = await generateImageAsBase64(params)
33
+ console.log("successful generation!", response.assetUrl.slice(0, 30))
34
+ if (!response.assetUrl?.length) {
35
+ throw new Error(`the generated image is empty`)
36
+ }
37
+ } catch (err) {
38
+ console.error(`failed to render.. but let's try again!`)
39
+ try {
40
+ response.assetUrl = await generateImageAsBase64(params)
41
+ console.log("successful generation!", response.assetUrl.slice(0, 30))
42
+ if (!response.assetUrl?.length) {
43
+ throw new Error(`the generated image is empty`)
44
+ }
45
+ } catch (err) {
46
+ console.error(`failed to generate the image, although ${err}`)
47
+ response.error = `failed to render scene: ${err}`
48
+ response.status = "error"
49
+ response.assetUrl = ""
50
+ }
51
+ }
52
+
53
+ return response
54
+ }
src/production/{renderStaticScene.mts β†’ renderImageSegmentation.mts} RENAMED
@@ -3,63 +3,17 @@ import path from "node:path"
3
  import { v4 as uuidv4 } from "uuid"
4
  import tmpDir from "temp-dir"
5
 
6
- import { ImageSegment, RenderedScene, RenderingJob, RenderRequest } from "../types.mts"
7
  import { segmentImage } from "../utils/segmentImage.mts"
8
- import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
9
  import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
10
 
11
 
12
- const pendingJobs: RenderingJob[] = []
 
 
 
13
 
14
- export async function renderStaticScene(scene: RenderRequest): Promise<RenderedScene> {
15
-
16
- let imageBase64 = ""
17
- let error = ""
18
-
19
- const width = 1024
20
- const height = 512
21
-
22
- const params = {
23
- positivePrompt: scene.prompt,
24
- seed: scene.seed || undefined,
25
- nbSteps: scene.nbSteps || undefined,
26
- width,
27
- height
28
- }
29
- console.log(`calling generateImageSDXLAsBase64 with: `, JSON.stringify(params, null, 2))
30
-
31
- try {
32
- imageBase64 = await generateImageSDXLAsBase64(params)
33
- console.log("successful generation!", imageBase64.slice(0, 30))
34
- error = ""
35
- if (!imageBase64?.length) {
36
- throw new Error(`the generated image is empty`)
37
- }
38
- } catch (err) {
39
- console.error(`failed to render.. but let's try again!`)
40
- try {
41
- imageBase64 = await generateImageSDXLAsBase64(params)
42
- console.log("successful generation!", imageBase64.slice(0, 30))
43
- error = ""
44
- if (!imageBase64?.length) {
45
- throw new Error(`the generated image is empty`)
46
- }
47
- } catch (err) {
48
- console.error(`failed to generate the image, although ${err}`)
49
- error = `failed to render scene: ${err}`
50
- return {
51
- assetUrl: imageBase64,
52
- error,
53
- maskBase64: "",
54
- segments: []
55
- } as RenderedScene
56
- }
57
- }
58
-
59
- const actionnables = Array.isArray(scene.actionnables) ? scene.actionnables : []
60
-
61
- let mask = ""
62
- let segments: ImageSegment[] = []
63
 
64
  if (actionnables.length > 0) {
65
  console.log("we have some actionnables:", actionnables)
@@ -68,35 +22,41 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
68
  const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
69
 
70
  // console.log("beginning:", imageBase64.slice(0, 100))
71
- await writeBase64ToFile(imageBase64, tmpImageFilePath)
72
  console.log("wrote the image to ", tmpImageFilePath)
73
 
74
  if (!tmpImageFilePath) {
75
  console.error("failed to get the image")
76
- error = "failed to segment the image"
 
77
  } else {
78
  console.log("got the first frame! segmenting..")
79
  try {
80
- const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
81
- mask = result.pngInBase64
82
- segments = result.segments
83
- console.log(`it worked the first time! got ${segments.length} segments`)
 
84
  } catch (err) {
85
  console.log("this takes too long :/ trying another server..")
86
  try {
87
- const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
88
- mask = result.pngInBase64
89
- segments = result.segments
90
- console.log(`it worked the second time! got ${segments.length} segments`)
 
91
  } catch (err) {
92
  console.log("trying one last time, on a 3rd server..")
93
  try {
94
- const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
95
- mask = result.pngInBase64
96
- segments = result.segments
97
- console.log(`it worked the third time! got ${segments.length} segments`)
 
98
  } catch (err) {
99
  console.log("yeah, all servers are busy it seems.. aborting")
 
 
100
  }
101
  }
102
  }
@@ -105,12 +65,5 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
105
  console.log("no actionnables: just returning the image, then")
106
  }
107
 
108
- error = ""
109
-
110
- return {
111
- assetUrl: imageBase64,
112
- error,
113
- maskBase64: mask,
114
- segments
115
- } as RenderedScene
116
  }
 
3
  import { v4 as uuidv4 } from "uuid"
4
  import tmpDir from "temp-dir"
5
 
6
+ import { RenderedScene, RenderRequest } from "../types.mts"
7
  import { segmentImage } from "../utils/segmentImage.mts"
 
8
  import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
9
 
10
 
11
+ export async function renderImageSegmentation(
12
+ request: RenderRequest,
13
+ response: RenderedScene,
14
+ ): Promise<RenderedScene> {
15
 
16
+ const actionnables = Array.isArray(request.actionnables) ? request.actionnables : []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
  if (actionnables.length > 0) {
19
  console.log("we have some actionnables:", actionnables)
 
22
  const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
23
 
24
  // console.log("beginning:", imageBase64.slice(0, 100))
25
+ await writeBase64ToFile(response.assetUrl, tmpImageFilePath)
26
  console.log("wrote the image to ", tmpImageFilePath)
27
 
28
  if (!tmpImageFilePath) {
29
  console.error("failed to get the image")
30
+ response.error = "failed to segment the image"
31
+ response.status = "error"
32
  } else {
33
  console.log("got the first frame! segmenting..")
34
  try {
35
+ const result = await segmentImage(tmpImageFilePath, actionnables, request.width, request.height)
36
+ response.maskBase64 = result.pngInBase64
37
+ response.segments = result.segments
38
+
39
+ console.log(`it worked the first time! got ${response.segments.length} segments`)
40
  } catch (err) {
41
  console.log("this takes too long :/ trying another server..")
42
  try {
43
+ const result = await segmentImage(tmpImageFilePath, actionnables, request.width, request.height)
44
+ response.maskBase64 = result.pngInBase64
45
+ response.segments = result.segments
46
+
47
+ console.log(`it worked the second time! got ${response.segments.length} segments`)
48
  } catch (err) {
49
  console.log("trying one last time, on a 3rd server..")
50
  try {
51
+ const result = await segmentImage(tmpImageFilePath, actionnables, request.width, request.height)
52
+ response.maskBase64 = result.pngInBase64
53
+ response.segments = result.segments
54
+
55
+ console.log(`it worked the third time! got ${response.segments.length} segments`)
56
  } catch (err) {
57
  console.log("yeah, all servers are busy it seems.. aborting")
58
+ response.error = "all servers are busy"
59
+ response.status = "error"
60
  }
61
  }
62
  }
 
65
  console.log("no actionnables: just returning the image, then")
66
  }
67
 
68
+ return response
 
 
 
 
 
 
 
69
  }
src/production/renderPipeline.mts ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import { RenderedScene, RenderRequest } from "../types.mts"
3
+
4
+ import { renderImage } from "./renderImage.mts"
5
+ import { renderVideo } from "./renderVideo.mts"
6
+ import { renderImageSegmentation } from "./renderImageSegmentation.mts"
7
+ import { renderVideoSegmentation } from "./renderVideoSegmentation.mts"
8
+
9
+ export async function renderPipeline(request: RenderRequest, response: RenderedScene) {
10
+ const isVideo = request?.nbFrames > 1
11
+
12
+ const renderContent = isVideo ? renderVideo : renderImage
13
+ const renderSegmentation = isVideo ? renderVideoSegmentation : renderImageSegmentation
14
+
15
+ if (isVideo) {
16
+ console.log(`rendering a video..`)
17
+ } else {
18
+ console.log(`rendering an image..`)
19
+ }
20
+ await renderContent(request, response)
21
+ await renderSegmentation(request, response)
22
+
23
+ /*
24
+ this is the optimized pipeline
25
+ However, right now it doesn't work because for some reason,
26
+ asking to generate the same seed + prompt on different nb of steps
27
+ doesn't generate the same image!
28
+
29
+ // first we need to wait for the low quality pre-render
30
+ await renderContent({
31
+ ...request,
32
+
33
+ // we are a bit more aggressive with the quality of the video preview
34
+ nbSteps: isVideo ? 8 : 16
35
+ }, response)
36
+
37
+ // then we can run both the segmentation and the high-res render at the same time
38
+ await Promise.all([
39
+ renderSegmentation(request, response),
40
+ renderContent(request, response)
41
+ ])
42
+ */
43
+
44
+ response.status = "completed"
45
+ response.error = ""
46
+ }
src/production/renderScene.mts CHANGED
@@ -1,13 +1,62 @@
 
 
1
  import { RenderedScene, RenderRequest } from "../types.mts"
2
- import { renderStaticScene } from "./renderStaticScene.mts"
3
- import { renderVideoScene } from "./renderVideoScene.mts"
 
 
 
 
 
 
 
 
 
 
 
4
 
5
- export async function renderScene(scene: RenderRequest): Promise<RenderedScene> {
6
- if (scene?.nbFrames === 1) {
7
- console.log(`calling renderStaticScene`)
8
- return renderStaticScene(scene)
 
 
 
 
 
9
  } else {
10
- console.log(`calling renderVideoScene`)
11
- return renderVideoScene(scene)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  }
 
13
  }
 
1
+ import { v4 as uuidv4 } from "uuid"
2
+
3
  import { RenderedScene, RenderRequest } from "../types.mts"
4
+ import { generateSeed } from "../utils/generateSeed.mts"
5
+ import { getValidNumber } from "../utils/getValidNumber.mts"
6
+ import { renderPipeline } from "./renderPipeline.mts"
7
+
8
+ const cache: Record<string, RenderedScene> = {}
9
+ const cacheQueue: string[] = []
10
+ const maxCacheSize = 1000
11
+
12
+ export async function renderScene(request: RenderRequest): Promise<RenderedScene> {
13
+ // const key = getCacheKey(scene)
14
+ const renderId = uuidv4()
15
+
16
+ request.nbFrames = getValidNumber(request.nbFrames, 1, 24, 16)
17
 
18
+ const isVideo = request?.nbFrames === 1
19
+
20
+ // important: we need a consistent seed for our multiple rendering passes
21
+ request.seed = getValidNumber(request.seed, 0, 2147483647, generateSeed())
22
+ request.nbSteps = getValidNumber(request.nbSteps, 5, 50, 10)
23
+
24
+ if (isVideo) {
25
+ request.width = getValidNumber(request.width, 256, 1024, 1024)
26
+ request.height = getValidNumber(request.width, 256, 1024, 512)
27
  } else {
28
+ request.width = getValidNumber(request.width, 256, 1280, 576)
29
+ request.height = getValidNumber(request.width, 256, 720, 320)
30
+ }
31
+
32
+ const response: RenderedScene = {
33
+ renderId,
34
+ status: "pending",
35
+ assetUrl: "",
36
+ error: "",
37
+ maskBase64: "",
38
+ segments: []
39
+ }
40
+
41
+ cache[renderId] = response
42
+ cacheQueue.push(renderId)
43
+ if (cacheQueue.length > maxCacheSize) {
44
+ const toRemove = cacheQueue.shift()
45
+ delete cache[toRemove]
46
+ }
47
+
48
+ // this is a fire-and-forget asynchronous pipeline:
49
+ // we start it, but we do not await for the response
50
+ renderPipeline(request, response)
51
+
52
+ console.log("renderScene: yielding the scene", response)
53
+ return response
54
+ }
55
+
56
+ export async function getRenderedScene(renderId: string): Promise<RenderedScene> {
57
+ const rendered = cache[renderId]
58
+ if (!rendered) {
59
+ throw new Error(`couldn't find any rendered scene with renderId ${renderId}`)
60
  }
61
+ return cache[renderId]
62
  }
src/production/renderVideo.mts ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { RenderedScene, RenderRequest } from "../types.mts"
2
+ import { generateVideo } from "./generateVideo.mts"
3
+
4
+ export async function renderVideo(
5
+ request: RenderRequest,
6
+ response: RenderedScene
7
+ ): Promise<RenderedScene> {
8
+
9
+ const params = {
10
+ seed: request.seed,
11
+ nbFrames: request.nbFrames,
12
+ nbSteps: request.nbSteps,
13
+ }
14
+
15
+ try {
16
+ response.assetUrl = await generateVideo(request.prompt, params)
17
+ // console.log("successfull generation")
18
+
19
+ if (!response.assetUrl?.length) {
20
+ throw new Error(`url for the generated video is empty`)
21
+ }
22
+ } catch (err) {
23
+ console.error(`failed to render the video scene.. but let's try again!`)
24
+
25
+ try {
26
+ response.assetUrl = await generateVideo(request.prompt, params)
27
+ // console.log("successfull generation")
28
+
29
+ if (!response.assetUrl?.length) {
30
+ throw new Error(`url for the generated video is empty`)
31
+ }
32
+
33
+ } catch (err) {
34
+ console.error(`it failed the video for second time ${err}`)
35
+ response.error = `failed to render video scene: ${err}`
36
+ response.status = "error"
37
+ }
38
+ }
39
+
40
+ return response
41
+ }
src/production/{renderVideoScene.mts β†’ renderVideoSegmentation.mts} RENAMED
@@ -1,76 +1,36 @@
1
  import { v4 as uuidv4 } from "uuid"
2
 
3
- import { ImageSegment, RenderedScene, RenderRequest } from "../types.mts"
4
  import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
5
- import { generateSeed } from "../utils/generateSeed.mts"
6
- import { getValidNumber } from "../utils/getValidNumber.mts"
7
- import { generateVideo } from "./generateVideo.mts"
8
  import { getFirstVideoFrame } from "../utils/getFirstVideoFrame.mts"
9
  import { segmentImage } from "../utils/segmentImage.mts"
10
 
11
- export async function renderVideoScene(scene: RenderRequest): Promise<RenderedScene> {
12
-
13
- let url = ""
14
- let error = ""
15
-
16
- const width = 576
17
- const height = 320
18
-
19
- const params = {
20
- seed: getValidNumber(scene.seed, 0, 2147483647, generateSeed()),
21
- nbFrames: getValidNumber(scene.nbFrames, 8, 24, 16), // 2 seconds by default
22
- nbSteps: getValidNumber(scene.nbSteps, 1, 50, 10), // use 10 by default to go fast, but not too sloppy
23
- }
24
-
25
- try {
26
- url = await generateVideo(scene.prompt, params)
27
- // console.log("successfull generation")
28
- error = ""
29
- if (!url?.length) {
30
- throw new Error(`url for the generated image is empty`)
31
- }
32
- } catch (err) {
33
- console.error(`failed to render the scene.. but let's try again!`)
34
-
35
- try {
36
- url = await generateVideo(scene.prompt, params)
37
- // console.log("successfull generation")
38
- error = ""
39
-
40
- if (!url?.length) {
41
- throw new Error(`url for the generated image is empty`)
42
- }
43
-
44
- } catch (err) {
45
- console.error(`it failed the second time ${err}`)
46
- error = `failed to render scene: ${err}`
47
- }
48
- }
49
-
50
-
51
- // TODO add segmentation here
52
- const actionnables = Array.isArray(scene.actionnables) ? scene.actionnables : []
53
-
54
- let mask = ""
55
- let segments: ImageSegment[] = []
56
 
57
  if (actionnables.length > 0) {
58
  console.log("we have some actionnables:", actionnables)
59
- if (scene.segmentation === "firstframe") {
60
  console.log("going to grab the first frame")
61
- const tmpVideoFilePath = await downloadFileToTmp(url, `${uuidv4()}`)
62
  console.log("downloaded the first frame to ", tmpVideoFilePath)
63
  const firstFrameFilePath = await getFirstVideoFrame(tmpVideoFilePath)
64
  console.log("downloaded the first frame to ", firstFrameFilePath)
65
 
66
  if (!firstFrameFilePath) {
67
  console.error("failed to get the image")
68
- error = "failed to segment the image"
 
69
  } else {
70
  console.log("got the first frame! segmenting..")
71
- const result = await segmentImage(firstFrameFilePath, actionnables, width, height)
72
- mask = result.pngInBase64
73
- segments = result.segments
 
74
  // console.log("success!", { segments })
75
  }
76
  /*
@@ -88,12 +48,5 @@ export async function renderVideoScene(scene: RenderRequest): Promise<RenderedSc
88
  }
89
  }
90
 
91
- error = ""
92
-
93
- return {
94
- assetUrl: url,
95
- error,
96
- maskBase64: mask,
97
- segments
98
- } as RenderedScene
99
  }
 
1
  import { v4 as uuidv4 } from "uuid"
2
 
3
+ import { RenderedScene, RenderRequest } from "../types.mts"
4
  import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
 
 
 
5
  import { getFirstVideoFrame } from "../utils/getFirstVideoFrame.mts"
6
  import { segmentImage } from "../utils/segmentImage.mts"
7
 
8
+ export async function renderVideoSegmentation(
9
+ request: RenderRequest,
10
+ response: RenderedScene
11
+ ): Promise<RenderedScene> {
12
+
13
+ const actionnables = Array.isArray(request.actionnables) ? request.actionnables : []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
 
15
  if (actionnables.length > 0) {
16
  console.log("we have some actionnables:", actionnables)
17
+ if (request.segmentation === "firstframe") {
18
  console.log("going to grab the first frame")
19
+ const tmpVideoFilePath = await downloadFileToTmp(response.assetUrl, `${uuidv4()}`)
20
  console.log("downloaded the first frame to ", tmpVideoFilePath)
21
  const firstFrameFilePath = await getFirstVideoFrame(tmpVideoFilePath)
22
  console.log("downloaded the first frame to ", firstFrameFilePath)
23
 
24
  if (!firstFrameFilePath) {
25
  console.error("failed to get the image")
26
+ response.error = "failed to segment the image"
27
+ response.status = "error"
28
  } else {
29
  console.log("got the first frame! segmenting..")
30
+ const result = await segmentImage(firstFrameFilePath, actionnables, request.width, request.height)
31
+ response.maskBase64 = result.pngInBase64
32
+ response.segments = result.segments
33
+
34
  // console.log("success!", { segments })
35
  }
36
  /*
 
48
  }
49
  }
50
 
51
+ return response
 
 
 
 
 
 
 
52
  }
src/types.mts CHANGED
@@ -269,6 +269,8 @@ export type Video = VideoSequence & {
269
  shots: VideoShot[]
270
  }
271
 
 
 
272
  export interface RenderRequest {
273
  prompt: string
274
 
@@ -292,6 +294,11 @@ export interface RenderRequest {
292
  nbSteps: number // min: 1, max: 50
293
 
294
  seed: number
 
 
 
 
 
295
  }
296
 
297
  export interface ImageSegmentationRequest {
@@ -307,7 +314,11 @@ export interface ImageSegment {
307
  score: number
308
  }
309
 
 
 
310
  export interface RenderedScene {
 
 
311
  assetUrl: string
312
  error: string
313
  maskBase64: string
 
269
  shots: VideoShot[]
270
  }
271
 
272
+ export type ProjectionMode = 'cartesian' | 'spherical'
273
+
274
  export interface RenderRequest {
275
  prompt: string
276
 
 
294
  nbSteps: number // min: 1, max: 50
295
 
296
  seed: number
297
+
298
+ width: number
299
+ height: number
300
+
301
+ projection: ProjectionMode
302
  }
303
 
304
  export interface ImageSegmentationRequest {
 
314
  score: number
315
  }
316
 
317
+ export type RenderedSceneStatus = 'pending' | 'completed' | 'error'
318
+
319
  export interface RenderedScene {
320
+ renderId: string
321
+ status: RenderedSceneStatus
322
  assetUrl: string
323
  error: string
324
  maskBase64: string
src/utils/generateImageSDXL.mts CHANGED
@@ -27,6 +27,7 @@ export async function generateImageSDXLAsBase64(options: {
27
  const width = getValidNumber(options?.width, 256, 1024, 512)
28
  const height = getValidNumber(options?.height, 256, 1024, 512)
29
  const nbSteps = getValidNumber(options?.nbSteps, 5, 100, 20)
 
30
 
31
  const instance = instances.shift()
32
  instances.push(instance)
@@ -68,8 +69,8 @@ export async function generateImageSDXLAsBase64(options: {
68
  seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component
69
  width, // number (numeric value between 256 and 1024) in 'Width' Slider component
70
  height, // number (numeric value between 256 and 1024) in 'Height' Slider component
71
- 7, // number (numeric value between 1 and 20) in 'Guidance scale for base' Slider component
72
- 7, // number (numeric value between 1 and 20) in 'Guidance scale for refiner' Slider component
73
  nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for base' Slider component
74
  nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for refiner' Slider component
75
  true, // boolean in 'Apply refiner' Checkbox component
 
27
  const width = getValidNumber(options?.width, 256, 1024, 512)
28
  const height = getValidNumber(options?.height, 256, 1024, 512)
29
  const nbSteps = getValidNumber(options?.nbSteps, 5, 100, 20)
30
+ console.log("SEED:", seed)
31
 
32
  const instance = instances.shift()
33
  instances.push(instance)
 
69
  seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component
70
  width, // number (numeric value between 256 and 1024) in 'Width' Slider component
71
  height, // number (numeric value between 256 and 1024) in 'Height' Slider component
72
+ 8, // number (numeric value between 1 and 20) in 'Guidance scale for base' Slider component
73
+ 8, // number (numeric value between 1 and 20) in 'Guidance scale for refiner' Slider component
74
  nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for base' Slider component
75
  nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for refiner' Slider component
76
  true, // boolean in 'Apply refiner' Checkbox component
src/utils/generateImageSDXL360.mts ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { client } from "@gradio/client"
2
+
3
+ import { generateSeed } from "../utils/generateSeed.mts"
4
+ import { getValidNumber } from "./getValidNumber.mts"
5
+
6
+ // we don't use replicas yet, because it ain't easy to get their hostname
7
+ const instances: string[] = [
8
+ `${process.env.VC_SDXL_360_SPACE_API_URL_1 || ""}`,
9
+ // `${process.env.VC_SDXL_SPACE_API_URL_2 || ""}`,
10
+ ].filter(instance => instance?.length > 0)
11
+
12
+ export async function generateImageSDXL360AsBase64(options: {
13
+ positivePrompt: string;
14
+ negativePrompt?: string;
15
+ seed?: number;
16
+ width?: number;
17
+ height?: number;
18
+ nbSteps?: number;
19
+ }) {
20
+
21
+ const positivePrompt = options?.positivePrompt || ""
22
+ if (!positivePrompt) {
23
+ throw new Error("missing prompt")
24
+ }
25
+ const negativePrompt = options?.negativePrompt || ""
26
+ const seed = getValidNumber(options?.seed, 0, 2147483647, generateSeed())
27
+ const width = getValidNumber(options?.width, 256, 1024, 512)
28
+ const height = getValidNumber(options?.height, 256, 1024, 512)
29
+ const nbSteps = getValidNumber(options?.nbSteps, 5, 100, 20)
30
+ console.log("SEED FOR 360:", seed)
31
+
32
+ const instance = instances.shift()
33
+ instances.push(instance)
34
+
35
+ const positive = [
36
+ "360 view",
37
+ positivePrompt,
38
+ "beautiful",
39
+ "intricate details",
40
+ "award winning",
41
+ "high resolution"
42
+ ].filter(word => word)
43
+ .join(", ")
44
+
45
+ const negative = [
46
+ negativePrompt,
47
+ "watermark",
48
+ "copyright",
49
+ "blurry",
50
+ // "artificial",
51
+ // "cropped",
52
+ "low quality",
53
+ "ugly"
54
+ ].filter(word => word)
55
+ .join(", ")
56
+
57
+ const api = await client(instance, {
58
+ hf_token: `${process.env.VC_HF_API_TOKEN}` as any
59
+ })
60
+
61
+
62
+ const rawResponse = (await api.predict("/run", [
63
+ positive, // string in 'Prompt' Textbox component
64
+ negative, // string in 'Negative prompt' Textbox component
65
+ positive, // string in 'Prompt 2' Textbox component
66
+ negative, // string in 'Negative prompt 2' Textbox component
67
+ true, // boolean in 'Use negative prompt' Checkbox component
68
+ false, // boolean in 'Use prompt 2' Checkbox component
69
+ false, // boolean in 'Use negative prompt 2' Checkbox component
70
+ seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component
71
+ width, // number (numeric value between 256 and 1024) in 'Width' Slider component
72
+ height, // number (numeric value between 256 and 1024) in 'Height' Slider component
73
+ 8, // number (numeric value between 1 and 20) in 'Guidance scale for base' Slider component
74
+ 8, // number (numeric value between 1 and 20) in 'Guidance scale for refiner' Slider component
75
+ nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for base' Slider component
76
+ nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for refiner' Slider component
77
+ true, // boolean in 'Apply refiner' Checkbox component
78
+ ])) as any
79
+
80
+ return rawResponse?.data?.[0] as string
81
+ }