Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Commit
Β·
096584a
1
Parent(s):
f7ac73c
added code for async processing
Browse files- src/index.mts +63 -10
- src/production/renderImage.mts +54 -0
- src/production/{renderStaticScene.mts β renderImageSegmentation.mts} +27 -74
- src/production/renderPipeline.mts +46 -0
- src/production/renderScene.mts +57 -8
- src/production/renderVideo.mts +41 -0
- src/production/{renderVideoScene.mts β renderVideoSegmentation.mts} +16 -63
- src/types.mts +11 -0
- src/utils/generateImageSDXL.mts +3 -2
- src/utils/generateImageSDXL360.mts +81 -0
src/index.mts
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
import { createReadStream, existsSync } from "node:fs"
|
2 |
import path from "node:path"
|
3 |
|
4 |
-
import { validate as uuidValidate } from "uuid"
|
5 |
import express from "express"
|
6 |
|
7 |
import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene } from "./types.mts"
|
@@ -21,7 +21,7 @@ import { initFolders } from "./initFolders.mts"
|
|
21 |
import { sortVideosByYoungestFirst } from "./utils/sortVideosByYoungestFirst.mts"
|
22 |
import { generateVideo } from "./production/generateVideo.mts"
|
23 |
import { generateSeed } from "./utils/generateSeed.mts"
|
24 |
-
import { renderScene } from "./production/renderScene.mts"
|
25 |
|
26 |
initFolders()
|
27 |
// to disable all processing (eg. to debug)
|
@@ -48,7 +48,9 @@ app.post("/render", async (req, res) => {
|
|
48 |
return
|
49 |
}
|
50 |
|
51 |
-
let
|
|
|
|
|
52 |
assetUrl: "",
|
53 |
maskBase64: "",
|
54 |
error: "",
|
@@ -56,33 +58,84 @@ app.post("/render", async (req, res) => {
|
|
56 |
}
|
57 |
|
58 |
try {
|
59 |
-
|
60 |
} catch (err) {
|
61 |
// console.log("failed to render scene!")
|
62 |
-
|
63 |
}
|
64 |
|
65 |
-
if (
|
66 |
console.log("server busy")
|
67 |
res.status(200)
|
68 |
-
res.write(JSON.stringify(
|
69 |
res.end()
|
70 |
return
|
71 |
-
} else if (
|
72 |
// console.log("server error")
|
73 |
res.status(500)
|
74 |
-
res.write(JSON.stringify(
|
75 |
res.end()
|
76 |
return
|
77 |
} else {
|
78 |
// console.log("all good")
|
79 |
res.status(200)
|
80 |
-
res.write(JSON.stringify(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
81 |
res.end()
|
82 |
return
|
83 |
}
|
84 |
})
|
85 |
|
|
|
86 |
// a "fast track" pipeline
|
87 |
/*
|
88 |
app.post("/segment", async (req, res) => {
|
|
|
1 |
import { createReadStream, existsSync } from "node:fs"
|
2 |
import path from "node:path"
|
3 |
|
4 |
+
import { v4 as uuidv4, validate as uuidValidate } from "uuid"
|
5 |
import express from "express"
|
6 |
|
7 |
import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene } from "./types.mts"
|
|
|
21 |
import { sortVideosByYoungestFirst } from "./utils/sortVideosByYoungestFirst.mts"
|
22 |
import { generateVideo } from "./production/generateVideo.mts"
|
23 |
import { generateSeed } from "./utils/generateSeed.mts"
|
24 |
+
import { getRenderedScene, renderScene } from "./production/renderScene.mts"
|
25 |
|
26 |
initFolders()
|
27 |
// to disable all processing (eg. to debug)
|
|
|
48 |
return
|
49 |
}
|
50 |
|
51 |
+
let response: RenderedScene = {
|
52 |
+
renderId: "",
|
53 |
+
status: "pending",
|
54 |
assetUrl: "",
|
55 |
maskBase64: "",
|
56 |
error: "",
|
|
|
58 |
}
|
59 |
|
60 |
try {
|
61 |
+
response = await renderScene(request)
|
62 |
} catch (err) {
|
63 |
// console.log("failed to render scene!")
|
64 |
+
response.error = `failed to render scene: ${err}`
|
65 |
}
|
66 |
|
67 |
+
if (response.error === "already rendering") {
|
68 |
console.log("server busy")
|
69 |
res.status(200)
|
70 |
+
res.write(JSON.stringify(response))
|
71 |
res.end()
|
72 |
return
|
73 |
+
} else if (response.error.length > 0) {
|
74 |
// console.log("server error")
|
75 |
res.status(500)
|
76 |
+
res.write(JSON.stringify(response))
|
77 |
res.end()
|
78 |
return
|
79 |
} else {
|
80 |
// console.log("all good")
|
81 |
res.status(200)
|
82 |
+
res.write(JSON.stringify(response))
|
83 |
+
res.end()
|
84 |
+
return
|
85 |
+
}
|
86 |
+
})
|
87 |
+
|
88 |
+
// a "fast track" pipeline
|
89 |
+
app.get("/render/:renderId", async (req, res) => {
|
90 |
+
|
91 |
+
const renderId = `${req.params.renderId}`
|
92 |
+
|
93 |
+
if (!uuidValidate(renderId)) {
|
94 |
+
console.error("invalid render id")
|
95 |
+
res.status(400)
|
96 |
+
res.write(JSON.stringify({ error: `invalid render id` }))
|
97 |
+
res.end()
|
98 |
+
return
|
99 |
+
}
|
100 |
+
|
101 |
+
let response: RenderedScene = {
|
102 |
+
renderId: "",
|
103 |
+
status: "pending",
|
104 |
+
assetUrl: "",
|
105 |
+
error: "",
|
106 |
+
maskBase64: "",
|
107 |
+
segments: []
|
108 |
+
}
|
109 |
+
|
110 |
+
try {
|
111 |
+
response = await getRenderedScene(renderId)
|
112 |
+
} catch (err) {
|
113 |
+
// console.log("failed to render scene!")
|
114 |
+
response.error = `failed to render scene: ${err}`
|
115 |
+
}
|
116 |
+
|
117 |
+
if (response.error === "already rendering") {
|
118 |
+
console.log("server busy")
|
119 |
+
res.status(200)
|
120 |
+
res.write(JSON.stringify(response))
|
121 |
+
res.end()
|
122 |
+
return
|
123 |
+
} else if (response.error.length > 0) {
|
124 |
+
// console.log("server error")
|
125 |
+
res.status(500)
|
126 |
+
res.write(JSON.stringify(response))
|
127 |
+
res.end()
|
128 |
+
return
|
129 |
+
} else {
|
130 |
+
// console.log("all good")
|
131 |
+
res.status(200)
|
132 |
+
res.write(JSON.stringify(response))
|
133 |
res.end()
|
134 |
return
|
135 |
}
|
136 |
})
|
137 |
|
138 |
+
|
139 |
// a "fast track" pipeline
|
140 |
/*
|
141 |
app.post("/segment", async (req, res) => {
|
src/production/renderImage.mts
ADDED
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { RenderedScene, RenderRequest } from "../types.mts"
|
2 |
+
import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
|
3 |
+
import { generateImageSDXL360AsBase64 } from "../utils/generateImageSDXL360.mts"
|
4 |
+
import { generateSeed } from "../utils/generateSeed.mts"
|
5 |
+
|
6 |
+
export async function renderImage(
|
7 |
+
request: RenderRequest,
|
8 |
+
response: RenderedScene,
|
9 |
+
): Promise<RenderedScene> {
|
10 |
+
|
11 |
+
const isSpherical = request.projection === 'spherical'
|
12 |
+
|
13 |
+
const generateImageAsBase64 = isSpherical
|
14 |
+
? generateImageSDXL360AsBase64
|
15 |
+
: generateImageSDXLAsBase64
|
16 |
+
|
17 |
+
console.log(`going to generate an image using ${request.projection || "default (cartesian)"} projection`)
|
18 |
+
|
19 |
+
const params = {
|
20 |
+
positivePrompt: request.prompt,
|
21 |
+
seed: request.seed,
|
22 |
+
nbSteps: request.nbSteps,
|
23 |
+
width: request.width,
|
24 |
+
request: request.height
|
25 |
+
}
|
26 |
+
|
27 |
+
console.log(`calling generateImageAsBase64 with: `, JSON.stringify(params, null, 2))
|
28 |
+
|
29 |
+
|
30 |
+
// first we generate a quick low quality version
|
31 |
+
try {
|
32 |
+
response.assetUrl = await generateImageAsBase64(params)
|
33 |
+
console.log("successful generation!", response.assetUrl.slice(0, 30))
|
34 |
+
if (!response.assetUrl?.length) {
|
35 |
+
throw new Error(`the generated image is empty`)
|
36 |
+
}
|
37 |
+
} catch (err) {
|
38 |
+
console.error(`failed to render.. but let's try again!`)
|
39 |
+
try {
|
40 |
+
response.assetUrl = await generateImageAsBase64(params)
|
41 |
+
console.log("successful generation!", response.assetUrl.slice(0, 30))
|
42 |
+
if (!response.assetUrl?.length) {
|
43 |
+
throw new Error(`the generated image is empty`)
|
44 |
+
}
|
45 |
+
} catch (err) {
|
46 |
+
console.error(`failed to generate the image, although ${err}`)
|
47 |
+
response.error = `failed to render scene: ${err}`
|
48 |
+
response.status = "error"
|
49 |
+
response.assetUrl = ""
|
50 |
+
}
|
51 |
+
}
|
52 |
+
|
53 |
+
return response
|
54 |
+
}
|
src/production/{renderStaticScene.mts β renderImageSegmentation.mts}
RENAMED
@@ -3,63 +3,17 @@ import path from "node:path"
|
|
3 |
import { v4 as uuidv4 } from "uuid"
|
4 |
import tmpDir from "temp-dir"
|
5 |
|
6 |
-
import {
|
7 |
import { segmentImage } from "../utils/segmentImage.mts"
|
8 |
-
import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
|
9 |
import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
|
10 |
|
11 |
|
12 |
-
|
|
|
|
|
|
|
13 |
|
14 |
-
|
15 |
-
|
16 |
-
let imageBase64 = ""
|
17 |
-
let error = ""
|
18 |
-
|
19 |
-
const width = 1024
|
20 |
-
const height = 512
|
21 |
-
|
22 |
-
const params = {
|
23 |
-
positivePrompt: scene.prompt,
|
24 |
-
seed: scene.seed || undefined,
|
25 |
-
nbSteps: scene.nbSteps || undefined,
|
26 |
-
width,
|
27 |
-
height
|
28 |
-
}
|
29 |
-
console.log(`calling generateImageSDXLAsBase64 with: `, JSON.stringify(params, null, 2))
|
30 |
-
|
31 |
-
try {
|
32 |
-
imageBase64 = await generateImageSDXLAsBase64(params)
|
33 |
-
console.log("successful generation!", imageBase64.slice(0, 30))
|
34 |
-
error = ""
|
35 |
-
if (!imageBase64?.length) {
|
36 |
-
throw new Error(`the generated image is empty`)
|
37 |
-
}
|
38 |
-
} catch (err) {
|
39 |
-
console.error(`failed to render.. but let's try again!`)
|
40 |
-
try {
|
41 |
-
imageBase64 = await generateImageSDXLAsBase64(params)
|
42 |
-
console.log("successful generation!", imageBase64.slice(0, 30))
|
43 |
-
error = ""
|
44 |
-
if (!imageBase64?.length) {
|
45 |
-
throw new Error(`the generated image is empty`)
|
46 |
-
}
|
47 |
-
} catch (err) {
|
48 |
-
console.error(`failed to generate the image, although ${err}`)
|
49 |
-
error = `failed to render scene: ${err}`
|
50 |
-
return {
|
51 |
-
assetUrl: imageBase64,
|
52 |
-
error,
|
53 |
-
maskBase64: "",
|
54 |
-
segments: []
|
55 |
-
} as RenderedScene
|
56 |
-
}
|
57 |
-
}
|
58 |
-
|
59 |
-
const actionnables = Array.isArray(scene.actionnables) ? scene.actionnables : []
|
60 |
-
|
61 |
-
let mask = ""
|
62 |
-
let segments: ImageSegment[] = []
|
63 |
|
64 |
if (actionnables.length > 0) {
|
65 |
console.log("we have some actionnables:", actionnables)
|
@@ -68,35 +22,41 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
|
|
68 |
const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
|
69 |
|
70 |
// console.log("beginning:", imageBase64.slice(0, 100))
|
71 |
-
await writeBase64ToFile(
|
72 |
console.log("wrote the image to ", tmpImageFilePath)
|
73 |
|
74 |
if (!tmpImageFilePath) {
|
75 |
console.error("failed to get the image")
|
76 |
-
error = "failed to segment the image"
|
|
|
77 |
} else {
|
78 |
console.log("got the first frame! segmenting..")
|
79 |
try {
|
80 |
-
const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
|
81 |
-
|
82 |
-
segments = result.segments
|
83 |
-
|
|
|
84 |
} catch (err) {
|
85 |
console.log("this takes too long :/ trying another server..")
|
86 |
try {
|
87 |
-
const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
|
88 |
-
|
89 |
-
segments = result.segments
|
90 |
-
|
|
|
91 |
} catch (err) {
|
92 |
console.log("trying one last time, on a 3rd server..")
|
93 |
try {
|
94 |
-
const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
|
95 |
-
|
96 |
-
segments = result.segments
|
97 |
-
|
|
|
98 |
} catch (err) {
|
99 |
console.log("yeah, all servers are busy it seems.. aborting")
|
|
|
|
|
100 |
}
|
101 |
}
|
102 |
}
|
@@ -105,12 +65,5 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
|
|
105 |
console.log("no actionnables: just returning the image, then")
|
106 |
}
|
107 |
|
108 |
-
|
109 |
-
|
110 |
-
return {
|
111 |
-
assetUrl: imageBase64,
|
112 |
-
error,
|
113 |
-
maskBase64: mask,
|
114 |
-
segments
|
115 |
-
} as RenderedScene
|
116 |
}
|
|
|
3 |
import { v4 as uuidv4 } from "uuid"
|
4 |
import tmpDir from "temp-dir"
|
5 |
|
6 |
+
import { RenderedScene, RenderRequest } from "../types.mts"
|
7 |
import { segmentImage } from "../utils/segmentImage.mts"
|
|
|
8 |
import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
|
9 |
|
10 |
|
11 |
+
export async function renderImageSegmentation(
|
12 |
+
request: RenderRequest,
|
13 |
+
response: RenderedScene,
|
14 |
+
): Promise<RenderedScene> {
|
15 |
|
16 |
+
const actionnables = Array.isArray(request.actionnables) ? request.actionnables : []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
17 |
|
18 |
if (actionnables.length > 0) {
|
19 |
console.log("we have some actionnables:", actionnables)
|
|
|
22 |
const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
|
23 |
|
24 |
// console.log("beginning:", imageBase64.slice(0, 100))
|
25 |
+
await writeBase64ToFile(response.assetUrl, tmpImageFilePath)
|
26 |
console.log("wrote the image to ", tmpImageFilePath)
|
27 |
|
28 |
if (!tmpImageFilePath) {
|
29 |
console.error("failed to get the image")
|
30 |
+
response.error = "failed to segment the image"
|
31 |
+
response.status = "error"
|
32 |
} else {
|
33 |
console.log("got the first frame! segmenting..")
|
34 |
try {
|
35 |
+
const result = await segmentImage(tmpImageFilePath, actionnables, request.width, request.height)
|
36 |
+
response.maskBase64 = result.pngInBase64
|
37 |
+
response.segments = result.segments
|
38 |
+
|
39 |
+
console.log(`it worked the first time! got ${response.segments.length} segments`)
|
40 |
} catch (err) {
|
41 |
console.log("this takes too long :/ trying another server..")
|
42 |
try {
|
43 |
+
const result = await segmentImage(tmpImageFilePath, actionnables, request.width, request.height)
|
44 |
+
response.maskBase64 = result.pngInBase64
|
45 |
+
response.segments = result.segments
|
46 |
+
|
47 |
+
console.log(`it worked the second time! got ${response.segments.length} segments`)
|
48 |
} catch (err) {
|
49 |
console.log("trying one last time, on a 3rd server..")
|
50 |
try {
|
51 |
+
const result = await segmentImage(tmpImageFilePath, actionnables, request.width, request.height)
|
52 |
+
response.maskBase64 = result.pngInBase64
|
53 |
+
response.segments = result.segments
|
54 |
+
|
55 |
+
console.log(`it worked the third time! got ${response.segments.length} segments`)
|
56 |
} catch (err) {
|
57 |
console.log("yeah, all servers are busy it seems.. aborting")
|
58 |
+
response.error = "all servers are busy"
|
59 |
+
response.status = "error"
|
60 |
}
|
61 |
}
|
62 |
}
|
|
|
65 |
console.log("no actionnables: just returning the image, then")
|
66 |
}
|
67 |
|
68 |
+
return response
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
69 |
}
|
src/production/renderPipeline.mts
ADDED
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
import { RenderedScene, RenderRequest } from "../types.mts"
|
3 |
+
|
4 |
+
import { renderImage } from "./renderImage.mts"
|
5 |
+
import { renderVideo } from "./renderVideo.mts"
|
6 |
+
import { renderImageSegmentation } from "./renderImageSegmentation.mts"
|
7 |
+
import { renderVideoSegmentation } from "./renderVideoSegmentation.mts"
|
8 |
+
|
9 |
+
export async function renderPipeline(request: RenderRequest, response: RenderedScene) {
|
10 |
+
const isVideo = request?.nbFrames > 1
|
11 |
+
|
12 |
+
const renderContent = isVideo ? renderVideo : renderImage
|
13 |
+
const renderSegmentation = isVideo ? renderVideoSegmentation : renderImageSegmentation
|
14 |
+
|
15 |
+
if (isVideo) {
|
16 |
+
console.log(`rendering a video..`)
|
17 |
+
} else {
|
18 |
+
console.log(`rendering an image..`)
|
19 |
+
}
|
20 |
+
await renderContent(request, response)
|
21 |
+
await renderSegmentation(request, response)
|
22 |
+
|
23 |
+
/*
|
24 |
+
this is the optimized pipeline
|
25 |
+
However, right now it doesn't work because for some reason,
|
26 |
+
asking to generate the same seed + prompt on different nb of steps
|
27 |
+
doesn't generate the same image!
|
28 |
+
|
29 |
+
// first we need to wait for the low quality pre-render
|
30 |
+
await renderContent({
|
31 |
+
...request,
|
32 |
+
|
33 |
+
// we are a bit more aggressive with the quality of the video preview
|
34 |
+
nbSteps: isVideo ? 8 : 16
|
35 |
+
}, response)
|
36 |
+
|
37 |
+
// then we can run both the segmentation and the high-res render at the same time
|
38 |
+
await Promise.all([
|
39 |
+
renderSegmentation(request, response),
|
40 |
+
renderContent(request, response)
|
41 |
+
])
|
42 |
+
*/
|
43 |
+
|
44 |
+
response.status = "completed"
|
45 |
+
response.error = ""
|
46 |
+
}
|
src/production/renderScene.mts
CHANGED
@@ -1,13 +1,62 @@
|
|
|
|
|
|
1 |
import { RenderedScene, RenderRequest } from "../types.mts"
|
2 |
-
import {
|
3 |
-
import {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
4 |
|
5 |
-
|
6 |
-
|
7 |
-
|
8 |
-
|
|
|
|
|
|
|
|
|
|
|
9 |
} else {
|
10 |
-
|
11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
12 |
}
|
|
|
13 |
}
|
|
|
1 |
+
import { v4 as uuidv4 } from "uuid"
|
2 |
+
|
3 |
import { RenderedScene, RenderRequest } from "../types.mts"
|
4 |
+
import { generateSeed } from "../utils/generateSeed.mts"
|
5 |
+
import { getValidNumber } from "../utils/getValidNumber.mts"
|
6 |
+
import { renderPipeline } from "./renderPipeline.mts"
|
7 |
+
|
8 |
+
const cache: Record<string, RenderedScene> = {}
|
9 |
+
const cacheQueue: string[] = []
|
10 |
+
const maxCacheSize = 1000
|
11 |
+
|
12 |
+
export async function renderScene(request: RenderRequest): Promise<RenderedScene> {
|
13 |
+
// const key = getCacheKey(scene)
|
14 |
+
const renderId = uuidv4()
|
15 |
+
|
16 |
+
request.nbFrames = getValidNumber(request.nbFrames, 1, 24, 16)
|
17 |
|
18 |
+
const isVideo = request?.nbFrames === 1
|
19 |
+
|
20 |
+
// important: we need a consistent seed for our multiple rendering passes
|
21 |
+
request.seed = getValidNumber(request.seed, 0, 2147483647, generateSeed())
|
22 |
+
request.nbSteps = getValidNumber(request.nbSteps, 5, 50, 10)
|
23 |
+
|
24 |
+
if (isVideo) {
|
25 |
+
request.width = getValidNumber(request.width, 256, 1024, 1024)
|
26 |
+
request.height = getValidNumber(request.width, 256, 1024, 512)
|
27 |
} else {
|
28 |
+
request.width = getValidNumber(request.width, 256, 1280, 576)
|
29 |
+
request.height = getValidNumber(request.width, 256, 720, 320)
|
30 |
+
}
|
31 |
+
|
32 |
+
const response: RenderedScene = {
|
33 |
+
renderId,
|
34 |
+
status: "pending",
|
35 |
+
assetUrl: "",
|
36 |
+
error: "",
|
37 |
+
maskBase64: "",
|
38 |
+
segments: []
|
39 |
+
}
|
40 |
+
|
41 |
+
cache[renderId] = response
|
42 |
+
cacheQueue.push(renderId)
|
43 |
+
if (cacheQueue.length > maxCacheSize) {
|
44 |
+
const toRemove = cacheQueue.shift()
|
45 |
+
delete cache[toRemove]
|
46 |
+
}
|
47 |
+
|
48 |
+
// this is a fire-and-forget asynchronous pipeline:
|
49 |
+
// we start it, but we do not await for the response
|
50 |
+
renderPipeline(request, response)
|
51 |
+
|
52 |
+
console.log("renderScene: yielding the scene", response)
|
53 |
+
return response
|
54 |
+
}
|
55 |
+
|
56 |
+
export async function getRenderedScene(renderId: string): Promise<RenderedScene> {
|
57 |
+
const rendered = cache[renderId]
|
58 |
+
if (!rendered) {
|
59 |
+
throw new Error(`couldn't find any rendered scene with renderId ${renderId}`)
|
60 |
}
|
61 |
+
return cache[renderId]
|
62 |
}
|
src/production/renderVideo.mts
ADDED
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { RenderedScene, RenderRequest } from "../types.mts"
|
2 |
+
import { generateVideo } from "./generateVideo.mts"
|
3 |
+
|
4 |
+
export async function renderVideo(
|
5 |
+
request: RenderRequest,
|
6 |
+
response: RenderedScene
|
7 |
+
): Promise<RenderedScene> {
|
8 |
+
|
9 |
+
const params = {
|
10 |
+
seed: request.seed,
|
11 |
+
nbFrames: request.nbFrames,
|
12 |
+
nbSteps: request.nbSteps,
|
13 |
+
}
|
14 |
+
|
15 |
+
try {
|
16 |
+
response.assetUrl = await generateVideo(request.prompt, params)
|
17 |
+
// console.log("successfull generation")
|
18 |
+
|
19 |
+
if (!response.assetUrl?.length) {
|
20 |
+
throw new Error(`url for the generated video is empty`)
|
21 |
+
}
|
22 |
+
} catch (err) {
|
23 |
+
console.error(`failed to render the video scene.. but let's try again!`)
|
24 |
+
|
25 |
+
try {
|
26 |
+
response.assetUrl = await generateVideo(request.prompt, params)
|
27 |
+
// console.log("successfull generation")
|
28 |
+
|
29 |
+
if (!response.assetUrl?.length) {
|
30 |
+
throw new Error(`url for the generated video is empty`)
|
31 |
+
}
|
32 |
+
|
33 |
+
} catch (err) {
|
34 |
+
console.error(`it failed the video for second time ${err}`)
|
35 |
+
response.error = `failed to render video scene: ${err}`
|
36 |
+
response.status = "error"
|
37 |
+
}
|
38 |
+
}
|
39 |
+
|
40 |
+
return response
|
41 |
+
}
|
src/production/{renderVideoScene.mts β renderVideoSegmentation.mts}
RENAMED
@@ -1,76 +1,36 @@
|
|
1 |
import { v4 as uuidv4 } from "uuid"
|
2 |
|
3 |
-
import {
|
4 |
import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
|
5 |
-
import { generateSeed } from "../utils/generateSeed.mts"
|
6 |
-
import { getValidNumber } from "../utils/getValidNumber.mts"
|
7 |
-
import { generateVideo } from "./generateVideo.mts"
|
8 |
import { getFirstVideoFrame } from "../utils/getFirstVideoFrame.mts"
|
9 |
import { segmentImage } from "../utils/segmentImage.mts"
|
10 |
|
11 |
-
export async function
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
const
|
17 |
-
const height = 320
|
18 |
-
|
19 |
-
const params = {
|
20 |
-
seed: getValidNumber(scene.seed, 0, 2147483647, generateSeed()),
|
21 |
-
nbFrames: getValidNumber(scene.nbFrames, 8, 24, 16), // 2 seconds by default
|
22 |
-
nbSteps: getValidNumber(scene.nbSteps, 1, 50, 10), // use 10 by default to go fast, but not too sloppy
|
23 |
-
}
|
24 |
-
|
25 |
-
try {
|
26 |
-
url = await generateVideo(scene.prompt, params)
|
27 |
-
// console.log("successfull generation")
|
28 |
-
error = ""
|
29 |
-
if (!url?.length) {
|
30 |
-
throw new Error(`url for the generated image is empty`)
|
31 |
-
}
|
32 |
-
} catch (err) {
|
33 |
-
console.error(`failed to render the scene.. but let's try again!`)
|
34 |
-
|
35 |
-
try {
|
36 |
-
url = await generateVideo(scene.prompt, params)
|
37 |
-
// console.log("successfull generation")
|
38 |
-
error = ""
|
39 |
-
|
40 |
-
if (!url?.length) {
|
41 |
-
throw new Error(`url for the generated image is empty`)
|
42 |
-
}
|
43 |
-
|
44 |
-
} catch (err) {
|
45 |
-
console.error(`it failed the second time ${err}`)
|
46 |
-
error = `failed to render scene: ${err}`
|
47 |
-
}
|
48 |
-
}
|
49 |
-
|
50 |
-
|
51 |
-
// TODO add segmentation here
|
52 |
-
const actionnables = Array.isArray(scene.actionnables) ? scene.actionnables : []
|
53 |
-
|
54 |
-
let mask = ""
|
55 |
-
let segments: ImageSegment[] = []
|
56 |
|
57 |
if (actionnables.length > 0) {
|
58 |
console.log("we have some actionnables:", actionnables)
|
59 |
-
if (
|
60 |
console.log("going to grab the first frame")
|
61 |
-
const tmpVideoFilePath = await downloadFileToTmp(
|
62 |
console.log("downloaded the first frame to ", tmpVideoFilePath)
|
63 |
const firstFrameFilePath = await getFirstVideoFrame(tmpVideoFilePath)
|
64 |
console.log("downloaded the first frame to ", firstFrameFilePath)
|
65 |
|
66 |
if (!firstFrameFilePath) {
|
67 |
console.error("failed to get the image")
|
68 |
-
error = "failed to segment the image"
|
|
|
69 |
} else {
|
70 |
console.log("got the first frame! segmenting..")
|
71 |
-
const result = await segmentImage(firstFrameFilePath, actionnables, width, height)
|
72 |
-
|
73 |
-
segments = result.segments
|
|
|
74 |
// console.log("success!", { segments })
|
75 |
}
|
76 |
/*
|
@@ -88,12 +48,5 @@ export async function renderVideoScene(scene: RenderRequest): Promise<RenderedSc
|
|
88 |
}
|
89 |
}
|
90 |
|
91 |
-
|
92 |
-
|
93 |
-
return {
|
94 |
-
assetUrl: url,
|
95 |
-
error,
|
96 |
-
maskBase64: mask,
|
97 |
-
segments
|
98 |
-
} as RenderedScene
|
99 |
}
|
|
|
1 |
import { v4 as uuidv4 } from "uuid"
|
2 |
|
3 |
+
import { RenderedScene, RenderRequest } from "../types.mts"
|
4 |
import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
|
|
|
|
|
|
|
5 |
import { getFirstVideoFrame } from "../utils/getFirstVideoFrame.mts"
|
6 |
import { segmentImage } from "../utils/segmentImage.mts"
|
7 |
|
8 |
+
export async function renderVideoSegmentation(
|
9 |
+
request: RenderRequest,
|
10 |
+
response: RenderedScene
|
11 |
+
): Promise<RenderedScene> {
|
12 |
+
|
13 |
+
const actionnables = Array.isArray(request.actionnables) ? request.actionnables : []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
14 |
|
15 |
if (actionnables.length > 0) {
|
16 |
console.log("we have some actionnables:", actionnables)
|
17 |
+
if (request.segmentation === "firstframe") {
|
18 |
console.log("going to grab the first frame")
|
19 |
+
const tmpVideoFilePath = await downloadFileToTmp(response.assetUrl, `${uuidv4()}`)
|
20 |
console.log("downloaded the first frame to ", tmpVideoFilePath)
|
21 |
const firstFrameFilePath = await getFirstVideoFrame(tmpVideoFilePath)
|
22 |
console.log("downloaded the first frame to ", firstFrameFilePath)
|
23 |
|
24 |
if (!firstFrameFilePath) {
|
25 |
console.error("failed to get the image")
|
26 |
+
response.error = "failed to segment the image"
|
27 |
+
response.status = "error"
|
28 |
} else {
|
29 |
console.log("got the first frame! segmenting..")
|
30 |
+
const result = await segmentImage(firstFrameFilePath, actionnables, request.width, request.height)
|
31 |
+
response.maskBase64 = result.pngInBase64
|
32 |
+
response.segments = result.segments
|
33 |
+
|
34 |
// console.log("success!", { segments })
|
35 |
}
|
36 |
/*
|
|
|
48 |
}
|
49 |
}
|
50 |
|
51 |
+
return response
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
52 |
}
|
src/types.mts
CHANGED
@@ -269,6 +269,8 @@ export type Video = VideoSequence & {
|
|
269 |
shots: VideoShot[]
|
270 |
}
|
271 |
|
|
|
|
|
272 |
export interface RenderRequest {
|
273 |
prompt: string
|
274 |
|
@@ -292,6 +294,11 @@ export interface RenderRequest {
|
|
292 |
nbSteps: number // min: 1, max: 50
|
293 |
|
294 |
seed: number
|
|
|
|
|
|
|
|
|
|
|
295 |
}
|
296 |
|
297 |
export interface ImageSegmentationRequest {
|
@@ -307,7 +314,11 @@ export interface ImageSegment {
|
|
307 |
score: number
|
308 |
}
|
309 |
|
|
|
|
|
310 |
export interface RenderedScene {
|
|
|
|
|
311 |
assetUrl: string
|
312 |
error: string
|
313 |
maskBase64: string
|
|
|
269 |
shots: VideoShot[]
|
270 |
}
|
271 |
|
272 |
+
export type ProjectionMode = 'cartesian' | 'spherical'
|
273 |
+
|
274 |
export interface RenderRequest {
|
275 |
prompt: string
|
276 |
|
|
|
294 |
nbSteps: number // min: 1, max: 50
|
295 |
|
296 |
seed: number
|
297 |
+
|
298 |
+
width: number
|
299 |
+
height: number
|
300 |
+
|
301 |
+
projection: ProjectionMode
|
302 |
}
|
303 |
|
304 |
export interface ImageSegmentationRequest {
|
|
|
314 |
score: number
|
315 |
}
|
316 |
|
317 |
+
export type RenderedSceneStatus = 'pending' | 'completed' | 'error'
|
318 |
+
|
319 |
export interface RenderedScene {
|
320 |
+
renderId: string
|
321 |
+
status: RenderedSceneStatus
|
322 |
assetUrl: string
|
323 |
error: string
|
324 |
maskBase64: string
|
src/utils/generateImageSDXL.mts
CHANGED
@@ -27,6 +27,7 @@ export async function generateImageSDXLAsBase64(options: {
|
|
27 |
const width = getValidNumber(options?.width, 256, 1024, 512)
|
28 |
const height = getValidNumber(options?.height, 256, 1024, 512)
|
29 |
const nbSteps = getValidNumber(options?.nbSteps, 5, 100, 20)
|
|
|
30 |
|
31 |
const instance = instances.shift()
|
32 |
instances.push(instance)
|
@@ -68,8 +69,8 @@ export async function generateImageSDXLAsBase64(options: {
|
|
68 |
seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component
|
69 |
width, // number (numeric value between 256 and 1024) in 'Width' Slider component
|
70 |
height, // number (numeric value between 256 and 1024) in 'Height' Slider component
|
71 |
-
|
72 |
-
|
73 |
nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for base' Slider component
|
74 |
nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for refiner' Slider component
|
75 |
true, // boolean in 'Apply refiner' Checkbox component
|
|
|
27 |
const width = getValidNumber(options?.width, 256, 1024, 512)
|
28 |
const height = getValidNumber(options?.height, 256, 1024, 512)
|
29 |
const nbSteps = getValidNumber(options?.nbSteps, 5, 100, 20)
|
30 |
+
console.log("SEED:", seed)
|
31 |
|
32 |
const instance = instances.shift()
|
33 |
instances.push(instance)
|
|
|
69 |
seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component
|
70 |
width, // number (numeric value between 256 and 1024) in 'Width' Slider component
|
71 |
height, // number (numeric value between 256 and 1024) in 'Height' Slider component
|
72 |
+
8, // number (numeric value between 1 and 20) in 'Guidance scale for base' Slider component
|
73 |
+
8, // number (numeric value between 1 and 20) in 'Guidance scale for refiner' Slider component
|
74 |
nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for base' Slider component
|
75 |
nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for refiner' Slider component
|
76 |
true, // boolean in 'Apply refiner' Checkbox component
|
src/utils/generateImageSDXL360.mts
ADDED
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { client } from "@gradio/client"
|
2 |
+
|
3 |
+
import { generateSeed } from "../utils/generateSeed.mts"
|
4 |
+
import { getValidNumber } from "./getValidNumber.mts"
|
5 |
+
|
6 |
+
// we don't use replicas yet, because it ain't easy to get their hostname
|
7 |
+
const instances: string[] = [
|
8 |
+
`${process.env.VC_SDXL_360_SPACE_API_URL_1 || ""}`,
|
9 |
+
// `${process.env.VC_SDXL_SPACE_API_URL_2 || ""}`,
|
10 |
+
].filter(instance => instance?.length > 0)
|
11 |
+
|
12 |
+
export async function generateImageSDXL360AsBase64(options: {
|
13 |
+
positivePrompt: string;
|
14 |
+
negativePrompt?: string;
|
15 |
+
seed?: number;
|
16 |
+
width?: number;
|
17 |
+
height?: number;
|
18 |
+
nbSteps?: number;
|
19 |
+
}) {
|
20 |
+
|
21 |
+
const positivePrompt = options?.positivePrompt || ""
|
22 |
+
if (!positivePrompt) {
|
23 |
+
throw new Error("missing prompt")
|
24 |
+
}
|
25 |
+
const negativePrompt = options?.negativePrompt || ""
|
26 |
+
const seed = getValidNumber(options?.seed, 0, 2147483647, generateSeed())
|
27 |
+
const width = getValidNumber(options?.width, 256, 1024, 512)
|
28 |
+
const height = getValidNumber(options?.height, 256, 1024, 512)
|
29 |
+
const nbSteps = getValidNumber(options?.nbSteps, 5, 100, 20)
|
30 |
+
console.log("SEED FOR 360:", seed)
|
31 |
+
|
32 |
+
const instance = instances.shift()
|
33 |
+
instances.push(instance)
|
34 |
+
|
35 |
+
const positive = [
|
36 |
+
"360 view",
|
37 |
+
positivePrompt,
|
38 |
+
"beautiful",
|
39 |
+
"intricate details",
|
40 |
+
"award winning",
|
41 |
+
"high resolution"
|
42 |
+
].filter(word => word)
|
43 |
+
.join(", ")
|
44 |
+
|
45 |
+
const negative = [
|
46 |
+
negativePrompt,
|
47 |
+
"watermark",
|
48 |
+
"copyright",
|
49 |
+
"blurry",
|
50 |
+
// "artificial",
|
51 |
+
// "cropped",
|
52 |
+
"low quality",
|
53 |
+
"ugly"
|
54 |
+
].filter(word => word)
|
55 |
+
.join(", ")
|
56 |
+
|
57 |
+
const api = await client(instance, {
|
58 |
+
hf_token: `${process.env.VC_HF_API_TOKEN}` as any
|
59 |
+
})
|
60 |
+
|
61 |
+
|
62 |
+
const rawResponse = (await api.predict("/run", [
|
63 |
+
positive, // string in 'Prompt' Textbox component
|
64 |
+
negative, // string in 'Negative prompt' Textbox component
|
65 |
+
positive, // string in 'Prompt 2' Textbox component
|
66 |
+
negative, // string in 'Negative prompt 2' Textbox component
|
67 |
+
true, // boolean in 'Use negative prompt' Checkbox component
|
68 |
+
false, // boolean in 'Use prompt 2' Checkbox component
|
69 |
+
false, // boolean in 'Use negative prompt 2' Checkbox component
|
70 |
+
seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component
|
71 |
+
width, // number (numeric value between 256 and 1024) in 'Width' Slider component
|
72 |
+
height, // number (numeric value between 256 and 1024) in 'Height' Slider component
|
73 |
+
8, // number (numeric value between 1 and 20) in 'Guidance scale for base' Slider component
|
74 |
+
8, // number (numeric value between 1 and 20) in 'Guidance scale for refiner' Slider component
|
75 |
+
nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for base' Slider component
|
76 |
+
nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for refiner' Slider component
|
77 |
+
true, // boolean in 'Apply refiner' Checkbox component
|
78 |
+
])) as any
|
79 |
+
|
80 |
+
return rawResponse?.data?.[0] as string
|
81 |
+
}
|