Spaces:
Build error
Build error
bill-jiang
commited on
Commit
•
f0a0274
1
Parent(s):
90b88a3
Update render method
Browse files- .gitignore +0 -1
- app.py +9 -56
- mGPT/render/blender/render.py +177 -0
- render.py +76 -0
.gitignore
CHANGED
@@ -8,7 +8,6 @@ __pycache__/
|
|
8 |
.DS_Store
|
9 |
pyglet
|
10 |
app2.py
|
11 |
-
render.py
|
12 |
cache
|
13 |
|
14 |
# Distribution / packaging
|
|
|
8 |
.DS_Store
|
9 |
pyglet
|
10 |
app2.py
|
|
|
11 |
cache
|
12 |
|
13 |
# Distribution / packaging
|
app.py
CHANGED
@@ -7,27 +7,18 @@ os.system('pip install pyglet==1.4.0a1')
|
|
7 |
os.system('pip install triangle==20220202')
|
8 |
|
9 |
import gradio as gr
|
10 |
-
import random
|
11 |
import torch
|
12 |
import time
|
13 |
-
import cv2
|
14 |
import numpy as np
|
15 |
-
import OpenGL.GL as gl
|
16 |
-
import imageio
|
17 |
import pytorch_lightning as pl
|
18 |
-
import
|
19 |
from pathlib import Path
|
20 |
from mGPT.data.build_data import build_data
|
21 |
from mGPT.models.build_model import build_model
|
22 |
from mGPT.config import parse_args
|
23 |
-
from scipy.spatial.transform import Rotation as RRR
|
24 |
-
import mGPT.render.matplot.plot_3d_global as plot_3d
|
25 |
-
from mGPT.render.pyrender.hybrik_loc2rot import HybrIKJointsToRotmat
|
26 |
-
from mGPT.render.pyrender.smpl_render import SMPLRender
|
27 |
from transformers import WhisperProcessor, WhisperForConditionalGeneration
|
28 |
import librosa
|
29 |
from huggingface_hub import snapshot_download
|
30 |
-
import eventlet
|
31 |
|
32 |
# Load model
|
33 |
cfg = parse_args(phase="webui") # parse config file
|
@@ -105,55 +96,17 @@ def render_motion(data, feats, method='fast'):
|
|
105 |
fname = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(
|
106 |
time.time())) + str(np.random.randint(10000, 99999))
|
107 |
video_fname = fname + '.mp4'
|
108 |
-
feats_fname = fname + '.npy'
|
|
|
109 |
output_npy_path = os.path.join(output_dir, feats_fname)
|
|
|
110 |
output_mp4_path = os.path.join(output_dir, video_fname)
|
111 |
np.save(output_npy_path, feats)
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
pose_generator = HybrIKJointsToRotmat()
|
118 |
-
pose = pose_generator(data)
|
119 |
-
pose = np.concatenate([
|
120 |
-
pose,
|
121 |
-
np.stack([np.stack([np.eye(3)] * pose.shape[0], 0)] * 2, 1)
|
122 |
-
], 1)
|
123 |
-
shape = [768, 768]
|
124 |
-
render = SMPLRender(cfg.RENDER.SMPL_MODEL_PATH)
|
125 |
-
|
126 |
-
r = RRR.from_rotvec(np.array([np.pi, 0.0, 0.0]))
|
127 |
-
pose[:, 0] = np.matmul(r.as_matrix().reshape(1, 3, 3), pose[:, 0])
|
128 |
-
vid = []
|
129 |
-
aroot = data[:, 0]
|
130 |
-
aroot[:, 1:] = -aroot[:, 1:]
|
131 |
-
params = dict(pred_shape=np.zeros([1, 10]),
|
132 |
-
pred_root=aroot,
|
133 |
-
pred_pose=pose)
|
134 |
-
render.init_renderer([shape[0], shape[1], 3], params)
|
135 |
-
for i in range(data.shape[0]):
|
136 |
-
renderImg = render.render(i)
|
137 |
-
vid.append(renderImg)
|
138 |
-
|
139 |
-
out = np.stack(vid, axis=0)
|
140 |
-
output_gif_path = output_mp4_path[:-4] + '.gif'
|
141 |
-
imageio.mimwrite(output_gif_path, out, duration=50)
|
142 |
-
out_video = mp.VideoFileClip(output_gif_path)
|
143 |
-
out_video.write_videofile(output_mp4_path)
|
144 |
-
del out, render
|
145 |
-
|
146 |
-
elif method == 'fast':
|
147 |
-
output_gif_path = output_mp4_path[:-4] + '.gif'
|
148 |
-
if len(data.shape) == 3:
|
149 |
-
data = data[None]
|
150 |
-
if isinstance(data, torch.Tensor):
|
151 |
-
data = data.cpu().numpy()
|
152 |
-
pose_vis = plot_3d.draw_to_batch(data, [''], [output_gif_path])
|
153 |
-
out_video = mp.VideoFileClip(output_gif_path)
|
154 |
-
out_video.write_videofile(output_mp4_path)
|
155 |
-
del pose_vis
|
156 |
-
|
157 |
return output_mp4_path, video_fname, output_npy_path, feats_fname
|
158 |
|
159 |
|
|
|
7 |
os.system('pip install triangle==20220202')
|
8 |
|
9 |
import gradio as gr
|
|
|
10 |
import torch
|
11 |
import time
|
|
|
12 |
import numpy as np
|
|
|
|
|
13 |
import pytorch_lightning as pl
|
14 |
+
import subprocess
|
15 |
from pathlib import Path
|
16 |
from mGPT.data.build_data import build_data
|
17 |
from mGPT.models.build_model import build_model
|
18 |
from mGPT.config import parse_args
|
|
|
|
|
|
|
|
|
19 |
from transformers import WhisperProcessor, WhisperForConditionalGeneration
|
20 |
import librosa
|
21 |
from huggingface_hub import snapshot_download
|
|
|
22 |
|
23 |
# Load model
|
24 |
cfg = parse_args(phase="webui") # parse config file
|
|
|
96 |
fname = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(
|
97 |
time.time())) + str(np.random.randint(10000, 99999))
|
98 |
video_fname = fname + '.mp4'
|
99 |
+
feats_fname = f"{fname}_feats" + '.npy'
|
100 |
+
data_fname = f"{fname}_joints" + '.npy'
|
101 |
output_npy_path = os.path.join(output_dir, feats_fname)
|
102 |
+
output_joints_path = os.path.join(output_dir, data_fname)
|
103 |
output_mp4_path = os.path.join(output_dir, video_fname)
|
104 |
np.save(output_npy_path, feats)
|
105 |
+
np.save(output_joints_path, data)
|
106 |
+
|
107 |
+
cmd3 = ["xvfb-run", "python", "-m", "render", "--joints_path", output_joints_path, "--method", method, "--output_mp4_path", output_mp4_path, "--smpl_model_path", cfg.RENDER.SMPL_MODEL_PATH]
|
108 |
+
subprocess.run(cmd3)
|
109 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
110 |
return output_mp4_path, video_fname, output_npy_path, feats_fname
|
111 |
|
112 |
|
mGPT/render/blender/render.py
ADDED
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import math
|
2 |
+
import os
|
3 |
+
import sys
|
4 |
+
import smplx
|
5 |
+
# import bpy
|
6 |
+
import numpy as np
|
7 |
+
|
8 |
+
from .camera import Camera
|
9 |
+
from .floor import get_trajectory, plot_floor, show_traj
|
10 |
+
from .sampler import get_frameidx
|
11 |
+
from .scene import setup_scene # noqa
|
12 |
+
from .tools import delete_objs, load_numpy_vertices_into_blender, style_detect
|
13 |
+
from .vertices import prepare_vertices
|
14 |
+
from mGPT.utils.joints import smplh_to_mmm_scaling_factor
|
15 |
+
|
16 |
+
|
17 |
+
def prune_begin_end(data, perc):
|
18 |
+
to_remove = int(len(data) * perc)
|
19 |
+
if to_remove == 0:
|
20 |
+
return data
|
21 |
+
return data[to_remove:-to_remove]
|
22 |
+
|
23 |
+
|
24 |
+
def render_current_frame(path):
|
25 |
+
bpy.context.scene.render.filepath = path
|
26 |
+
bpy.ops.render.render(use_viewport=True, write_still=True)
|
27 |
+
|
28 |
+
|
29 |
+
def render(npydata,
|
30 |
+
frames_folder,
|
31 |
+
*,
|
32 |
+
mode,
|
33 |
+
model_path,
|
34 |
+
faces_path,
|
35 |
+
gt=False,
|
36 |
+
exact_frame=None,
|
37 |
+
num=8,
|
38 |
+
downsample=True,
|
39 |
+
canonicalize=True,
|
40 |
+
always_on_floor=False,
|
41 |
+
denoising=True,
|
42 |
+
oldrender=True,
|
43 |
+
res="high",
|
44 |
+
init=True,
|
45 |
+
accelerator='gpu',
|
46 |
+
device=[0]):
|
47 |
+
if init:
|
48 |
+
# Setup the scene (lights / render engine / resolution etc)
|
49 |
+
setup_scene(res=res,
|
50 |
+
denoising=denoising,
|
51 |
+
oldrender=oldrender,
|
52 |
+
accelerator=accelerator,
|
53 |
+
device=device)
|
54 |
+
|
55 |
+
is_mesh, is_smplx, jointstype = style_detect(npydata)
|
56 |
+
|
57 |
+
if not is_mesh:
|
58 |
+
npydata = npydata * smplh_to_mmm_scaling_factor
|
59 |
+
|
60 |
+
if is_smplx:
|
61 |
+
smplx_model_male = smplx.create(model_path,
|
62 |
+
model_type='smplx',
|
63 |
+
gender='male',
|
64 |
+
ext='npz',
|
65 |
+
num_betas=10,
|
66 |
+
flat_hand_mean=True,
|
67 |
+
use_pca=False)
|
68 |
+
faces_path = smplx_model_male.faces
|
69 |
+
|
70 |
+
|
71 |
+
|
72 |
+
# Put everything in this folder
|
73 |
+
if mode == "video":
|
74 |
+
if always_on_floor:
|
75 |
+
frames_folder += "_of"
|
76 |
+
os.makedirs(frames_folder, exist_ok=True)
|
77 |
+
# if it is a mesh, it is already downsampled
|
78 |
+
if downsample and not is_mesh:
|
79 |
+
npydata = npydata[::8]
|
80 |
+
elif mode == "sequence":
|
81 |
+
img_name, ext = os.path.splitext(frames_folder)
|
82 |
+
if always_on_floor:
|
83 |
+
img_name += "_of"
|
84 |
+
img_path = f"{img_name}{ext}"
|
85 |
+
|
86 |
+
elif mode == "frame":
|
87 |
+
img_name, ext = os.path.splitext(frames_folder)
|
88 |
+
if always_on_floor:
|
89 |
+
img_name += "_of"
|
90 |
+
img_path = f"{img_name}_{exact_frame}{ext}"
|
91 |
+
|
92 |
+
# remove X% of begining and end
|
93 |
+
# as it is almost always static
|
94 |
+
# in this part
|
95 |
+
if mode == "sequence":
|
96 |
+
perc = 0.2
|
97 |
+
npydata = prune_begin_end(npydata, perc)
|
98 |
+
|
99 |
+
if is_mesh:
|
100 |
+
from .meshes import Meshes
|
101 |
+
data = Meshes(npydata,
|
102 |
+
gt=gt,
|
103 |
+
mode=mode,
|
104 |
+
faces_path=faces_path,
|
105 |
+
canonicalize=canonicalize,
|
106 |
+
always_on_floor=always_on_floor,
|
107 |
+
is_smplx=is_smplx)
|
108 |
+
else:
|
109 |
+
from .joints import Joints
|
110 |
+
data = Joints(npydata,
|
111 |
+
gt=gt,
|
112 |
+
mode=mode,
|
113 |
+
canonicalize=canonicalize,
|
114 |
+
always_on_floor=always_on_floor,
|
115 |
+
jointstype=jointstype)
|
116 |
+
|
117 |
+
# Number of frames possible to render
|
118 |
+
nframes = len(data)
|
119 |
+
|
120 |
+
# Show the trajectory
|
121 |
+
show_traj(data.trajectory)
|
122 |
+
|
123 |
+
# Create a floor
|
124 |
+
plot_floor(data.data, big_plane=False)
|
125 |
+
|
126 |
+
# initialize the camera
|
127 |
+
camera = Camera(first_root=data.get_root(0), mode=mode, is_mesh=is_mesh)
|
128 |
+
|
129 |
+
frameidx = get_frameidx(mode=mode,
|
130 |
+
nframes=nframes,
|
131 |
+
exact_frame=exact_frame,
|
132 |
+
frames_to_keep=num)
|
133 |
+
|
134 |
+
nframes_to_render = len(frameidx)
|
135 |
+
|
136 |
+
# center the camera to the middle
|
137 |
+
if mode == "sequence":
|
138 |
+
camera.update(data.get_mean_root())
|
139 |
+
|
140 |
+
imported_obj_names = []
|
141 |
+
for index, frameidx in enumerate(frameidx):
|
142 |
+
if mode == "sequence":
|
143 |
+
frac = index / (nframes_to_render - 1)
|
144 |
+
mat = data.get_sequence_mat(frac)
|
145 |
+
else:
|
146 |
+
mat = data.mat
|
147 |
+
camera.update(data.get_root(frameidx))
|
148 |
+
|
149 |
+
islast = index == (nframes_to_render - 1)
|
150 |
+
|
151 |
+
objname = data.load_in_blender(frameidx, mat)
|
152 |
+
name = f"{str(index).zfill(4)}"
|
153 |
+
|
154 |
+
if mode == "video":
|
155 |
+
path = os.path.join(frames_folder, f"frame_{name}.png")
|
156 |
+
else:
|
157 |
+
path = img_path
|
158 |
+
|
159 |
+
if mode == "sequence":
|
160 |
+
imported_obj_names.extend(objname)
|
161 |
+
elif mode == "frame":
|
162 |
+
camera.update(data.get_root(frameidx))
|
163 |
+
|
164 |
+
if mode != "sequence" or islast:
|
165 |
+
render_current_frame(path)
|
166 |
+
delete_objs(objname)
|
167 |
+
|
168 |
+
bpy.ops.wm.save_as_mainfile(filepath=frames_folder.replace('.png','.blend').replace('_frames','.blend'))
|
169 |
+
|
170 |
+
# remove every object created
|
171 |
+
delete_objs(imported_obj_names)
|
172 |
+
delete_objs(["Plane", "myCurve", "Cylinder"])
|
173 |
+
|
174 |
+
if mode == "video":
|
175 |
+
return frames_folder
|
176 |
+
else:
|
177 |
+
return img_path
|
render.py
ADDED
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
from argparse import ArgumentParser
|
3 |
+
import numpy as np
|
4 |
+
import OpenGL.GL as gl
|
5 |
+
import imageio
|
6 |
+
import cv2
|
7 |
+
import random
|
8 |
+
import torch
|
9 |
+
import moviepy.editor as mp
|
10 |
+
from scipy.spatial.transform import Rotation as RRR
|
11 |
+
import mGPT.render.matplot.plot_3d_global as plot_3d
|
12 |
+
from mGPT.render.pyrender.hybrik_loc2rot import HybrIKJointsToRotmat
|
13 |
+
from mGPT.render.pyrender.smpl_render import SMPLRender
|
14 |
+
|
15 |
+
|
16 |
+
if __name__ == '__main__':
|
17 |
+
|
18 |
+
parser = ArgumentParser()
|
19 |
+
parser.add_argument('--joints_path', type=str, help='Path to joints data')
|
20 |
+
parser.add_argument('--method', type=str, help='Method for rendering')
|
21 |
+
parser.add_argument('--output_mp4_path', type=str, help='Path to output MP4 file')
|
22 |
+
parser.add_argument('--smpl_model_path', type=str, help='Path to SMPL model')
|
23 |
+
|
24 |
+
args = parser.parse_args()
|
25 |
+
|
26 |
+
joints_path = args.joints_path
|
27 |
+
method = args.method
|
28 |
+
output_mp4_path = args.output_mp4_path
|
29 |
+
smpl_model_path = args.smpl_model_path
|
30 |
+
|
31 |
+
|
32 |
+
data = np.load(joints_path)
|
33 |
+
|
34 |
+
if method == 'slow':
|
35 |
+
if len(data.shape) == 4:
|
36 |
+
data = data[0]
|
37 |
+
data = data - data[0, 0]
|
38 |
+
pose_generator = HybrIKJointsToRotmat()
|
39 |
+
pose = pose_generator(data)
|
40 |
+
pose = np.concatenate([
|
41 |
+
pose,
|
42 |
+
np.stack([np.stack([np.eye(3)] * pose.shape[0], 0)] * 2, 1)
|
43 |
+
], 1)
|
44 |
+
shape = [768, 768]
|
45 |
+
render = SMPLRender(smpl_model_path)
|
46 |
+
|
47 |
+
r = RRR.from_rotvec(np.array([np.pi, 0.0, 0.0]))
|
48 |
+
pose[:, 0] = np.matmul(r.as_matrix().reshape(1, 3, 3), pose[:, 0])
|
49 |
+
vid = []
|
50 |
+
aroot = data[:, 0]
|
51 |
+
aroot[:, 1:] = -aroot[:, 1:]
|
52 |
+
params = dict(pred_shape=np.zeros([1, 10]),
|
53 |
+
pred_root=aroot,
|
54 |
+
pred_pose=pose)
|
55 |
+
render.init_renderer([shape[0], shape[1], 3], params)
|
56 |
+
for i in range(data.shape[0]):
|
57 |
+
renderImg = render.render(i)
|
58 |
+
vid.append(renderImg)
|
59 |
+
|
60 |
+
out = np.stack(vid, axis=0)
|
61 |
+
output_gif_path = output_mp4_path[:-4] + '.gif'
|
62 |
+
imageio.mimwrite(output_gif_path, out, duration=50)
|
63 |
+
out_video = mp.VideoFileClip(output_gif_path)
|
64 |
+
out_video.write_videofile(output_mp4_path)
|
65 |
+
del out, render
|
66 |
+
|
67 |
+
elif method == 'fast':
|
68 |
+
output_gif_path = output_mp4_path[:-4] + '.gif'
|
69 |
+
if len(data.shape) == 3:
|
70 |
+
data = data[None]
|
71 |
+
if isinstance(data, torch.Tensor):
|
72 |
+
data = data.cpu().numpy()
|
73 |
+
pose_vis = plot_3d.draw_to_batch(data, [''], [output_gif_path])
|
74 |
+
out_video = mp.VideoFileClip(output_gif_path)
|
75 |
+
out_video.write_videofile(output_mp4_path)
|
76 |
+
del pose_vis
|