import os import imageio import numpy as np import torch from tqdm import tqdm from pytorch3d.renderer import ( PerspectiveCameras, TexturesVertex, PointLights, Materials, RasterizationSettings, MeshRenderer, MeshRasterizer, SoftPhongShader, ) from pytorch3d.renderer.mesh.shader import ShaderBase from pytorch3d.structures import Meshes class NormalShader(ShaderBase): def __init__(self, device = "cpu", **kwargs): super().__init__(device=device, **kwargs) def forward(self, fragments, meshes, **kwargs): blend_params = kwargs.get("blend_params", self.blend_params) texels = fragments.bary_coords.clone() texels = texels.permute(0, 3, 1, 2, 4) texels = texels * 2 - 1 # 将 bary_coords 映射到 [-1, 1] # 获取法线 verts_normals = meshes.verts_normals_packed() faces_normals = verts_normals[meshes.faces_packed()] bary_coords = fragments.bary_coords pixel_normals = (bary_coords[..., None] * faces_normals[fragments.pix_to_face]).sum(dim=-2) pixel_normals = pixel_normals / pixel_normals.norm(dim=-1, keepdim=True) # 将法线映射到颜色空间 # colors = (pixel_normals + 1) / 2 # 将法线映射到 [0, 1] colors = torch.clamp(pixel_normals, -1, 1) print(colors.shape) mask = (fragments.pix_to_face > 0).float() colors = torch.cat([colors, mask.unsqueeze(-1)], dim=-1) # colors[fragments.pix_to_face < 0] = 0 # 混合颜色 # images = self.blend(texels, colors, fragments, blend_params) return colors def overlay_image_onto_background(image, mask, bbox, background): if isinstance(image, torch.Tensor): image = image.detach().cpu().numpy() if isinstance(mask, torch.Tensor): mask = mask.detach().cpu().numpy() out_image = background.copy() bbox = bbox[0].int().cpu().numpy().copy() roi_image = out_image[bbox[1]:bbox[3], bbox[0]:bbox[2]] if len(roi_image) < 1 or len(roi_image[1]) < 1: return out_image try: roi_image[mask] = image[mask] except Exception as e: raise e out_image[bbox[1]:bbox[3], bbox[0]:bbox[2]] = roi_image return out_image def update_intrinsics_from_bbox(K_org, bbox): ''' update intrinsics for cropped images ''' device, dtype = K_org.device, K_org.dtype K = torch.zeros((K_org.shape[0], 4, 4) ).to(device=device, dtype=dtype) K[:, :3, :3] = K_org.clone() K[:, 2, 2] = 0 K[:, 2, -1] = 1 K[:, -1, 2] = 1 image_sizes = [] for idx, bbox in enumerate(bbox): left, upper, right, lower = bbox cx, cy = K[idx, 0, 2], K[idx, 1, 2] new_cx = cx - left new_cy = cy - upper new_height = max(lower - upper, 1) new_width = max(right - left, 1) new_cx = new_width - new_cx new_cy = new_height - new_cy K[idx, 0, 2] = new_cx K[idx, 1, 2] = new_cy image_sizes.append((int(new_height), int(new_width))) return K, image_sizes def perspective_projection(x3d, K, R=None, T=None): if R != None: x3d = torch.matmul(R, x3d.transpose(1, 2)).transpose(1, 2) if T != None: x3d = x3d + T.transpose(1, 2) x2d = torch.div(x3d, x3d[..., 2:]) x2d = torch.matmul(K, x2d.transpose(-1, -2)).transpose(-1, -2)[..., :2] return x2d def compute_bbox_from_points(X, img_w, img_h, scaleFactor=1.2): left = torch.clamp(X.min(1)[0][:, 0], min=0, max=img_w) right = torch.clamp(X.max(1)[0][:, 0], min=0, max=img_w) top = torch.clamp(X.min(1)[0][:, 1], min=0, max=img_h) bottom = torch.clamp(X.max(1)[0][:, 1], min=0, max=img_h) cx = (left + right) / 2 cy = (top + bottom) / 2 width = (right - left) height = (bottom - top) new_left = torch.clamp(cx - width/2 * scaleFactor, min=0, max=img_w-1) new_right = torch.clamp(cx + width/2 * scaleFactor, min=1, max=img_w) new_top = torch.clamp(cy - height / 2 * scaleFactor, min=0, max=img_h-1) new_bottom = torch.clamp(cy + height / 2 * scaleFactor, min=1, max=img_h) bbox = torch.stack((new_left.detach(), new_top.detach(), new_right.detach(), new_bottom.detach())).int().float().T return bbox class Renderer(): def __init__(self, width, height, K, device, faces=None): self.width = width self.height = height self.K = K self.device = device if faces is not None: self.faces = torch.from_numpy( (faces).astype('int') ).unsqueeze(0).to(self.device) self.initialize_camera_params() self.lights = PointLights(device=device, location=[[0.0, 0.0, -10.0]]) self.create_renderer() def create_camera(self, R=None, T=None): if R is not None: self.R = R.clone().view(1, 3, 3).to(self.device) if T is not None: self.T = T.clone().view(1, 3).to(self.device) return PerspectiveCameras( device=self.device, R=self.R.mT, T=self.T, K=self.K_full, image_size=self.image_sizes, in_ndc=False) def create_renderer(self): self.renderer = MeshRenderer( rasterizer=MeshRasterizer( raster_settings=RasterizationSettings( image_size=self.image_sizes[0], blur_radius=1e-5,), ), shader=SoftPhongShader( device=self.device, lights=self.lights, ) ) def create_normal_renderer(self): normal_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=RasterizationSettings( image_size=self.image_sizes[0], ), ), shader=NormalShader(device=self.device), ) return normal_renderer def initialize_camera_params(self): """Hard coding for camera parameters TODO: Do some soft coding""" # Extrinsics self.R = torch.diag( torch.tensor([1, 1, 1]) ).float().to(self.device).unsqueeze(0) self.T = torch.tensor( [0, 0, 0] ).unsqueeze(0).float().to(self.device) # Intrinsics self.K = self.K.unsqueeze(0).float().to(self.device) self.bboxes = torch.tensor([[0, 0, self.width, self.height]]).float() self.K_full, self.image_sizes = update_intrinsics_from_bbox(self.K, self.bboxes) self.cameras = self.create_camera() def render_normal(self, vertices): vertices = vertices.unsqueeze(0) mesh = Meshes(verts=vertices, faces=self.faces) normal_renderer = self.create_normal_renderer() results = normal_renderer(mesh) results = torch.flip(results, [1, 2]) return results def render_mesh(self, vertices, background, colors=[0.8, 0.8, 0.8]): self.update_bbox(vertices[::50], scale=1.2) vertices = vertices.unsqueeze(0) if colors[0] > 1: colors = [c / 255. for c in colors] verts_features = torch.tensor(colors).reshape(1, 1, 3).to(device=vertices.device, dtype=vertices.dtype) verts_features = verts_features.repeat(1, vertices.shape[1], 1) textures = TexturesVertex(verts_features=verts_features) mesh = Meshes(verts=vertices, faces=self.faces, textures=textures,) materials = Materials( device=self.device, specular_color=(colors, ), shininess=0 ) results = torch.flip( self.renderer(mesh, materials=materials, cameras=self.cameras, lights=self.lights), [1, 2] ) image = results[0, ..., :3] * 255 mask = results[0, ..., -1] > 1e-3 image = overlay_image_onto_background(image, mask, self.bboxes, background.copy()) self.reset_bbox() return image def update_bbox(self, x3d, scale=2.0, mask=None): """ Update bbox of cameras from the given 3d points x3d: input 3D keypoints (or vertices), (num_frames, num_points, 3) """ if x3d.size(-1) != 3: x2d = x3d.unsqueeze(0) else: x2d = perspective_projection(x3d.unsqueeze(0), self.K, self.R, self.T.reshape(1, 3, 1)) if mask is not None: x2d = x2d[:, ~mask] bbox = compute_bbox_from_points(x2d, self.width, self.height, scale) self.bboxes = bbox self.K_full, self.image_sizes = update_intrinsics_from_bbox(self.K, bbox) self.cameras = self.create_camera() self.create_renderer() def reset_bbox(self,): bbox = torch.zeros((1, 4)).float().to(self.device) bbox[0, 2] = self.width bbox[0, 3] = self.height self.bboxes = bbox self.K_full, self.image_sizes = update_intrinsics_from_bbox(self.K, bbox) self.cameras = self.create_camera() self.create_renderer() class RendererUtil(): def __init__(self, K, w, h, device, faces, keep_origin=True): self.keep_origin = keep_origin self.default_R = torch.eye(3) self.default_T = torch.zeros(3) self.device = device self.renderer = Renderer(w, h, K, device, faces) def set_extrinsic(self, R, T): self.default_R = R self.default_T = T def render_normal(self, verts_list): if not len(verts_list) == 1: return None self.renderer.create_camera(self.default_R, self.default_T) normal_map = self.renderer.render_normal(verts_list[0]) return normal_map[0, :, :, 0] def render_frame(self, humans, pred_rend_array, verts_list=None, color_list=None): if not isinstance(pred_rend_array, np.ndarray): pred_rend_array = np.asarray(pred_rend_array) self.renderer.create_camera(self.default_R, self.default_T) _img = pred_rend_array if humans is not None: for human in humans: _img = self.renderer.render_mesh(human['v3d'].to(self.device), _img) else: for i, verts in enumerate(verts_list): if color_list is None: _img = self.renderer.render_mesh(verts.to(self.device), _img) else: _img = self.renderer.render_mesh(verts.to(self.device), _img, color_list[i]) if self.keep_origin: _img = np.concatenate([np.asarray(pred_rend_array), _img],1).astype(np.uint8) return _img def render_video(self, results, pil_bis_frames, fps, out_path): writer = imageio.get_writer( out_path, fps=fps, mode='I', format='FFMPEG', macro_block_size=1 ) for i, humans in enumerate(tqdm(results)): pred_rend_array = pil_bis_frames[i] _img = self.render_frame( humans, pred_rend_array) try: writer.append_data(_img) except: print('Error in writing video') print(type(_img)) writer.close() def render_frame(renderer, humans, pred_rend_array, default_R, default_T, device, keep_origin=True): if not isinstance(pred_rend_array, np.ndarray): pred_rend_array = np.asarray(pred_rend_array) renderer.create_camera(default_R, default_T) _img = pred_rend_array if humans is None: humans = [] if isinstance(humans, dict): humans = [humans] for human in humans: if isinstance(human, dict): v3d = human['v3d'].to(device) else: v3d = human _img = renderer.render_mesh(v3d, _img) if keep_origin: _img = np.concatenate([np.asarray(pred_rend_array), _img],1).astype(np.uint8) return _img def render_video(results, faces, K, pil_bis_frames, fps, out_path, device, keep_origin=True): # results [F, N, ...] if isinstance(pil_bis_frames[0], np.ndarray): height, width, _ = pil_bis_frames[0].shape else: shape = pil_bis_frames[0].size width, height = shape[1], shape[0] renderer = Renderer(width, height, K[0], device, faces) # build default camera default_R, default_T = torch.eye(3), torch.zeros(3) writer = imageio.get_writer( out_path, fps=fps, mode='I', format='FFMPEG', macro_block_size=1 ) for i, humans in enumerate(tqdm(results)): pred_rend_array = pil_bis_frames[i] _img = render_frame(renderer, humans, pred_rend_array, default_R, default_T, device, keep_origin) try: writer.append_data(_img) except: print('Error in writing video') print(type(_img)) writer.close()