# Multi-HMR # Copyright (c) 2024-present NAVER Corp. # CC BY-NC-SA 4.0 license import os import sys sys.path.append("./") sys.path.append("./engine") sys.path.append("./engine/pose_estimation") import copy import einops import numpy as np import roma import torch import torch.nn as nn from blocks import ( Dinov2Backbone, FourierPositionEncoding, SMPL_Layer, TransformerDecoder, ) from pose_utils import ( inverse_perspective_projection, pad_to_max, rebatch, rot6d_to_rotmat, undo_focal_length_normalization, undo_log_depth, unpatch, ) from torch import nn def unravel_index(index, shape): out = [] for dim in reversed(shape): out.append(index % dim) index = index // dim return tuple(reversed(out)) class Model(nn.Module): """A ViT backbone followed by a "HPH" head (stack of cross attention layers with queries corresponding to detected humans.)""" def __init__( self, backbone="dinov2_vitb14", pretrained_backbone=False, img_size=896, camera_embedding="geometric", # geometric encodes viewing directions with fourrier encoding camera_embedding_num_bands=16, # increase the size of the camera embedding camera_embedding_max_resolution=64, # does not increase the size of the camera embedding nearness=True, # regress log(1/z) xat_depth=2, # number of cross attention block (SA, CA, MLP) in the HPH head. xat_num_heads=8, # Number of attention heads dict_smpl_layer=None, person_center="head", clip_dist=True, num_betas=10, smplx_dir=None, *args, **kwargs, ): super().__init__() # Save options self.img_size = img_size self.nearness = nearness self.clip_dist = (clip_dist,) self.xat_depth = xat_depth self.xat_num_heads = xat_num_heads self.num_betas = num_betas self.output_mesh = True # Setup backbone self.backbone = Dinov2Backbone(backbone, pretrained=pretrained_backbone) self.embed_dim = self.backbone.embed_dim self.patch_size = self.backbone.patch_size assert self.img_size % self.patch_size == 0, "Invalid img size" # Camera instrinsics self.fovn = 60 self.camera_embedding = camera_embedding self.camera_embed_dim = 0 if self.camera_embedding is not None: if not self.camera_embedding == "geometric": raise NotImplementedError( "Only geometric camera embedding is implemented" ) self.camera = FourierPositionEncoding( n=3, num_bands=camera_embedding_num_bands, max_resolution=camera_embedding_max_resolution, ) # import pdb # pdb.set_trace() self.camera_embed_dim = self.camera.channels # Heads - Detection self.mlp_classif = regression_mlp( [self.embed_dim, self.embed_dim, 1] ) # bg or human # Heads - Human properties self.mlp_offset = regression_mlp([self.embed_dim, self.embed_dim, 2]) # offset # SMPL Layers self.nrot = 53 dict_smpl_layer = { "neutral": { 10: SMPL_Layer( smplx_dir, type="smplx", gender="neutral", num_betas=10, kid=False, person_center=person_center, ), 11: SMPL_Layer( smplx_dir, type="smplx", gender="neutral", num_betas=11, kid=False, person_center=person_center, ), } } _moduleDict = [] for k, _smpl_layer in dict_smpl_layer.items(): for x, y in _smpl_layer.items(): _moduleDict.append([f"{k}_{x}", copy.deepcopy(y)]) self.smpl_layer = nn.ModuleDict(_moduleDict) self.x_attention_head = HPH( num_body_joints=self.nrot - 1, # 23, context_dim=self.embed_dim + self.camera_embed_dim, dim=1024, depth=self.xat_depth, heads=self.xat_num_heads, mlp_dim=1024, dim_head=32, dropout=0.0, emb_dropout=0.0, at_token_res=self.img_size // self.patch_size, num_betas=self.num_betas, smplx_dir=smplx_dir, ) print(f"person center is {person_center}") # set whether do filter def set_filter(self, apply_filter): self.apply_filter = apply_filter def detection( self, z, nms_kernel_size, det_thresh, N, idx=None, max_dist=None, is_training=False, ): """Detection score on the entire low res image""" scores = _sigmoid(self.mlp_classif(z)) # per token detection score. # Restore Height and Width dimensions. scores = unpatch( scores, patch_size=1, c=scores.shape[2], img_size=int(np.sqrt(N)) ) pseudo_idx = idx if not is_training: if ( nms_kernel_size > 1 ): # Easy nms: supress adjacent high scores with max pooling. scores = _nms(scores, kernel=nms_kernel_size) _scores = torch.permute(scores, (0, 2, 3, 1)) # Binary decision (keep confident detections) idx = apply_threshold(det_thresh, _scores) if pseudo_idx is not None: max_dist = 4 if max_dist is None else max_dist mask = (torch.abs(idx[1] - pseudo_idx[1]) <= max_dist) & ( torch.abs(idx[2] - pseudo_idx[2]) <= max_dist ) idx_num = torch.sum(mask) if idx_num < 1: top = torch.clamp( pseudo_idx[1] - max_dist, min=0, max=_scores.shape[1] - 1 ) bottom = torch.clamp( pseudo_idx[1] + max_dist, min=0, max=_scores.shape[1] ) left = torch.clamp( pseudo_idx[2] - max_dist, min=0, max=_scores.shape[2] - 1 ) right = torch.clamp( pseudo_idx[2] + max_dist, min=0, max=_scores.shape[2] ) neigborhoods = _scores[:, top:bottom, left:right, :] idx = torch.argmax(neigborhoods) try: idx = unravel_index(idx, neigborhoods.shape) except Exception as e: print(pseudo_idx) raise e idx = ( pseudo_idx[0], idx[1] + pseudo_idx[1] - max_dist, idx[2] + pseudo_idx[2] - max_dist, pseudo_idx[3], ) elif idx_num > 1: # TODO idx = (idx[0][mask], idx[1][mask], idx[2][mask], idx[3][mask]) else: idx = (idx[0][mask], idx[1][mask], idx[2][mask], idx[3][mask]) # elif bbox is not None: # mask = (idx[1] >= bbox[1]) & (idx[1] >= bbox[3]) & (idx[2] >= bbox[0]) & (idx[2] <= bbox[2]) # idx_num = torch.sum(mask) # if idx_num < 1: # top = torch.clamp(bbox[1], min=0, max=_scores.shape[1]-1) # bottom = torch.clamp(bbox[3], min=0, max=_scores.shape[1]-1) # left = torch.clamp(bbox[0], min=0, max=_scores.shape[2]-1) # right = torch.clamp(bbox[2], min=0, max=_scores.shape[2]-1) # neigborhoods = _scores[:, top:bottom, left:right, :] # idx = torch.argmax(neigborhoods) # try: # idx = unravel_index(idx, neigborhoods.shape) # except Exception as e: # print(pseudo_idx) # raise e # idx = (idx[0], idx[1] + top, idx[2] + left, idx[3]) # else: # idx = (idx[0][mask], idx[1][mask], idx[2][mask], idx[3][mask]) else: assert idx is not None # training time # Scores scores_detected = scores[ idx[0], idx[3], idx[1], idx[2] ] # scores of the detected humans only scores = torch.permute(scores, (0, 2, 3, 1)) return scores, scores_detected, idx def embedd_camera(self, K, z): """Embed viewing directions using fourrier encoding.""" bs = z.shape[0] _h, _w = list(z.shape[-2:]) points = ( torch.stack( [ torch.arange(0, _h, 1).reshape(-1, 1).repeat(1, _w), torch.arange(0, _w, 1).reshape(1, -1).repeat(_h, 1), ], -1, ) .to(z.device) .float() ) # [h,w,2] points = ( points * self.patch_size + self.patch_size // 2 ) # move to pixel space - we give the pixel center of each token points = points.reshape(1, -1, 2).repeat(bs, 1, 1) # (bs, N, 2): 2D points distance = torch.ones(bs, points.shape[1], 1).to( K.device ) # (bs, N, 1): distance in the 3D world rays = inverse_perspective_projection(points, K, distance) # (bs, N, 3) rays_embeddings = self.camera(pos=rays) # Repeat for each element of the batch z_K = rays_embeddings.reshape(bs, _h, _w, self.camera_embed_dim) # [bs,h,w,D] return z_K def to_euclidean_dist(self, x, dist, _K): # Focal length normalization focal = _K[:, [0], [0]] dist = undo_focal_length_normalization( dist, focal, fovn=self.fovn, img_size=x.shape[-1] ) # log space if self.nearness: dist = undo_log_depth(dist) # Clamping if self.clip_dist: dist = torch.clamp(dist, 0, 50) return dist def get_smpl(self): return self.smpl_layer[f"neutral_{self.num_betas}"] def generate_meshes(self, out): """ Generates meshes for each person detected in the image. This function processes the output of the detection model, which includes rotation vectors, shapes, locations, distances, expressions, and other information related to SMPL-X parameters. Parameters: out (dict): A dictionary containing detection results and SMPL-X related parameters. Returns: list: A list of dictionaries, each containing information about a detected person's mesh. """ # Neutral persons = [] rotvec, shape, loc, dist, expression, K_det = ( out["rotvec"], out["shape"], out["loc"], out["dist"], out["expression"], out["K_det"], ) scores_det = out["scores_det"] idx = out["idx"] smpl_out = self.smpl_layer[f"neutral_{self.num_betas}"]( rotvec, shape, loc, dist, None, K=K_det, expression=expression ) out.update(smpl_out) for i in range(idx[0].shape[0]): person = { # Detection "scores": scores_det[i], # detection scores "loc": out["loc"][i], # 2d pixel location of the primary keypoints # SMPL-X params "transl": out["transl"][i], # from the primary keypoint i.e. the head "transl_pelvis": out["transl_pelvis"][i], # of the pelvis joint "rotvec": out["rotvec"][i], "expression": out["expression"][i], "shape": out["shape"][i], # SMPL-X meshs "v3d": out["v3d"][i], "j3d": out["j3d"][i], "j2d": out["j2d"][i], } persons.append(person) return persons def forward( self, x, idx=None, max_dist=None, det_thresh=0.3, nms_kernel_size=3, K=None, is_training=False, *args, **kwargs, ): """ Forward pass of the model and compute the loss according to the groundtruth Args: - x: RGB image - [bs,3,224,224] - idx: GT location of persons - tuple of 3 tensor of shape [p] - idx_j2d: GT location of 2d-kpts for each detected humans - tensor of shape [bs',14,2] - location in pixel space Return: - y: [bs,D,16,16] """ persons = [] out = {} # Feature extraction z = self.backbone(x) B, N, C = z.size() # [bs,256,768] # Detection scores, scores_det, idx = self.detection( z, nms_kernel_size=nms_kernel_size, det_thresh=det_thresh, N=N, idx=idx, max_dist=max_dist, is_training=is_training, ) if torch.any(scores_det < 0.1): return persons if len(idx[1]) == 0 and not is_training: # no humans detected in the frame return persons # Map of Dense Feature z = unpatch( z, patch_size=1, c=z.shape[2], img_size=int(np.sqrt(N)) ) # [bs,D,16,16] z_all = z # Extract the 'central' features z = torch.reshape( z, (z.shape[0], 1, z.shape[1] // 1, z.shape[2], z.shape[3]) ) # [bs,stack_K,D,16,16] z_central = z[idx[0], idx[3], :, idx[1], idx[2]] # dense vectors # 2D offset regression offset = self.mlp_offset(z_central) # Camera instrincs K_det = K[idx[0]] # cameras for detected person z_K = self.embedd_camera(K, z) # Embed viewing directions. z_central = torch.cat( [z_central, z_K[idx[0], idx[1], idx[2]]], 1 ) # Add to query tokens. z_all = torch.cat( [z_all, z_K.permute(0, 3, 1, 2)], 1 ) # for the cross-attention only z = torch.cat([z, z_K.permute(0, 3, 1, 2).unsqueeze(1)], 2) # Distance for estimating the 3D location in 3D space loc = torch.stack([idx[2], idx[1]]).permute( 1, 0 ) # Moving from higher resolution the location of the pelvis loc = (loc + 0.5 + offset) * self.patch_size # SMPL parameter regression kv = z_all[ idx[0] ] # retrieving dense features associated to each central vector pred_smpl_params, pred_cam = self.x_attention_head( z_central, kv, idx_0=idx[0], idx_det=idx ) # Get outputs from the SMPL layer. shape = pred_smpl_params["betas"] rotmat = torch.cat( [pred_smpl_params["global_orient"], pred_smpl_params["body_pose"]], 1 ) expression = pred_smpl_params["expression"] rotvec = roma.rotmat_to_rotvec(rotmat) # Distance dist = pred_cam[:, 0][:, None] out["dist_postprocessed"] = ( dist # before applying any post-processing such as focal length normalization, inverse or log ) dist = self.to_euclidean_dist(x, dist, K_det) # Populate output dictionnary out.update( { "scores": scores, "offset": offset, "dist": dist, "expression": expression, "rotmat": rotmat, "shape": shape, "rotvec": rotvec, "loc": loc, } ) assert ( rotvec.shape[0] == shape.shape[0] == loc.shape[0] == dist.shape[0] ), "Incoherent shapes" if not self.output_mesh: out.update( { "K_det": K_det, "scores_det": scores_det, "idx": idx, } ) return out # Neutral smpl_out = self.smpl_layer[f"neutral_{self.num_betas}"]( rotvec, shape, loc, dist, None, K=K_det, expression=expression ) out.update(smpl_out) # Return if is_training: return out else: # Populate a dictionnary for each person for i in range(idx[0].shape[0]): person = { # Detection "scores": scores_det[i], # detection scores "loc": out["loc"][i], # 2d pixel location of the primary keypoints # SMPL-X params "transl": out["transl"][ i ], # from the primary keypoint i.e. the head "transl_pelvis": out["transl_pelvis"][i], # of the pelvis joint "rotvec": out["rotvec"][i], "expression": out["expression"][i], "shape": out["shape"][i], # SMPL-X meshs "v3d": out["v3d"][i], "j3d": out["j3d"][i], "j2d": out["j2d"][i], "dist": out["dist"][i], "offset": out["offset"][i], } persons.append(person) return persons class HPH(nn.Module): """Cross-attention based SMPL Transformer decoder Code modified from: https://github.com/shubham-goel/4D-Humans/blob/a0def798c7eac811a63c8220fcc22d983b39785e/hmr2/models/heads/smpl_head.py#L17 https://github.com/shubham-goel/4D-Humans/blob/a0def798c7eac811a63c8220fcc22d983b39785e/hmr2/models/components/pose_transformer.py#L301 """ def __init__( self, num_body_joints=52, context_dim=1280, dim=1024, depth=2, heads=8, mlp_dim=1024, dim_head=64, dropout=0.0, emb_dropout=0.0, at_token_res=32, num_betas=10, smplx_dir=None, ): super().__init__() self.joint_rep_type, self.joint_rep_dim = "6d", 6 self.num_body_joints = num_body_joints self.nrot = self.num_body_joints + 1 npose = self.joint_rep_dim * (self.num_body_joints + 1) self.npose = npose self.depth = (depth,) self.heads = (heads,) self.res = at_token_res self.input_is_mean_shape = True _context_dim = context_dim # for the central features self.num_betas = num_betas assert num_betas in [10, 11] # Transformer Decoder setup. # Based on https://github.com/shubham-goel/4D-Humans/blob/8830bb330558eea2395b7f57088ef0aae7f8fa22/hmr2/configs_hydra/experiment/hmr_vit_transformer.yaml#L35 transformer_args = dict( num_tokens=1, token_dim=( (npose + self.num_betas + 3 + _context_dim) if self.input_is_mean_shape else 1 ), dim=dim, depth=depth, heads=heads, mlp_dim=mlp_dim, dim_head=dim_head, dropout=dropout, emb_dropout=emb_dropout, context_dim=context_dim, ) self.transformer = TransformerDecoder(**transformer_args) dim = transformer_args["dim"] # Final decoders to regress targets self.decpose, self.decshape, self.deccam, self.decexpression = [ nn.Linear(dim, od) for od in [npose, num_betas, 3, 10] ] # Register bufffers for the smpl layer. self.set_smpl_init(smplx_dir) # Init learned embeddings for the cross attention queries self.init_learned_queries(context_dim) def init_learned_queries(self, context_dim, std=0.2): """Init learned embeddings for queries""" self.cross_queries_x = nn.Parameter(torch.zeros(self.res, context_dim)) torch.nn.init.normal_(self.cross_queries_x, std=std) self.cross_queries_y = nn.Parameter(torch.zeros(self.res, context_dim)) torch.nn.init.normal_(self.cross_queries_y, std=std) self.cross_values_x = nn.Parameter(torch.zeros(self.res, context_dim)) torch.nn.init.normal_(self.cross_values_x, std=std) self.cross_values_y = nn.Parameter( nn.Parameter(torch.zeros(self.res, context_dim)) ) torch.nn.init.normal_(self.cross_values_y, std=std) def set_smpl_init(self, smplx_dir): """Fetch saved SMPL parameters and register buffers.""" mean_params = np.load(os.path.join(smplx_dir, "smpl_mean_params.npz")) if self.nrot == 53: init_body_pose = ( torch.eye(3) .reshape(1, 3, 3) .repeat(self.nrot, 1, 1)[:, :, :2] .flatten(1) .reshape(1, -1) ) init_body_pose[:, : 24 * 6] = torch.from_numpy( mean_params["pose"][:] ).float() # global_orient+body_pose from SMPL else: init_body_pose = torch.from_numpy( mean_params["pose"].astype(np.float32) ).unsqueeze(0) init_betas = torch.from_numpy(mean_params["shape"].astype("float32")).unsqueeze( 0 ) init_cam = torch.from_numpy(mean_params["cam"].astype(np.float32)).unsqueeze(0) init_betas_kid = torch.cat( [init_betas, torch.zeros_like(init_betas[:, [0]])], 1 ) init_expression = 0.0 * torch.from_numpy( mean_params["shape"].astype("float32") ).unsqueeze(0) if self.num_betas == 11: init_betas = torch.cat([init_betas, torch.zeros_like(init_betas[:, :1])], 1) self.register_buffer("init_body_pose", init_body_pose) self.register_buffer("init_betas", init_betas) self.register_buffer("init_betas_kid", init_betas_kid) self.register_buffer("init_cam", init_cam) self.register_buffer("init_expression", init_expression) def cross_attn_inputs(self, x, x_central, idx_0, idx_det): """Reshape and pad x_central to have the right shape for Cross-attention processing. Inject learned embeddings to query and key inputs at the location of detected people. """ h, w = x.shape[2], x.shape[3] x = einops.rearrange(x, "b c h w -> b (h w) c") assert idx_0 is not None, "Learned cross queries only work with multicross" if idx_0.shape[0] > 0: # reconstruct the batch/nb_people dimensions: pad for images with fewer people than max. counts, idx_det_0 = rebatch(idx_0, idx_det) old_shape = x_central.shape # Legacy check for old versions assert idx_det is not None, "idx_det needed for learned_attention" # xx is the tensor with all features xx = einops.rearrange(x, "b (h w) c -> b c h w", h=h, w=w) # Get learned embeddings for queries, at positions with detected people. queries_xy = ( self.cross_queries_x[idx_det[1]] + self.cross_queries_y[idx_det[2]] ) # Add the embedding to the central features. x_central = x_central + queries_xy assert x_central.shape == old_shape, "Problem with shape" # Make it a tensor of dim. [batch, max_ppl_along_batch, ...] x_central, mask = pad_to_max(x_central, counts) # xx = einops.rearrange(x, 'b (h w) c -> b c h w', h=h, w=w) xx = xx[torch.cumsum(counts, dim=0) - 1] # Inject leared embeddings for key/values at detected locations. values_xy = ( self.cross_values_x[idx_det[1]] + self.cross_values_y[idx_det[2]] ) xx[idx_det_0, :, idx_det[1], idx_det[2]] += values_xy x = einops.rearrange(xx, "b c h w -> b (h w) c") num_ppl = x_central.shape[1] else: mask = None num_ppl = 1 counts = None return x, x_central, mask, num_ppl, counts def forward(self, x_central, x, idx_0=None, idx_det=None, **kwargs): """ " Forward the HPH module. """ batch_size = x.shape[0] # Reshape inputs for cross attention and inject learned embeddings for queries and values. x, x_central, mask, num_ppl, counts = self.cross_attn_inputs( x, x_central, idx_0, idx_det ) # Add init (mean smpl params) to the query for each quantity being regressed. bs = x_central.shape[0] if idx_0.shape[0] else batch_size expand = lambda x: x.expand(bs, num_ppl, -1) pred_body_pose, pred_betas, pred_cam, pred_expression = [ expand(x) for x in [ self.init_body_pose, self.init_betas, self.init_cam, self.init_expression, ] ] token = torch.cat([x_central, pred_body_pose, pred_betas, pred_cam], dim=-1) if len(token.shape) == 2: token = token[:, None, :] # Process query and inputs with the cross-attention module. token_out = self.transformer(token, context=x, mask=mask) # Reshape outputs from [batch_size, nmax_ppl, ...] to [total_ppl, ...] if mask is not None: # Stack along batch axis. token_out_list = [token_out[i, :c, ...] for i, c in enumerate(counts)] token_out = torch.concat(token_out_list, dim=0) else: token_out = token_out.squeeze(1) # (B, C) # Decoded output token and add to init for each quantity to regress. reshape = ( (lambda x: x) if idx_0.shape[0] == 0 else (lambda x: x[0, 0, ...][None, ...]) ) decoders = [self.decpose, self.decshape, self.deccam, self.decexpression] inits = [pred_body_pose, pred_betas, pred_cam, pred_expression] pred_body_pose, pred_betas, pred_cam, pred_expression = [ d(token_out) + reshape(i) for d, i in zip(decoders, inits) ] # Convert self.joint_rep_type -> rotmat joint_conversion_fn = rot6d_to_rotmat # conversion pred_body_pose = joint_conversion_fn(pred_body_pose).view( batch_size, self.num_body_joints + 1, 3, 3 ) # Build the output dict pred_smpl_params = { "global_orient": pred_body_pose[:, [0]], "body_pose": pred_body_pose[:, 1:], "betas": pred_betas, #'betas_kid': pred_betas_kid, "expression": pred_expression, } return pred_smpl_params, pred_cam # , pred_smpl_params_list def regression_mlp(layers_sizes): """ Return a fully connected network. """ assert len(layers_sizes) >= 2 in_features = layers_sizes[0] layers = [] for i in range(1, len(layers_sizes) - 1): out_features = layers_sizes[i] layers.append(torch.nn.Linear(in_features, out_features)) layers.append(torch.nn.ReLU()) in_features = out_features layers.append(torch.nn.Linear(in_features, layers_sizes[-1])) return torch.nn.Sequential(*layers) def apply_threshold(det_thresh, _scores): """Apply thresholding to detection scores; if stack_K is used and det_thresh is a list, apply to each channel separately""" if isinstance(det_thresh, list): det_thresh = det_thresh[0] idx = torch.where(_scores >= det_thresh) return idx def _nms(heat, kernel=3): """easy non maximal supression (as in CenterNet)""" if kernel not in [2, 4]: pad = (kernel - 1) // 2 else: if kernel == 2: pad = 1 else: pad = 2 hmax = nn.functional.max_pool2d(heat, (kernel, kernel), stride=1, padding=pad) if hmax.shape[2] > heat.shape[2]: hmax = hmax[:, :, : heat.shape[2], : heat.shape[3]] keep = (hmax == heat).float() return heat * keep def _sigmoid(x): y = torch.clamp(x.sigmoid_(), min=1e-4, max=1 - 1e-4) return y if __name__ == "__main__": Model()