dynamic_predictor/dust3r/cloud_opt/camera_estimator.py

import numpy as np
import torch
import torch.nn as nn
import cv2

from dust3r.utils.geometry import inv, geotrf, depthmap_to_absolute_camera_coordinates
from dust3r.post_process import estimate_focal_knowing_depth


def camera_parameter_estimation(view_n, view_m, pred_n, pred_m, im_conf):
    

    # for frame n and m.
    # pair_n2m: n->m
    conf            = float(pred_n['conf'].mean() * pred_m['conf'].mean())
    K, focal        = estimate_intrinsic(view_n['true_shape'], pred_n['pts3d'], pred_n['conf'])
    
    try:
        rel_pose_n2m = estimate_extrinsic(view_m['true_shape'], pred_m['pts3d_in_other_view'], im_conf.cpu().numpy(), K)
        R_mn, T_mn = rel_pose_n2m[:3, :3], rel_pose_n2m[:3, 3]
    except Exception as e:
        print(f"Error estimating extrinsic parameters: {e}")
        rel_pose_n2m = torch.eye(4).cuda()
        R_mn, T_mn = rel_pose_n2m[:3, :3], rel_pose_n2m[:3, 3]
    

    # ptcloud is expressed in camera n
    

    depth_n, depth_m = pred_n['pts3d'][..., 2], geotrf(inv(rel_pose_n2m), pred_m['pts3d_in_other_view'])[..., 2]
    
    return conf, K, focal, R_mn, T_mn, depth_n, depth_m


def estimate_intrinsic(img_shape, pts3d, conf):
    H, W = img_shape
    pts3d = pts3d.cpu()
    pp = torch.tensor((W/2, H/2))
    focal = float(estimate_focal_knowing_depth(pts3d[None], pp, focal_mode='weiszfeld')) 
    K = np.float32([(focal, 0, pp[0]), (0, focal, pp[1]), (0, 0, 1)])

    return K, focal

def estimate_extrinsic(img_shape, pts3d, conf, K):
    min_conf_thr = 3

    H, W = img_shape
    H, W = H.item(), W.item()
    # estimate the pose of pts1 in image 2
    pts3d = pts3d.cpu().numpy()
    pixels = np.mgrid[:W, :H].T.astype(np.float32)
    msk = (conf > min_conf_thr)

    res = cv2.solvePnPRansac(pts3d[msk], pixels[msk], K, None,
                                iterationsCount=100, reprojectionError=5, flags=cv2.SOLVEPNP_SQPNP)
    success, R, T, inliers = res
    assert success

    R = cv2.Rodrigues(R)[0]  # world to cam
    pose = inv(np.r_[np.c_[R, T], [(0, 0, 0, 1)]])  # cam to world

    return torch.from_numpy(pose.astype(np.float32)).cuda()