scripts/utils.py

import torch
import numpy as np
from PIL import Image
import pymeshlab
import trimesh
from pytorch3d.renderer import TexturesVertex
from pytorch3d.structures import Meshes
from rembg import new_session, remove
import torch.nn.functional as F
from typing import List, Tuple
from pygltflib import GLTF2, Material, PbrMetallicRoughness
import time


# Constants

providers = [
    ('CUDAExecutionProvider', {
        'device_id': 0,
        'arena_extend_strategy': 'kSameAsRequested',
        'gpu_mem_limit': 8 * 1024 * 1024 * 1024,
        'cudnn_conv_algo_search': 'HEURISTIC',
    })
]

session = new_session(providers=providers)

NEG_PROMPT = "sketch, sculpture, hand drawing, outline, single color, NSFW, lowres, bad anatomy,bad hands, text, error, missing fingers, yellow sleeves, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry,(worst quality:1.4),(low quality:1.4)"

# Helper functions


def load_mesh_with_trimesh(file_name, file_type=None):
    mesh = trimesh.load(file_name, file_type=file_type)
    if isinstance(mesh, trimesh.Scene):
        mesh = _process_trimesh_scene(mesh)

    vertices, faces, colors = _extract_mesh_data(mesh)
    return vertices, faces, colors


def _process_trimesh_scene(mesh):
    from io import BytesIO
    with BytesIO() as f:
        mesh.export(f, file_type="obj")
        f.seek(0)
        mesh = trimesh.load(f, file_type="obj")
    if isinstance(mesh, trimesh.Scene):
        mesh = trimesh.util.concatenate(
            tuple(trimesh.Trimesh(vertices=g.vertices, faces=g.faces)
                  for g in mesh.geometry.values()))
    return mesh


def _extract_mesh_data(mesh):
    vertices = torch.from_numpy(mesh.vertices).T
    faces = torch.from_numpy(mesh.faces).T
    colors = _get_mesh_colors(mesh)
    return vertices, faces, colors


def _get_mesh_colors(mesh):
    if mesh.visual is not None and hasattr(mesh.visual, 'vertex_colors'):
        return torch.from_numpy(mesh.visual.vertex_colors)[..., :3].T / 255.
    return torch.ones_like(mesh.vertices).T * 0.5


def meshlab_mesh_to_py3dmesh(mesh: pymeshlab.Mesh) -> Meshes:
    verts = torch.from_numpy(mesh.vertex_matrix()).float()
    faces = torch.from_numpy(mesh.face_matrix()).long()
    colors = torch.from_numpy(mesh.vertex_color_matrix()[..., :3]).float()
    textures = TexturesVertex(verts_features=[colors])
    return Meshes(verts=[verts], faces=[faces], textures=textures)


def py3dmesh_to_meshlab_mesh(meshes: Meshes) -> pymeshlab.Mesh:
    colors_in = F.pad(meshes.textures.verts_features_packed().cpu().float(), [
                      0, 1], value=1).numpy().astype(np.float64)
    return pymeshlab.Mesh(
        vertex_matrix=meshes.verts_packed().cpu().float().numpy().astype(np.float64),
        face_matrix=meshes.faces_packed().cpu().long().numpy().astype(np.int32),
        v_normals_matrix=meshes.verts_normals_packed(
        ).cpu().float().numpy().astype(np.float64),
        v_color_matrix=colors_in)


def to_pyml_mesh(vertices, faces):
    return pymeshlab.Mesh(
        vertex_matrix=vertices.cpu().float().numpy().astype(np.float64),
        face_matrix=faces.cpu().long().numpy().astype(np.int32),
    )


def to_py3d_mesh(vertices, faces, normals=None):
    mesh = Meshes(verts=[vertices], faces=[faces], textures=None)
    if normals is None:
        normals = mesh.verts_normals_packed()
    mesh.textures = TexturesVertex(verts_features=[normals / 2 + 0.5])
    return mesh


def from_py3d_mesh(mesh):
    return mesh.verts_list()[0], mesh.faces_list()[0], mesh.textures.verts_features_packed()

# Normal map rotation functions


def rotate_normalmap_by_angle(normal_map: np.ndarray, angle: float):
    angle_rad = np.radians(angle)
    R = np.array([
        [np.cos(angle_rad), 0, np.sin(angle_rad)],
        [0, 1, 0],
        [-np.sin(angle_rad), 0, np.cos(angle_rad)]
    ])
    return np.dot(normal_map.reshape(-1, 3), R.T).reshape(normal_map.shape)


def rotate_normals(normal_pils, return_types='np', rotate_direction=1) -> np.ndarray:
    n_views = len(normal_pils)
    ret = []
    for idx, rgba_normal in enumerate(normal_pils):
        normal_np, alpha_np = _process_normal_image(rgba_normal)
        normal_np = rotate_normalmap_by_angle(
            normal_np, rotate_direction * idx * (360 / n_views))
        rgba_normal_np = _combine_normal_and_alpha(normal_np, alpha_np)
        ret.append(_format_output(rgba_normal_np, return_types))
    return ret


def _process_normal_image(rgba_normal):
    normal_np = np.array(rgba_normal)[:, :, :3] / 255 * 2 - 1
    alpha_np = np.array(rgba_normal)[:, :, 3] / 255
    return normal_np, alpha_np


def _combine_normal_and_alpha(normal_np, alpha_np):
    normal_np = (normal_np + 1) / 2
    normal_np = normal_np * alpha_np[..., None]
    return np.concatenate([normal_np * 255, alpha_np[:, :, None] * 255], axis=-1)


def _format_output(rgba_normal_np, return_types):
    if return_types == 'np':
        return rgba_normal_np
    elif return_types == 'pil':
        return Image.fromarray(rgba_normal_np.astype(np.uint8))
    else:
        raise ValueError(
            f"return_types should be 'np' or 'pil', but got {return_types}")


def rotate_normalmap_by_angle_torch(normal_map, angle):
    angle_rad = torch.tensor(np.radians(angle)).to(normal_map)
    R = torch.tensor([
        [torch.cos(angle_rad), 0, torch.sin(angle_rad)],
        [0, 1, 0],
        [-torch.sin(angle_rad), 0, torch.cos(angle_rad)]
    ]).to(normal_map)
    return torch.matmul(normal_map.view(-1, 3), R.T).view(normal_map.shape)


def do_rotate(rgba_normal, angle):
    rgba_normal = torch.from_numpy(rgba_normal).float().cuda() / 255
    rotated_normal_tensor = rotate_normalmap_by_angle_torch(
        rgba_normal[..., :3] * 2 - 1, angle)
    rotated_normal_tensor = (rotated_normal_tensor + 1) / 2
    rotated_normal_tensor = rotated_normal_tensor * rgba_normal[:, :, [3]]
    return torch.cat([rotated_normal_tensor * 255, rgba_normal[:, :, [3]] * 255], dim=-1).cpu().numpy()


def rotate_normals_torch(normal_pils, return_types='np', rotate_direction=1):
    n_views = len(normal_pils)
    ret = []
    for idx, rgba_normal in enumerate(normal_pils):
        angle = rotate_direction * idx * (360 / n_views)
        rgba_normal_np = do_rotate(np.array(rgba_normal), angle)
        ret.append(_format_output(rgba_normal_np, return_types))
    return ret

# Background change functions


def change_bkgd(img_pils, new_bkgd=(0., 0., 0.)):
    new_bkgd = np.array(new_bkgd).reshape(1, 1, 3)
    return [_change_single_image_bkgd(rgba_img, new_bkgd) for rgba_img in img_pils]


def _change_single_image_bkgd(rgba_img, new_bkgd):
    img_np, alpha_np = np.array(
        rgba_img)[:, :, :3] / 255, np.array(rgba_img)[:, :, 3] / 255
    ori_bkgd = img_np[:1, :1]
    alpha_np_clamp = np.clip(alpha_np, 1e-6, 1)
    ori_img_np = (img_np - ori_bkgd *
                  (1 - alpha_np[..., None])) / alpha_np_clamp[..., None]
    img_np = np.where(alpha_np[..., None] > 0.05, ori_img_np *
                      alpha_np[..., None] + new_bkgd * (1 - alpha_np[..., None]), new_bkgd)
    rgba_img_np = np.concatenate(
        [img_np * 255, alpha_np[..., None] * 255], axis=-1)
    return Image.fromarray(rgba_img_np.astype(np.uint8))


def change_bkgd_to_normal(normal_pils) -> List[Image.Image]:
    n_views = len(normal_pils)
    return [_change_single_normal_bkgd(rgba_normal, idx, n_views) for idx, rgba_normal in enumerate(normal_pils)]


def _change_single_normal_bkgd(rgba_normal, idx, n_views):
    target_bkgd = rotate_normalmap_by_angle(
        np.array([[[0., 0., 1.]]]), idx * (360 / n_views))
    normal_np, alpha_np = np.array(
        rgba_normal)[:, :, :3] / 255 * 2 - 1, np.array(rgba_normal)[:, :, 3] / 255
    old_bkgd = normal_np[:1, :1]
    normal_np[alpha_np > 0.05] = (normal_np[alpha_np > 0.05] - old_bkgd * (
        1 - alpha_np[alpha_np > 0.05][..., None])) / alpha_np[alpha_np > 0.05][..., None]
    normal_np = normal_np * alpha_np[..., None] + \
        target_bkgd * (1 - alpha_np[..., None])
    normal_np = (normal_np + 1) / 2
    rgba_normal_np = np.concatenate(
        [normal_np * 255, alpha_np[..., None] * 255], axis=-1)
    return Image.fromarray(rgba_normal_np.astype(np.uint8))

# Mesh and GLB handling functions


def fix_vert_color_glb(mesh_path):
    obj1 = GLTF2().load(mesh_path)
    obj1.meshes[0].primitives[0].material = 0
    obj1.materials.append(Material(
        pbrMetallicRoughness=PbrMetallicRoughness(
            baseColorFactor=[1.0, 1.0, 1.0, 1.0],
            metallicFactor=0.,
            roughnessFactor=1.0,
        ),
        emissiveFactor=[0.0, 0.0, 0.0],
        doubleSided=True,
    ))
    obj1.save(mesh_path)


def srgb_to_linear(c_srgb):
    return np.where(c_srgb <= 0.04045, c_srgb / 12.92, ((c_srgb + 0.055) / 1.055) ** 2.4).clip(0, 1.)


def save_py3dmesh_with_trimesh_fast(meshes: Meshes, save_glb_path, apply_sRGB_to_LinearRGB=True):
    vertices, triangles, np_color = _extract_mesh_data_for_trimesh(meshes)

    if save_glb_path.endswith(".glb"):
        vertices[:, [0, 2]] = -vertices[:, [0, 2]]

    if apply_sRGB_to_LinearRGB:
        np_color = srgb_to_linear(np_color)

    mesh = trimesh.Trimesh(
        vertices=vertices, faces=triangles, vertex_colors=np_color)
    mesh.remove_unreferenced_vertices()
    mesh.export(save_glb_path)

    if save_glb_path.endswith(".glb"):
        fix_vert_color_glb(save_glb_path)
    print(f"saving to {save_glb_path}")


def _extract_mesh_data_for_trimesh(meshes):
    vertices = meshes.verts_packed().cpu().float().numpy()
    triangles = meshes.faces_packed().cpu().long().numpy()
    np_color = meshes.textures.verts_features_packed().cpu().float().numpy()
    assert vertices.shape[0] == np_color.shape[0]
    assert np_color.shape[1] == 3
    assert 0 <= np_color.min() and np_color.max(
    ) <= 1, f"min={np_color.min()}, max={np_color.max()}"
    return vertices, triangles, np_color


def save_glb_and_video(save_mesh_prefix: str, meshes: Meshes, with_timestamp=True, dist=3.5, azim_offset=180, resolution=512, fov_in_degrees=1 / 1.15, cam_type="ortho", view_padding=60, export_video=True) -> Tuple[str, str]:
    import time
    if '.' in save_mesh_prefix:
        save_mesh_prefix = ".".join(save_mesh_prefix.split('.')[:-1])
    if with_timestamp:
        save_mesh_prefix = save_mesh_prefix + f"_{int(time.time())}"
    ret_mesh = save_mesh_prefix + ".glb"
    save_py3dmesh_with_trimesh_fast(meshes, ret_mesh)
    return ret_mesh, None

# Mesh cleaning and preprocessing functions (continued)


def simple_clean_mesh(pyml_mesh: pymeshlab.Mesh, apply_smooth=True, stepsmoothnum=1, apply_sub_divide=False, sub_divide_threshold=0.25):
    ms = pymeshlab.MeshSet()
    ms.add_mesh(pyml_mesh, "cube_mesh")

    if apply_smooth:
        ms.apply_filter("apply_coord_laplacian_smoothing",
                        stepsmoothnum=stepsmoothnum, cotangentweight=False)

    if apply_sub_divide:
        ms.apply_filter("meshing_repair_non_manifold_vertices")
        ms.apply_filter("meshing_repair_non_manifold_edges",
                        method='Remove Faces')
        ms.apply_filter("meshing_surface_subdivision_loop", iterations=2,
                        threshold=pymeshlab.PercentageValue(sub_divide_threshold))

    return meshlab_mesh_to_py3dmesh(ms.current_mesh())


def expand2square(pil_img, background_color):
    width, height = pil_img.size
    if width == height:
        return pil_img

    new_size = max(width, height)
    result = Image.new(pil_img.mode, (new_size, new_size), background_color)

    if width > height:
        result.paste(pil_img, (0, (width - height) // 2))
    else:
        result.paste(pil_img, ((height - width) // 2, 0))

    return result


def simple_preprocess(input_image, rembg_session=session, background_color=255):
    RES = 2048
    input_image.thumbnail([RES, RES], Image.Resampling.LANCZOS)

    if input_image.mode != 'RGBA':
        image_rem = input_image.convert('RGBA')
        input_image = remove(
            image_rem, alpha_matting=False, session=rembg_session)

    arr = np.asarray(input_image)
    alpha = arr[:, :, -1]

    x_nonzero, y_nonzero = (alpha > 60).sum(axis=1).nonzero()[
        0], (alpha > 60).sum(axis=0).nonzero()[0]
    x_min, x_max = int(x_nonzero.min()), int(x_nonzero.max())
    y_min, y_max = int(y_nonzero.min()), int(y_nonzero.max())

    arr = arr[x_min:x_max, y_min:y_max]
    input_image = Image.fromarray(arr)
    return expand2square(input_image, (background_color, background_color, background_color, 0))


def init_target(img_pils, new_bkgd=(0., 0., 0.), device="cuda"):
    new_bkgd = torch.tensor(
        new_bkgd, dtype=torch.float32).view(1, 1, 3).to(device)
    imgs = torch.stack([torch.from_numpy(np.array(img, dtype=np.float32))
                       for img in img_pils]).to(device) / 255

    img_nps = imgs[..., :3]
    alpha_nps = imgs[..., 3]
    ori_bkgds = img_nps[:, :1, :1]

    alpha_nps_clamp = torch.clamp(alpha_nps, 1e-6, 1)
    ori_img_nps = (img_nps - ori_bkgds * (1 - alpha_nps.unsqueeze(-1))
                   ) / alpha_nps_clamp.unsqueeze(-1)
    ori_img_nps = torch.clamp(ori_img_nps, 0, 1)

    img_nps = torch.where(alpha_nps.unsqueeze(-1) > 0.05,
                          ori_img_nps *
                          alpha_nps.unsqueeze(-1) + new_bkgd *
                          (1 - alpha_nps.unsqueeze(-1)),
                          new_bkgd)

    return torch.cat([img_nps, alpha_nps.unsqueeze(-1)], dim=-1)


def save_obj_and_video(save_mesh_prefix: str, meshes: Meshes, with_timestamp=True, **kwargs) -> Tuple[str, str]:
    if '.' in save_mesh_prefix:
        save_mesh_prefix = ".".join(save_mesh_prefix.split('.')[:-1])
    if with_timestamp:
        save_mesh_prefix = save_mesh_prefix + f"_{int(time.time())}"
    ret_mesh = save_mesh_prefix + ".obj"
    
    vertices = meshes.verts_packed().cpu().float().numpy()
    triangles = meshes.faces_packed().cpu().long().numpy()
    np_color = meshes.textures.verts_features_packed().cpu().float().numpy()
    
    # Apply sRGB to LinearRGB conversion
    np_color = srgb_to_linear(np_color)
    
    mesh = trimesh.Trimesh(vertices=vertices, faces=triangles, vertex_colors=np_color)
    mesh.remove_unreferenced_vertices()
    mesh.export(ret_mesh)
    
    print(f"Saved to {ret_mesh}")
    
    return ret_mesh, None