interface.py

import os
import random
import torch
import hydra
import numpy as np
import zipfile
import time
import uuid

from typing import Any
from hydra import compose, initialize
from omegaconf import DictConfig, OmegaConf
from huggingface_hub import hf_hub_download

from utils.misc import compute_model_dim
from datasets.base import create_dataset
from datasets.misc import collate_fn_general, collate_fn_squeeze_pcd_batch
from models.base import create_model
from models.visualizer import create_visualizer
from models.environment import create_enviroment

def pretrain_pointtrans_weight_path():
    return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/POINTTRANS_C_32768/model.pth')

def model_weight_path(task, has_observation=False):
    if task == 'pose_gen':
        return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-09_11-22-52_PoseGen_ddm4_lr1e-4_ep100/ckpts/model.pth')
    elif task == 'motion_gen' and has_observation == True:
        return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-09_14-28-12_MotionGen_ddm_T200_lr1e-4_ep300_obser/ckpts/model.pth')
    elif task == 'motion_gen' and has_observation == False:
        return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-09_12-54-50_MotionGen_ddm_T200_lr1e-4_ep300/ckpts/model.pth')
    elif task == 'path_planning':
        return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-25_20-57-28_Path_ddm4_LR1e-4_E100_REL/ckpts/model.pth')
    else:
        raise Exception('Unexcepted task.')

def pose_motion_data_path():
    zip_path = hf_hub_download('SceneDiffuser/SceneDiffuser', 'hf_data/pose_motion.zip')
    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
        zip_ref.extractall(os.path.dirname(zip_path))
    
    rpath = os.path.join(os.path.dirname(zip_path), 'pose_motion')

    return (
        os.path.join(rpath, 'PROXD_temp'),
        os.path.join(rpath, 'models_smplx_v1_1/models/'),
        os.path.join(rpath, 'PROX'),
        os.path.join(rpath, 'PROX/V02_05')
    )

def path_planning_data_path():
    zip_path = hf_hub_download('SceneDiffuser/SceneDiffuser', 'hf_data/path_planning.zip')
    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
        zip_ref.extractall(os.path.dirname(zip_path))
    
    return os.path.join(os.path.dirname(zip_path), 'path_planning')

def load_ckpt(model: torch.nn.Module, path: str) -> None:
    """ load ckpt for current model

    Args:
        model: current model
        path: save path
    """
    assert os.path.exists(path), 'Can\'t find provided ckpt.'

    saved_state_dict = torch.load(path)['model']
    model_state_dict = model.state_dict()

    for key in model_state_dict:
        if key in saved_state_dict:
            model_state_dict[key] = saved_state_dict[key]
        ## model is trained with ddm
        if 'module.'+key in saved_state_dict:
            model_state_dict[key] = saved_state_dict['module.'+key]
    
    model.load_state_dict(model_state_dict)

def _sampling(cfg: DictConfig, scene: str) -> Any:
    ## compute modeling dimension according to task
    cfg.model.d_x = compute_model_dim(cfg.task)
    
    if cfg.gpu is not None:
        device = f'cuda:{cfg.gpu}'
    else:
        device = 'cpu'
    
    dataset = create_dataset(cfg.task.dataset, 'test', cfg.slurm, case_only=True, specific_scene=scene)
    
    if cfg.model.scene_model.name == 'PointTransformer':
        collate_fn = collate_fn_squeeze_pcd_batch
    else:
        collate_fn = collate_fn_general
    
    dataloader = dataset.get_dataloader(
        batch_size=1,
        collate_fn=collate_fn,
        shuffle=True,
    )
    
    ## create model and load ckpt
    model = create_model(cfg, slurm=cfg.slurm, device=device)
    model.to(device=device)
    load_ckpt(model, path=model_weight_path(cfg.task.name, cfg.task.has_observation if 'has_observation' in cfg.task else False))
    
    ## create visualizer and visualize
    visualizer = create_visualizer(cfg.task.visualizer)
    results = visualizer.visualize(model, dataloader)
    return results

def _planning(cfg: DictConfig, scene: str) -> Any:
    ## compute modeling dimension according to task
    cfg.model.d_x = compute_model_dim(cfg.task)
    
    if cfg.gpu is not None:
        device = f'cuda:{cfg.gpu}'
    else:
        device = 'cpu'
    
    dataset = create_dataset(cfg.task.dataset, 'test', cfg.slurm, case_only=True, specific_scene=scene)
    
    if cfg.model.scene_model.name == 'PointTransformer':
        collate_fn = collate_fn_squeeze_pcd_batch
    else:
        collate_fn = collate_fn_general
    
    dataloader = dataset.get_dataloader(
        batch_size=1,
        collate_fn=collate_fn,
        shuffle=True,
    )
    
    ## create model and load ckpt
    model = create_model(cfg, slurm=cfg.slurm, device=device)
    model.to(device=device)
    load_ckpt(model, path=model_weight_path(cfg.task.name, cfg.task.has_observation if 'has_observation' in cfg.task else False))
    
    ## create environment for planning task and run
    env = create_enviroment(cfg.task.env)
    results = env.run(model, dataloader)
    return results


## interface for five task
## real-time model:
##   - pose generation
##   - motion generation
##   - path planning
def pose_generation(scene, count, seed, opt, scale) -> Any:
    scene_model_weight_path = pretrain_pointtrans_weight_path()
    data_dir, smpl_dir, prox_dir, vposer_dir = pose_motion_data_path()
    override_config = [
        "diffuser=ddpm",
        "model=unet",
        f"model.scene_model.pretrained_weights={scene_model_weight_path}",
        "task=pose_gen",
        "task.visualizer.name=PoseGenVisualizerHF",
        f"task.visualizer.ksample={count}",
        f"task.dataset.data_dir={data_dir}",
        f"task.dataset.smpl_dir={smpl_dir}",
        f"task.dataset.prox_dir={prox_dir}",
        f"task.dataset.vposer_dir={vposer_dir}",
    ]

    if opt == True:
        override_config += [
            "optimizer=pose_in_scene",
            "optimizer.scale_type=div_var",
            f"optimizer.scale={scale}",
            "optimizer.vposer=false",
            "optimizer.contact_weight=0.02",
            "optimizer.collision_weight=1.0"
        ]
    
    initialize(config_path="./scenediffuser/configs", version_base=None)
    config = compose(config_name="default", overrides=override_config)

    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

    res = _sampling(config, scene)

    hydra.core.global_hydra.GlobalHydra.instance().clear()
    return res

def motion_generation(scene, count, seed, withstart, opt, scale) -> Any:
    scene_model_weight_path = pretrain_pointtrans_weight_path()
    data_dir, smpl_dir, prox_dir, vposer_dir = pose_motion_data_path()
    override_config = [
        "diffuser=ddpm",
        "diffuser.steps=200",
        "model=unet",
        "model.use_position_embedding=true",
        f"model.scene_model.pretrained_weights={scene_model_weight_path}",
        "task=motion_gen",
        f"task.has_observation={withstart}",
        "task.dataset.repr_type=absolute",
        "task.dataset.frame_interval_test=20",
        "task.visualizer.name=MotionGenVisualizerHF",
        f"task.visualizer.ksample={count}",
        f"task.dataset.data_dir={data_dir}",
        f"task.dataset.smpl_dir={smpl_dir}",
        f"task.dataset.prox_dir={prox_dir}",
        f"task.dataset.vposer_dir={vposer_dir}",
    ]
    if opt == True:
        override_config += [
            "optimizer=motion_in_scene",
            "optimizer.scale_type=div_var",
            f"optimizer.scale={scale}",
            "optimizer.vposer=false",
            "optimizer.contact_weight=0.02",
            "optimizer.collision_weight=1.0",
            "optimizer.smoothness_weight=0.001",
            "optimizer.frame_interval=1",
        ]
    
    initialize(config_path="./scenediffuser/configs", version_base=None)
    config = compose(config_name="default", overrides=override_config)

    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

    res_gifs = _sampling(config, scene)

    ## save sampled motion as .gif file
    datestr = time.strftime("%Y-%m-%d", time.localtime(time.time()))
    target_dir = os.path.join('./results/motion_generation/', f'd-{datestr}')
    os.makedirs(target_dir, exist_ok=True)
    res = []
    uuid_str = uuid.uuid4()
    for i, imgs in enumerate(res_gifs):
        target_path = os.path.join(target_dir, f'{uuid_str}--{i}.gif')
        imgs = [im.resize((720, 405)) for im in imgs] # resize image for low resolution to save space
        img, *img_rest = imgs
        img.save(fp=target_path, format='GIF', append_images=img_rest, save_all=True, duration=33.33, loop=0)
        res.append(target_path)

    hydra.core.global_hydra.GlobalHydra.instance().clear()
    return res

def grasp_generation(case_id):
    assert isinstance(case_id, str)
    res = f"./results/grasp_generation/results/{case_id}/{random.randint(0, 19)}.glb"
    if not os.path.exists(res):
        results_path = hf_hub_download('SceneDiffuser/SceneDiffuser', 'results/grasp_generation/results.zip')
        os.makedirs('./results/grasp_generation/', exist_ok=True)
        with zipfile.ZipFile(results_path, 'r') as zip_ref:
            zip_ref.extractall('./results/grasp_generation/')
    
    return res

def path_planning(scene, mode, count, seed, opt, scale_opt, pla, scale_pla):

    scene_model_weight_path = pretrain_pointtrans_weight_path()
    data_dir = path_planning_data_path()

    override_config = [
        "diffuser=ddpm",
        "model=unet",
        "model.use_position_embedding=true",
        f"model.scene_model.pretrained_weights={scene_model_weight_path}",
        "task=path_planning",
        "task.visualizer.name=PathPlanningRenderingVisualizerHF",
        f"task.visualizer.ksample={count}",
        f"task.dataset.data_dir={data_dir}",
        "task.dataset.repr_type=relative",
        "task.env.name=PathPlanningEnvWrapperHF",
        "task.env.inpainting_horizon=16",
        "task.env.robot_top=3.0",
        "task.env.env_adaption=false"
    ]

    if opt == True:
        override_config += [
            "optimizer=path_in_scene",
            "optimizer.scale_type=div_var",
            "optimizer.continuity=false",
            f"optimizer.scale={scale_opt}",
        ]
    if pla == True:
        override_config += [
            "planner=greedy_path_planning",
            f"planner.scale={scale_pla}",
            "planner.scale_type=div_var",
            "planner.greedy_type=all_frame_exp"
        ]
    
    initialize(config_path="./scenediffuser/configs", version_base=None)
    config = compose(config_name="default", overrides=override_config)

    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

    if mode == 'Sampling':
        img = _sampling(config, scene)
        res = (img, 0)
    elif mode == 'Planning':
        res = _planning(config, scene)
    else:
        res = (None, 0)

    hydra.core.global_hydra.GlobalHydra.instance().clear()
    return res