dataset/communal/base_dataset.py

"""
@Date: 2021/07/26
@description:
"""
import numpy as np
import torch

from utils.boundary import corners2boundary, visibility_corners, get_heat_map
from utils.conversion import xyz2depth, uv2xyz, uv2pixel
from dataset.communal.data_augmentation import PanoDataAugmentation


class BaseDataset(torch.utils.data.Dataset):
    def __init__(self, mode, shape=None, max_wall_num=999, aug=None, camera_height=1.6, patch_num=256, keys=None):
        if keys is None or len(keys) == 0:
            keys = ['image', 'depth', 'ratio', 'id', 'corners']
        if shape is None:
            shape = [512, 1024]

        assert mode == 'train' or mode == 'val' or mode == 'test' or mode is None, 'unknown mode!'
        self.mode = mode
        self.keys = keys
        self.shape = shape
        self.pano_aug = None if aug is None or mode == 'val' else PanoDataAugmentation(aug)
        self.camera_height = camera_height
        self.max_wall_num = max_wall_num
        self.patch_num = patch_num
        self.data = None

    def __len__(self):
        return len(self.data)

    @staticmethod
    def get_depth(corners, plan_y=1, length=256, visible=True):
        visible_floor_boundary = corners2boundary(corners, length=length, visible=visible)
        # The horizon-depth relative to plan_y
        visible_depth = xyz2depth(uv2xyz(visible_floor_boundary, plan_y), plan_y)
        return visible_depth

    def process_data(self, label, image, patch_num):
        """
        :param label:
        :param image:
        :param patch_num:
        :return:
        """
        corners = label['corners']
        if self.pano_aug is not None:
            corners, image = self.pano_aug.execute_aug(corners, image if 'image' in self.keys else None)
        eps = 1e-3
        corners[:, 1] = np.clip(corners[:, 1], 0.5+eps, 1-eps)

        output = {}
        if 'image' in self.keys:
            image = image.transpose(2, 0, 1)
            output['image'] = image

        visible_corners = None
        if 'corner_class' in self.keys or 'depth' in self.keys:
            visible_corners = visibility_corners(corners)

        if 'depth' in self.keys:
            depth = self.get_depth(visible_corners, length=patch_num, visible=False)
            assert len(depth) == patch_num, f"{label['id']}, {len(depth)}, {self.pano_aug.parameters}, {corners}"
            output['depth'] = depth

        if 'ratio' in self.keys:
            # Why use ratio? Because when floor_height =y_plan=1, we only need to predict ceil_height(ratio).
            output['ratio'] = label['ratio']

        if 'id' in self.keys:
            output['id'] = label['id']

        if 'corners' in self.keys:
            # all corners for evaluating Full_IoU
            assert len(label['corners']) <= 32, "len(label['corners']):"+len(label['corners'])
            output['corners'] = np.zeros((32, 2), dtype=np.float32)
            output['corners'][:len(label['corners'])] = label['corners']

        if 'corner_heat_map' in self.keys:
            output['corner_heat_map'] = get_heat_map(visible_corners[..., 0])

        if 'object' in self.keys and 'objects' in label:
            output[f'object_heat_map'] = np.zeros((3, patch_num), dtype=np.float32)
            output['object_size'] = np.zeros((3, patch_num), dtype=np.float32)  # width, height, bottom_height
            for i, type in enumerate(label['objects']):
                if len(label['objects'][type]) == 0:
                    continue

                u_s = []
                for obj in label['objects'][type]:
                    center_u = obj['center_u']
                    u_s.append(center_u)
                    center_pixel_u = uv2pixel(np.array([center_u]), w=patch_num, axis=0)[0]
                    output['object_size'][0, center_pixel_u] = obj['width_u']
                    output['object_size'][1, center_pixel_u] = obj['height_v']
                    output['object_size'][2, center_pixel_u] = obj['boundary_v']
                output[f'object_heat_map'][i] = get_heat_map(np.array(u_s))

        return output


if __name__ == '__main__':
    from dataset.communal.read import read_image, read_label
    from visualization.boundary import draw_boundaries
    from utils.boundary import depth2boundaries
    from tqdm import trange

    # np.random.seed(0)
    dataset = BaseDataset()
    dataset.pano_aug = PanoDataAugmentation(aug={
        'STRETCH': True,
        'ROTATE': True,
        'FLIP': True,
    })
    # pano_img = read_image("../src/demo.png")
    # label = read_label("../src/demo.json")
    pano_img_path = "../../src/dataset/mp3d/image/yqstnuAEVhm_6589ad7a5a0444b59adbf501c0f0fe53.png"
    label_path = "../../src/dataset/mp3d/label/yqstnuAEVhm_6589ad7a5a0444b59adbf501c0f0fe53.json"
    pano_img = read_image(pano_img_path)
    label = read_label(label_path)

    # batch test
    for i in trange(1):
        output = dataset.process_data(label, pano_img, 256)
        boundary_list = depth2boundaries(output['ratio'], output['depth'], step=None)
        draw_boundaries(output['image'].transpose(1, 2, 0), boundary_list=boundary_list, show=True)