|
from typing import *
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
from ...modules import sparse as sp
|
|
from ...utils.random_utils import hammersley_sequence
|
|
from .base import SparseTransformerBase
|
|
from ...representations import Gaussian
|
|
|
|
|
|
class SLatGaussianDecoder(SparseTransformerBase):
|
|
def __init__(
|
|
self,
|
|
resolution: int,
|
|
model_channels: int,
|
|
latent_channels: int,
|
|
num_blocks: int,
|
|
num_heads: Optional[int] = None,
|
|
num_head_channels: Optional[int] = 64,
|
|
mlp_ratio: float = 4,
|
|
attn_mode: Literal["full", "shift_window", "shift_sequence", "shift_order", "swin"] = "swin",
|
|
window_size: int = 8,
|
|
pe_mode: Literal["ape", "rope"] = "ape",
|
|
use_fp16: bool = False,
|
|
use_checkpoint: bool = False,
|
|
qk_rms_norm: bool = False,
|
|
representation_config: dict = None,
|
|
):
|
|
super().__init__(
|
|
in_channels=latent_channels,
|
|
model_channels=model_channels,
|
|
num_blocks=num_blocks,
|
|
num_heads=num_heads,
|
|
num_head_channels=num_head_channels,
|
|
mlp_ratio=mlp_ratio,
|
|
attn_mode=attn_mode,
|
|
window_size=window_size,
|
|
pe_mode=pe_mode,
|
|
use_fp16=use_fp16,
|
|
use_checkpoint=use_checkpoint,
|
|
qk_rms_norm=qk_rms_norm,
|
|
)
|
|
self.resolution = resolution
|
|
self.rep_config = representation_config
|
|
self._calc_layout()
|
|
self.out_layer = sp.SparseLinear(model_channels, self.out_channels)
|
|
self._build_perturbation()
|
|
|
|
self.initialize_weights()
|
|
if use_fp16:
|
|
self.convert_to_fp16()
|
|
|
|
def initialize_weights(self) -> None:
|
|
super().initialize_weights()
|
|
|
|
nn.init.constant_(self.out_layer.weight, 0)
|
|
nn.init.constant_(self.out_layer.bias, 0)
|
|
|
|
def _build_perturbation(self) -> None:
|
|
perturbation = [hammersley_sequence(3, i, self.rep_config['num_gaussians']) for i in range(self.rep_config['num_gaussians'])]
|
|
perturbation = torch.tensor(perturbation).float() * 2 - 1
|
|
perturbation = perturbation / self.rep_config['voxel_size']
|
|
perturbation = torch.atanh(perturbation).to(self.device)
|
|
self.register_buffer('offset_perturbation', perturbation)
|
|
|
|
def _calc_layout(self) -> None:
|
|
self.layout = {
|
|
'_xyz' : {'shape': (self.rep_config['num_gaussians'], 3), 'size': self.rep_config['num_gaussians'] * 3},
|
|
'_features_dc' : {'shape': (self.rep_config['num_gaussians'], 1, 3), 'size': self.rep_config['num_gaussians'] * 3},
|
|
'_scaling' : {'shape': (self.rep_config['num_gaussians'], 3), 'size': self.rep_config['num_gaussians'] * 3},
|
|
'_rotation' : {'shape': (self.rep_config['num_gaussians'], 4), 'size': self.rep_config['num_gaussians'] * 4},
|
|
'_opacity' : {'shape': (self.rep_config['num_gaussians'], 1), 'size': self.rep_config['num_gaussians']},
|
|
}
|
|
start = 0
|
|
for k, v in self.layout.items():
|
|
v['range'] = (start, start + v['size'])
|
|
start += v['size']
|
|
self.out_channels = start
|
|
|
|
def to_representation(self, x: sp.SparseTensor) -> List[Gaussian]:
|
|
"""
|
|
Convert a batch of network outputs to 3D representations.
|
|
|
|
Args:
|
|
x: The [N x * x C] sparse tensor output by the network.
|
|
|
|
Returns:
|
|
list of representations
|
|
"""
|
|
ret = []
|
|
for i in range(x.shape[0]):
|
|
representation = Gaussian(
|
|
sh_degree=0,
|
|
aabb=[-0.5, -0.5, -0.5, 1.0, 1.0, 1.0],
|
|
mininum_kernel_size = self.rep_config['3d_filter_kernel_size'],
|
|
scaling_bias = self.rep_config['scaling_bias'],
|
|
opacity_bias = self.rep_config['opacity_bias'],
|
|
scaling_activation = self.rep_config['scaling_activation']
|
|
)
|
|
xyz = (x.coords[x.layout[i]][:, 1:].float() + 0.5) / self.resolution
|
|
for k, v in self.layout.items():
|
|
if k == '_xyz':
|
|
offset = x.feats[x.layout[i]][:, v['range'][0]:v['range'][1]].reshape(-1, *v['shape'])
|
|
offset = offset * self.rep_config['lr'][k]
|
|
if self.rep_config['perturb_offset']:
|
|
offset = offset + self.offset_perturbation
|
|
offset = torch.tanh(offset) / self.resolution * 0.5 * self.rep_config['voxel_size']
|
|
_xyz = xyz.unsqueeze(1) + offset
|
|
setattr(representation, k, _xyz.flatten(0, 1))
|
|
else:
|
|
feats = x.feats[x.layout[i]][:, v['range'][0]:v['range'][1]].reshape(-1, *v['shape']).flatten(0, 1)
|
|
feats = feats * self.rep_config['lr'][k]
|
|
setattr(representation, k, feats)
|
|
ret.append(representation)
|
|
return ret
|
|
|
|
def forward(self, x: sp.SparseTensor) -> List[Gaussian]:
|
|
h = super().forward(x)
|
|
h = h.type(x.dtype)
|
|
h = h.replace(F.layer_norm(h.feats, h.feats.shape[-1:]))
|
|
h = self.out_layer(h)
|
|
return self.to_representation(h)
|
|
|
