openrec/modeling/decoders/ctc_decoder.py

import torch
import torch.nn as nn
import torch.nn.functional as F

from openrec.modeling.encoders.svtrnet import (
    Block,
    ConvBNLayer,
    kaiming_normal_,
    trunc_normal_,
    zeros_,
    ones_,
)


class Swish(nn.Module):

    def __init__(self):
        super().__init__()

    def forward(self, x):
        return x * F.sigmoid(x)


class EncoderWithSVTR(nn.Module):

    def __init__(
        self,
        in_channels,
        dims=64,  # XS
        depth=2,
        hidden_dims=120,
        use_guide=False,
        num_heads=8,
        qkv_bias=True,
        mlp_ratio=2.0,
        drop_rate=0.1,
        attn_drop_rate=0.1,
        drop_path=0.0,
        kernel_size=[3, 3],
        qk_scale=None,
        use_pool=True,
    ):
        super(EncoderWithSVTR, self).__init__()
        self.depth = depth
        self.use_guide = use_guide
        self.use_pool = use_pool
        self.conv1 = ConvBNLayer(
            in_channels,
            in_channels // 8,
            kernel_size=kernel_size,
            padding=[kernel_size[0] // 2, kernel_size[1] // 2],
            act=Swish,
            bias=False)
        self.conv2 = ConvBNLayer(in_channels // 8,
                                 hidden_dims,
                                 kernel_size=1,
                                 act=Swish,
                                 bias=False)

        self.svtr_block = nn.ModuleList([
            Block(
                dim=hidden_dims,
                num_heads=num_heads,
                mixer='Global',
                HW=None,
                mlp_ratio=mlp_ratio,
                qkv_bias=qkv_bias,
                qk_scale=qk_scale,
                drop=drop_rate,
                act_layer=Swish,
                attn_drop=attn_drop_rate,
                drop_path=drop_path,
                norm_layer='nn.LayerNorm',
                eps=1e-05,
                prenorm=False,
            ) for i in range(depth)
        ])
        self.norm = nn.LayerNorm(hidden_dims, eps=1e-6)
        self.conv3 = ConvBNLayer(hidden_dims,
                                 in_channels,
                                 kernel_size=1,
                                 act=Swish,
                                 bias=False)
        # last conv-nxn, the input is concat of input tensor and conv3 output tensor
        self.conv4 = ConvBNLayer(
            2 * in_channels,
            in_channels // 8,
            kernel_size=kernel_size,
            padding=[kernel_size[0] // 2, kernel_size[1] // 2],
            act=Swish,
            bias=False)

        self.conv1x1 = ConvBNLayer(in_channels // 8,
                                   dims,
                                   kernel_size=1,
                                   act=Swish,
                                   bias=False)
        self.out_channels = dims
        self.apply(self._init_weights)

    def _init_weights(self, m):
        if isinstance(m, nn.Linear):
            trunc_normal_(m.weight, mean=0, std=0.02)
            if isinstance(m, nn.Linear) and m.bias is not None:
                zeros_(m.bias)
        if isinstance(m, nn.LayerNorm):
            zeros_(m.bias)
            ones_(m.weight)
        if isinstance(m, nn.Conv2d):
            kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')

    def pool_h_2(self, x):
        # x: B, C, H, W
        x = x.mean(dim=2, keepdim=True)
        x = F.avg_pool2d(x, kernel_size=(1, 2))
        return x  # B, C, 1, W//2

    def forward(self, x):

        if self.use_pool:
            x = self.pool_h_2(x)

        # for use guide
        if self.use_guide:
            z = x.detach()
        else:
            z = x
        # for short cut
        h = z
        # reduce dim
        z = self.conv1(z)
        z = self.conv2(z)
        # SVTR global block
        B, C, H, W = z.shape
        z = z.flatten(2).transpose(1, 2)
        for blk in self.svtr_block:
            z = blk(z)
        z = self.norm(z)
        # last stage
        z = z.reshape(-1, H, W, C).permute(0, 3, 1, 2)
        z = self.conv3(z)
        z = torch.concat((h, z), dim=1)
        z = self.conv1x1(self.conv4(z))
        return z


class CTCDecoder(nn.Module):

    def __init__(self,
                 in_channels,
                 out_channels=6625,
                 mid_channels=None,
                 return_feats=False,
                 svtr_encoder=None,
                 **kwargs):
        super(CTCDecoder, self).__init__()
        if svtr_encoder is not None:
            svtr_encoder['in_channels'] = in_channels
            self.svtr_encoder = EncoderWithSVTR(**svtr_encoder)
            in_channels = self.svtr_encoder.out_channels
        else:
            self.svtr_encoder = None
        if mid_channels is None:
            self.fc = nn.Linear(
                in_channels,
                out_channels,
            )
        else:
            self.fc1 = nn.Linear(
                in_channels,
                mid_channels,
            )
            self.fc2 = nn.Linear(
                mid_channels,
                out_channels,
            )

        self.out_channels = out_channels
        self.mid_channels = mid_channels
        self.return_feats = return_feats

    def forward(self, x, data=None):

        if self.svtr_encoder is not None:
            x = self.svtr_encoder(x)
            x = x.flatten(2).transpose(1, 2)

        if self.mid_channels is None:
            predicts = self.fc(x)
        else:
            x = self.fc1(x)
            predicts = self.fc2(x)

        if self.return_feats:
            result = (x, predicts)
        else:
            result = predicts

        if not self.training:
            predicts = F.softmax(predicts, dim=2)
            result = predicts

        return result