import math import torch import torch.nn as nn import torch.nn.functional as F def length_to_mask(length, max_len=None, dtype=None, device=None): assert len(length.shape) == 1 if max_len is None: max_len = length.max().long().item() mask = torch.arange(max_len, device=length.device, dtype=length.dtype).expand(len(length), max_len) < length.unsqueeze(1) if dtype is None: dtype = length.dtype if device is None: device = length.device return torch.as_tensor(mask, dtype=dtype, device=device) def get_padding_elem(L_in, stride, kernel_size, dilation): if stride > 1: padding = [math.floor(kernel_size / 2), math.floor(kernel_size / 2)] else: L_out = (math.floor((L_in - dilation * (kernel_size - 1) - 1) / stride) + 1) padding = [math.floor((L_in - L_out) / 2), math.floor((L_in - L_out) / 2)] return padding class _BatchNorm1d(nn.Module): def __init__(self, input_shape=None, input_size=None, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True, combine_batch_time=False, skip_transpose=False): super().__init__() self.combine_batch_time = combine_batch_time self.skip_transpose = skip_transpose if input_size is None and skip_transpose: input_size = input_shape[1] elif input_size is None: input_size = input_shape[-1] self.norm = nn.BatchNorm1d(input_size, eps=eps, momentum=momentum, affine=affine, track_running_stats=track_running_stats) def forward(self, x): shape_or = x.shape if self.combine_batch_time:x = x.reshape(shape_or[0] * shape_or[1], shape_or[2]) if x.ndim == 3 else x.reshape(shape_or[0] * shape_or[1], shape_or[3], shape_or[2]) elif not self.skip_transpose: x = x.transpose(-1, 1) x_n = self.norm(x) if self.combine_batch_time: x_n = x_n.reshape(shape_or) elif not self.skip_transpose: x_n = x_n.transpose(1, -1) return x_n class _Conv1d(nn.Module): def __init__(self, out_channels, kernel_size, input_shape=None, in_channels=None, stride=1, dilation=1, padding="same", groups=1, bias=True, padding_mode="reflect", skip_transpose=False, weight_norm=False, conv_init=None, default_padding=0): super().__init__() self.kernel_size = kernel_size self.stride = stride self.dilation = dilation self.padding = padding self.padding_mode = padding_mode self.unsqueeze = False self.skip_transpose = skip_transpose if input_shape is None and in_channels is None: raise ValueError if in_channels is None: in_channels = self._check_input_shape(input_shape) self.in_channels = in_channels self.conv = nn.Conv1d(in_channels, out_channels, self.kernel_size, stride=self.stride, dilation=self.dilation, padding=default_padding, groups=groups, bias=bias) if conv_init == "kaiming": nn.init.kaiming_normal_(self.conv.weight) elif conv_init == "zero": nn.init.zeros_(self.conv.weight) elif conv_init == "normal": nn.init.normal_(self.conv.weight, std=1e-6) if weight_norm: self.conv = nn.utils.weight_norm(self.conv) def forward(self, x): if not self.skip_transpose: x = x.transpose(1, -1) if self.unsqueeze: x = x.unsqueeze(1) if self.padding == "same": x = self._manage_padding(x, self.kernel_size, self.dilation, self.stride) elif self.padding == "causal": x = F.pad(x, ((self.kernel_size - 1) * self.dilation, 0)) elif self.padding == "valid": pass else: raise ValueError wx = self.conv(x) if self.unsqueeze: wx = wx.squeeze(1) if not self.skip_transpose: wx = wx.transpose(1, -1) return wx def _manage_padding(self, x, kernel_size, dilation, stride): return F.pad(x, get_padding_elem(self.in_channels, stride, kernel_size, dilation), mode=self.padding_mode) def _check_input_shape(self, shape): if len(shape) == 2: self.unsqueeze = True in_channels = 1 elif self.skip_transpose: in_channels = shape[1] elif len(shape) == 3: in_channels = shape[2] else: raise ValueError if not self.padding == "valid" and self.kernel_size % 2 == 0: raise ValueError return in_channels def remove_weight_norm(self): self.conv = nn.utils.remove_weight_norm(self.conv) class Linear(torch.nn.Module): def __init__(self, n_neurons, input_shape=None, input_size=None, bias=True, max_norm=None, combine_dims=False): super().__init__() self.max_norm = max_norm self.combine_dims = combine_dims if input_shape is None and input_size is None: raise ValueError if input_size is None: input_size = input_shape[-1] if len(input_shape) == 4 and self.combine_dims: input_size = input_shape[2] * input_shape[3] self.w = nn.Linear(input_size, n_neurons, bias=bias) def forward(self, x): if x.ndim == 4 and self.combine_dims: x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]) if self.max_norm is not None: self.w.weight.data = torch.renorm(self.w.weight.data, p=2, dim=0, maxnorm=self.max_norm) return self.w(x) class Conv1d(_Conv1d): def __init__(self, *args, **kwargs): super().__init__(skip_transpose=True, *args, **kwargs) class BatchNorm1d(_BatchNorm1d): def __init__(self, *args, **kwargs): super().__init__(skip_transpose=True, *args, **kwargs) class TDNNBlock(nn.Module): def __init__(self, in_channels, out_channels, kernel_size, dilation, activation=nn.ReLU, groups=1, dropout=0.0): super().__init__() self.conv = Conv1d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, dilation=dilation, groups=groups) self.activation = activation() self.norm = BatchNorm1d(input_size=out_channels) self.dropout = nn.Dropout1d(p=dropout) def forward(self, x): return self.dropout(self.norm(self.activation(self.conv(x)))) class Res2NetBlock(torch.nn.Module): def __init__(self, in_channels, out_channels, scale=8, kernel_size=3, dilation=1, dropout=0.0): super().__init__() assert in_channels % scale == 0 assert out_channels % scale == 0 in_channel = in_channels // scale hidden_channel = out_channels // scale self.blocks = nn.ModuleList([TDNNBlock(in_channel, hidden_channel, kernel_size=kernel_size, dilation=dilation, dropout=dropout) for _ in range(scale - 1)]) self.scale = scale def forward(self, x): y = [] for i, x_i in enumerate(torch.chunk(x, self.scale, dim=1)): if i == 0: y_i = x_i elif i == 1: y_i = self.blocks[i - 1](x_i) else: y_i = self.blocks[i - 1](x_i + y_i) y.append(y_i) return torch.cat(y, dim=1) class SEBlock(nn.Module): def __init__(self, in_channels, se_channels, out_channels): super().__init__() self.conv1 = Conv1d(in_channels=in_channels, out_channels=se_channels, kernel_size=1) self.relu = torch.nn.ReLU(inplace=True) self.conv2 = Conv1d(in_channels=se_channels, out_channels=out_channels, kernel_size=1) self.sigmoid = torch.nn.Sigmoid() def forward(self, x, lengths=None): L = x.shape[-1] if lengths is not None: mask = length_to_mask(lengths * L, max_len=L, device=x.device).unsqueeze(1) s = (x * mask).sum(dim=2, keepdim=True) / mask.sum(dim=2, keepdim=True) else: s = x.mean(dim=2, keepdim=True) return self.sigmoid(self.conv2(self.relu(self.conv1(s)))) * x class AttentiveStatisticsPooling(nn.Module): def __init__(self, channels, attention_channels=128, global_context=True): super().__init__() self.eps = 1e-12 self.global_context = global_context self.tdnn = TDNNBlock(channels * 3, attention_channels, 1, 1) if global_context else TDNNBlock(channels, attention_channels, 1, 1) self.tanh = nn.Tanh() self.conv = Conv1d(in_channels=attention_channels, out_channels=channels, kernel_size=1) def forward(self, x, lengths=None): L = x.shape[-1] def _compute_statistics(x, m, dim=2, eps=self.eps): mean = (m * x).sum(dim) return mean, torch.sqrt((m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(eps)) if lengths is None: lengths = torch.ones(x.shape[0], device=x.device) mask = length_to_mask(lengths * L, max_len=L, device=x.device).unsqueeze(1) if self.global_context: mean, std = _compute_statistics(x, mask / mask.sum(dim=2, keepdim=True).float()) attn = torch.cat([x, mean.unsqueeze(2).repeat(1, 1, L), std.unsqueeze(2).repeat(1, 1, L)], dim=1) else: attn = x mean, std = _compute_statistics(x, F.softmax(self.conv(self.tanh(self.tdnn(attn))).masked_fill(mask == 0, float("-inf")), dim=2)) return torch.cat((mean, std), dim=1).unsqueeze(2) class SERes2NetBlock(nn.Module): def __init__(self, in_channels, out_channels, res2net_scale=8, se_channels=128, kernel_size=1, dilation=1, activation=torch.nn.ReLU, groups=1, dropout=0.0): super().__init__() self.out_channels = out_channels self.tdnn1 = TDNNBlock(in_channels, out_channels, kernel_size=1, dilation=1, activation=activation, groups=groups, dropout=dropout) self.res2net_block = Res2NetBlock(out_channels, out_channels, res2net_scale, kernel_size, dilation) self.tdnn2 = TDNNBlock(out_channels, out_channels, kernel_size=1, dilation=1, activation=activation, groups=groups, dropout=dropout) self.se_block = SEBlock(out_channels, se_channels, out_channels) self.shortcut = None if in_channels != out_channels: self.shortcut = Conv1d(in_channels=in_channels, out_channels=out_channels, kernel_size=1) def forward(self, x, lengths=None): residual = x if self.shortcut: residual = self.shortcut(x) return self.se_block(self.tdnn2(self.res2net_block(self.tdnn1(x))), lengths) + residual class ECAPA_TDNN(torch.nn.Module): def __init__(self, input_size, device="cpu", lin_neurons=192, activation=torch.nn.ReLU, channels=[512, 512, 512, 512, 1536], kernel_sizes=[5, 3, 3, 3, 1], dilations=[1, 2, 3, 4, 1], attention_channels=128, res2net_scale=8, se_channels=128, global_context=True, groups=[1, 1, 1, 1, 1], dropout=0.0): super().__init__() assert len(channels) == len(kernel_sizes) assert len(channels) == len(dilations) self.channels = channels self.blocks = nn.ModuleList() self.blocks.append(TDNNBlock(input_size, channels[0], kernel_sizes[0], dilations[0], activation, groups[0], dropout)) for i in range(1, len(channels) - 1): self.blocks.append(SERes2NetBlock(channels[i - 1], channels[i], res2net_scale=res2net_scale, se_channels=se_channels, kernel_size=kernel_sizes[i], dilation=dilations[i], activation=activation, groups=groups[i], dropout=dropout)) self.mfa = TDNNBlock(channels[-2] * (len(channels) - 2), channels[-1], kernel_sizes[-1], dilations[-1], activation, groups=groups[-1], dropout=dropout) self.asp = AttentiveStatisticsPooling(channels[-1], attention_channels=attention_channels, global_context=global_context) self.asp_bn = BatchNorm1d(input_size=channels[-1] * 2) self.fc = Conv1d(in_channels=channels[-1] * 2, out_channels=lin_neurons, kernel_size=1) def forward(self, x, lengths=None): x = x.transpose(1, 2) xl = [] for layer in self.blocks: try: x = layer(x, lengths=lengths) except TypeError: x = layer(x) xl.append(x) return self.fc(self.asp_bn(self.asp(self.mfa(torch.cat(xl[1:], dim=1)), lengths=lengths))).transpose(1, 2) class Classifier(torch.nn.Module): def __init__(self, input_size, device="cpu", lin_blocks=0, lin_neurons=192, out_neurons=1211): super().__init__() self.blocks = nn.ModuleList() for _ in range(lin_blocks): self.blocks.extend([_BatchNorm1d(input_size=input_size), Linear(input_size=input_size, n_neurons=lin_neurons)]) input_size = lin_neurons self.weight = nn.Parameter(torch.FloatTensor(out_neurons, input_size, device=device)) nn.init.xavier_uniform_(self.weight) def forward(self, x): for layer in self.blocks: x = layer(x) return F.linear(F.normalize(x.squeeze(1)), F.normalize(self.weight)).unsqueeze(1)