|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.init as init
|
|
|
|
__all__ = ['BatchNormReimpl']
|
|
|
|
|
|
class BatchNorm2dReimpl(nn.Module):
|
|
"""
|
|
A re-implementation of batch normalization, used for testing the numerical
|
|
stability.
|
|
|
|
Author: acgtyrant
|
|
See also:
|
|
https://github.com/vacancy/Synchronized-BatchNorm-PyTorch/issues/14
|
|
"""
|
|
def __init__(self, num_features, eps=1e-5, momentum=0.1):
|
|
super().__init__()
|
|
|
|
self.num_features = num_features
|
|
self.eps = eps
|
|
self.momentum = momentum
|
|
self.weight = nn.Parameter(torch.empty(num_features))
|
|
self.bias = nn.Parameter(torch.empty(num_features))
|
|
self.register_buffer('running_mean', torch.zeros(num_features))
|
|
self.register_buffer('running_var', torch.ones(num_features))
|
|
self.reset_parameters()
|
|
|
|
def reset_running_stats(self):
|
|
self.running_mean.zero_()
|
|
self.running_var.fill_(1)
|
|
|
|
def reset_parameters(self):
|
|
self.reset_running_stats()
|
|
init.uniform_(self.weight)
|
|
init.zeros_(self.bias)
|
|
|
|
def forward(self, input_):
|
|
batchsize, channels, height, width = input_.size()
|
|
numel = batchsize * height * width
|
|
input_ = input_.permute(1, 0, 2, 3).contiguous().view(channels, numel)
|
|
sum_ = input_.sum(1)
|
|
sum_of_square = input_.pow(2).sum(1)
|
|
mean = sum_ / numel
|
|
sumvar = sum_of_square - sum_ * mean
|
|
|
|
self.running_mean = (
|
|
(1 - self.momentum) * self.running_mean
|
|
+ self.momentum * mean.detach()
|
|
)
|
|
unbias_var = sumvar / (numel - 1)
|
|
self.running_var = (
|
|
(1 - self.momentum) * self.running_var
|
|
+ self.momentum * unbias_var.detach()
|
|
)
|
|
|
|
bias_var = sumvar / numel
|
|
inv_std = 1 / (bias_var + self.eps).pow(0.5)
|
|
output = (
|
|
(input_ - mean.unsqueeze(1)) * inv_std.unsqueeze(1) *
|
|
self.weight.unsqueeze(1) + self.bias.unsqueeze(1))
|
|
|
|
return output.view(channels, batchsize, height, width).permute(1, 0, 2, 3).contiguous()
|
|
|
|
|
