import math import torch import torch.nn as nn # ----------------- Customed NormLayer Ops ----------------- class FrozenBatchNorm2d(torch.nn.Module): def __init__(self, n): super(FrozenBatchNorm2d, self).__init__() self.register_buffer("weight", torch.ones(n)) self.register_buffer("bias", torch.zeros(n)) self.register_buffer("running_mean", torch.zeros(n)) self.register_buffer("running_var", torch.ones(n)) def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs): num_batches_tracked_key = prefix + 'num_batches_tracked' if num_batches_tracked_key in state_dict: del state_dict[num_batches_tracked_key] super(FrozenBatchNorm2d, self)._load_from_state_dict( state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) def forward(self, x): # move reshapes to the beginning # to make it fuser-friendly w = self.weight.reshape(1, -1, 1, 1) b = self.bias.reshape(1, -1, 1, 1) rv = self.running_var.reshape(1, -1, 1, 1) rm = self.running_mean.reshape(1, -1, 1, 1) eps = 1e-5 scale = w * (rv + eps).rsqrt() bias = b - rm * scale return x * scale + bias class LayerNorm2D(nn.Module): def __init__(self, normalized_shape, norm_layer=nn.LayerNorm): super().__init__() self.ln = norm_layer(normalized_shape) if norm_layer is not None else nn.Identity() def forward(self, x): """ x: N C H W """ x = x.permute(0, 2, 3, 1) x = self.ln(x) x = x.permute(0, 3, 1, 2) return x # ----------------- Basic CNN Ops ----------------- def get_conv2d(c1, c2, k, p, s, g, bias=False): conv = nn.Conv2d(c1, c2, k, stride=s, padding=p, groups=g, bias=bias) return conv def get_activation(act_type=None): if act_type == 'relu': return nn.ReLU(inplace=True) elif act_type == 'lrelu': return nn.LeakyReLU(0.1, inplace=True) elif act_type == 'mish': return nn.Mish(inplace=True) elif act_type == 'silu': return nn.SiLU(inplace=True) elif act_type == 'gelu': return nn.GELU() elif act_type is None: return nn.Identity() else: raise NotImplementedError def get_norm(norm_type, dim): if norm_type == 'BN': return nn.BatchNorm2d(dim) elif norm_type == 'GN': return nn.GroupNorm(num_groups=32, num_channels=dim) elif norm_type is None: return nn.Identity() else: raise NotImplementedError class BasicConv(nn.Module): def __init__(self, in_dim, # in channels out_dim, # out channels kernel_size=1, # kernel size padding=0, # padding stride=1, # padding act_type :str = 'lrelu', # activation norm_type :str = 'BN', # normalization ): super(BasicConv, self).__init__() add_bias = False if norm_type else True self.conv = get_conv2d(in_dim, out_dim, k=kernel_size, p=padding, s=stride, g=1, bias=add_bias) self.norm = get_norm(norm_type, out_dim) self.act = get_activation(act_type) def forward(self, x): return self.act(self.norm(self.conv(x))) class UpSampleWrapper(nn.Module): """Upsample last feat map to specific stride.""" def __init__(self, in_dim, upsample_factor): super(UpSampleWrapper, self).__init__() # ---------- Basic parameters ---------- self.upsample_factor = upsample_factor # ---------- Network parameters ---------- if upsample_factor == 1: self.upsample = nn.Identity() else: scale = int(math.log2(upsample_factor)) dim = in_dim layers = [] for _ in range(scale-1): layers += [ nn.ConvTranspose2d(dim, dim // 2, kernel_size=2, stride=2), LayerNorm2D(dim // 2), nn.GELU() ] dim = dim // 2 layers += [nn.ConvTranspose2d(dim, dim // 2, kernel_size=2, stride=2)] dim = dim // 2 self.upsample = nn.Sequential(*layers) self.out_dim = dim def forward(self, x): x = self.upsample(x) return x # ----------------- MLP modules ----------------- class MLP(nn.Module): def __init__(self, in_dim, hidden_dim, out_dim, num_layers): super().__init__() self.num_layers = num_layers h = [hidden_dim] * (num_layers - 1) self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([in_dim] + h, h + [out_dim])) def forward(self, x): for i, layer in enumerate(self.layers): x = nn.functional.relu(layer(x)) if i < self.num_layers - 1 else layer(x) return x class FFN(nn.Module): def __init__(self, d_model=256, mlp_ratio=4.0, dropout=0., act_type='relu'): super().__init__() self.fpn_dim = round(d_model * mlp_ratio) self.linear1 = nn.Linear(d_model, self.fpn_dim) self.activation = get_activation(act_type) self.dropout2 = nn.Dropout(dropout) self.linear2 = nn.Linear(self.fpn_dim, d_model) self.dropout3 = nn.Dropout(dropout) self.norm = nn.LayerNorm(d_model) def forward(self, src): src2 = self.linear2(self.dropout2(self.activation(self.linear1(src)))) src = src + self.dropout3(src2) src = self.norm(src) return src # ----------------- Basic CNN Ops ----------------- class FrozenBatchNorm2d(torch.nn.Module): def __init__(self, n): super(FrozenBatchNorm2d, self).__init__() self.register_buffer("weight", torch.ones(n)) self.register_buffer("bias", torch.zeros(n)) self.register_buffer("running_mean", torch.zeros(n)) self.register_buffer("running_var", torch.ones(n)) def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs): num_batches_tracked_key = prefix + 'num_batches_tracked' if num_batches_tracked_key in state_dict: del state_dict[num_batches_tracked_key] super(FrozenBatchNorm2d, self)._load_from_state_dict( state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) def forward(self, x): # move reshapes to the beginning # to make it fuser-friendly w = self.weight.reshape(1, -1, 1, 1) b = self.bias.reshape(1, -1, 1, 1) rv = self.running_var.reshape(1, -1, 1, 1) rm = self.running_mean.reshape(1, -1, 1, 1) eps = 1e-5 scale = w * (rv + eps).rsqrt() bias = b - rm * scale return x * scale + bias