import numpy as np import torch import torch.nn as nn # --------------------- Basic modules --------------------- def get_conv2d(c1, c2, k, p, s, d, g, bias=False): conv = nn.Conv2d(c1, c2, k, stride=s, padding=p, dilation=d, groups=g, bias=bias) return conv def get_activation(act_type=None): if act_type == 'relu': return nn.ReLU(inplace=True) elif act_type == 'lrelu': return nn.LeakyReLU(0.1, inplace=True) elif act_type == 'mish': return nn.Mish(inplace=True) elif act_type == 'silu': return nn.SiLU(inplace=True) elif act_type is None: return nn.Identity() else: raise NotImplementedError def get_norm(norm_type, dim): if norm_type == 'BN': return nn.BatchNorm2d(dim) elif norm_type == 'GN': return nn.GroupNorm(num_groups=32, num_channels=dim) elif norm_type is None: return nn.Identity() else: raise NotImplementedError class BasicConv(nn.Module): def __init__(self, in_dim, # in channels out_dim, # out channels kernel_size=1, # kernel size padding=0, # padding stride=1, # padding dilation=1, # dilation act_type :str = 'lrelu', # activation norm_type :str = 'BN', # normalization depthwise :bool = False ): super(BasicConv, self).__init__() self.depthwise = depthwise if not depthwise: self.conv = get_conv2d(in_dim, out_dim, k=kernel_size, p=padding, s=stride, d=dilation, g=1, bias=True) self.norm = get_norm(norm_type, out_dim) else: self.conv1 = get_conv2d(in_dim, in_dim, k=kernel_size, p=padding, s=stride, d=dilation, g=in_dim, bias=True) self.norm1 = get_norm(norm_type, in_dim) self.conv2 = get_conv2d(in_dim, out_dim, k=1, p=0, s=1, d=1, g=1, bias=True) self.norm2 = get_norm(norm_type, out_dim) self.act = get_activation(act_type) def forward(self, x): if not self.depthwise: return self.act(self.norm(self.conv(x))) else: # Depthwise conv x = self.norm1(self.conv1(x)) # Pointwise conv x = self.act(self.norm2(self.conv2(x))) return x class RepVGGBlock(nn.Module): def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1, dilation=1, groups=1, deploy=False, ): super(RepVGGBlock, self).__init__() assert kernel_size == 3 assert padding == 1 # --------- Basic parameters --------- self.deploy = deploy self.groups = groups self.in_channels = in_channels self.out_channels = out_channels padding_11 = padding - kernel_size // 2 # --------- Model parameters --------- if deploy: self.rbr_reparam = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups, bias=True) else: self.rbr_identity = nn.BatchNorm2d(num_features=in_channels) if out_channels == in_channels and stride == 1 else None self.rbr_dense = BasicConv(in_channels, out_channels, kernel_size=kernel_size, padding=padding, stride=stride, act_type=None) self.rbr_1x1 = BasicConv(in_channels, out_channels, kernel_size=1, padding=padding_11, stride=stride, act_type=None) self.nonlinearity = nn.ReLU() def forward(self, inputs): '''Forward process''' if hasattr(self, 'rbr_reparam'): return self.nonlinearity(self.rbr_reparam(inputs)) if self.rbr_identity is None: id_out = 0 else: id_out = self.rbr_identity(inputs) return self.nonlinearity(self.rbr_dense(inputs) + self.rbr_1x1(inputs) + id_out) def get_equivalent_kernel_bias(self): kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense) kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1) kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity) return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid def _avg_to_3x3_tensor(self, avgp): channels = self.in_channels groups = self.groups kernel_size = avgp.kernel_size input_dim = channels // groups k = torch.zeros((channels, input_dim, kernel_size, kernel_size)) k[np.arange(channels), np.tile(np.arange(input_dim), groups), :, :] = 1.0 / kernel_size ** 2 return k def _pad_1x1_to_3x3_tensor(self, kernel1x1): if kernel1x1 is None: return 0 else: return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1]) def _fuse_bn_tensor(self, branch): if branch is None: return 0, 0 if isinstance(branch, BasicConv): kernel = branch.conv.weight bias = branch.conv.bias return kernel, bias elif isinstance(branch, nn.BatchNorm2d): if not hasattr(self, 'id_tensor'): input_dim = self.in_channels // self.groups kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), dtype=np.float32) for i in range(self.in_channels): kernel_value[i, i % input_dim, 1, 1] = 1 self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device) kernel = self.id_tensor running_mean = branch.running_mean running_var = branch.running_var gamma = branch.weight beta = branch.bias eps = branch.eps std = (running_var + eps).sqrt() t = (gamma / std).reshape(-1, 1, 1, 1) return kernel * t, beta - running_mean * gamma / std def switch_to_deploy(self): if hasattr(self, 'rbr_reparam'): return kernel, bias = self.get_equivalent_kernel_bias() self.rbr_reparam = nn.Conv2d(in_channels=self.rbr_dense.conv.in_channels, out_channels=self.rbr_dense.conv.out_channels, kernel_size=self.rbr_dense.conv.kernel_size, stride=self.rbr_dense.conv.stride, padding=self.rbr_dense.conv.padding, dilation=self.rbr_dense.conv.dilation, groups=self.rbr_dense.conv.groups, bias=True) self.rbr_reparam.weight.data = kernel self.rbr_reparam.bias.data = bias for para in self.parameters(): para.detach_() self.__delattr__('rbr_dense') self.__delattr__('rbr_1x1') if hasattr(self, 'rbr_identity'): self.__delattr__('rbr_identity') if hasattr(self, 'id_tensor'): self.__delattr__('id_tensor') self.deploy = True # ---------------------------- Basic Modules ---------------------------- class RepBlock(nn.Module): def __init__(self, in_channels, out_channels, num_blocks=1, block=RepVGGBlock): super().__init__() self.conv1 = block(in_channels, out_channels) self.block = nn.Sequential(*(block(out_channels, out_channels) for _ in range(num_blocks - 1))) if num_blocks > 1 else nn.Identity() if block == BottleRep: self.conv1 = BottleRep(in_channels, out_channels, weight=True) num_blocks = num_blocks // 2 self.block = nn.Sequential(*(BottleRep(out_channels, out_channels, weight=True) for _ in range(num_blocks - 1))) if num_blocks > 1 else None def forward(self, x): x = self.conv1(x) if self.block is not None: x = self.block(x) return x class BottleRep(nn.Module): def __init__(self, in_channels, out_channels, weight=False): super().__init__() self.conv1 = RepVGGBlock(in_channels, out_channels, kernel_size=3, padding=1, stride=1) self.conv2 = RepVGGBlock(out_channels, out_channels, kernel_size=3, padding=1, stride=1) if in_channels != out_channels: self.shortcut = False else: self.shortcut = True if weight: self.alpha = nn.Parameter(torch.ones(1)) else: self.alpha = 1.0 def forward(self, x): outputs = self.conv1(x) outputs = self.conv2(outputs) return outputs + self.alpha * x if self.shortcut else outputs class RepCSPBlock(nn.Module): def __init__(self, in_channels, out_channels, num_blocks=1, expansion=0.5): super().__init__() inter_dim = round(out_channels * expansion) # hidden channels self.cv1 = BasicConv(in_channels, inter_dim, kernel_size=1, act_type='relu') self.cv2 = BasicConv(in_channels, inter_dim, kernel_size=1, act_type='relu') self.cv3 = BasicConv(2 * inter_dim, out_channels, kernel_size=1, act_type='relu') self.module = RepBlock(inter_dim, inter_dim, num_blocks, block=BottleRep) def forward(self, x): x1 = self.cv1(x) x2 = self.module(self.cv2(x)) out = self.cv3(torch.cat((x1, x2), dim=1)) return out