import torch import torch.nn as nn from utils.box_ops import bbox_iou # -------------------------- Task Aligned Assigner -------------------------- class TaskAlignedAssigner(nn.Module): def __init__(self, num_classes = 80, topk_candidates = 10, alpha = 0.5, beta = 6.0, eps = 1e-9): super(TaskAlignedAssigner, self).__init__() self.topk_candidates = topk_candidates self.num_classes = num_classes self.bg_idx = num_classes self.alpha = alpha self.beta = beta self.eps = eps @torch.no_grad() def forward(self, pd_scores, pd_bboxes, anc_points, gt_labels, gt_bboxes): self.bs = pd_scores.size(0) self.n_max_boxes = gt_bboxes.size(1) mask_pos, align_metric, overlaps = self.get_pos_mask( pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points) target_gt_idx, fg_mask, mask_pos = select_highest_overlaps( mask_pos, overlaps, self.n_max_boxes) # Assigned target target_labels, target_bboxes, target_scores = self.get_targets( gt_labels, gt_bboxes, target_gt_idx, fg_mask) # normalize align_metric *= mask_pos pos_align_metrics = align_metric.amax(axis=-1, keepdim=True) # b, max_num_obj pos_overlaps = (overlaps * mask_pos).amax(axis=-1, keepdim=True) # b, max_num_obj norm_align_metric = (align_metric * pos_overlaps / (pos_align_metrics + self.eps)).amax(-2).unsqueeze(-1) target_scores = target_scores * norm_align_metric return target_labels, target_bboxes, target_scores, fg_mask.bool(), target_gt_idx def get_pos_mask(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points): # get in_gts mask, (b, max_num_obj, h*w) mask_in_gts = select_candidates_in_gts(anc_points, gt_bboxes) # get anchor_align metric, (b, max_num_obj, h*w) align_metric, overlaps = self.get_box_metrics(pd_scores, pd_bboxes, gt_labels, gt_bboxes, mask_in_gts) # get topk_metric mask, (b, max_num_obj, h*w) mask_topk = self.select_topk_candidates(align_metric) # merge all mask to a final mask, (b, max_num_obj, h*w) mask_pos = mask_topk * mask_in_gts return mask_pos, align_metric, overlaps def get_box_metrics(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes, mask_in_gts): """Compute alignment metric given predicted and ground truth bounding boxes.""" na = pd_bboxes.shape[-2] mask_in_gts = mask_in_gts.bool() # b, max_num_obj, h*w overlaps = torch.zeros([self.bs, self.n_max_boxes, na], dtype=pd_bboxes.dtype, device=pd_bboxes.device) bbox_scores = torch.zeros([self.bs, self.n_max_boxes, na], dtype=pd_scores.dtype, device=pd_scores.device) ind = torch.zeros([2, self.bs, self.n_max_boxes], dtype=torch.long) # 2, b, max_num_obj ind[0] = torch.arange(end=self.bs).view(-1, 1).expand(-1, self.n_max_boxes) # b, max_num_obj ind[1] = gt_labels.squeeze(-1) # b, max_num_obj # Get the scores of each grid for each gt cls bbox_scores[mask_in_gts] = pd_scores[ind[0], :, ind[1]][mask_in_gts] # b, max_num_obj, h*w # (b, max_num_obj, 1, 4), (b, 1, h*w, 4) pd_boxes = pd_bboxes.unsqueeze(1).expand(-1, self.n_max_boxes, -1, -1)[mask_in_gts] gt_boxes = gt_bboxes.unsqueeze(2).expand(-1, -1, na, -1)[mask_in_gts] overlaps[mask_in_gts] = bbox_iou(gt_boxes, pd_boxes, xywh=False, CIoU=True).squeeze(-1).clamp_(0) align_metric = bbox_scores.pow(self.alpha) * overlaps.pow(self.beta) return align_metric, overlaps def select_topk_candidates(self, metrics, largest=True): """ Args: metrics: (b, max_num_obj, h*w). topk_mask: (b, max_num_obj, topk) or None """ # (b, max_num_obj, topk) topk_metrics, topk_idxs = torch.topk(metrics, self.topk_candidates, dim=-1, largest=largest) topk_mask = (topk_metrics.max(-1, keepdim=True)[0] > self.eps).expand_as(topk_idxs) # (b, max_num_obj, topk) topk_idxs.masked_fill_(~topk_mask, 0) # (b, max_num_obj, topk, h*w) -> (b, max_num_obj, h*w) count_tensor = torch.zeros(metrics.shape, dtype=torch.int8, device=topk_idxs.device) ones = torch.ones_like(topk_idxs[:, :, :1], dtype=torch.int8, device=topk_idxs.device) for k in range(self.topk_candidates): # Expand topk_idxs for each value of k and add 1 at the specified positions count_tensor.scatter_add_(-1, topk_idxs[:, :, k:k + 1], ones) # count_tensor.scatter_add_(-1, topk_idxs, torch.ones_like(topk_idxs, dtype=torch.int8, device=topk_idxs.device)) # Filter invalid bboxes count_tensor.masked_fill_(count_tensor > 1, 0) return count_tensor.to(metrics.dtype) def get_targets(self, gt_labels, gt_bboxes, target_gt_idx, fg_mask): # Assigned target labels, (b, 1) batch_ind = torch.arange(end=self.bs, dtype=torch.int64, device=gt_labels.device)[..., None] target_gt_idx = target_gt_idx + batch_ind * self.n_max_boxes # (b, h*w) target_labels = gt_labels.long().flatten()[target_gt_idx] # (b, h*w) # Assigned target boxes, (b, max_num_obj, 4) -> (b, h*w, 4) target_bboxes = gt_bboxes.view(-1, 4)[target_gt_idx] # Assigned target scores target_labels.clamp_(0) # 10x faster than F.one_hot() target_scores = torch.zeros((target_labels.shape[0], target_labels.shape[1], self.num_classes), dtype=torch.int64, device=target_labels.device) # (b, h*w, 80) target_scores.scatter_(2, target_labels.unsqueeze(-1), 1) fg_scores_mask = fg_mask[:, :, None].repeat(1, 1, self.num_classes) # (b, h*w, 80) target_scores = torch.where(fg_scores_mask > 0, target_scores, 0) return target_labels, target_bboxes, target_scores # -------------------------- Basic Functions -------------------------- def select_candidates_in_gts(xy_centers, gt_bboxes, eps=1e-9): """select the positive anchors's center in gt Args: xy_centers (Tensor): shape(bs*n_max_boxes, num_total_anchors, 4) gt_bboxes (Tensor): shape(bs, n_max_boxes, 4) Return: (Tensor): shape(bs, n_max_boxes, num_total_anchors) """ n_anchors = xy_centers.size(0) bs, n_max_boxes, _ = gt_bboxes.size() _gt_bboxes = gt_bboxes.reshape([-1, 4]) xy_centers = xy_centers.unsqueeze(0).repeat(bs * n_max_boxes, 1, 1) gt_bboxes_lt = _gt_bboxes[:, 0:2].unsqueeze(1).repeat(1, n_anchors, 1) gt_bboxes_rb = _gt_bboxes[:, 2:4].unsqueeze(1).repeat(1, n_anchors, 1) b_lt = xy_centers - gt_bboxes_lt b_rb = gt_bboxes_rb - xy_centers bbox_deltas = torch.cat([b_lt, b_rb], dim=-1) bbox_deltas = bbox_deltas.reshape([bs, n_max_boxes, n_anchors, -1]) return (bbox_deltas.min(axis=-1)[0] > eps).to(gt_bboxes.dtype) def select_highest_overlaps(mask_pos, overlaps, n_max_boxes): """if an anchor box is assigned to multiple gts, the one with the highest iou will be selected. Args: mask_pos (Tensor): shape(bs, n_max_boxes, num_total_anchors) overlaps (Tensor): shape(bs, n_max_boxes, num_total_anchors) Return: target_gt_idx (Tensor): shape(bs, num_total_anchors) fg_mask (Tensor): shape(bs, num_total_anchors) mask_pos (Tensor): shape(bs, n_max_boxes, num_total_anchors) """ fg_mask = mask_pos.sum(-2) if fg_mask.max() > 1: # one anchor is assigned to multiple gt_bboxes mask_multi_gts = (fg_mask.unsqueeze(1) > 1).expand(-1, n_max_boxes, -1) # (b, n_max_boxes, h*w) max_overlaps_idx = overlaps.argmax(1) # (b, h*w) is_max_overlaps = torch.zeros(mask_pos.shape, dtype=mask_pos.dtype, device=mask_pos.device) is_max_overlaps.scatter_(1, max_overlaps_idx.unsqueeze(1), 1) mask_pos = torch.where(mask_multi_gts, is_max_overlaps, mask_pos).float() # (b, n_max_boxes, h*w) fg_mask = mask_pos.sum(-2) # Find each grid serve which gt(index) target_gt_idx = mask_pos.argmax(-2) # (b, h*w) return target_gt_idx, fg_mask, mask_pos def iou_calculator(box1, box2, eps=1e-9): """Calculate iou for batch Args: box1 (Tensor): shape(bs, n_max_boxes, 1, 4) box2 (Tensor): shape(bs, 1, num_total_anchors, 4) Return: (Tensor): shape(bs, n_max_boxes, num_total_anchors) """ box1 = box1.unsqueeze(2) # [N, M1, 4] -> [N, M1, 1, 4] box2 = box2.unsqueeze(1) # [N, M2, 4] -> [N, 1, M2, 4] px1y1, px2y2 = box1[:, :, :, 0:2], box1[:, :, :, 2:4] gx1y1, gx2y2 = box2[:, :, :, 0:2], box2[:, :, :, 2:4] x1y1 = torch.maximum(px1y1, gx1y1) x2y2 = torch.minimum(px2y2, gx2y2) overlap = (x2y2 - x1y1).clip(0).prod(-1) area1 = (px2y2 - px1y1).clip(0).prod(-1) area2 = (gx2y2 - gx1y1).clip(0).prod(-1) union = area1 + area2 - overlap + eps return overlap / union