RT-ODLab/engine.py

import torch
import torch.distributed as dist

import time
import os
import math
import numpy as np
import random

from utils import distributed_utils
from utils.vis_tools import vis_data


def rescale_image_targets(images, targets, stride, min_box_size):
    """
        Deployed for Multi scale trick.
    """
    if isinstance(stride, int):
        max_stride = stride
    elif isinstance(stride, list):
        max_stride = max(stride)

    # During training phase, the shape of input image is square.
    old_img_size = images.shape[-1]
    new_img_size = random.randrange(old_img_size * 0.5, old_img_size * 1.5 + max_stride) // max_stride * max_stride  # size
    if new_img_size / old_img_size != 1:
        # interpolate
        images = torch.nn.functional.interpolate(
                            input=images, 
                            size=new_img_size, 
                            mode='bilinear', 
                            align_corners=False)
        # rescale targets
        for tgt in targets:
            boxes = tgt["boxes"].clone()
            labels = tgt["labels"].clone()
            boxes = torch.clamp(boxes, 0, old_img_size)
            # rescale box
            boxes[:, [0, 2]] = boxes[:, [0, 2]] / old_img_size * new_img_size
            boxes[:, [1, 3]] = boxes[:, [1, 3]] / old_img_size * new_img_size
            # refine tgt
            tgt_boxes_wh = boxes[..., 2:] - boxes[..., :2]
            min_tgt_size = torch.min(tgt_boxes_wh, dim=-1)[0]
            keep = (min_tgt_size >= min_box_size)

            tgt["boxes"] = boxes[keep]
            tgt["labels"] = labels[keep]

    return images, targets, new_img_size


def train_one_epoch(epoch,
                    total_epochs,
                    args, 
                    device, 
                    ema,
                    model,
                    criterion,
                    cfg, 
                    dataloader, 
                    optimizer,
                    scheduler,
                    lf,
                    scaler,
                    last_opt_step):
    epoch_size = len(dataloader)
    img_size = args.img_size
    t0 = time.time()
    nw = epoch_size * args.wp_epoch
    accumulate = accumulate = max(1, round(64 / args.batch_size))

    # train one epoch
    for iter_i, (images, targets) in enumerate(dataloader):
        ni = iter_i + epoch * epoch_size
        # Warmup
        if ni <= nw:
            xi = [0, nw]  # x interp
            accumulate = max(1, np.interp(ni, xi, [1, 64 / args.batch_size]).round())
            for j, x in enumerate(optimizer.param_groups):
                # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0
                x['lr'] = np.interp(
                    ni, xi, [cfg['warmup_bias_lr'] if j == 0 else 0.0, x['initial_lr'] * lf(epoch)])
                if 'momentum' in x:
                    x['momentum'] = np.interp(ni, xi, [cfg['warmup_momentum'], cfg['momentum']])
                            
        # visualize train targets
        if args.vis_tgt:
            vis_data(images, targets)

        # to device
        images = images.to(device, non_blocking=True).float() / 255.

        # multi scale
        if args.multi_scale:
            images, targets, img_size = rescale_image_targets(
                images, targets, model.stride, args.min_box_size)
            
        # inference
        with torch.cuda.amp.autocast(enabled=args.fp16):
            outputs = model(images)
            # loss
            loss_dict = criterion(outputs=outputs, targets=targets)
            losses = loss_dict['losses']
            losses *= images.shape[0]  # loss * bs

            # reduce            
            loss_dict_reduced = distributed_utils.reduce_dict(loss_dict)

            if args.distributed:
                # gradient averaged between devices in DDP mode
                losses *= distributed_utils.get_world_size()

        # check loss
        try:
            if torch.isnan(losses):
                print('loss is NAN !!')
                continue
        except:
            print(loss_dict)

        # backward
        scaler.scale(losses).backward()

        # Optimize
        if ni - last_opt_step >= accumulate:
            if cfg['clip_grad'] > 0:
                # unscale gradients
                scaler.unscale_(optimizer)
                # clip gradients
                torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=cfg['clip_grad'])
            # optimizer.step
            scaler.step(optimizer)
            scaler.update()
            optimizer.zero_grad()
            # ema
            if ema:
                ema.update(model)
            last_opt_step = ni

        # display
        if distributed_utils.is_main_process() and iter_i % 10 == 0:
            t1 = time.time()
            cur_lr = [param_group['lr']  for param_group in optimizer.param_groups]
            # basic infor
            log =  '[Epoch: {}/{}]'.format(epoch+1, total_epochs)
            log += '[Iter: {}/{}]'.format(iter_i, epoch_size)
            log += '[lr: {:.6f}]'.format(cur_lr[2])
            # loss infor
            for k in loss_dict_reduced.keys():
                if k == 'losses' and args.distributed:
                    world_size = distributed_utils.get_world_size()
                    log += '[{}: {:.2f}]'.format(k, loss_dict[k] / world_size)
                else:
                    log += '[{}: {:.2f}]'.format(k, loss_dict[k])

            # other infor
            log += '[time: {:.2f}]'.format(t1 - t0)
            log += '[size: {}]'.format(img_size)

            # print log infor
            print(log, flush=True)
            
            t0 = time.time()
    
    scheduler.step()

    return last_opt_step


def val_one_epoch(args, 
                  model, 
                  evaluator,
                  optimizer,
                  epoch,
                  best_map,
                  path_to_save):
    # check evaluator
    if distributed_utils.is_main_process():
        if evaluator is None:
            print('No evaluator ... save model and go on training.')
            print('Saving state, epoch: {}'.format(epoch + 1))
            weight_name = '{}_epoch_{}.pth'.format(args.model, epoch + 1)
            checkpoint_path = os.path.join(path_to_save, weight_name)
            torch.save({'model': model.state_dict(),
                        'mAP': -1.,
                        'optimizer': optimizer.state_dict(),
                        'epoch': epoch,
                        'args': args}, 
                        checkpoint_path)                      
            
        else:
            print('eval ...')
            # set eval mode
            model.trainable = False
            model.eval()

            # evaluate
            evaluator.evaluate(model)

            cur_map = evaluator.map
            if cur_map > best_map:
                # update best-map
                best_map = cur_map
                # save model
                print('Saving state, epoch:', epoch + 1)
                weight_name = '{}_epoch_{}_{:.2f}.pth'.format(args.model, epoch + 1, best_map*100)
                checkpoint_path = os.path.join(path_to_save, weight_name)
                torch.save({'model': model.state_dict(),
                            'mAP': round(best_map*100, 1),
                            'optimizer': optimizer.state_dict(),
                            'epoch': epoch,
                            'args': args}, 
                            checkpoint_path)                      

            # set train mode.
            model.trainable = True
            model.train()

    if args.distributed:
        # wait for all processes to synchronize
        dist.barrier()

    return best_map