samH98/LungCancerDetection

1import numpy as np2import random3import torch4import torch.nn as nn5 6from models.common import Conv, DWConv7from utils.google_utils import attempt_download8 9 10class CrossConv(nn.Module):11    # Cross Convolution Downsample12    def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False):13        # ch_in, ch_out, kernel, stride, groups, expansion, shortcut14        super(CrossConv, self).__init__()15        c_ = int(c2 * e)  # hidden channels16        self.cv1 = Conv(c1, c_, (1, k), (1, s))17        self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g)18        self.add = shortcut and c1 == c219 20    def forward(self, x):21        return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))22 23 24class Sum(nn.Module):25    # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.0907026    def __init__(self, n, weight=False):  # n: number of inputs27        super(Sum, self).__init__()28        self.weight = weight  # apply weights boolean29        self.iter = range(n - 1)  # iter object30        if weight:31            self.w = nn.Parameter(-torch.arange(1., n) / 2, requires_grad=True)  # layer weights32 33    def forward(self, x):34        y = x[0]  # no weight35        if self.weight:36            w = torch.sigmoid(self.w) * 237            for i in self.iter:38                y = y + x[i + 1] * w[i]39        else:40            for i in self.iter:41                y = y + x[i + 1]42        return y43 44 45class MixConv2d(nn.Module):46    # Mixed Depthwise Conv https://arxiv.org/abs/1907.0959547    def __init__(self, c1, c2, k=(1, 3), s=1, equal_ch=True):48        super(MixConv2d, self).__init__()49        groups = len(k)50        if equal_ch:  # equal c_ per group51            i = torch.linspace(0, groups - 1E-6, c2).floor()  # c2 indices52            c_ = [(i == g).sum() for g in range(groups)]  # intermediate channels53        else:  # equal weight.numel() per group54            b = [c2] + [0] * groups55            a = np.eye(groups + 1, groups, k=-1)56            a -= np.roll(a, 1, axis=1)57            a *= np.array(k) ** 258            a[0] = 159            c_ = np.linalg.lstsq(a, b, rcond=None)[0].round()  # solve for equal weight indices, ax = b60 61        self.m = nn.ModuleList([nn.Conv2d(c1, int(c_[g]), k[g], s, k[g] // 2, bias=False) for g in range(groups)])62        self.bn = nn.BatchNorm2d(c2)63        self.act = nn.LeakyReLU(0.1, inplace=True)64 65    def forward(self, x):66        return x + self.act(self.bn(torch.cat([m(x) for m in self.m], 1)))67 68 69class Ensemble(nn.ModuleList):70    # Ensemble of models71    def __init__(self):72        super(Ensemble, self).__init__()73 74    def forward(self, x, augment=False):75        y = []76        for module in self:77            y.append(module(x, augment)[0])78        # y = torch.stack(y).max(0)[0]  # max ensemble79        # y = torch.stack(y).mean(0)  # mean ensemble80        y = torch.cat(y, 1)  # nms ensemble81        return y, None  # inference, train output82 83 84 85 86 87class ORT_NMS(torch.autograd.Function):88    '''ONNX-Runtime NMS operation'''89    @staticmethod90    def forward(ctx,91                boxes,92                scores,93                max_output_boxes_per_class=torch.tensor([100]),94                iou_threshold=torch.tensor([0.45]),95                score_threshold=torch.tensor([0.25])):96        device = boxes.device97        batch = scores.shape[0]98        num_det = random.randint(0, 100)99        batches = torch.randint(0, batch, (num_det,)).sort()[0].to(device)100        idxs = torch.arange(100, 100 + num_det).to(device)101        zeros = torch.zeros((num_det,), dtype=torch.int64).to(device)102        selected_indices = torch.cat([batches[None], zeros[None], idxs[None]], 0).T.contiguous()103        selected_indices = selected_indices.to(torch.int64)104        return selected_indices105 106    @staticmethod107    def symbolic(g, boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold):108        return g.op("NonMaxSuppression", boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold)109 110 111class TRT_NMS(torch.autograd.Function):112    '''TensorRT NMS operation'''113    @staticmethod114    def forward(115        ctx,116        boxes,117        scores,118        background_class=-1,119        box_coding=1,120        iou_threshold=0.45,121        max_output_boxes=100,122        plugin_version="1",123        score_activation=0,124        score_threshold=0.25,125    ):126        batch_size, num_boxes, num_classes = scores.shape127        num_det = torch.randint(0, max_output_boxes, (batch_size, 1), dtype=torch.int32)128        det_boxes = torch.randn(batch_size, max_output_boxes, 4)129        det_scores = torch.randn(batch_size, max_output_boxes)130        det_classes = torch.randint(0, num_classes, (batch_size, max_output_boxes), dtype=torch.int32)131        return num_det, det_boxes, det_scores, det_classes132 133    @staticmethod134    def symbolic(g,135                 boxes,136                 scores,137                 background_class=-1,138                 box_coding=1,139                 iou_threshold=0.45,140                 max_output_boxes=100,141                 plugin_version="1",142                 score_activation=0,143                 score_threshold=0.25):144        out = g.op("TRT::EfficientNMS_TRT",145                   boxes,146                   scores,147                   background_class_i=background_class,148                   box_coding_i=box_coding,149                   iou_threshold_f=iou_threshold,150                   max_output_boxes_i=max_output_boxes,151                   plugin_version_s=plugin_version,152                   score_activation_i=score_activation,153                   score_threshold_f=score_threshold,154                   outputs=4)155        nums, boxes, scores, classes = out156        return nums, boxes, scores, classes157 158 159class ONNX_ORT(nn.Module):160    '''onnx module with ONNX-Runtime NMS operation.'''161    def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=640, device=None, n_classes=80):162        super().__init__()163        self.device = device if device else torch.device("cpu")164        self.max_obj = torch.tensor([max_obj]).to(device)165        self.iou_threshold = torch.tensor([iou_thres]).to(device)166        self.score_threshold = torch.tensor([score_thres]).to(device)167        self.max_wh = max_wh # if max_wh != 0 : non-agnostic else : agnostic168        self.convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],169                                           dtype=torch.float32,170                                           device=self.device)171        self.n_classes=n_classes172 173    def forward(self, x):174        boxes = x[:, :, :4]175        conf = x[:, :, 4:5]176        scores = x[:, :, 5:]177        if self.n_classes == 1:178            scores = conf # for models with one class, cls_loss is 0 and cls_conf is always 0.5,179                                 # so there is no need to multiplicate.180        else:181            scores *= conf  # conf = obj_conf * cls_conf182        boxes @= self.convert_matrix183        max_score, category_id = scores.max(2, keepdim=True)184        dis = category_id.float() * self.max_wh185        nmsbox = boxes + dis186        max_score_tp = max_score.transpose(1, 2).contiguous()187        selected_indices = ORT_NMS.apply(nmsbox, max_score_tp, self.max_obj, self.iou_threshold, self.score_threshold)188        X, Y = selected_indices[:, 0], selected_indices[:, 2]189        selected_boxes = boxes[X, Y, :]190        selected_categories = category_id[X, Y, :].float()191        selected_scores = max_score[X, Y, :]192        X = X.unsqueeze(1).float()193        return torch.cat([X, selected_boxes, selected_categories, selected_scores], 1)194 195class ONNX_TRT(nn.Module):196    '''onnx module with TensorRT NMS operation.'''197    def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None ,device=None, n_classes=80):198        super().__init__()199        assert max_wh is None200        self.device = device if device else torch.device('cpu')201        self.background_class = -1,202        self.box_coding = 1,203        self.iou_threshold = iou_thres204        self.max_obj = max_obj205        self.plugin_version = '1'206        self.score_activation = 0207        self.score_threshold = score_thres208        self.n_classes=n_classes209 210    def forward(self, x):211        boxes = x[:, :, :4]212        conf = x[:, :, 4:5]213        scores = x[:, :, 5:]214        if self.n_classes == 1:215            scores = conf # for models with one class, cls_loss is 0 and cls_conf is always 0.5,216                                 # so there is no need to multiplicate.217        else:218            scores *= conf  # conf = obj_conf * cls_conf219        num_det, det_boxes, det_scores, det_classes = TRT_NMS.apply(boxes, scores, self.background_class, self.box_coding,220                                                                    self.iou_threshold, self.max_obj,221                                                                    self.plugin_version, self.score_activation,222                                                                    self.score_threshold)223        return num_det, det_boxes, det_scores, det_classes224 225 226class End2End(nn.Module):227    '''export onnx or tensorrt model with NMS operation.'''228    def __init__(self, model, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None, device=None, n_classes=80):229        super().__init__()230        device = device if device else torch.device('cpu')231        assert isinstance(max_wh,(int)) or max_wh is None232        self.model = model.to(device)233        self.model.model[-1].end2end = True234        self.patch_model = ONNX_TRT if max_wh is None else ONNX_ORT235        self.end2end = self.patch_model(max_obj, iou_thres, score_thres, max_wh, device, n_classes)236        self.end2end.eval()237 238    def forward(self, x):239        x = self.model(x)240        x = self.end2end(x)241        return x242 243 244 245 246 247def attempt_load(weights, map_location=None):248    # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a249    model = Ensemble()250    for w in weights if isinstance(weights, list) else [weights]:251        attempt_download(w)252        ckpt = torch.load(w, map_location=map_location)  # load253        model.append(ckpt['ema' if ckpt.get('ema') else 'model'].float().fuse().eval())  # FP32 model254    255    # Compatibility updates256    for m in model.modules():257        if type(m) in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]:258            m.inplace = True  # pytorch 1.7.0 compatibility259        elif type(m) is nn.Upsample:260            m.recompute_scale_factor = None  # torch 1.11.0 compatibility261        elif type(m) is Conv:262            m._non_persistent_buffers_set = set()  # pytorch 1.6.0 compatibility263    264    if len(model) == 1:265        return model[-1]  # return model266    else:267        print('Ensemble created with %s\n' % weights)268        for k in ['names', 'stride']:269            setattr(model, k, getattr(model[-1], k))270        return model  # return ensemble271 272 273