xdecoder/Instruct-X-Decoder

1# --------------------------------------------------------2# X-Decoder -- Generalized Decoding for Pixel, Image, and Language3# Copyright (c) 2022 Microsoft4# Licensed under The MIT License [see LICENSE for details]5# Written by Xueyan Zou (xueyan@cs.wisc.edu)6# --------------------------------------------------------7 8import cv29import torch10import numpy as np11from PIL import Image12from torchvision import transforms13 14 15t = []16t.append(transforms.Resize(224, interpolation=Image.BICUBIC))17transform = transforms.Compose(t)18 19t = []20t.append(transforms.Resize(512, interpolation=Image.BICUBIC))21transform_v = transforms.Compose(t)22 23def image_captioning(model, image, texts, inpainting_text, *args, **kwargs):24    with torch.no_grad():25        image_ori = transform_v(image)26        width = image_ori.size[0]27        height = image_ori.size[1]28        image_ori = np.asarray(image_ori)29 30        image = transform(image)31        image = np.asarray(image)32        images = torch.from_numpy(image.copy()).permute(2,0,1).cuda()33 34        batch_inputs = [{'image': images, 'height': height, 'width': width, 'image_id': 0}]35        outputs = model.model.evaluate_captioning(batch_inputs)36        text = outputs[-1]['captioning_text']37 38        image_ori = image_ori.copy()39        cv2.rectangle(image_ori, (0, height-60), (width, height), (0,0,0), -1)40        font                   = cv2.FONT_HERSHEY_DUPLEX41        fontScale              = 1.242        thickness              = 243        lineType               = 244        bottomLeftCornerOfText = (10, height-20)45        fontColor              = [255,255,255]46        cv2.putText(image_ori, text,47            bottomLeftCornerOfText,48            font, 49            fontScale,50            fontColor,51            thickness,52            lineType)53    torch.cuda.empty_cache()54    return Image.fromarray(image_ori), text, None55 56