taskswithcode/DeticChatGPT

1import os2from pyChatGPT import ChatGPT3 4os.system("pip install -U gradio")5 6import sys7import gradio as gr8 9os.system(10    "pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.9/index.html"11)12 13# clone and install Detic14os.system(15    "git clone https://github.com/facebookresearch/Detic.git --recurse-submodules"16)17os.chdir("Detic")18 19# Install detectron220import torch21 22# Some basic setup:23# Setup detectron2 logger24import detectron225from detectron2.utils.logger import setup_logger26 27setup_logger()28 29# import some common libraries30import sys31import numpy as np32import os, json, cv2, random33 34# import some common detectron2 utilities35from detectron2 import model_zoo36from detectron2.engine import DefaultPredictor37from detectron2.config import get_cfg38from detectron2.utils.visualizer import Visualizer39from detectron2.data import MetadataCatalog, DatasetCatalog40 41# Detic libraries42sys.path.insert(0, "third_party/CenterNet2/projects/CenterNet2/")43sys.path.insert(0, "third_party/CenterNet2/")44from centernet.config import add_centernet_config45from detic.config import add_detic_config46from detic.modeling.utils import reset_cls_test47 48from PIL import Image49 50# Build the detector and download our pretrained weights51cfg = get_cfg()52add_centernet_config(cfg)53add_detic_config(cfg)54cfg.MODEL.DEVICE = "cpu"55cfg.merge_from_file("configs/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.yaml")56cfg.MODEL.WEIGHTS = "https://dl.fbaipublicfiles.com/detic/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.pth"57cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model58cfg.MODEL.ROI_BOX_HEAD.ZEROSHOT_WEIGHT_PATH = "rand"59cfg.MODEL.ROI_HEADS.ONE_CLASS_PER_PROPOSAL = (60    True  # For better visualization purpose. Set to False for all classes.61)62predictor = DefaultPredictor(cfg)63 64# Setup the model's vocabulary using build-in datasets65 66BUILDIN_CLASSIFIER = {67    "lvis": "datasets/metadata/lvis_v1_clip_a+cname.npy",68    "objects365": "datasets/metadata/o365_clip_a+cnamefix.npy",69    "openimages": "datasets/metadata/oid_clip_a+cname.npy",70    "coco": "datasets/metadata/coco_clip_a+cname.npy",71}72 73BUILDIN_METADATA_PATH = {74    "lvis": "lvis_v1_val",75    "objects365": "objects365_v2_val",76    "openimages": "oid_val_expanded",77    "coco": "coco_2017_val",78}79 80vocabulary = "lvis"  # change to 'lvis', 'objects365', 'openimages', or 'coco'81metadata = MetadataCatalog.get(BUILDIN_METADATA_PATH[vocabulary])82classifier = BUILDIN_CLASSIFIER[vocabulary]83num_classes = len(metadata.thing_classes)84reset_cls_test(predictor.model, classifier, num_classes)85 86 87 88def inference(img,unique_only):89 90    im = cv2.imread(img)91 92    outputs = predictor(im)93    v = Visualizer(im[:, :, ::-1], metadata)94    out = v.draw_instance_predictions(outputs["instances"].to("cpu"))95 96    detected_objects = []97    object_list_str = []98 99    box_locations = outputs["instances"].pred_boxes100    box_loc_screen = box_locations.tensor.cpu().numpy()101    unique_object_dict = {}102    for i, box_coord in enumerate(box_loc_screen):103        x0, y0, x1, y1 = box_coord104        width = x1 - x0105        height = y1 - y0106        predicted_label = metadata.thing_classes[outputs["instances"].pred_classes[i]]107        detected_objects.append(108            {109                "prediction": predicted_label,110                "x": int(x0),111                "y": int(y0),112                "w": int(width),113                "h": int(height),114            }115        )116        if ((not unique_only) or  (unique_only and predicted_label not in unique_object_dict)):117            object_list_str.append(118            f"{predicted_label} - X:{int(x0)} Y: {int(y0)} Width: {int(width)} Height: {int(height)}"119            )120            unique_object_dict[predicted_label] = 1121 122 123    output_str = "Imagine you are a blind but intelligent image captioner who is only given the X,Y coordinates and width, height of  each object in a scene with no specific attributes of the objects themselves. Create a description of the scene using the relative positions and sizes of objects\n"124    for line in object_list_str:125        output_str += line + "\n"126 127    return (128        Image.fromarray(np.uint8(out.get_image())).convert("RGB"),129        output_str130    )131 132 133with gr.Blocks() as demo:134    gr.Markdown("<div style=\"font-size:22; color: #2f2f2f; text-align: center\"><b>Detic for ChatGPT</b></div> <i>")135    gr.Markdown("<div style=\"font-size:12; color: #6f6f6f; text-align: center\"><i>A duplicated tweak of  <a href=\"https://huggingface.co/spaces/taesiri/DeticChatGPT\">taesiri's Dectic/ChatGPT demo</a></i>")136    gr.Markdown("Use Detic to detect objects in an image and then copy/paste output text into your ChatGPT playground.")137    138    with gr.Column():139        inp = gr.Image(label="Input Image", type="filepath")140        chk = gr.Checkbox(label="Unique Objects only? (useful to reduce ChatGPT input to speed up its reponse and also eliminate timeouts")141        btn_detic = gr.Button("Run Detic for ChatGPT")142    with gr.Column():143        outviz = gr.Image(label="Visualization", type="pil")144        output_desc = gr.Textbox(label="Description for using in ChatGPT", lines=5)145        # outputjson = gr.JSON(label="Detected Objects")146 147    btn_detic.click(fn=inference, inputs=[inp,chk], outputs=[outviz, output_desc])148 149demo.launch()150