chendl/compositional_test

1import argparse
2import os
3import random
4
5import numpy as np
6import torch
7import torch.backends.cudnn as cudnn
8import gradio as gr
9
10from minigpt4.common.config import Config
11from minigpt4.common.dist_utils import get_rank
12from minigpt4.common.registry import registry
13from minigpt4.conversation.conversation import Chat, CONV_VISION
14
15# imports modules for registration
16from minigpt4.datasets.builders import *
17from minigpt4.models import *
18from minigpt4.processors import *
19from minigpt4.runners import *
20from minigpt4.tasks import *
21
22
23def parse_args():
24    parser = argparse.ArgumentParser(description="Demo")
25    parser.add_argument("--cfg-path", type=str, default='eval_configs/minigpt4.yaml',
26                        help="path to configuration file.")
27    parser.add_argument(
28        "--options",
29        nargs="+",
30        help="override some settings in the used config, the key-value pair "
31             "in xxx=yyy format will be merged into config file (deprecate), "
32             "change to --cfg-options instead.",
33    )
34    args = parser.parse_args()
35    return args
36
37
38def setup_seeds(config):
39    seed = config.run_cfg.seed + get_rank()
40
41    random.seed(seed)
42    np.random.seed(seed)
43    torch.manual_seed(seed)
44
45    cudnn.benchmark = False
46    cudnn.deterministic = True
47
48
49# ========================================
50#             Model Initialization
51# ========================================
52
53SHARED_UI_WARNING = f'''### [NOTE] It is possible that you are waiting in a lengthy queue.
54You can duplicate and use it with a paid private GPU.
55<a class="duplicate-button" style="display:inline-block" target="_blank" href="https://huggingface.co/spaces/Vision-CAIR/minigpt4?duplicate=true"><img style="margin-top:0;margin-bottom:0" src="https://huggingface.co/datasets/huggingface/badges/raw/main/duplicate-this-space-xl-dark.svg" alt="Duplicate Space"></a>
56Alternatively, you can also use the demo on our [project page](https://minigpt-4.github.io).
57'''
58
59print('Initializing Chat')
60cfg = Config(parse_args())
61
62model_config = cfg.model_cfg
63model_cls = registry.get_model_class(model_config.arch)
64model = model_cls.from_config(model_config).to('cuda:0')
65
66vis_processor_cfg = cfg.datasets_cfg.cc_align.vis_processor.train
67vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg)
68chat = Chat(model, vis_processor)
69print('Initialization Finished')
70
71
72# ========================================
73#             Gradio Setting
74# ========================================
75
76def gradio_reset(chat_state, img_list):
77    if chat_state is not None:
78        chat_state.messages = []
79    if img_list is not None:
80        img_list = []
81    return None, gr.update(value=None, interactive=True), gr.update(placeholder='Please upload your image first',
82                                                                    interactive=False), gr.update(
83        value="Upload & Start Chat", interactive=True), chat_state, img_list
84
85
86def upload_img(gr_img, text_input, chat_state):
87    if gr_img is None:
88        return None, None, gr.update(interactive=True), chat_state, None
89    chat_state = CONV_VISION.copy()
90    img_list = []
91    llm_message = chat.upload_img(gr_img, chat_state, img_list)
92    return gr.update(interactive=False), gr.update(interactive=True, placeholder='Type and press Enter'), gr.update(
93        value="Start Chatting", interactive=False), chat_state, img_list
94
95def ask(text, conv):
96    if len(conv.messages) > 0 and conv.messages[-1][0] == conv.roles[0] \
97            and conv.messages[-1][1][-6:] == '</Img>':  # last message is image.
98        conv.messages[-1][1] = ' '.join([conv.messages[-1][1], text])
99    else:
100        conv.append_message(conv.roles[0], text)
101            
102def gradio_ask(user_message, chatbot, chat_state):
103    if len(user_message) == 0:
104        return gr.update(interactive=True, placeholder='Input should not be empty!'), chatbot, chat_state
105    chat.ask(user_message, chat_state)
106    chatbot = chatbot + [[user_message, None]]
107    return '', chatbot, chat_state
108
109
110def gradio_answer(chatbot, chat_state, img_list, num_beams, temperature):
111    llm_message = chat.answer(conv=chat_state, img_list=img_list, max_new_tokens=300, num_beams=1, temperature=temperature, max_length=2000)[0]
112    chatbot[-1][1] = llm_message
113    return chatbot, chat_state, img_list
114
115
116title = """<h1 align="center">Demo of Compositional-VLM</h1>"""
117description = """<h3>This is the demo of Compositional-VLM. Upload your images and start chatting!</h3>"""
118article = """<div style='display:flex; gap: 0.25rem; '><a href='https://compositionalvlm.github.io/'><img src='https://img.shields.io/badge/Project-Page-Green'></a><a href='https://github.com/Vision-CAIR/MiniGPT-4'><img src='https://img.shields.io/badge/Github-Code-blue'></a><a href='https://github.com/TsuTikgiau/blip2-llm/blob/release_prepare/MiniGPT_4.pdf'><img src='https://img.shields.io/badge/Paper-PDF-red'></a></div>
119"""
120
121# TODO show examples below
122
123with gr.Blocks() as demo:
124    gr.Markdown(title)
125    gr.Markdown(SHARED_UI_WARNING)
126    gr.Markdown(description)
127    gr.Markdown(article)
128
129    with gr.Row():
130        with gr.Column(scale=0.5):
131            image = gr.Image(type="pil")
132            upload_button = gr.Button(value="Upload & Start Chat", interactive=True, variant="primary")
133            clear = gr.Button("Restart")
134
135            num_beams = gr.Slider(
136                minimum=1,
137                maximum=5,
138                value=1,
139                step=1,
140                interactive=True,
141                label="beam search numbers)",
142            )
143
144            temperature = gr.Slider(
145                minimum=0.1,
146                maximum=2.0,
147                value=1.0,
148                step=0.1,
149                interactive=True,
150                label="Temperature",
151            )
152
153        with gr.Column():
154            chat_state = gr.State()
155            img_list = gr.State()
156            chatbot = gr.Chatbot(label='Compositional-VLM')
157            text_input = gr.Textbox(label='User', placeholder='Please upload your image first', interactive=False)
158
159    upload_button.click(upload_img, [image, text_input, chat_state],
160                        [image, text_input, upload_button, chat_state, img_list])
161
162    text_input.submit(gradio_ask, [text_input, chatbot, chat_state], [text_input, chatbot, chat_state]).then(
163        gradio_answer, [chatbot, chat_state, img_list, num_beams, temperature], [chatbot, chat_state, img_list]
164    )
165    clear.click(gradio_reset, [chat_state, img_list], [chatbot, image, text_input, upload_button, chat_state, img_list],
166                queue=False)
167
168demo.launch(enable_queue=True)