OpenGVLab/InternVL
510
1import spaces2import argparse3from ast import parse4import datetime5import json6import os7import time8import hashlib9import re10 11import gradio as gr12import requests13import random14from filelock import FileLock15from io import BytesIO16from PIL import Image, ImageDraw, ImageFont17 18from constants import LOGDIR19from utils import (20 build_logger,21 server_error_msg,22 violates_moderation,23 moderation_msg,24 load_image_from_base64,25 get_log_filename,26)27from conversation import Conversation28 29logger = build_logger("gradio_web_server", "gradio_web_server.log")30 31headers = {"User-Agent": "InternVL-Chat Client"}32 33no_change_btn = gr.Button()34enable_btn = gr.Button(interactive=True)35disable_btn = gr.Button(interactive=False)36 37 38@spaces.GPU(duration=10)39def make_zerogpu_happy():40 pass41 42 43def write2file(path, content):44 lock = FileLock(f"{path}.lock")45 with lock:46 with open(path, "a") as fout:47 fout.write(content)48 49 50get_window_url_params = """51function() {52 const params = new URLSearchParams(window.location.search);53 url_params = Object.fromEntries(params);54 console.log(url_params);55 return url_params;56 }57"""58 59 60def init_state(state=None):61 if state is not None:62 del state63 return Conversation()64 65 66def find_bounding_boxes(state, response):67 pattern = re.compile(r"<ref>\s*(.*?)\s*</ref>\s*<box>\s*(\[\[.*?\]\])\s*</box>")68 matches = pattern.findall(response)69 results = []70 for match in matches:71 results.append((match[0], eval(match[1])))72 returned_image = None73 latest_image = state.get_images(source=state.USER)[-1]74 returned_image = latest_image.copy()75 width, height = returned_image.size76 draw = ImageDraw.Draw(returned_image)77 for result in results:78 line_width = max(1, int(min(width, height) / 200))79 random_color = (80 random.randint(0, 128),81 random.randint(0, 128),82 random.randint(0, 128),83 )84 category_name, coordinates = result85 coordinates = [86 (87 float(x[0]) / 1000,88 float(x[1]) / 1000,89 float(x[2]) / 1000,90 float(x[3]) / 1000,91 )92 for x in coordinates93 ]94 coordinates = [95 (96 int(x[0] * width),97 int(x[1] * height),98 int(x[2] * width),99 int(x[3] * height),100 )101 for x in coordinates102 ]103 for box in coordinates:104 draw.rectangle(box, outline=random_color, width=line_width)105 font = ImageFont.truetype("assets/SimHei.ttf", int(20 * line_width / 2))106 text_size = font.getbbox(category_name)107 text_width, text_height = (108 text_size[2] - text_size[0],109 text_size[3] - text_size[1],110 )111 text_position = (box[0], max(0, box[1] - text_height))112 draw.rectangle(113 [114 text_position,115 (text_position[0] + text_width, text_position[1] + text_height),116 ],117 fill=random_color,118 )119 draw.text(text_position, category_name, fill="white", font=font)120 return returned_image if len(matches) > 0 else None121 122 123def vote_last_response(state, liked, request: gr.Request):124 conv_data = {125 "tstamp": round(time.time(), 4),126 "like": liked,127 "model": 'InternVL2.5-78B',128 "state": state.dict(),129 "ip": request.client.host,130 }131 write2file(get_log_filename(), json.dumps(conv_data) + "\n")132 133 134def upvote_last_response(state, request: gr.Request):135 logger.info(f"upvote. ip: {request.client.host}")136 vote_last_response(state, True, request)137 textbox = gr.MultimodalTextbox(value=None, interactive=True)138 return (textbox,) + (disable_btn,) * 3139 140 141def downvote_last_response(state, request: gr.Request):142 logger.info(f"downvote. ip: {request.client.host}")143 vote_last_response(state, False, request)144 textbox = gr.MultimodalTextbox(value=None, interactive=True)145 return (textbox,) + (disable_btn,) * 3146 147 148def vote_selected_response(149 state, request: gr.Request, data: gr.LikeData150):151 logger.info(152 f"Vote: {data.liked}, index: {data.index}, value: {data.value} , ip: {request.client.host}"153 )154 conv_data = {155 "tstamp": round(time.time(), 4),156 "like": data.liked,157 "index": data.index,158 "model": 'InternVL2.5-78B',159 "state": state.dict(),160 "ip": request.client.host,161 }162 write2file(get_log_filename(), json.dumps(conv_data) + "\n")163 return164 165 166def flag_last_response(state, request: gr.Request):167 logger.info(f"flag. ip: {request.client.host}")168 vote_last_response(state, "flag", request)169 textbox = gr.MultimodalTextbox(value=None, interactive=True)170 return (textbox,) + (disable_btn,) * 3171 172 173def regenerate(state, image_process_mode, request: gr.Request):174 logger.info(f"regenerate. ip: {request.client.host}")175 # state.messages[-1][-1] = None176 state.update_message(Conversation.ASSISTANT, content='', image=None, idx=-1)177 prev_human_msg = state.messages[-2]178 if type(prev_human_msg[1]) in (tuple, list):179 prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)180 state.skip_next = False181 textbox = gr.MultimodalTextbox(value=None, interactive=True)182 return (state, state.to_gradio_chatbot(), textbox) + (disable_btn,) * 5183 184 185def clear_history(request: gr.Request):186 logger.info(f"clear_history. ip: {request.client.host}")187 state = init_state()188 textbox = gr.MultimodalTextbox(value=None, interactive=True)189 return (state, state.to_gradio_chatbot(), textbox) + (disable_btn,) * 5190 191 192def add_text(state, message, system_prompt, request: gr.Request):193 print(f"state: {state}")194 if not state:195 state = init_state()196 images = message.get("files", [])197 text = message.get("text", "").strip()198 logger.info(f"add_text. ip: {request.client.host}. len: {len(text)}")199 # import pdb; pdb.set_trace()200 textbox = gr.MultimodalTextbox(value=None, interactive=False)201 if len(text) <= 0 and len(images) == 0:202 state.skip_next = True203 return (state, state.to_gradio_chatbot(), textbox) + (no_change_btn,) * 5204 if args.moderate:205 flagged = violates_moderation(text)206 if flagged:207 state.skip_next = True208 textbox = gr.MultimodalTextbox(209 value={"text": moderation_msg}, interactive=True210 )211 return (state, state.to_gradio_chatbot(), textbox) + (no_change_btn,) * 5212 images = [Image.open(path).convert("RGB") for path in images]213 214 if len(images) > 0 and len(state.get_images(source=state.USER)) > 0:215 state = init_state(state)216 state.set_system_message(system_prompt)217 state.append_message(Conversation.USER, text, images)218 state.skip_next = False219 return (state, state.to_gradio_chatbot(), textbox) + (220 disable_btn,221 ) * 5222 223 224def http_bot(225 state,226 temperature,227 top_p,228 repetition_penalty,229 max_new_tokens,230 max_input_tiles,231 request: gr.Request,232):233 model_name = 'InternVL2.5-78B'234 logger.info(f"http_bot. ip: {request.client.host}")235 start_tstamp = time.time()236 if hasattr(state, "skip_next") and state.skip_next:237 # This generate call is skipped due to invalid inputs238 yield (239 state,240 state.to_gradio_chatbot(),241 gr.MultimodalTextbox(interactive=False),242 ) + (no_change_btn,) * 5243 return244 245 worker_addr = os.environ.get("WORKER_ADDR", "")246 api_token = os.environ.get("API_TOKEN", "")247 # headers = {"Authorization": f"{api_token}", "Content-Type": "application/json"}248 headers = {249 "Authorization": f"Bearer {api_token}", 250 "Content-Type": "application/json", 251 "Cookie": "acw_tc=65efca1a-119e-4da4-8fd6-338f5b70f29afaa1be67470ba113e38318b0208a00b2"252 }253 254 # No available worker255 if worker_addr == "":256 # state.messages[-1][-1] = server_error_msg257 state.update_message(Conversation.ASSISTANT, server_error_msg)258 yield (259 state,260 state.to_gradio_chatbot(),261 gr.MultimodalTextbox(interactive=False),262 disable_btn,263 disable_btn,264 disable_btn,265 enable_btn,266 enable_btn,267 )268 return269 270 all_images = state.get_images(source=state.USER)271 all_image_paths = [state.save_image(image) for image in all_images]272 273 # Make requests274 pload = {275 "model": model_name,276 "messages": state.get_prompt_v2(inlude_image=True, max_dynamic_patch=max_input_tiles),277 "temperature": float(temperature),278 "top_p": float(top_p),279 "max_tokens": max_new_tokens,280 "repetition_penalty": repetition_penalty,281 "stream": True282 }283 logger.info(f"==== request ====\n{pload}")284 state.append_message(Conversation.ASSISTANT, state.streaming_placeholder)285 yield (286 state,287 state.to_gradio_chatbot(),288 gr.MultimodalTextbox(interactive=False),289 ) + (disable_btn,) * 5290 291 try:292 # Stream output293 response = requests.post(worker_addr, json=pload, headers=headers, stream=True, timeout=300)294 finnal_output = ''295 for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\n"):296 if chunk:297 chunk = chunk.decode()298 if chunk == 'data: [DONE]':299 break300 if chunk.startswith("data:"):301 chunk = chunk[5:]302 chunk = json.loads(chunk)303 output = chunk['choices'][0]['delta']['content']304 finnal_output += output305 306 state.update_message(Conversation.ASSISTANT, finnal_output + state.streaming_placeholder, None)307 yield (308 state,309 state.to_gradio_chatbot(),310 gr.MultimodalTextbox(interactive=False),311 ) + (disable_btn,) * 5312 except requests.exceptions.RequestException as e:313 state.update_message(Conversation.ASSISTANT, server_error_msg, None)314 yield (315 state,316 state.to_gradio_chatbot(),317 gr.MultimodalTextbox(interactive=True),318 ) + (319 disable_btn,320 disable_btn,321 disable_btn,322 enable_btn,323 enable_btn,324 )325 return326 327 ai_response = state.return_last_message()328 if "<ref>" in ai_response:329 returned_image = find_bounding_boxes(state, ai_response)330 returned_image = [returned_image] if returned_image else []331 state.update_message(Conversation.ASSISTANT, ai_response, returned_image)332 333 state.end_of_current_turn()334 335 yield (336 state,337 state.to_gradio_chatbot(),338 gr.MultimodalTextbox(interactive=True),339 ) + (enable_btn,) * 5340 341 finish_tstamp = time.time()342 logger.info(f"{finnal_output}")343 data = {344 "tstamp": round(finish_tstamp, 4),345 "like": None,346 "model": model_name,347 "start": round(start_tstamp, 4),348 "finish": round(start_tstamp, 4),349 "state": state.dict(),350 "images": all_image_paths,351 "ip": request.client.host,352 }353 write2file(get_log_filename(), json.dumps(data) + "\n")354 355# <h1 style="font-size: 28px; font-weight: bold;">Expanding Performance Boundaries of Open-Source Multimodal Models with Model, Data, and Test-Time Scaling</h1>356title_html = """357<img src="https://internvl.opengvlab.com/assets/logo-47b364d3.jpg" style="width: 280px; height: 70px;">358<p>InternVL2.5: Expanding Performance Boundaries of Open-Source Multimodal Models with Model, Data, and Test-Time Scaling</p>359<a href="https://internvl.github.io/blog/2024-12-05-InternVL-2.5/">[🆕 InternVL Blog]</a> 360<a href="https://huggingface.co/papers/2412.05271">[📖 InternVL Paper]</a>361<a href="https://github.com/OpenGVLab/InternVL">[🌟 Github]</a><br>362<a href="https://internvl.readthedocs.io/en/latest/">[📜 Document]</a> 363<a href="https://internvl.opengvlab.com/">[🗨️ Official Demo]</a> 364"""365 366 367tos_markdown = """368### Terms of use369By using this service, users are required to agree to the following terms:370The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.371Please click the "Flag" button if you get any inappropriate answer! We will collect those to keep improving our moderator.372For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.373"""374 375 376# .gradio-container {margin: 5px 10px 0 10px !important};377block_css = """378.gradio-container {margin: 0.1% 1% 0 1% !important; max-width: 98% !important;};379#buttons button {380 min-width: min(120px,100%);381}382 383.gradient-text {384 font-size: 28px;385 width: auto;386 font-weight: bold;387 background: linear-gradient(45deg, red, orange, yellow, green, blue, indigo, violet);388 background-clip: text;389 -webkit-background-clip: text;390 color: transparent;391}392 393.plain-text {394 font-size: 22px;395 width: auto;396 font-weight: bold;397}398"""399 400js = """401function createWaveAnimation() {402 const text = document.getElementById('text');403 var i = 0;404 setInterval(function() {405 const colors = [406 'red, orange, yellow, green, blue, indigo, violet, purple',407 'orange, yellow, green, blue, indigo, violet, purple, red',408 'yellow, green, blue, indigo, violet, purple, red, orange',409 'green, blue, indigo, violet, purple, red, orange, yellow',410 'blue, indigo, violet, purple, red, orange, yellow, green',411 'indigo, violet, purple, red, orange, yellow, green, blue',412 'violet, purple, red, orange, yellow, green, blue, indigo',413 'purple, red, orange, yellow, green, blue, indigo, violet',414 ];415 const angle = 45;416 const colorIndex = i % colors.length;417 text.style.background = `linear-gradient(${angle}deg, ${colors[colorIndex]})`;418 text.style.webkitBackgroundClip = 'text';419 text.style.backgroundClip = 'text';420 text.style.color = 'transparent';421 text.style.fontSize = '28px';422 text.style.width = 'auto';423 text.textContent = 'InternVL2';424 text.style.fontWeight = 'bold';425 i += 1;426 }, 200);427 const params = new URLSearchParams(window.location.search);428 url_params = Object.fromEntries(params);429 // console.log(url_params);430 // console.log('hello world...');431 // console.log(window.location.search);432 // console.log('hello world...');433 // alert(window.location.search)434 // alert(url_params);435 return url_params;436}437 438"""439 440 441def build_demo():442 textbox = gr.MultimodalTextbox(443 interactive=True,444 file_types=["image", "video"],445 placeholder="Enter message or upload file...",446 show_label=False,447 )448 449 with gr.Blocks(450 title="InternVL-Chat",451 theme=gr.themes.Default(),452 css=block_css,453 ) as demo:454 state = gr.State()455 456 with gr.Row():457 with gr.Column(scale=2):458 # gr.Image('./gallery/logo-47b364d3.jpg')459 gr.HTML(title_html)460 461 with gr.Accordion("Settings", open=False) as setting_row:462 system_prompt = gr.Textbox(463 value="请尽可能详细地回答用户的问题。",464 label="System Prompt",465 interactive=True,466 )467 temperature = gr.Slider(468 minimum=0.0,469 maximum=1.0,470 value=0.2,471 step=0.1,472 interactive=True,473 label="Temperature",474 )475 top_p = gr.Slider(476 minimum=0.0,477 maximum=1.0,478 value=0.7,479 step=0.1,480 interactive=True,481 label="Top P",482 )483 repetition_penalty = gr.Slider(484 minimum=1.0,485 maximum=1.5,486 value=1.1,487 step=0.02,488 interactive=True,489 label="Repetition penalty",490 )491 max_output_tokens = gr.Slider(492 minimum=0,493 maximum=4096,494 value=1024,495 step=64,496 interactive=True,497 label="Max output tokens",498 )499 max_input_tiles = gr.Slider(500 minimum=1,501 maximum=32,502 value=12,503 step=1,504 interactive=True,505 label="Max input tiles (control the image size)",506 )507 examples = gr.Examples(508 examples=[509 [510 {511 "files": [512 "gallery/14.jfif",513 ],514 "text": "Please help me analyze this picture.",515 }516 ],517 [518 {519 "files": [520 "gallery/1-2.PNG",521 ],522 "text": "Implement this flow chart using python",523 }524 ],525 [526 {527 "files": [528 "gallery/15.PNG",529 ],530 "text": "Please help me analyze this picture.",531 }532 ],533 ],534 inputs=[textbox],535 )536 537 with gr.Column(scale=8):538 chatbot = gr.Chatbot(539 elem_id="chatbot",540 label="InternVL",541 height=580,542 show_copy_button=True,543 show_share_button=True,544 avatar_images=[545 "assets/human.png",546 "assets/assistant.png",547 ],548 bubble_full_width=False,549 )550 with gr.Row():551 with gr.Column(scale=8):552 textbox.render()553 with gr.Column(scale=1, min_width=50):554 submit_btn = gr.Button(value="Send", variant="primary")555 with gr.Row(elem_id="buttons") as button_row:556 upvote_btn = gr.Button(value="👍 Upvote", interactive=False)557 downvote_btn = gr.Button(value="👎 Downvote", interactive=False)558 flag_btn = gr.Button(value="⚠️ Flag", interactive=False)559 # stop_btn = gr.Button(value="⏹️ Stop Generation", interactive=False)560 regenerate_btn = gr.Button(561 value="🔄 Regenerate", interactive=False562 )563 clear_btn = gr.Button(value="🗑️ Clear", interactive=False)564 565 gr.Markdown(tos_markdown)566 url_params = gr.JSON(visible=False)567 568 # Register listeners569 btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]570 upvote_btn.click(571 upvote_last_response,572 [state],573 [textbox, upvote_btn, downvote_btn, flag_btn],574 )575 downvote_btn.click(576 downvote_last_response,577 [state],578 [textbox, upvote_btn, downvote_btn, flag_btn],579 )580 chatbot.like(581 vote_selected_response,582 [state],583 [],584 )585 flag_btn.click(586 flag_last_response,587 [state],588 [textbox, upvote_btn, downvote_btn, flag_btn],589 )590 regenerate_btn.click(591 regenerate,592 [state, system_prompt],593 [state, chatbot, textbox] + btn_list,594 ).then(595 http_bot,596 [597 state,598 temperature,599 top_p,600 repetition_penalty,601 max_output_tokens,602 max_input_tiles,603 ],604 [state, chatbot, textbox] + btn_list,605 )606 clear_btn.click(clear_history, None, [state, chatbot, textbox] + btn_list)607 608 textbox.submit(609 add_text,610 [state, textbox, system_prompt],611 [state, chatbot, textbox] + btn_list,612 ).then(613 http_bot,614 [615 state,616 temperature,617 top_p,618 repetition_penalty,619 max_output_tokens,620 max_input_tiles,621 ],622 [state, chatbot, textbox] + btn_list,623 )624 submit_btn.click(625 add_text,626 [state, textbox, system_prompt],627 [state, chatbot, textbox] + btn_list,628 ).then(629 http_bot,630 [631 state,632 temperature,633 top_p,634 repetition_penalty,635 max_output_tokens,636 max_input_tiles,637 ],638 [state, chatbot, textbox] + btn_list,639 )640 641 return demo642 643 644if __name__ == "__main__":645 parser = argparse.ArgumentParser()646 parser.add_argument("--host", type=str, default="0.0.0.0")647 parser.add_argument("--port", type=int, default=7860)648 parser.add_argument("--concurrency-count", type=int, default=10)649 parser.add_argument("--share", action="store_true")650 parser.add_argument("--moderate", action="store_true")651 args = parser.parse_args()652 logger.info(f"args: {args}")653 654 logger.info(args)655 demo = build_demo()656 demo.queue(api_open=False).launch(657 server_name=args.host,658 server_port=args.port,659 share=args.share,660 max_threads=args.concurrency_count,661 )662