xnetba/xnetAgent
0
1import concurrent2import functools3import logging4import os5import random6import re7import traceback8import uuid9import datetime10from collections import deque11import itertools12 13from collections import defaultdict14from time import sleep15from typing import Generator, Tuple, List, Dict16 17import boto318import gradio as gr19import requests20from datasets import load_dataset21 22logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))23logging.getLogger("httpx").setLevel(logging.WARNING)24 25# Create a DynamoDB client26dynamodb = boto3.resource('dynamodb', region_name='us-east-1')27# Get a reference to the table28table = dynamodb.Table('oaaic_chatbot_arena')29 30 31def prompt_human_instruct(system_msg, history):32 return system_msg.strip() + "\n" + \33 "\n".join(["\n".join(["###Human: "+item[0], "###Assistant: "+item[1]])34 for item in history])35 36 37def prompt_instruct(system_msg, history):38 return system_msg.strip() + "\n" + \39 "\n".join(["\n".join(["### Instruction: "+item[0], "### Response: "+item[1]])40 for item in history])41 42 43def prompt_chat(system_msg, history):44 return system_msg.strip() + "\n" + \45 "\n".join(["\n".join(["USER: "+item[0], "ASSISTANT: "+item[1]])46 for item in history])47 48 49def prompt_roleplay(system_msg, history):50 return "<|system|>" + system_msg.strip() + "\n" + \51 "\n".join(["\n".join(["<|user|>"+item[0], "<|model|>"+item[1]])52 for item in history])53 54 55class Pipeline:56 prefer_async = True57 58 def __init__(self, endpoint_id, name, prompt_fn, stop_tokens=None):59 self.endpoint_id = endpoint_id60 self.name = name61 self.prompt_fn = prompt_fn62 stop_tokens = stop_tokens or []63 self.generation_config = {64 "max_new_tokens": 1024,65 "top_k": 40,66 "top_p": 0.90,67 "temperature": 0.72,68 "repetition_penalty": 1.22,69 "last_n_tokens": 64,70 "seed": -1,71 "batch_size": 8,72 "threads": -1,73 "stop": ["</s>", "USER:", "### Instruction:"] + stop_tokens,74 }75 76 def get_generation_config(self):77 return self.generation_config.copy()78 79 def __call__(self, prompt, config=None) -> Generator[List[Dict[str, str]], None, None]:80 input = config if config else self.generation_config.copy()81 input["prompt"] = prompt82 83 if self.prefer_async:84 url = f"https://api.runpod.ai/v2/{self.endpoint_id}/run"85 else:86 url = f"https://api.runpod.ai/v2/{self.endpoint_id}/runsync"87 headers = {88 "Authorization": f"Bearer {os.environ['RUNPOD_AI_API_KEY']}"89 }90 response = requests.post(url, headers=headers, json={"input": input})91 92 if response.status_code == 200:93 data = response.json()94 task_id = data.get('id')95 return self.stream_output(task_id)96 97 def stream_output(self,task_id) -> Generator[List[Dict[str, str]], None, None]:98 url = f"https://api.runpod.ai/v2/{self.endpoint_id}/stream/{task_id}"99 headers = {100 "Authorization": f"Bearer {os.environ['RUNPOD_AI_API_KEY']}"101 }102 103 while True:104 try:105 response = requests.get(url, headers=headers)106 if response.status_code == 200:107 data = response.json()108 yield [{"generated_text": "".join([s["output"] for s in data["stream"]])}]109 if data.get('status') == 'COMPLETED':110 return111 elif response.status_code >= 400:112 logging.error(response.json())113 except ConnectionError:114 pass115 116 def poll_for_status(self, task_id):117 url = f"https://api.runpod.ai/v2/{self.endpoint_id}/status/{task_id}"118 headers = {119 "Authorization": f"Bearer {os.environ['RUNPOD_AI_API_KEY']}"120 }121 122 while True:123 response = requests.get(url, headers=headers)124 if response.status_code == 200:125 data = response.json()126 if data.get('status') == 'COMPLETED':127 return [{"generated_text": data["output"]}]128 elif response.status_code >= 400:129 logging.error(response.json())130 # Sleep for 3 seconds between each request131 sleep(3)132 133 def transform_prompt(self, system_msg, history):134 return self.prompt_fn(system_msg, history)135 136 137AVAILABLE_MODELS = {138 "hermes-13b": ("p0zqb2gkcwp0ww", prompt_instruct),139 "manticore-13b-chat": ("u6tv84bpomhfei", prompt_chat),140 "airoboros-13b": ("rglzxnk80660ja", prompt_chat),141 "wizard-vicuna-13b": ("9vvpikt4ttyqos", prompt_chat),142 "lmsys-vicuna-13b": ("2nlb32ydkaz6yd", prompt_chat),143 "supercot-13b": ("0be7865dwxpwqk", prompt_instruct, ["Instruction:"]),144 "mpt-7b-instruct": ("jpqbvnyluj18b0", prompt_instruct),145 "guanaco-13b": ("yxl8w98z017mw2", prompt_instruct),146 # "minotaur-13b": ("6f1baphxjpjk7b", prompt_chat),147 "minotaur-13b-fixed": ("sjnkstd3e40ojj", prompt_roleplay),148 "wizardlm-13b": ("k0chcxsgukov8x", prompt_instruct),149 "selfee-13b": ("50rnvxln9bmf4c", prompt_instruct),150 "robin-v2-13b": ("4cw4vwzzhsl5pq", prompt_human_instruct, ["###Human"]),151 "minotaur-15b-8k": ("zdk804d2txtt68", prompt_chat),152}153 154OAAIC_MODELS = [155 "minotaur-15b-8k",156 "minotaur-13b-fixed",157 "manticore-13b-chat",158 # "minotaur-mpt-7b",159]160OAAIC_MODELS_ROLEPLAY = {161 "manticore-13b-chat-roleplay": ("u6tv84bpomhfei", prompt_roleplay),162 "minotaur-13b-roleplay": ("6f1baphxjpjk7b", prompt_roleplay),163 "minotaur-13b-fixed-roleplay": ("sjnkstd3e40ojj", prompt_roleplay),164 "minotaur-15b-8k-roleplay": ("zdk804d2txtt68", prompt_roleplay),165 # "minotaur-mpt-7b": ("vm1wcsje126x1x", prompt_chat),166}167 168_memoized_models = defaultdict()169 170 171def get_model_pipeline(model_name):172 if not _memoized_models.get(model_name):173 kwargs = {}174 if model_name in AVAILABLE_MODELS:175 if len(AVAILABLE_MODELS[model_name]) >= 3:176 kwargs["stop_tokens"] = AVAILABLE_MODELS[model_name][2]177 _memoized_models[model_name] = Pipeline(AVAILABLE_MODELS[model_name][0], model_name, AVAILABLE_MODELS[model_name][1], **kwargs)178 elif model_name in OAAIC_MODELS_ROLEPLAY:179 _memoized_models[model_name] = Pipeline(OAAIC_MODELS_ROLEPLAY[model_name][0], model_name, OAAIC_MODELS_ROLEPLAY[model_name][1], **kwargs)180 return _memoized_models.get(model_name)181 182start_message = """Below is a dialogue between a USER and an ASSISTANT. The USER may ask questions, request information, or provide instructions for a task, often supplementing with additional context. The ASSISTANT responds accurately and effectively, offering insights, answering questions, or executing tasks to the best of its ability based on the given information. 183"""184 185 186def user(message, nudge_msg, history1, history2):187 history1 = history1 or []188 history2 = history2 or []189 # Append the user's message to the conversation history190 history1.append([message, nudge_msg])191 history2.append([message, nudge_msg])192 193 return "", nudge_msg, history1, history2194 195 196def token_generator(generator1, generator2, mapping_fn=None, fillvalue=None):197 if not fillvalue:198 fillvalue = ''199 if not mapping_fn:200 mapping_fn = lambda x: x201 for output1, output2 in itertools.zip_longest(generator1, generator2, fillvalue=fillvalue):202 tokens1 = re.findall(r'(.*?)(\s|$)', mapping_fn(output1))203 tokens2 = re.findall(r'(.*?)(\s|$)', mapping_fn(output2))204 205 for token1, token2 in itertools.zip_longest(tokens1, tokens2, fillvalue=''):206 yield "".join(token1), "".join(token2)207 208 209def chat(history1, history2, system_msg, state):210 history1 = history1 or []211 history2 = history2 or []212 213 arena_bots = None214 if state and "models" in state and state['models']:215 arena_bots = state['models']216 if not arena_bots:217 arena_bots = list(AVAILABLE_MODELS.keys())218 random.shuffle(arena_bots)219 # bootstrap a new bot into the arena more often220 if "minotaur-15b-8k" not in arena_bots[0:2] and random.choice([True, False, False]):221 arena_bots.insert(random.choice([0,1]), "minotaur-15b-8k")222 223 battle = arena_bots[0:2]224 model1 = get_model_pipeline(battle[0])225 model2 = get_model_pipeline(battle[1])226 227 messages1 = model1.transform_prompt(system_msg, history1)228 messages2 = model2.transform_prompt(system_msg, history2)229 230 # remove last space from assistant, some models output a ZWSP if you leave a space231 messages1 = messages1.rstrip()232 messages2 = messages2.rstrip()233 234 model1_res = model1(messages1) # type: Generator[str, None, None]235 model2_res = model2(messages2) # type: Generator[str, None, None]236 res = token_generator(model1_res, model2_res, lambda x: x[0]['generated_text'], fillvalue=[{'generated_text': ''}]) # type: Generator[Tuple[str, str], None, None]237 logging.info({"models": [model1.name, model2.name]})238 for t1, t2 in res:239 if t1 is not None:240 history1[-1][1] += t1241 if t2 is not None:242 history2[-1][1] += t2243 # stream the response244 # [arena_chatbot1, arena_chatbot2, arena_message, reveal1, reveal2, arena_state]245 yield history1, history2, "", gr.update(value=battle[0]), gr.update(value=battle[1]), {"models": [model1.name, model2.name]}246 sleep(0.05)247 248 249def chosen_one(label, choice1_history, choice2_history, system_msg, nudge_msg, rlhf_persona, state):250 if not state:251 logging.error("missing state!!!")252 # Generate a uuid for each submission253 arena_battle_id = str(uuid.uuid4())254 255 # Get the current timestamp256 timestamp = datetime.datetime.now().isoformat()257 258 # Put the item in the table259 table.put_item(260 Item={261 'arena_battle_id': arena_battle_id,262 'timestamp': timestamp,263 'system_msg': system_msg,264 'nudge_prefix': nudge_msg,265 'choice1_name': state["models"][0],266 'choice1': choice1_history,267 'choice2_name': state["models"][1],268 'choice2': choice2_history,269 'label': label,270 'rlhf_persona': rlhf_persona,271 }272 )273 274chosen_one_first = functools.partial(chosen_one, 1)275chosen_one_second = functools.partial(chosen_one, 2)276chosen_one_tie = functools.partial(chosen_one, 0)277chosen_one_suck = functools.partial(chosen_one, 1)278 279leaderboard_intro = """### TBD280- This is very much a work-in-progress, if you'd like to help build this out, join us on [Discord](https://discord.gg/QYF8QrtEUm)281 282"""283elo_scores = load_dataset("openaccess-ai-collective/chatbot-arena-elo-scores")284elo_scores = elo_scores["train"].sort("elo_score", reverse=True)285 286 287def refresh_md():288 return leaderboard_intro + "\n" + dataset_to_markdown()289 290 291def fetch_elo_scores():292 elo_scores = load_dataset("openaccess-ai-collective/chatbot-arena-elo-scores")293 elo_scores = elo_scores["train"].sort("elo_score", reverse=True)294 return elo_scores295 296 297def dataset_to_markdown():298 dataset = fetch_elo_scores()299 # Get column names (dataset features)300 columns = list(dataset.features.keys())301 # Start markdown string with table headers302 markdown_string = "| " + " | ".join(columns) + " |\n"303 # Add markdown table row separator for headers304 markdown_string += "| " + " | ".join("---" for _ in columns) + " |\n"305 306 # Add each row from dataset to the markdown string307 for i in range(len(dataset)):308 row = dataset[i]309 markdown_string += "| " + " | ".join(str(row[column]) for column in columns) + " |\n"310 311 return markdown_string312 313 314"""315OpenAccess AI Chatbots chat316"""317 318def open_clear_chat(chat_history_state, chat_message, nudge_msg):319 chat_history_state = []320 chat_message = ''321 nudge_msg = ''322 return chat_history_state, chat_message, nudge_msg323 324 325def open_user(message, nudge_msg, history):326 history = history or []327 # Append the user's message to the conversation history328 history.append([message, nudge_msg])329 return "", nudge_msg, history330 331 332def open_chat(model_name, history, system_msg, max_new_tokens, temperature, top_p, top_k, repetition_penalty):333 history = history or []334 335 model = get_model_pipeline(model_name)336 config = model.get_generation_config()337 config["max_new_tokens"] = max_new_tokens338 config["temperature"] = temperature339 config["temperature"] = temperature340 config["top_p"] = top_p341 config["top_k"] = top_k342 config["repetition_penalty"] = repetition_penalty343 344 messages = model.transform_prompt(system_msg, history)345 346 # remove last space from assistant, some models output a ZWSP if you leave a space347 messages = messages.rstrip()348 349 model_res = model(messages, config=config) # type: Generator[List[Dict[str, str]], None, None]350 for res in model_res:351 # tokens = re.findall(r'\s*\S+\s*', res[0]['generated_text'])352 tokens = re.findall(r'(.*?)(\s|$)', res[0]['generated_text'])353 for subtoken in tokens:354 subtoken = "".join(subtoken)355 history[-1][1] += subtoken356 # stream the response357 yield history, history, ""358 sleep(0.01)359 360 361def open_rp_chat(model_name, history, system_msg, max_new_tokens, temperature, top_p, top_k, repetition_penalty):362 history = history or []363 364 model = get_model_pipeline(f"{model_name}-roleplay")365 config = model.get_generation_config()366 config["max_new_tokens"] = max_new_tokens367 config["temperature"] = temperature368 config["temperature"] = temperature369 config["top_p"] = top_p370 config["top_k"] = top_k371 config["repetition_penalty"] = repetition_penalty372 373 messages = model.transform_prompt(system_msg, history)374 375 # remove last space from assistant, some models output a ZWSP if you leave a space376 messages = messages.rstrip()377 378 model_res = model(messages, config=config) # type: Generator[List[Dict[str, str]], None, None]379 for res in model_res:380 tokens = re.findall(r'(.*?)(\s|$)', res[0]['generated_text'])381 # tokens = re.findall(r'\s*\S+\s*', res[0]['generated_text'])382 for subtoken in tokens:383 subtoken = "".join(subtoken)384 history[-1][1] += subtoken385 # stream the response386 yield history, history, ""387 sleep(0.01)388 389 390with gr.Blocks() as arena:391 with gr.Row():392 with gr.Column():393 gr.Markdown(f"""394 ### brought to you by OpenAccess AI Collective395 - Checkout out [our writeup on how this was built.](https://medium.com/@winglian/inference-any-llm-with-serverless-in-15-minutes-69eeb548a41d)396 - This Space runs on CPU only, and uses GGML with GPU support via Runpod Serverless.397 - Responses may not stream immediately due to cold starts on Serverless.398 - Some responses WILL take AT LEAST 20 seconds to respond 399 - The Chatbot Arena (for now), is single turn only. Responses will be cleared after submission. 400 - Responses from the Arena will be used for building reward models. These reward models can be bucketed by Personas.401 - [💵 Consider Donating on our Patreon](http://patreon.com/OpenAccessAICollective) or become a [GitHub Sponsor](https://github.com/sponsors/OpenAccess-AI-Collective)402 - Join us on [Discord](https://discord.gg/PugNNHAF5r) 403 """)404 with gr.Tab("Chatbot Arena"):405 with gr.Row():406 with gr.Column():407 arena_chatbot1 = gr.Chatbot(label="Chatbot A")408 with gr.Column():409 arena_chatbot2 = gr.Chatbot(label="Chatbot B")410 with gr.Row():411 choose1 = gr.Button(value="👈 Prefer left (A)", variant="secondary", visible=False).style(full_width=True)412 choose2 = gr.Button(value="👉 Prefer right (B)", variant="secondary", visible=False).style(full_width=True)413 choose3 = gr.Button(value="🤝 Tie", variant="secondary", visible=False).style(full_width=True)414 choose4 = gr.Button(value="🤮 Both are bad", variant="secondary", visible=False).style(full_width=True)415 with gr.Row():416 reveal1 = gr.Textbox(label="Model Name", value="", interactive=False, visible=False).style(full_width=True)417 reveal2 = gr.Textbox(label="Model Name", value="", interactive=False, visible=False).style(full_width=True)418 with gr.Row():419 dismiss_reveal = gr.Button(value="Dismiss & Continue", variant="secondary", visible=False).style(full_width=True)420 with gr.Row():421 with gr.Column():422 arena_message = gr.Textbox(423 label="What do you want to ask?",424 placeholder="Ask me anything.",425 lines=3,426 )427 with gr.Column():428 arena_rlhf_persona = gr.Textbox(429 "", label="Persona Tags", interactive=True, visible=True, placeholder="Tell us about how you are judging the quality. ex: #CoT #SFW #NSFW #helpful #ethical #creativity", lines=2)430 arena_system_msg = gr.Textbox(431 start_message, label="System Message", interactive=True, visible=True, placeholder="system prompt", lines=8)432 433 arena_nudge_msg = gr.Textbox(434 "", label="Assistant Nudge", interactive=True, visible=True, placeholder="the first words of the assistant response to nudge them in the right direction.", lines=2)435 with gr.Row():436 arena_submit = gr.Button(value="Send message", variant="secondary").style(full_width=True)437 arena_clear = gr.Button(value="New topic", variant="secondary").style(full_width=False)438 # arena_regenerate = gr.Button(value="Regenerate", variant="secondary").style(full_width=False)439 arena_state = gr.State({})440 441 arena_clear.click(lambda: None, None, arena_chatbot1, queue=False)442 arena_clear.click(lambda: None, None, arena_chatbot2, queue=False)443 arena_clear.click(lambda: None, None, arena_message, queue=False)444 arena_clear.click(lambda: None, None, arena_nudge_msg, queue=False)445 arena_clear.click(lambda: None, None, arena_state, queue=False)446 447 submit_click_event = arena_submit.click(448 lambda *args: (449 gr.update(visible=False, interactive=False),450 gr.update(visible=False),451 gr.update(visible=False),452 ),453 inputs=[], outputs=[arena_message, arena_clear, arena_submit], queue=True454 ).then(455 fn=user, inputs=[arena_message, arena_nudge_msg, arena_chatbot1, arena_chatbot2], outputs=[arena_message, arena_nudge_msg, arena_chatbot1, arena_chatbot2], queue=True456 ).then(457 fn=chat, inputs=[arena_chatbot1, arena_chatbot2, arena_system_msg, arena_state], outputs=[arena_chatbot1, arena_chatbot2, arena_message, reveal1, reveal2, arena_state], queue=True458 ).then(459 lambda *args: (460 gr.update(visible=False, interactive=False),461 gr.update(visible=True),462 gr.update(visible=True),463 gr.update(visible=True),464 gr.update(visible=True),465 gr.update(visible=False),466 gr.update(visible=False),467 ),468 inputs=[arena_message, arena_nudge_msg, arena_system_msg], outputs=[arena_message, choose1, choose2, choose3, choose4, arena_clear, arena_submit], queue=True469 )470 471 choose1_click_event = choose1.click(472 fn=chosen_one_first, inputs=[arena_chatbot1, arena_chatbot2, arena_system_msg, arena_nudge_msg, arena_rlhf_persona, arena_state], outputs=[], queue=True473 ).then(474 lambda *args: (475 gr.update(visible=False),476 gr.update(visible=False),477 gr.update(visible=False),478 gr.update(visible=False),479 gr.update(visible=True),480 gr.update(visible=True),481 gr.update(visible=True),482 ),483 inputs=[], outputs=[choose1, choose2, choose3, choose4, dismiss_reveal, reveal1, reveal2], queue=True484 )485 486 choose2_click_event = choose2.click(487 fn=chosen_one_second, inputs=[arena_chatbot1, arena_chatbot2, arena_system_msg, arena_nudge_msg, arena_rlhf_persona, arena_state], outputs=[], queue=True488 ).then(489 lambda *args: (490 gr.update(visible=False),491 gr.update(visible=False),492 gr.update(visible=False),493 gr.update(visible=False),494 gr.update(visible=True),495 gr.update(visible=True),496 gr.update(visible=True),497 ),498 inputs=[], outputs=[choose1, choose2, choose3, choose4, dismiss_reveal, reveal1, reveal2], queue=True499 )500 501 choose3_click_event = choose3.click(502 fn=chosen_one_tie, inputs=[arena_chatbot1, arena_chatbot2, arena_system_msg, arena_nudge_msg, arena_rlhf_persona, arena_state], outputs=[], queue=True503 ).then(504 lambda *args: (505 gr.update(visible=False),506 gr.update(visible=False),507 gr.update(visible=False),508 gr.update(visible=False),509 gr.update(visible=True),510 gr.update(visible=True),511 gr.update(visible=True),512 ),513 inputs=[], outputs=[choose1, choose2, choose3, choose4, dismiss_reveal, reveal1, reveal2], queue=True514 )515 516 choose4_click_event = choose4.click(517 fn=chosen_one_suck, inputs=[arena_chatbot1, arena_chatbot2, arena_system_msg, arena_nudge_msg, arena_rlhf_persona, arena_state], outputs=[], queue=True518 ).then(519 lambda *args: (520 gr.update(visible=False),521 gr.update(visible=False),522 gr.update(visible=False),523 gr.update(visible=False),524 gr.update(visible=True),525 gr.update(visible=True),526 gr.update(visible=True),527 ),528 inputs=[], outputs=[choose1, choose2, choose3, choose4, dismiss_reveal, reveal1, reveal2], queue=True529 )530 531 dismiss_click_event = dismiss_reveal.click(532 lambda *args: (533 gr.update(visible=True, interactive=True),534 gr.update(visible=False),535 gr.update(visible=True),536 gr.update(visible=True),537 gr.update(visible=False),538 gr.update(visible=False),539 None,540 None,541 None,542 ),543 inputs=[], outputs=[544 arena_message,545 dismiss_reveal,546 arena_clear, arena_submit,547 reveal1, reveal2,548 arena_chatbot1, arena_chatbot2,549 arena_state,550 ], queue=True551 )552 with gr.Tab("Leaderboard"):553 with gr.Column():554 leaderboard_markdown = gr.Markdown(f"""{leaderboard_intro}555{dataset_to_markdown()}556""")557 leaderboad_refresh = gr.Button(value="Refresh Leaderboard", variant="secondary").style(full_width=True)558 leaderboad_refresh.click(fn=refresh_md, inputs=[], outputs=[leaderboard_markdown])559 with gr.Tab("OAAIC Chatbots"):560 gr.Markdown("# GGML Spaces Chatbot Demo")561 open_model_choice = gr.Dropdown(label="Model", choices=OAAIC_MODELS, value=OAAIC_MODELS[0])562 open_chatbot = gr.Chatbot().style(height=400)563 with gr.Row():564 open_message = gr.Textbox(565 label="What do you want to chat about?",566 placeholder="Ask me anything.",567 lines=3,568 )569 with gr.Row():570 open_submit = gr.Button(value="Send message", variant="secondary").style(full_width=True)571 open_roleplay = gr.Button(value="Roleplay", variant="secondary").style(full_width=True)572 open_clear = gr.Button(value="New topic", variant="secondary").style(full_width=False)573 open_stop = gr.Button(value="Stop", variant="secondary").style(full_width=False)574 with gr.Row():575 with gr.Column():576 open_max_tokens = gr.Slider(20, 1000, label="Max Tokens", step=20, value=300)577 open_temperature = gr.Slider(0.2, 2.0, label="Temperature", step=0.1, value=0.8)578 open_top_p = gr.Slider(0.0, 1.0, label="Top P", step=0.05, value=0.95)579 open_top_k = gr.Slider(0, 100, label="Top K", step=1, value=40)580 open_repetition_penalty = gr.Slider(0.0, 2.0, label="Repetition Penalty", step=0.1, value=1.1)581 582 open_system_msg = gr.Textbox(583 start_message, label="System Message", interactive=True, visible=True, placeholder="system prompt, useful for RP", lines=5)584 585 open_nudge_msg = gr.Textbox(586 "", label="Assistant Nudge", interactive=True, visible=True, placeholder="the first words of the assistant response to nudge them in the right direction.", lines=1)587 588 open_chat_history_state = gr.State()589 open_clear.click(open_clear_chat, inputs=[open_chat_history_state, open_message, open_nudge_msg], outputs=[open_chat_history_state, open_message, open_nudge_msg], queue=False)590 open_clear.click(lambda: None, None, open_chatbot, queue=False)591 592 open_submit_click_event = open_submit.click(593 fn=open_user, inputs=[open_message, open_nudge_msg, open_chat_history_state], outputs=[open_message, open_nudge_msg, open_chat_history_state], queue=True594 ).then(595 fn=open_chat, inputs=[open_model_choice, open_chat_history_state, open_system_msg, open_max_tokens, open_temperature, open_top_p, open_top_k, open_repetition_penalty], outputs=[open_chatbot, open_chat_history_state, open_message], queue=True596 )597 open_roleplay_click_event = open_roleplay.click(598 fn=open_user, inputs=[open_message, open_nudge_msg, open_chat_history_state], outputs=[open_message, open_nudge_msg, open_chat_history_state], queue=True599 ).then(600 fn=open_rp_chat, inputs=[open_model_choice, open_chat_history_state, open_system_msg, open_max_tokens, open_temperature, open_top_p, open_top_k, open_repetition_penalty], outputs=[open_chatbot, open_chat_history_state, open_message], queue=True601 )602 open_stop.click(fn=None, inputs=None, outputs=None, cancels=[open_submit_click_event, open_roleplay_click_event], queue=False)603 604arena.queue(concurrency_count=5, max_size=16).launch(debug=True, server_name="0.0.0.0", server_port=7860)