nxphi47/MultiPurpose-Chatbot-DEMO
1
1import os2from gradio.themes import ThemeClass as Theme3import numpy as np4import argparse5import gradio as gr6from typing import Any, Iterator7from typing import Iterator, List, Optional, Tuple8import filelock9import glob10import json11import time12from gradio.routes import Request13from gradio.utils import SyncToAsyncIterator, async_iteration14from gradio.helpers import special_args15import anyio16from typing import AsyncGenerator, Callable, Literal, Union, cast, Generator17 18from gradio_client.documentation import document, set_documentation_group19from gradio.components import Button, Component20from gradio.events import Dependency, EventListenerMethod21from typing import List, Optional, Union, Dict, Tuple22from tqdm.auto import tqdm23from huggingface_hub import snapshot_download24 25 26import inspect27from typing import AsyncGenerator, Callable, Literal, Union, cast28 29import anyio30from gradio_client import utils as client_utils31from gradio_client.documentation import document32 33from gradio.blocks import Blocks34from gradio.components import (35 Button,36 Chatbot,37 Component,38 Markdown,39 State,40 Textbox,41 get_component_instance,42)43from gradio.events import Dependency, on44from gradio.helpers import create_examples as Examples # noqa: N81245from gradio.helpers import special_args46from gradio.layouts import Accordion, Group, Row47from gradio.routes import Request48from gradio.themes import ThemeClass as Theme49from gradio.utils import SyncToAsyncIterator, async_iteration50 51 52from .base_demo import register_demo, get_demo_class, BaseDemo53from ..configs import (54 SYSTEM_PROMPT,55 MODEL_NAME,56 MAX_TOKENS,57 TEMPERATURE,58 USE_PANEL,59 CHATBOT_HEIGHT,60)61 62from ..globals import MODEL_ENGINE63 64CHAT_EXAMPLES = [65 ["Explain general relativity."],66]67DATETIME_FORMAT = "Current date time: {cur_datetime}."68 69 70def gradio_history_to_openai_conversations(message=None, history=None, system_prompt=None):71 conversations = []72 system_prompt = system_prompt or SYSTEM_PROMPT73 if history is not None and len(history) > 0:74 for i, (prompt, res) in enumerate(history):75 if prompt is not None:76 conversations.append({"role": "user", "content": prompt.strip()})77 if res is not None:78 conversations.append({"role": "assistant", "content": res.strip()})79 if message is not None:80 if len(message.strip()) == 0:81 raise gr.Error("The message cannot be empty!")82 conversations.append({"role": "user", "content": message.strip()})83 if conversations[0]['role'] != 'system':84 conversations = [{"role": "system", "content": system_prompt}] + conversations85 return conversations86 87 88def gradio_history_to_conversation_prompt(message=None, history=None, system_prompt=None):89 global MODEL_ENGINE90 full_prompt = MODEL_ENGINE.apply_chat_template(91 gradio_history_to_openai_conversations(92 message, history=history, system_prompt=system_prompt),93 add_generation_prompt=True94 )95 return full_prompt96 97 98 99def get_datetime_string():100 from datetime import datetime101 now = datetime.now()102 # dd/mm/YY H:M:S103 dt_string = now.strftime("%B %d, %Y, %H:%M:%S")104 return dt_string105 106 107def format_conversation(history, system_prompt=None):108 _str = '\n'.join([109 (110 f'<<<User>>> {h[0]}\n'111 f'<<<Asst>>> {h[1]}'112 )113 for h in history114 ])115 _str = ""116 for mes, res in history:117 if mes is not None:118 _str += f'<<<User>>> {mes}\n'119 if res is not None:120 _str += f'<<<Asst>>> {res}\n'121 if system_prompt is not None:122 _str = f"<<<Syst>>> {system_prompt}\n" + _str123 return _str124 125 126def chat_response_stream_multiturn_engine(127 message: str, 128 history: List[Tuple[str, str]], 129 temperature: float, 130 max_tokens: int, 131 system_prompt: Optional[str] = SYSTEM_PROMPT,132):133 global MODEL_ENGINE134 temperature = float(temperature)135 # ! remove frequency_penalty136 # frequency_penalty = float(frequency_penalty)137 max_tokens = int(max_tokens)138 message = message.strip()139 if len(message) == 0:140 raise gr.Error("The message cannot be empty!")141 # ! skip safety142 if DATETIME_FORMAT in system_prompt:143 # ! This sometime works sometimes dont144 system_prompt = system_prompt.format(cur_datetime=get_datetime_string())145 full_prompt = gradio_history_to_conversation_prompt(message.strip(), history=history, system_prompt=system_prompt)146 # ! length checked147 num_tokens = len(MODEL_ENGINE.tokenizer.encode(full_prompt))148 if num_tokens >= MODEL_ENGINE.max_position_embeddings - 128:149 raise gr.Error(f"Conversation or prompt is too long ({num_tokens} toks), please clear the chatbox or try shorter input.")150 print(full_prompt)151 outputs = None152 response = None153 num_tokens = -1154 for j, outputs in enumerate(MODEL_ENGINE.generate_yield_string(155 prompt=full_prompt,156 temperature=temperature,157 max_tokens=max_tokens,158 )):159 if isinstance(outputs, tuple):160 response, num_tokens = outputs161 else:162 response, num_tokens = outputs, -1163 yield response, num_tokens164 165 print(format_conversation(history + [[message, response]]))166 167 if response is not None:168 yield response, num_tokens169 170 171class CustomizedChatInterface(gr.ChatInterface):172 """173 Fixing some issue with chatinterace174 """175 176 def __init__(177 self,178 fn: Callable,179 *,180 chatbot: Chatbot | None = None,181 textbox: Textbox | None = None,182 additional_inputs: str | Component | list[str | Component] | None = None,183 additional_inputs_accordion_name: str | None = None,184 additional_inputs_accordion: str | Accordion | None = None,185 examples: list[str] | None = None,186 cache_examples: bool | None = None,187 title: str | None = None,188 description: str | None = None,189 theme: Theme | str | None = None,190 css: str | None = None,191 js: str | None = None,192 head: str | None = None,193 analytics_enabled: bool | None = None,194 submit_btn: str | None | Button = "Submit",195 stop_btn: str | None | Button = "Stop",196 retry_btn: str | None | Button = "🔄 Retry",197 undo_btn: str | None | Button = "↩️ Undo",198 clear_btn: str | None | Button = "🗑️ Clear",199 autofocus: bool = True,200 concurrency_limit: int | None | Literal["default"] = "default",201 fill_height: bool = True,202 ):203 """204 Parameters:205 fn: The function to wrap the chat interface around. Should accept two parameters: a string input message and list of two-element lists of the form [[user_message, bot_message], ...] representing the chat history, and return a string response. See the Chatbot documentation for more information on the chat history format.206 chatbot: An instance of the gr.Chatbot component to use for the chat interface, if you would like to customize the chatbot properties. If not provided, a default gr.Chatbot component will be created.207 textbox: An instance of the gr.Textbox component to use for the chat interface, if you would like to customize the textbox properties. If not provided, a default gr.Textbox component will be created.208 additional_inputs: An instance or list of instances of gradio components (or their string shortcuts) to use as additional inputs to the chatbot. If components are not already rendered in a surrounding Blocks, then the components will be displayed under the chatbot, in an accordion.209 additional_inputs_accordion_name: Deprecated. Will be removed in a future version of Gradio. Use the `additional_inputs_accordion` parameter instead.210 additional_inputs_accordion: If a string is provided, this is the label of the `gr.Accordion` to use to contain additional inputs. A `gr.Accordion` object can be provided as well to configure other properties of the container holding the additional inputs. Defaults to a `gr.Accordion(label="Additional Inputs", open=False)`. This parameter is only used if `additional_inputs` is provided.211 examples: Sample inputs for the function; if provided, appear below the chatbot and can be clicked to populate the chatbot input.212 cache_examples: If True, caches examples in the server for fast runtime in examples. The default option in HuggingFace Spaces is True. The default option elsewhere is False.213 title: a title for the interface; if provided, appears above chatbot in large font. Also used as the tab title when opened in a browser window.214 description: a description for the interface; if provided, appears above the chatbot and beneath the title in regular font. Accepts Markdown and HTML content.215 theme: Theme to use, loaded from gradio.themes.216 css: Custom css as a string or path to a css file. This css will be included in the demo webpage.217 js: Custom js or path to js file to run when demo is first loaded. This javascript will be included in the demo webpage.218 head: Custom html to insert into the head of the demo webpage. This can be used to add custom meta tags, scripts, stylesheets, etc. to the page.219 analytics_enabled: Whether to allow basic telemetry. If None, will use GRADIO_ANALYTICS_ENABLED environment variable if defined, or default to True.220 submit_btn: Text to display on the submit button. If None, no button will be displayed. If a Button object, that button will be used.221 stop_btn: Text to display on the stop button, which replaces the submit_btn when the submit_btn or retry_btn is clicked and response is streaming. Clicking on the stop_btn will halt the chatbot response. If set to None, stop button functionality does not appear in the chatbot. If a Button object, that button will be used as the stop button.222 retry_btn: Text to display on the retry button. If None, no button will be displayed. If a Button object, that button will be used.223 undo_btn: Text to display on the delete last button. If None, no button will be displayed. If a Button object, that button will be used.224 clear_btn: Text to display on the clear button. If None, no button will be displayed. If a Button object, that button will be used.225 autofocus: If True, autofocuses to the textbox when the page loads.226 concurrency_limit: If set, this is the maximum number of chatbot submissions that can be running simultaneously. Can be set to None to mean no limit (any number of chatbot submissions can be running simultaneously). Set to "default" to use the default concurrency limit (defined by the `default_concurrency_limit` parameter in `.queue()`, which is 1 by default).227 fill_height: If True, the chat interface will expand to the height of window.228 """229 try:230 super(gr.ChatInterface, self).__init__(231 analytics_enabled=analytics_enabled,232 mode="chat_interface",233 css=css,234 title=title or "Gradio",235 theme=theme,236 js=js,237 head=head,238 fill_height=fill_height,239 )240 except Exception as e:241 # Handling some old gradio version with out fill_height242 super(gr.ChatInterface, self).__init__(243 analytics_enabled=analytics_enabled,244 mode="chat_interface",245 css=css,246 title=title or "Gradio",247 theme=theme,248 js=js,249 head=head,250 # fill_height=fill_height,251 )252 self.concurrency_limit = concurrency_limit253 self.fn = fn254 self.is_async = inspect.iscoroutinefunction(255 self.fn256 ) or inspect.isasyncgenfunction(self.fn)257 self.is_generator = inspect.isgeneratorfunction(258 self.fn259 ) or inspect.isasyncgenfunction(self.fn)260 self.examples = examples261 if self.space_id and cache_examples is None:262 self.cache_examples = True263 else:264 self.cache_examples = cache_examples or False265 self.buttons: list[Button | None] = []266 267 if additional_inputs:268 if not isinstance(additional_inputs, list):269 additional_inputs = [additional_inputs]270 self.additional_inputs = [271 get_component_instance(i)272 for i in additional_inputs # type: ignore273 ]274 else:275 self.additional_inputs = []276 if additional_inputs_accordion_name is not None:277 print(278 "The `additional_inputs_accordion_name` parameter is deprecated and will be removed in a future version of Gradio. Use the `additional_inputs_accordion` parameter instead."279 )280 self.additional_inputs_accordion_params = {281 "label": additional_inputs_accordion_name282 }283 if additional_inputs_accordion is None:284 self.additional_inputs_accordion_params = {285 "label": "Additional Inputs",286 "open": False,287 }288 elif isinstance(additional_inputs_accordion, str):289 self.additional_inputs_accordion_params = {290 "label": additional_inputs_accordion291 }292 elif isinstance(additional_inputs_accordion, Accordion):293 self.additional_inputs_accordion_params = (294 additional_inputs_accordion.recover_kwargs(295 additional_inputs_accordion.get_config()296 )297 )298 else:299 raise ValueError(300 f"The `additional_inputs_accordion` parameter must be a string or gr.Accordion, not {type(additional_inputs_accordion)}"301 )302 303 with self:304 if title:305 Markdown(306 f"<h1 style='text-align: center; margin-bottom: 1rem'>{self.title}</h1>"307 )308 if description:309 Markdown(description)310 311 if chatbot:312 self.chatbot = chatbot.render()313 else:314 self.chatbot = Chatbot(315 label="Chatbot", scale=1, height=200 if fill_height else None316 )317 318 with Row():319 for btn in [retry_btn, undo_btn, clear_btn]:320 if btn is not None:321 if isinstance(btn, Button):322 btn.render()323 elif isinstance(btn, str):324 btn = Button(btn, variant="secondary", size="sm")325 else:326 raise ValueError(327 f"All the _btn parameters must be a gr.Button, string, or None, not {type(btn)}"328 )329 self.buttons.append(btn) # type: ignore330 331 with Group():332 with Row():333 if textbox:334 textbox.container = False335 textbox.show_label = False336 textbox_ = textbox.render()337 assert isinstance(textbox_, Textbox)338 self.textbox = textbox_339 else:340 self.textbox = Textbox(341 container=False,342 show_label=False,343 label="Message",344 placeholder="Type a message...",345 scale=7,346 autofocus=autofocus,347 )348 if submit_btn is not None:349 if isinstance(submit_btn, Button):350 submit_btn.render()351 elif isinstance(submit_btn, str):352 submit_btn = Button(353 submit_btn,354 variant="primary",355 scale=2,356 min_width=150,357 )358 else:359 raise ValueError(360 f"The submit_btn parameter must be a gr.Button, string, or None, not {type(submit_btn)}"361 )362 if stop_btn is not None:363 if isinstance(stop_btn, Button):364 stop_btn.visible = False365 stop_btn.render()366 elif isinstance(stop_btn, str):367 stop_btn = Button(368 stop_btn,369 variant="stop",370 visible=False,371 scale=2,372 min_width=150,373 )374 else:375 raise ValueError(376 f"The stop_btn parameter must be a gr.Button, string, or None, not {type(stop_btn)}"377 )378 self.num_tokens = Textbox(379 container=False,380 show_label=False,381 label="num_tokens",382 placeholder="0 tokens",383 scale=1,384 interactive=False,385 # autofocus=autofocus,386 min_width=10387 )388 self.buttons.extend([submit_btn, stop_btn]) # type: ignore389 390 self.fake_api_btn = Button("Fake API", visible=False)391 self.fake_response_textbox = Textbox(label="Response", visible=False)392 (393 self.retry_btn,394 self.undo_btn,395 self.clear_btn,396 self.submit_btn,397 self.stop_btn,398 ) = self.buttons399 400 if examples:401 if self.is_generator:402 examples_fn = self._examples_stream_fn403 else:404 examples_fn = self._examples_fn405 406 self.examples_handler = Examples(407 examples=examples,408 inputs=[self.textbox] + self.additional_inputs,409 outputs=self.chatbot,410 fn=examples_fn,411 )412 413 any_unrendered_inputs = any(414 not inp.is_rendered for inp in self.additional_inputs415 )416 if self.additional_inputs and any_unrendered_inputs:417 with Accordion(**self.additional_inputs_accordion_params): # type: ignore418 for input_component in self.additional_inputs:419 if not input_component.is_rendered:420 input_component.render()421 422 # The example caching must happen after the input components have rendered423 if cache_examples:424 client_utils.synchronize_async(self.examples_handler.cache)425 426 self.saved_input = State()427 self.chatbot_state = (428 State(self.chatbot.value) if self.chatbot.value else State([])429 )430 431 self._setup_events()432 self._setup_api()433 434 # replace events so that submit button is disabled during generation, if stop_btn not found435 # this prevent weird behavior436 def _setup_stop_events(437 self, event_triggers: list[EventListenerMethod], event_to_cancel: Dependency438 ) -> None:439 from gradio.components import State440 event_triggers = event_triggers if isinstance(event_triggers, (list, tuple)) else [event_triggers]441 if self.stop_btn and self.is_generator:442 if self.submit_btn:443 for event_trigger in event_triggers:444 event_trigger(445 lambda: (446 Button(visible=False),447 Button(visible=True),448 ),449 None,450 [self.submit_btn, self.stop_btn],451 api_name=False,452 queue=False,453 )454 event_to_cancel.then(455 lambda: (Button(visible=True), Button(visible=False)),456 None,457 [self.submit_btn, self.stop_btn],458 api_name=False,459 queue=False,460 )461 else:462 for event_trigger in event_triggers:463 event_trigger(464 lambda: Button(visible=True),465 None,466 [self.stop_btn],467 api_name=False,468 queue=False,469 )470 event_to_cancel.then(471 lambda: Button(visible=False),472 None,473 [self.stop_btn],474 api_name=False,475 queue=False,476 )477 self.stop_btn.click(478 None,479 None,480 None,481 cancels=event_to_cancel,482 api_name=False,483 )484 else:485 if self.submit_btn:486 for event_trigger in event_triggers:487 event_trigger(488 lambda: Button(interactive=False),489 None,490 [self.submit_btn],491 api_name=False,492 queue=False,493 )494 event_to_cancel.then(495 lambda: Button(interactive=True),496 None,497 [self.submit_btn],498 api_name=False,499 queue=False,500 )501 # upon clear, cancel the submit event as well502 if self.clear_btn:503 self.clear_btn.click(504 lambda: ([], [], None, Button(interactive=True)),505 None,506 [self.chatbot, self.chatbot_state, self.saved_input, self.submit_btn],507 queue=False,508 api_name=False,509 cancels=event_to_cancel,510 )511 512 def _setup_events(self) -> None:513 from gradio.components import State514 has_on = False515 try:516 from gradio.events import Dependency, EventListenerMethod, on517 has_on = True518 except ImportError as ie:519 has_on = False520 submit_fn = self._stream_fn if self.is_generator else self._submit_fn521 if not self.is_generator:522 raise NotImplementedError(f'should use generator')523 524 if has_on:525 # new version526 submit_triggers = (527 [self.textbox.submit, self.submit_btn.click]528 if self.submit_btn529 else [self.textbox.submit]530 )531 submit_event = (532 on(533 submit_triggers,534 self._clear_and_save_textbox,535 [self.textbox],536 [self.textbox, self.saved_input],537 api_name=False,538 queue=False,539 )540 .then(541 self._display_input,542 [self.saved_input, self.chatbot_state],543 [self.chatbot, self.chatbot_state],544 api_name=False,545 queue=False,546 )547 .then(548 submit_fn,549 [self.saved_input, self.chatbot_state] + self.additional_inputs,550 [self.chatbot, self.chatbot_state, self.num_tokens],551 api_name=False,552 )553 )554 self._setup_stop_events(submit_triggers, submit_event)555 else:556 raise ValueError(f'Better install new gradio version than 3.44.0')557 558 if self.retry_btn:559 retry_event = (560 self.retry_btn.click(561 self._delete_prev_fn,562 [self.chatbot_state],563 [self.chatbot, self.saved_input, self.chatbot_state],564 api_name=False,565 queue=False,566 )567 .then(568 self._display_input,569 [self.saved_input, self.chatbot_state],570 [self.chatbot, self.chatbot_state],571 api_name=False,572 queue=False,573 )574 .then(575 submit_fn,576 [self.saved_input, self.chatbot_state] + self.additional_inputs,577 [self.chatbot, self.chatbot_state, self.num_tokens],578 api_name=False,579 )580 )581 self._setup_stop_events([self.retry_btn.click], retry_event)582 583 if self.undo_btn:584 self.undo_btn.click(585 self._delete_prev_fn,586 [self.chatbot_state],587 [self.chatbot, self.saved_input, self.chatbot_state],588 api_name=False,589 queue=False,590 ).then(591 lambda x: x,592 [self.saved_input],593 [self.textbox],594 api_name=False,595 queue=False,596 )597 # Reconfigure clear_btn to stop and clear text box598 599 def _clear_and_save_textbox(self, message: str) -> tuple[str, str]:600 return "", message601 602 def _display_input(603 self, message: str, history: List[List[Union[str, None]]]604 ) -> Tuple[List[List[Union[str, None]]], List[List[list[Union[str, None]]]]]:605 if message is not None and message.strip() != "":606 history.append([message, None])607 return history, history608 609 async def _stream_fn(610 self,611 message: str,612 history_with_input,613 request: Request,614 *args,615 ) -> AsyncGenerator:616 history = history_with_input[:-1]617 inputs, _, _ = special_args(618 self.fn, inputs=[message, history, *args], request=request619 )620 621 if self.is_async:622 generator = self.fn(*inputs)623 else:624 generator = await anyio.to_thread.run_sync(625 self.fn, *inputs, limiter=self.limiter626 )627 generator = SyncToAsyncIterator(generator, self.limiter)628 629 # ! In case of error, yield the previous history & undo any generation before raising error630 try:631 first_response_pack = await async_iteration(generator)632 if isinstance(first_response_pack, (tuple, list)):633 first_response, num_tokens = first_response_pack634 else:635 first_response, num_tokens = first_response_pack, -1636 update = history + [[message, first_response]]637 yield update, update, f"{num_tokens} toks"638 except StopIteration:639 update = history + [[message, None]]640 yield update, update, "NaN toks"641 except Exception as e:642 yield history, history, "NaN toks"643 raise e644 645 try:646 async for response_pack in generator:647 if isinstance(response_pack, (tuple, list)):648 response, num_tokens = response_pack649 else:650 response, num_tokens = response_pack, "NaN toks"651 update = history + [[message, response]]652 yield update, update, f"{num_tokens} toks"653 except Exception as e:654 yield history, history, "NaN toks"655 raise e656 657@register_demo658class ChatInterfaceDemo(BaseDemo):659 @property660 def tab_name(self):661 return "Chat"662 663 def create_demo(664 self, 665 title: str | None = None, 666 description: str | None = None, 667 **kwargs668 ) -> gr.Blocks:669 system_prompt = kwargs.get("system_prompt", SYSTEM_PROMPT)670 max_tokens = kwargs.get("max_tokens", MAX_TOKENS)671 temperature = kwargs.get("temperature", TEMPERATURE)672 model_name = kwargs.get("model_name", MODEL_NAME)673 # frequence_penalty = FREQUENCE_PENALTY674 # presence_penalty = PRESENCE_PENALTY675 676 demo_chat = CustomizedChatInterface(677 chat_response_stream_multiturn_engine,678 chatbot=gr.Chatbot(679 label=model_name,680 bubble_full_width=False,681 latex_delimiters=[682 { "left": "$", "right": "$", "display": False},683 { "left": "$$", "right": "$$", "display": True},684 ],685 show_copy_button=True,686 layout="panel" if USE_PANEL else "bubble",687 height=CHATBOT_HEIGHT,688 ),689 textbox=gr.Textbox(placeholder='Type message', lines=1, max_lines=128, min_width=200, scale=8),690 submit_btn=gr.Button(value='Submit', variant="primary", scale=0),691 title=title,692 description=description,693 additional_inputs=[694 gr.Number(value=temperature, label='Temperature (higher -> more random)'), 695 gr.Number(value=max_tokens, label='Max generated tokens (increase if want more generation)'), 696 # gr.Number(value=frequence_penalty, label='Frequency penalty (> 0 encourage new tokens over repeated tokens)'), 697 # gr.Number(value=presence_penalty, label='Presence penalty (> 0 encourage new tokens, < 0 encourage existing tokens)'), 698 gr.Textbox(value=system_prompt, label='System prompt', lines=4)699 ], 700 examples=CHAT_EXAMPLES,701 cache_examples=False702 )703 return demo_chat704 705