sonic-coder/CPU-LLM-Inference
0
1import os2import time3import gc4import sys5import threading6from itertools import islice7from datetime import datetime8import re9from typing import List, Dict, Any, Optional, Tuple, Generator10from dataclasses import dataclass11import logging12import gradio as gr13import torch14from transformers import pipeline, TextIteratorStreamer15from transformers import AutoTokenizer16from bs4 import BeautifulSoup17import requests18from urllib.parse import quote_plus19import json20import urllib.parse21from config import MODELS22 23logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')24logger = logging.getLogger(__name__)25 26cancel_event = threading.Event()27 28ACCESS_TOKEN = os.environ.get('HF_TOKEN', '')29if ACCESS_TOKEN == '':30 ACCESS_TOKEN = None31 32PIPELINES = {}33SEARCH_TIMEOUT_DEFAULT = 5.034 35@dataclass36class SearchResult:37 title: str38 snippet: str39 url: Optional[str] = None40 41 def format(self, max_chars: int = 50) -> str:42 snippet = self.snippet[:max_chars] + "..." if len(self.snippet) > max_chars else self.snippet43 return f"{self.title} - {snippet}"44 45@dataclass46class GenerationConfig:47 max_tokens: int = 102448 temperature: float = 0.749 top_k: int = 4050 top_p: float = 0.951 repetition_penalty: float = 1.252 53 def to_dict(self) -> Dict[str, Any]:54 return {55 'max_new_tokens': self.max_tokens,56 'temperature': self.temperature,57 'top_k': self.top_k,58 'top_p': self.top_p,59 'repetition_penalty': self.repetition_penalty,60 }61 62class SearchEngine:63 USER_AGENTS = [64 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',65 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',66 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'67 ]68 69 @staticmethod70 def _get_headers() -> Dict[str, str]:71 return {72 'User-Agent': SearchEngine.USER_AGENTS[0],73 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',74 'Accept-Language': 'en-US,en;q=0.5',75 'Accept-Encoding': 'gzip, deflate',76 'Connection': 'keep-alive',77 'Upgrade-Insecure-Requests': '1',78 'Cache-Control': 'max-age=0'79 }80 81class GoogleSearch(SearchEngine):82 @staticmethod83 def search(query: str, max_results: int = 6, max_chars: int = 50) -> List[SearchResult]:84 encoded_query = quote_plus(query)85 search_urls = [86 f"https://www.google.com/search?q={encoded_query}&safe=off&num={max_results}",87 f"https://www.google.com/search?q={encoded_query}&safe=off&num={max_results}&hl=en",88 f"https://www.google.com/webhp?safe=off&q={encoded_query}&num={max_results}"89 ]90 91 for user_agent in SearchEngine.USER_AGENTS:92 headers = SearchEngine._get_headers()93 headers['User-Agent'] = user_agent94 95 for search_url in search_urls:96 try:97 response = requests.get(search_url, headers=headers, timeout=15, verify=True)98 response.raise_for_status()99 100 soup = BeautifulSoup(response.text, 'html.parser')101 102 selectors = [103 ('div', 'g'),104 ('div', 'tF2Cxc'),105 ('div', 'MjjYud'),106 ('div', 'yuRUbf')107 ]108 109 search_results = []110 for tag, class_name in selectors:111 search_results = soup.find_all(tag, class_=class_name)112 if search_results:113 break114 115 if not search_results:116 search_results = soup.find_all('div', class_=re.compile(r'^(g|tF2Cxc|MjjYud|yuRUbf)'))117 118 results = []119 for result in search_results[:max_results]:120 try:121 title_elem = result.find('h3') or result.find('h2')122 if not title_elem:123 continue124 125 snippet_elem = result.find('div', class_='VwiC3b') or \126 result.find('div', class_='IsZvec') or \127 result.find('div', class_='lEBKkf')128 129 link_elem = result.find('a')130 if not link_elem:131 continue132 133 link = link_elem.get('href', '')134 if link.startswith('/url?q='):135 link = urllib.parse.unquote(link.split('/url?q=')[1].split('&')[0])136 137 if not link.startswith('http'):138 continue139 140 title = title_elem.text.strip()141 snippet = snippet_elem.text.strip() if snippet_elem else ""142 snippet = ' '.join(snippet.split())143 144 if title and snippet:145 results.append(SearchResult(title=title, snippet=snippet, url=link))146 147 except Exception as e:148 logger.debug(f"Error parsing Google result: {e}")149 continue150 151 if results:152 return results153 154 except Exception as e:155 logger.debug(f"Google search attempt failed: {e}")156 continue157 158 return []159 160class DuckDuckGoSearch(SearchEngine):161 @staticmethod162 def search(query: str, max_results: int = 6, max_chars: int = 50) -> List[SearchResult]:163 try:164 from ddgs import DDGS165 with DDGS() as ddgs:166 results = []167 for r in islice(ddgs.text(query, region="wt-wt", safesearch="off", timelimit="y"), max_results):168 title = r.get('title', 'No Title')169 body = r.get('body', '')170 results.append(SearchResult(title=title, snippet=body))171 return results172 except Exception as e:173 logger.debug(f"DuckDuckGo search failed: {e}")174 return []175 176class BingSearch(SearchEngine):177 @staticmethod178 def search(query: str, max_results: int = 6, max_chars: int = 50) -> List[SearchResult]:179 try:180 headers = SearchEngine._get_headers()181 search_url = f"https://www.bing.com/search?q={quote_plus(query)}&safeSearch=off&count={max_results}"182 183 response = requests.get(search_url, headers=headers, timeout=10)184 response.raise_for_status()185 186 soup = BeautifulSoup(response.text, 'html.parser')187 results = []188 189 for result in soup.find_all('li', class_='b_algo')[:max_results]:190 try:191 title_elem = result.find('h2')192 snippet_elem = result.find('p')193 194 if title_elem and snippet_elem:195 title = title_elem.text.strip()196 snippet = snippet_elem.text.strip()197 results.append(SearchResult(title=title, snippet=snippet))198 199 except Exception as e:200 logger.debug(f"Error parsing Bing result: {e}")201 continue202 203 return results204 except Exception as e:205 logger.debug(f"Bing search failed: {e}")206 return []207 208class SearchManager:209 _engines = [GoogleSearch, DuckDuckGoSearch, BingSearch]210 211 @classmethod212 def search(cls, query: str, max_results: int = 6, max_chars: int = 50, timeout: float = 5.0) -> List[SearchResult]:213 for engine_cls in cls._engines:214 try:215 result_container = []216 search_thread = threading.Thread(217 target=lambda: result_container.extend(engine_cls.search(query, max_results, max_chars))218 )219 search_thread.daemon = True220 search_thread.start()221 search_thread.join(timeout=timeout)222 223 if result_container:224 logger.info(f"Search successful with {engine_cls.__name__}: {len(result_container)} results")225 return result_container226 227 except Exception as e:228 logger.warning(f"Search engine {engine_cls.__name__} failed: {e}")229 continue230 231 return []232 233class ModelManager:234 _pipelines = {}235 _lock = threading.Lock()236 237 @classmethod238 def load_pipeline(cls, model_name: str) -> pipeline:239 with cls._lock:240 if model_name in cls._pipelines:241 return cls._pipelines[model_name]242 243 repo = MODELS[model_name]["repo_id"]244 245 try:246 tokenizer = AutoTokenizer.from_pretrained(247 repo, 248 token=ACCESS_TOKEN if ACCESS_TOKEN else None249 )250 except Exception as e:251 logger.warning(f"Failed to load tokenizer with token, trying without: {e}")252 tokenizer = AutoTokenizer.from_pretrained(repo)253 254 for dtype in (torch.bfloat16, torch.float16, torch.float32):255 try:256 pipe_kwargs = {257 'task': "text-generation",258 'model': repo,259 'tokenizer': tokenizer,260 'trust_remote_code': True,261 'dtype': dtype,262 'device_map': "auto",263 'use_cache': True,264 }265 if ACCESS_TOKEN:266 pipe_kwargs['token'] = ACCESS_TOKEN267 268 pipe = pipeline(**pipe_kwargs)269 cls._pipelines[model_name] = pipe270 return pipe271 except Exception as e:272 logger.warning(f"Failed to load with {dtype}: {e}")273 continue274 275 pipe_kwargs = {276 'task': "text-generation",277 'model': repo,278 'tokenizer': tokenizer,279 'trust_remote_code': True,280 'device_map': "auto",281 'use_cache': True,282 }283 if ACCESS_TOKEN:284 pipe_kwargs['token'] = ACCESS_TOKEN285 286 pipe = pipeline(**pipe_kwargs)287 cls._pipelines[model_name] = pipe288 return pipe289 290class PromptBuilder:291 @staticmethod292 def format_conversation(history: List[Dict], system_prompt: str, tokenizer) -> str:293 if hasattr(tokenizer, "chat_template") and tokenizer.chat_template:294 messages = [{"role": "system", "content": system_prompt.strip()}] + history295 return tokenizer.apply_chat_template(296 messages, 297 tokenize=False, 298 add_generation_prompt=True, 299 enable_thinking=True300 )301 else:302 prompt = f"{system_prompt.strip()}\n"303 for msg in history:304 if msg['role'] == 'user':305 prompt += f"User: {msg['content'].strip()}\n"306 elif msg['role'] == 'assistant':307 prompt += f"Assistant: {msg['content'].strip()}\n"308 309 if not prompt.strip().endswith("Assistant:"):310 prompt += "Assistant: "311 return prompt312 313 @staticmethod314 def build_search_context(search_results: List[SearchResult], system_prompt: str, user_query: str) -> str:315 if not search_results:316 return system_prompt.strip()317 318 formatted_results = "\n".join(f"[{i+1}] {r.format()}" for i, r in enumerate(search_results))319 320 return f"""{system_prompt.strip()}321 322# SEARCH CONTEXT (TRUSTED SOURCES ONLY)323Below are search results. Treat them as the ONLY source of truth for answering.324{formatted_results}325 326RULES (VERY IMPORTANT):327- Do NOT use outside knowledge. Do NOT guess or fill missing information.328- If the answer is not clearly supported by the search results, say: "Not enough information in the provided sources."329- Every factual statement must be directly supported by at least one citation [citation:X].330- Do NOT add explanations, examples, or background that are not explicitly present in the sources.331- Do NOT paraphrase beyond what is necessary for clarity.332- If sources conflict, mention the conflict and cite both.333- If multiple sources are used, distribute citations per sentence, not only at the end.334 335CITATION RULES:336- Use inline citations like this: [citation:1]337- If multiple sources support a sentence: [citation:1][citation:3]338- Never place all citations only at the end.339 340ANSWER POLICY:341- Be concise and strictly grounded.342- No speculation, no assumptions, no "likely", no "probably".343- If the user requests a list, only include items explicitly found in sources.344- If sources are insufficient, stop and ask for more data instead of guessing.345 346DATE CONTEXT:347- Today is {datetime.now().strftime('%Y-%m-%d')} (use only for time reference, not for assumptions).348 349USER QUESTION:350{user_query}"""351 352class StreamProcessor:353 @staticmethod354 def process_stream(streamer: TextIteratorStreamer, history: List[Dict]) -> Generator[Tuple[List[Dict], str], None, None]:355 thought_buf = ''356 answer_buf = ''357 in_thought = False358 assistant_message_started = False359 360 for chunk in streamer:361 if cancel_event.is_set():362 if assistant_message_started and history and history[-1]['role'] == 'assistant':363 history[-1]['content'] += " [Generation Canceled]"364 yield history, "Generation canceled by user."365 break366 367 text = chunk368 369 if not in_thought and '<think>' in text:370 in_thought = True371 history.append({'role': 'assistant', 'content': '', 'metadata': {'title': '💭 Thought'}})372 assistant_message_started = True373 after = text.split('<think>', 1)[1]374 thought_buf += after375 376 if '</think>' in thought_buf:377 before, after2 = thought_buf.split('</think>', 1)378 history[-1]['content'] = before.strip()379 in_thought = False380 answer_buf = after2381 history.append({'role': 'assistant', 'content': answer_buf})382 else:383 history[-1]['content'] = thought_buf384 yield history, ""385 continue386 387 if in_thought:388 thought_buf += text389 if '</think>' in thought_buf:390 before, after2 = thought_buf.split('</think>', 1)391 history[-1]['content'] = before.strip()392 in_thought = False393 answer_buf = after2394 history.append({'role': 'assistant', 'content': answer_buf})395 else:396 history[-1]['content'] = thought_buf397 yield history, ""398 continue399 400 if not assistant_message_started:401 history.append({'role': 'assistant', 'content': ''})402 assistant_message_started = True403 404 answer_buf += text405 history[-1]['content'] = answer_buf.strip()406 yield history, ""407 408def chat_response(409 user_msg: str,410 chat_history: List[Dict],411 system_prompt: str,412 enable_search: bool,413 max_results: int,414 max_chars: int,415 model_name: str,416 max_tokens: int,417 temperature: float,418 top_k: int,419 top_p: float,420 repeat_penalty: float,421 search_timeout: float422) -> Generator[Tuple[List[Dict], str], None, None]:423 cancel_event.clear()424 history = list(chat_history or [])425 history.append({'role': 'user', 'content': user_msg})426 427 search_results: List[SearchResult] = []428 search_debug = "Web search disabled."429 430 if enable_search:431 search_debug = "🔍 Searching across multiple engines..."432 try:433 search_results = SearchManager.search(434 user_msg, 435 int(max_results), 436 int(max_chars), 437 float(search_timeout)438 )439 440 if search_results:441 search_debug = f"✅ Search completed - Found {len(search_results)} results\n\n" + "\n".join(442 f"- {r.format(int(max_chars))}" for r in search_results443 )444 else:445 search_debug = "❌ No search results found. Check internet connection or try again."446 except Exception as e:447 search_debug = f"❌ Search failed: {str(e)}"448 logger.error(f"Search error: {e}")449 450 try:451 if enable_search and search_results:452 enriched_prompt = PromptBuilder.build_search_context(453 search_results, 454 system_prompt, 455 user_msg456 )457 else:458 enriched_prompt = system_prompt.strip()459 460 pipe = ModelManager.load_pipeline(model_name)461 462 prompt = PromptBuilder.format_conversation(history, enriched_prompt, pipe.tokenizer)463 prompt_debug = f"\n\n--- Prompt Preview ---\n```\n{prompt[:500]}...\n```" if len(prompt) > 500 else f"\n\n--- Prompt Preview ---\n```\n{prompt}\n```"464 465 config = GenerationConfig(466 max_tokens=max_tokens,467 temperature=temperature,468 top_k=top_k,469 top_p=top_p,470 repetition_penalty=repeat_penalty471 )472 473 streamer = TextIteratorStreamer(474 pipe.tokenizer,475 skip_prompt=True,476 skip_special_tokens=True477 )478 479 gen_kwargs = config.to_dict()480 gen_kwargs['streamer'] = streamer481 gen_kwargs['return_full_text'] = False482 483 gen_thread = threading.Thread(484 target=pipe,485 args=(prompt,),486 kwargs=gen_kwargs487 )488 gen_thread.start()489 490 yield history, search_debug491 492 for history_update, debug_update in StreamProcessor.process_stream(streamer, history):493 yield history_update, debug_update494 495 gen_thread.join(timeout=5.0)496 yield history, search_debug + prompt_debug497 498 except GeneratorExit:499 logger.info("Generation cancelled by user")500 return501 except Exception as e:502 logger.error(f"Generation error: {e}")503 history.append({'role': 'assistant', 'content': f"Error: {str(e)}"})504 yield history, search_debug505 finally:506 gc.collect()507 508def get_model_size(model_name: str) -> float:509 return MODELS.get(model_name, {}).get("params_b", 4.0)510 511def get_duration_estimate(512 model_name: str,513 enable_search: bool,514 max_tokens: int,515 search_timeout: float516) -> float:517 model_size = get_model_size(model_name)518 use_aot = model_size >= 2519 520 base_duration = 20 if not use_aot else 40521 token_duration = max_tokens * 0.005522 search_duration = 10 if enable_search else 0523 aot_compilation = 20 if use_aot else 0524 525 return base_duration + token_duration + search_duration + aot_compilation526 527def update_duration_estimate(528 model_name: str,529 enable_search: bool,530 max_results: int,531 max_chars: int,532 max_tokens: int,533 search_timeout: float534) -> str:535 try:536 duration = get_duration_estimate(model_name, enable_search, max_tokens, search_timeout)537 model_size = get_model_size(model_name)538 539 return f"""⏱️ **Estimated GPU Time: {duration:.1f} seconds**540 541📊 **Model Size:** {model_size:.1f}B parameters542🔍 **Web Search:** {'Enabled (Multi-Engine)' if enable_search else 'Disabled'}"""543 except Exception as e:544 logger.error(f"Error calculating estimate: {e}")545 return f"⚠️ Error calculating estimate: {e}"546 547def update_default_prompt(enable_search: bool) -> str:548 return "You are a helpful assistant."549 550with gr.Blocks(551 title="LLM Inference",552 theme=gr.themes.Soft(553 primary_hue="blue",554 secondary_hue="blue",555 neutral_hue="slate",556 radius_size="lg",557 font=[gr.themes.GoogleFont("Syne"), "Arial", "sans-serif"]558 ),559 css="""560 .duration-estimate { background: linear-gradient(135deg, #667eea15 0%, #764ba215 100%); border-left: 4px solid #667eea; padding: 12px; border-radius: 8px; margin: 16px 0; }561 .chatbot { border-radius: 12px; box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1); }562 button.primary { font-weight: 600; }563 .gradio-accordion { margin-bottom: 12px; }564 """565) as demo:566 gr.Markdown("""567 # 🧠 LLM Inference with Multi-Engine Search568 """)569 570 with gr.Row():571 with gr.Column(scale=3):572 with gr.Group():573 gr.Markdown("### ⚙️ Core Settings")574 model_dd = gr.Dropdown(575 label="🤖 Model",576 choices=list(MODELS.keys()),577 value="Qwen3-1.7B",578 info="Select the language model to use"579 )580 search_chk = gr.Checkbox(581 label="🔍 Enable Web Search",582 value=False,583 info="Search across Google, DuckDuckGo, and Bing (no API required)"584 )585 sys_prompt = gr.Textbox(label="📝 System Prompt", lines=3, value=update_default_prompt(False), placeholder="Define the assistant's behavior and personality...")586 587 duration_display = gr.Markdown(588 value=update_duration_estimate("Qwen3-1.7B", False, 4, 50, 1024, 5.0),589 elem_classes="duration-estimate"590 )591 592 with gr.Accordion("🎛️ Advanced Generation Parameters", open=False):593 max_tok = gr.Slider(594 64, 16384, value=1024, step=32,595 label="Max Tokens",596 info="Maximum length of generated response"597 )598 temp = gr.Slider(599 0.1, 2.0, value=0.7, step=0.1,600 label="Temperature",601 info="Higher = more creative, Lower = more focused"602 )603 with gr.Row():604 k = gr.Slider(605 1, 100, value=40, step=1,606 label="Top-K",607 info="Number of top tokens to consider"608 )609 p = gr.Slider(610 0.1, 1.0, value=0.9, step=0.05,611 label="Top-P",612 info="Nucleus sampling threshold"613 )614 rp = gr.Slider(615 1.0, 2.0, value=1.2, step=0.1,616 label="Repetition Penalty",617 info="Penalize repeated tokens"618 )619 620 with gr.Accordion("🌐 Web Search Settings", open=False, visible=False) as search_settings:621 mr = gr.Number(622 value=4, precision=0,623 label="Max Results",624 info="Number of search results to retrieve"625 )626 mc = gr.Number(627 value=50, precision=0,628 label="Max Chars/Result",629 info="Character limit per search result"630 )631 st = gr.Slider(632 minimum=0.0, maximum=30.0, step=0.5, value=5.0,633 label="Search Timeout (s)",634 info="Maximum time to wait for search results"635 )636 gr.Markdown("""637 ⚠️ **Search Engines:**638 - Google (primary)639 - DuckDuckGo (fallback)640 - Bing (fallback)641 642 SafeSearch is **OFF** for comprehensive results.643 """)644 645 with gr.Row():646 clr = gr.Button("🗑️ Clear Chat", variant="secondary", scale=1)647 648 with gr.Column(scale=7):649 chat = gr.Chatbot(650 type="messages",651 height=600,652 label="💬 Conversation",653 show_copy_button=True,654 avatar_images=(655 "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='40' height='40'%3E%3Crect width='40' height='40' rx='20' fill='%23f093fb'/%3E%3Ctext x='20' y='28' text-anchor='middle' font-size='20' fill='white' font-family='Arial'%3E👤%3C/text%3E%3C/svg%3E",656 "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='40' height='40'%3E%3Crect width='40' height='40' rx='20' fill='%23667eea'/%3E%3Ctext x='20' y='28' text-anchor='middle' font-size='20' fill='white' font-family='Arial'%3E🤖%3C/text%3E%3C/svg%3E"657 ),658 bubble_full_width=False,659 render_markdown=True,660 sanitize_html=False661 )662 663 with gr.Row():664 txt = gr.Textbox(665 placeholder="💭 Type your message here... (Press Enter to send)",666 scale=9,667 container=False,668 show_label=False,669 lines=1,670 max_lines=5671 )672 with gr.Column(scale=1, min_width=120):673 submit_btn = gr.Button("📤 Send", variant="primary", size="lg")674 cancel_btn = gr.Button("⏹️ Stop", variant="stop", visible=False, size="lg")675 676 gr.Examples(677 examples=[678 ["Explain quantum computing in simple terms"],679 ["Write a Python function to calculate fibonacci numbers"],680 ["What are the latest developments in AI? (Enable web search)"],681 ["Tell me a creative story about a time traveler"],682 ["Help me debug this code: def add(a,b): return a+b+1"]683 ],684 inputs=txt,685 label="💡 Example Prompts"686 )687 688 with gr.Accordion("🔍 Debug Info", open=False):689 dbg = gr.Markdown()690 691 gr.Markdown("""692 ---693 💡 **Tips:** 694 - Use **Advanced Parameters** to fine-tune creativity and response length695 - Enable **Web Search** for real-time information (uses multiple search engines)696 - SafeSearch is **OFF** for comprehensive results697 - Try different **models** for various tasks (reasoning, coding, general chat)698 - Click the **Copy** button on responses to save them to your clipboard699 """, elem_classes="footer")700 701 chat_inputs = [txt, chat, sys_prompt, search_chk, mr, mc, model_dd, max_tok, temp, k, p, rp, st]702 ui_components = [chat, dbg, txt, submit_btn, cancel_btn]703 704 def submit_and_manage_ui(user_msg, chat_history, *args):705 if not user_msg.strip():706 yield {}707 return708 709 yield {710 txt: gr.update(value="", interactive=False),711 submit_btn: gr.update(interactive=False),712 cancel_btn: gr.update(visible=True),713 }714 715 cancelled = False716 try:717 backend_args = [user_msg, chat_history] + list(args)718 for response_chunk in chat_response(*backend_args):719 yield {720 chat: response_chunk[0],721 dbg: response_chunk[1],722 }723 except GeneratorExit:724 cancelled = True725 print("Generation cancelled by user.")726 raise727 except Exception as e:728 print(f"An error occurred during generation: {e}")729 error_history = (chat_history or []) + [730 {'role': 'user', 'content': user_msg},731 {'role': 'assistant', 'content': f"**An error occurred:** {str(e)}"}732 ]733 yield {chat: error_history}734 finally:735 if not cancelled:736 print("Resetting UI state.")737 yield {738 txt: gr.update(interactive=True),739 submit_btn: gr.update(interactive=True),740 cancel_btn: gr.update(visible=False),741 }742 743 def set_cancel_flag():744 cancel_event.set()745 print("Cancellation signal sent.")746 747 def reset_ui_after_cancel():748 cancel_event.clear()749 print("UI reset after cancellation.")750 return {751 txt: gr.update(interactive=True),752 submit_btn: gr.update(interactive=True),753 cancel_btn: gr.update(visible=False),754 }755 756 submit_event = txt.submit(757 fn=submit_and_manage_ui,758 inputs=chat_inputs,759 outputs=ui_components,760 )761 submit_btn.click(762 fn=submit_and_manage_ui,763 inputs=chat_inputs,764 outputs=ui_components,765 )766 767 cancel_btn.click(768 fn=set_cancel_flag,769 cancels=[submit_event]770 ).then(771 fn=reset_ui_after_cancel,772 outputs=ui_components773 )774 775 duration_inputs = [model_dd, search_chk, mr, mc, max_tok, st]776 for component in duration_inputs:777 component.change(fn=update_duration_estimate, inputs=duration_inputs, outputs=duration_display)778 779 def toggle_search_settings(enabled):780 return gr.update(visible=enabled)781 782 search_chk.change(783 fn=lambda enabled: (update_default_prompt(enabled), gr.update(visible=enabled)),784 inputs=search_chk,785 outputs=[sys_prompt, search_settings]786 )787 788 clr.click(fn=lambda: ([], "", ""), outputs=[chat, txt, dbg])789 790 demo.launch(share=True)