antonovmaxim/text-generation-webui-space
12
1import logging2import os3import requests4import warnings5import modules.logging_colors6 7os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'8os.environ['BITSANDBYTES_NOWELCOME'] = '1'9warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')10logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)11 12# This is a hack to prevent Gradio from phoning home when it gets imported13def my_get(url, **kwargs):14 logging.info('Gradio HTTP request redirected to localhost :)')15 kwargs.setdefault('allow_redirects', True)16 return requests.api.request('get', 'http://127.0.0.1/', **kwargs)17 18 19original_get = requests.get20requests.get = my_get21import gradio as gr22requests.get = original_get23 24import matplotlib25matplotlib.use('Agg') # This fixes LaTeX rendering on some systems26 27import importlib28import io29import json30import math31import os32import re33import sys34import time35import traceback36import zipfile37from datetime import datetime38from functools import partial39from pathlib import Path40 41import psutil42import torch43import yaml44from PIL import Image45 46import modules.extensions as extensions_module47from modules import chat, shared, training, ui, utils48from modules.extensions import apply_extensions49from modules.html_generator import chat_html_wrapper50from modules.LoRA import add_lora_to_model51from modules.models import load_model, load_soft_prompt, unload_model52from modules.text_generation import generate_reply_wrapper, get_encoded_length, stop_everything_event53 54 55def load_model_wrapper(selected_model, autoload=False):56 if not autoload:57 yield f"The settings for {selected_model} have been updated.\nClick on \"Load the model\" to load it."58 return59 60 if selected_model == 'None':61 yield "No model selected"62 else:63 try:64 yield f"Loading {selected_model}..."65 shared.model_name = selected_model66 unload_model()67 if selected_model != '':68 shared.model, shared.tokenizer = load_model(shared.model_name)69 70 yield f"Successfully loaded {selected_model}"71 except:72 yield traceback.format_exc()73 74 75def load_lora_wrapper(selected_loras):76 yield ("Applying the following LoRAs to {}:\n\n{}".format(shared.model_name, '\n'.join(selected_loras)))77 add_lora_to_model(selected_loras)78 yield ("Successfuly applied the LoRAs")79 80 81def load_preset_values(preset_menu, state, return_dict=False):82 generate_params = {83 'do_sample': True,84 'temperature': 1,85 'top_p': 1,86 'typical_p': 1,87 'repetition_penalty': 1,88 'encoder_repetition_penalty': 1,89 'top_k': 50,90 'num_beams': 1,91 'penalty_alpha': 0,92 'min_length': 0,93 'length_penalty': 1,94 'no_repeat_ngram_size': 0,95 'early_stopping': False,96 }97 with open(Path(f'presets/{preset_menu}.txt'), 'r') as infile:98 preset = infile.read()99 for i in preset.splitlines():100 i = i.rstrip(',').strip().split('=')101 if len(i) == 2 and i[0].strip() != 'tokens':102 generate_params[i[0].strip()] = eval(i[1].strip())103 generate_params['temperature'] = min(1.99, generate_params['temperature'])104 105 if return_dict:106 return generate_params107 else:108 state.update(generate_params)109 return state, *[generate_params[k] for k in ['do_sample', 'temperature', 'top_p', 'typical_p', 'repetition_penalty', 'encoder_repetition_penalty', 'top_k', 'min_length', 'no_repeat_ngram_size', 'num_beams', 'penalty_alpha', 'length_penalty', 'early_stopping']]110 111 112def upload_soft_prompt(file):113 with zipfile.ZipFile(io.BytesIO(file)) as zf:114 zf.extract('meta.json')115 j = json.loads(open('meta.json', 'r').read())116 name = j['name']117 Path('meta.json').unlink()118 119 with open(Path(f'softprompts/{name}.zip'), 'wb') as f:120 f.write(file)121 122 return name123 124 125def open_save_prompt():126 fname = f"{datetime.now().strftime('%Y-%m-%d-%H%M%S')}"127 return gr.update(value=fname, visible=True), gr.update(visible=False), gr.update(visible=True)128 129 130def save_prompt(text, fname):131 if fname != "":132 with open(Path(f'prompts/{fname}.txt'), 'w', encoding='utf-8') as f:133 f.write(text)134 135 message = f"Saved to prompts/{fname}.txt"136 else:137 message = "Error: No prompt name given."138 139 return message, gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)140 141 142def load_prompt(fname):143 if fname in ['None', '']:144 return ''145 elif fname.startswith('Instruct-'):146 fname = re.sub('^Instruct-', '', fname)147 with open(Path(f'characters/instruction-following/{fname}.yaml'), 'r', encoding='utf-8') as f:148 data = yaml.safe_load(f)149 output = ''150 if 'context' in data:151 output += data['context']152 153 replacements = {154 '<|user|>': data['user'],155 '<|bot|>': data['bot'],156 '<|user-message|>': 'Input',157 }158 159 output += utils.replace_all(data['turn_template'].split('<|bot-message|>')[0], replacements)160 return output.rstrip(' ')161 else:162 with open(Path(f'prompts/{fname}.txt'), 'r', encoding='utf-8') as f:163 text = f.read()164 if text[-1] == '\n':165 text = text[:-1]166 167 return text168 169 170def count_tokens(text):171 tokens = get_encoded_length(text)172 return f'{tokens} tokens in the input.'173 174 175def download_model_wrapper(repo_id):176 try:177 downloader = importlib.import_module("download-model")178 repo_id_parts = repo_id.split(":")179 model = repo_id_parts[0] if len(repo_id_parts) > 0 else repo_id180 branch = repo_id_parts[1] if len(repo_id_parts) > 1 else "main"181 check = False182 183 yield ("Cleaning up the model/branch names")184 model, branch = downloader.sanitize_model_and_branch_names(model, branch)185 186 yield ("Getting the download links from Hugging Face")187 links, sha256, is_lora = downloader.get_download_links_from_huggingface(model, branch, text_only=False)188 189 yield ("Getting the output folder")190 output_folder = downloader.get_output_folder(model, branch, is_lora)191 192 if check:193 yield ("Checking previously downloaded files")194 downloader.check_model_files(model, branch, links, sha256, output_folder)195 else:196 yield (f"Downloading files to {output_folder}")197 downloader.download_model_files(model, branch, links, sha256, output_folder, threads=1)198 yield ("Done!")199 except:200 yield traceback.format_exc()201 202 203# Update the command-line arguments based on the interface values204def update_model_parameters(state, initial=False):205 elements = ui.list_model_elements() # the names of the parameters206 gpu_memories = []207 208 for i, element in enumerate(elements):209 if element not in state:210 continue211 212 value = state[element]213 if element.startswith('gpu_memory'):214 gpu_memories.append(value)215 continue216 217 if initial and vars(shared.args)[element] != vars(shared.args_defaults)[element]:218 continue219 220 # Setting null defaults221 if element in ['wbits', 'groupsize', 'model_type'] and value == 'None':222 value = vars(shared.args_defaults)[element]223 elif element in ['cpu_memory'] and value == 0:224 value = vars(shared.args_defaults)[element]225 226 # Making some simple conversions227 if element in ['wbits', 'groupsize', 'pre_layer']:228 value = int(value)229 elif element == 'cpu_memory' and value is not None:230 value = f"{value}MiB"231 232 if element in ['pre_layer']:233 value = [value] if value > 0 else None234 235 setattr(shared.args, element, value)236 237 found_positive = False238 for i in gpu_memories:239 if i > 0:240 found_positive = True241 break242 243 if not (initial and vars(shared.args)['gpu_memory'] != vars(shared.args_defaults)['gpu_memory']):244 if found_positive:245 shared.args.gpu_memory = [f"{i}MiB" for i in gpu_memories]246 else:247 shared.args.gpu_memory = None248 249 250def get_model_specific_settings(model):251 settings = shared.model_config252 model_settings = {}253 254 for pat in settings:255 if re.match(pat.lower(), model.lower()):256 for k in settings[pat]:257 model_settings[k] = settings[pat][k]258 259 return model_settings260 261 262def load_model_specific_settings(model, state, return_dict=False):263 model_settings = get_model_specific_settings(model)264 for k in model_settings:265 if k in state:266 state[k] = model_settings[k]267 268 return state269 270 271def save_model_settings(model, state):272 if model == 'None':273 yield ("Not saving the settings because no model is loaded.")274 return275 276 with Path(f'{shared.args.model_dir}/config-user.yaml') as p:277 if p.exists():278 user_config = yaml.safe_load(open(p, 'r').read())279 else:280 user_config = {}281 282 model_regex = model + '$' # For exact matches283 if model_regex not in user_config:284 user_config[model_regex] = {}285 286 for k in ui.list_model_elements():287 user_config[model_regex][k] = state[k]288 289 with open(p, 'w') as f:290 f.write(yaml.dump(user_config))291 292 yield (f"Settings for {model} saved to {p}")293 294 295def create_model_menus():296 # Finding the default values for the GPU and CPU memories297 total_mem = []298 for i in range(torch.cuda.device_count()):299 total_mem.append(math.floor(torch.cuda.get_device_properties(i).total_memory / (1024 * 1024)))300 301 default_gpu_mem = []302 if shared.args.gpu_memory is not None and len(shared.args.gpu_memory) > 0:303 for i in shared.args.gpu_memory:304 if 'mib' in i.lower():305 default_gpu_mem.append(int(re.sub('[a-zA-Z ]', '', i)))306 else:307 default_gpu_mem.append(int(re.sub('[a-zA-Z ]', '', i)) * 1000)308 while len(default_gpu_mem) < len(total_mem):309 default_gpu_mem.append(0)310 311 total_cpu_mem = math.floor(psutil.virtual_memory().total / (1024 * 1024))312 if shared.args.cpu_memory is not None:313 default_cpu_mem = re.sub('[a-zA-Z ]', '', shared.args.cpu_memory)314 else:315 default_cpu_mem = 0316 317 with gr.Row():318 with gr.Column():319 with gr.Row():320 with gr.Column():321 with gr.Row():322 shared.gradio['model_menu'] = gr.Dropdown(choices=utils.get_available_models(), value=shared.model_name, label='Model')323 ui.create_refresh_button(shared.gradio['model_menu'], lambda: None, lambda: {'choices': utils.get_available_models()}, 'refresh-button')324 325 with gr.Column():326 with gr.Row():327 shared.gradio['lora_menu'] = gr.Dropdown(multiselect=True, choices=utils.get_available_loras(), value=shared.lora_names, label='LoRA(s)')328 ui.create_refresh_button(shared.gradio['lora_menu'], lambda: None, lambda: {'choices': utils.get_available_loras(), 'value': shared.lora_names}, 'refresh-button')329 330 with gr.Column():331 with gr.Row():332 shared.gradio['lora_menu_apply'] = gr.Button(value='Apply the selected LoRAs')333 with gr.Row():334 load = gr.Button("Load the model", visible=not shared.settings['autoload_model'])335 unload = gr.Button("Unload the model")336 reload = gr.Button("Reload the model")337 save_settings = gr.Button("Save settings for this model")338 339 with gr.Row():340 with gr.Column():341 with gr.Box():342 gr.Markdown('Transformers parameters')343 with gr.Row():344 with gr.Column():345 for i in range(len(total_mem)):346 shared.gradio[f'gpu_memory_{i}'] = gr.Slider(label=f"gpu-memory in MiB for device :{i}", maximum=total_mem[i], value=default_gpu_mem[i])347 shared.gradio['cpu_memory'] = gr.Slider(label="cpu-memory in MiB", maximum=total_cpu_mem, value=default_cpu_mem)348 349 with gr.Column():350 shared.gradio['auto_devices'] = gr.Checkbox(label="auto-devices", value=shared.args.auto_devices)351 shared.gradio['disk'] = gr.Checkbox(label="disk", value=shared.args.disk)352 shared.gradio['cpu'] = gr.Checkbox(label="cpu", value=shared.args.cpu)353 shared.gradio['bf16'] = gr.Checkbox(label="bf16", value=shared.args.bf16)354 shared.gradio['load_in_8bit'] = gr.Checkbox(label="load-in-8bit", value=shared.args.load_in_8bit)355 356 with gr.Column():357 with gr.Box():358 gr.Markdown('GPTQ parameters')359 with gr.Row():360 with gr.Column():361 shared.gradio['wbits'] = gr.Dropdown(label="wbits", choices=["None", 1, 2, 3, 4, 8], value=shared.args.wbits if shared.args.wbits > 0 else "None")362 shared.gradio['groupsize'] = gr.Dropdown(label="groupsize", choices=["None", 32, 64, 128, 1024], value=shared.args.groupsize if shared.args.groupsize > 0 else "None")363 364 with gr.Column():365 shared.gradio['model_type'] = gr.Dropdown(label="model_type", choices=["None", "llama", "opt", "gptj"], value=shared.args.model_type or "None")366 shared.gradio['pre_layer'] = gr.Slider(label="pre_layer", minimum=0, maximum=100, value=shared.args.pre_layer[0] if shared.args.pre_layer is not None else 0)367 368 with gr.Row():369 with gr.Column():370 with gr.Row():371 shared.gradio['autoload_model'] = gr.Checkbox(value=shared.settings['autoload_model'], label='Autoload the model', info='Whether to load the model as soon as it is selected in the Model dropdown.')372 373 shared.gradio['custom_model_menu'] = gr.Textbox(label="Download custom model or LoRA", info="Enter the Hugging Face username/model path, for instance: facebook/galactica-125m. To specify a branch, add it at the end after a \":\" character like this: facebook/galactica-125m:main")374 shared.gradio['download_model_button'] = gr.Button("Download")375 376 with gr.Column():377 with gr.Box():378 gr.Markdown('llama.cpp parameters')379 with gr.Row():380 with gr.Column():381 shared.gradio['threads'] = gr.Slider(label="threads", minimum=0, step=1, maximum=32, value=shared.args.threads)382 shared.gradio['n_batch'] = gr.Slider(label="n_batch", minimum=1, maximum=2048, value=shared.args.n_batch)383 shared.gradio['n_gpu_layers'] = gr.Slider(label="n-gpu-layers", minimum=0, maximum=128, value=shared.args.n_gpu_layers)384 385 with gr.Column():386 shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap)387 shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock)388 389 with gr.Row(): 390 shared.gradio['model_status'] = gr.Markdown('No model is loaded' if shared.model_name == 'None' else 'Ready')391 392 # In this event handler, the interface state is read and updated393 # with the model defaults (if any), and then the model is loaded394 # unless "autoload_model" is unchecked395 shared.gradio['model_menu'].change(396 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(397 load_model_specific_settings, [shared.gradio[k] for k in ['model_menu', 'interface_state']], shared.gradio['interface_state']).then(398 ui.apply_interface_values, shared.gradio['interface_state'], [shared.gradio[k] for k in ui.list_interface_input_elements(chat=shared.is_chat())], show_progress=False).then(399 update_model_parameters, shared.gradio['interface_state'], None).then(400 load_model_wrapper, [shared.gradio[k] for k in ['model_menu', 'autoload_model']], shared.gradio['model_status'], show_progress=False)401 402 load.click(403 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(404 update_model_parameters, shared.gradio['interface_state'], None).then(405 partial(load_model_wrapper, autoload=True), shared.gradio['model_menu'], shared.gradio['model_status'], show_progress=False)406 407 unload.click(408 unload_model, None, None).then(409 lambda: "Model unloaded", None, shared.gradio['model_status'])410 411 reload.click(412 unload_model, None, None).then(413 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(414 update_model_parameters, shared.gradio['interface_state'], None).then(415 partial(load_model_wrapper, autoload=True), shared.gradio['model_menu'], shared.gradio['model_status'], show_progress=False)416 417 save_settings.click(418 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(419 save_model_settings, [shared.gradio[k] for k in ['model_menu', 'interface_state']], shared.gradio['model_status'], show_progress=False)420 421 shared.gradio['lora_menu_apply'].click(load_lora_wrapper, shared.gradio['lora_menu'], shared.gradio['model_status'], show_progress=False)422 shared.gradio['download_model_button'].click(download_model_wrapper, shared.gradio['custom_model_menu'], shared.gradio['model_status'], show_progress=False)423 shared.gradio['autoload_model'].change(lambda x: gr.update(visible=not x), shared.gradio['autoload_model'], load)424 425 426def create_settings_menus(default_preset):427 428 generate_params = load_preset_values(default_preset if not shared.args.flexgen else 'Naive', {}, return_dict=True)429 430 with gr.Row():431 with gr.Column():432 with gr.Row():433 shared.gradio['preset_menu'] = gr.Dropdown(choices=utils.get_available_presets(), value=default_preset if not shared.args.flexgen else 'Naive', label='Generation parameters preset')434 ui.create_refresh_button(shared.gradio['preset_menu'], lambda: None, lambda: {'choices': utils.get_available_presets()}, 'refresh-button')435 with gr.Column():436 shared.gradio['seed'] = gr.Number(value=shared.settings['seed'], label='Seed (-1 for random)')437 438 with gr.Row():439 with gr.Column():440 with gr.Box():441 gr.Markdown('Custom generation parameters ([click here to view technical documentation](https://huggingface.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig))')442 with gr.Row():443 with gr.Column():444 shared.gradio['temperature'] = gr.Slider(0.01, 1.99, value=generate_params['temperature'], step=0.01, label='temperature', info='Primary factor to control randomness of outputs. 0 = deterministic (only the most likely token is used). Higher value = more randomness.')445 shared.gradio['top_p'] = gr.Slider(0.0, 1.0, value=generate_params['top_p'], step=0.01, label='top_p', info='If not set to 1, select tokens with probabilities adding up to less than this number. Higher value = higher range of possible random results.')446 shared.gradio['top_k'] = gr.Slider(0, 200, value=generate_params['top_k'], step=1, label='top_k', info='Similar to top_p, but select instead only the top_k most likely tokens. Higher value = higher range of possible random results.')447 shared.gradio['typical_p'] = gr.Slider(0.0, 1.0, value=generate_params['typical_p'], step=0.01, label='typical_p', info='If not set to 1, select only tokens that are at least this much more likely to appear than random tokens, given the prior text.')448 with gr.Column():449 shared.gradio['repetition_penalty'] = gr.Slider(1.0, 1.5, value=generate_params['repetition_penalty'], step=0.01, label='repetition_penalty', info='Exponential penalty factor for repeating prior tokens. 1 means no penalty, higher value = less repetition, lower value = more repetition.')450 shared.gradio['encoder_repetition_penalty'] = gr.Slider(0.8, 1.5, value=generate_params['encoder_repetition_penalty'], step=0.01, label='encoder_repetition_penalty', info='Also known as the "Hallucinations filter". Used to penalize tokens that are *not* in the prior text. Higher value = more likely to stay in context, lower value = more likely to diverge.')451 shared.gradio['no_repeat_ngram_size'] = gr.Slider(0, 20, step=1, value=generate_params['no_repeat_ngram_size'], label='no_repeat_ngram_size', info='If not set to 0, specifies the length of token sets that are completely blocked from repeating at all. Higher values = blocks larger phrases, lower values = blocks words or letters from repeating. Only 0 or high values are a good idea in most cases.')452 shared.gradio['min_length'] = gr.Slider(0, 2000, step=1, value=generate_params['min_length'], label='min_length', info='Minimum generation length in tokens.')453 shared.gradio['do_sample'] = gr.Checkbox(value=generate_params['do_sample'], label='do_sample')454 with gr.Column():455 with gr.Box():456 gr.Markdown('Contrastive search')457 shared.gradio['penalty_alpha'] = gr.Slider(0, 5, value=generate_params['penalty_alpha'], label='penalty_alpha')458 459 gr.Markdown('Beam search (uses a lot of VRAM)')460 with gr.Row():461 with gr.Column():462 shared.gradio['num_beams'] = gr.Slider(1, 20, step=1, value=generate_params['num_beams'], label='num_beams')463 shared.gradio['length_penalty'] = gr.Slider(-5, 5, value=generate_params['length_penalty'], label='length_penalty')464 with gr.Column():465 shared.gradio['early_stopping'] = gr.Checkbox(value=generate_params['early_stopping'], label='early_stopping')466 467 with gr.Box():468 with gr.Row():469 with gr.Column():470 shared.gradio['truncation_length'] = gr.Slider(value=shared.settings['truncation_length'], minimum=shared.settings['truncation_length_min'], maximum=shared.settings['truncation_length_max'], step=1, label='Truncate the prompt up to this length', info='The leftmost tokens are removed if the prompt exceeds this length. Most models require this to be at most 2048.')471 shared.gradio['custom_stopping_strings'] = gr.Textbox(lines=1, value=shared.settings["custom_stopping_strings"] or None, label='Custom stopping strings', info='In addition to the defaults. Written between "" and separated by commas. For instance: "\\nYour Assistant:", "\\nThe assistant:"')472 with gr.Column():473 shared.gradio['ban_eos_token'] = gr.Checkbox(value=shared.settings['ban_eos_token'], label='Ban the eos_token', info='Forces the model to never end the generation prematurely.')474 shared.gradio['add_bos_token'] = gr.Checkbox(value=shared.settings['add_bos_token'], label='Add the bos_token to the beginning of prompts', info='Disabling this can make the replies more creative.')475 476 shared.gradio['skip_special_tokens'] = gr.Checkbox(value=shared.settings['skip_special_tokens'], label='Skip special tokens', info='Some specific models need this unset.')477 shared.gradio['stream'] = gr.Checkbox(value=not shared.args.no_stream, label='Activate text streaming')478 479 with gr.Accordion('Soft prompt', open=False):480 with gr.Row():481 shared.gradio['softprompts_menu'] = gr.Dropdown(choices=utils.get_available_softprompts(), value='None', label='Soft prompt')482 ui.create_refresh_button(shared.gradio['softprompts_menu'], lambda: None, lambda: {'choices': utils.get_available_softprompts()}, 'refresh-button')483 484 gr.Markdown('Upload a soft prompt (.zip format):')485 with gr.Row():486 shared.gradio['upload_softprompt'] = gr.File(type='binary', file_types=['.zip'])487 488 shared.gradio['preset_menu'].change(load_preset_values, [shared.gradio[k] for k in ['preset_menu', 'interface_state']], [shared.gradio[k] for k in ['interface_state', 'do_sample', 'temperature', 'top_p', 'typical_p', 'repetition_penalty', 'encoder_repetition_penalty', 'top_k', 'min_length', 'no_repeat_ngram_size', 'num_beams', 'penalty_alpha', 'length_penalty', 'early_stopping']])489 shared.gradio['softprompts_menu'].change(load_soft_prompt, shared.gradio['softprompts_menu'], shared.gradio['softprompts_menu'], show_progress=True)490 shared.gradio['upload_softprompt'].upload(upload_soft_prompt, shared.gradio['upload_softprompt'], shared.gradio['softprompts_menu'])491 492 493def set_interface_arguments(interface_mode, extensions, bool_active):494 modes = ["default", "notebook", "chat", "cai_chat"]495 cmd_list = vars(shared.args)496 bool_list = [k for k in cmd_list if type(cmd_list[k]) is bool and k not in modes]497 498 shared.args.extensions = extensions499 for k in modes[1:]:500 setattr(shared.args, k, False)501 if interface_mode != "default":502 setattr(shared.args, interface_mode, True)503 504 for k in bool_list:505 setattr(shared.args, k, False)506 for k in bool_active:507 setattr(shared.args, k, True)508 509 shared.need_restart = True510 511 512def create_interface():513 514 # Defining some variables515 gen_events = []516 default_preset = shared.settings['presets'][next((k for k in shared.settings['presets'] if re.match(k.lower(), shared.model_name.lower())), 'default')]517 if len(shared.lora_names) == 1:518 default_text = load_prompt(shared.settings['prompts'][next((k for k in shared.settings['prompts'] if re.match(k.lower(), shared.lora_names[0].lower())), 'default')])519 else:520 default_text = load_prompt(shared.settings['prompts'][next((k for k in shared.settings['prompts'] if re.match(k.lower(), shared.model_name.lower())), 'default')])521 title = 'Text generation web UI'522 523 # Authentication variables524 auth = None525 if shared.args.gradio_auth_path is not None:526 gradio_auth_creds = []527 with open(shared.args.gradio_auth_path, 'r', encoding="utf8") as file:528 for line in file.readlines():529 gradio_auth_creds += [x.strip() for x in line.split(',') if x.strip()]530 auth = [tuple(cred.split(':')) for cred in gradio_auth_creds]531 532 # Importing the extension files and executing their setup() functions533 if shared.args.extensions is not None and len(shared.args.extensions) > 0:534 extensions_module.load_extensions()535 536 # css/js strings537 css = ui.css if not shared.is_chat() else ui.css + ui.chat_css538 js = ui.main_js if not shared.is_chat() else ui.main_js + ui.chat_js539 css += apply_extensions('css')540 js += apply_extensions('js')541 542 with gr.Blocks(css=css, analytics_enabled=False, title=title, theme=ui.theme) as shared.gradio['interface']:543 544 # Create chat mode interface545 if shared.is_chat():546 shared.input_elements = ui.list_interface_input_elements(chat=True)547 shared.gradio['interface_state'] = gr.State({k: None for k in shared.input_elements})548 shared.gradio['Chat input'] = gr.State()549 shared.gradio['dummy'] = gr.State()550 551 with gr.Tab('Text generation', elem_id='main'):552 shared.gradio['display'] = gr.HTML(value=chat_html_wrapper(shared.history['visible'], shared.settings['name1'], shared.settings['name2'], 'chat', 'cai-chat'))553 shared.gradio['textbox'] = gr.Textbox(label='Input')554 with gr.Row():555 shared.gradio['Stop'] = gr.Button('Stop', elem_id='stop')556 shared.gradio['Generate'] = gr.Button('Generate', elem_id='Generate', variant='primary')557 shared.gradio['Continue'] = gr.Button('Continue')558 559 with gr.Row():560 shared.gradio['Copy last reply'] = gr.Button('Copy last reply')561 shared.gradio['Regenerate'] = gr.Button('Regenerate')562 shared.gradio['Replace last reply'] = gr.Button('Replace last reply')563 564 with gr.Row():565 shared.gradio['Impersonate'] = gr.Button('Impersonate')566 shared.gradio['Send dummy message'] = gr.Button('Send dummy message')567 shared.gradio['Send dummy reply'] = gr.Button('Send dummy reply')568 569 with gr.Row():570 shared.gradio['Remove last'] = gr.Button('Remove last')571 shared.gradio['Clear history'] = gr.Button('Clear history')572 shared.gradio['Clear history-confirm'] = gr.Button('Confirm', variant='stop', visible=False)573 shared.gradio['Clear history-cancel'] = gr.Button('Cancel', visible=False)574 575 shared.gradio['mode'] = gr.Radio(choices=['chat', 'chat-instruct', 'instruct'], value=shared.settings['mode'] if shared.settings['mode'] in ['chat', 'instruct', 'chat-instruct'] else 'chat', label='Mode', info='Defines how the chat prompt is generated. In instruct and chat-instruct modes, the instruction template selected under "Chat settings" must match the current model.')576 shared.gradio['chat_style'] = gr.Dropdown(choices=utils.get_available_chat_styles(), label='Chat style', value=shared.settings['chat_style'], visible=shared.settings['mode'] != 'instruct')577 578 with gr.Tab('Chat settings', elem_id='chat-settings'):579 with gr.Row():580 shared.gradio['character_menu'] = gr.Dropdown(choices=utils.get_available_characters(), label='Character', elem_id='character-menu', info='Used in chat and chat-instruct modes.')581 ui.create_refresh_button(shared.gradio['character_menu'], lambda: None, lambda: {'choices': utils.get_available_characters()}, 'refresh-button')582 583 with gr.Row():584 with gr.Column(scale=8):585 shared.gradio['name1'] = gr.Textbox(value=shared.settings['name1'], lines=1, label='Your name')586 shared.gradio['name2'] = gr.Textbox(value=shared.settings['name2'], lines=1, label='Character\'s name')587 shared.gradio['context'] = gr.Textbox(value=shared.settings['context'], lines=4, label='Context')588 shared.gradio['greeting'] = gr.Textbox(value=shared.settings['greeting'], lines=4, label='Greeting')589 590 with gr.Column(scale=1):591 shared.gradio['character_picture'] = gr.Image(label='Character picture', type='pil')592 shared.gradio['your_picture'] = gr.Image(label='Your picture', type='pil', value=Image.open(Path('cache/pfp_me.png')) if Path('cache/pfp_me.png').exists() else None)593 594 shared.gradio['instruction_template'] = gr.Dropdown(choices=utils.get_available_instruction_templates(), label='Instruction template', value='None', info='Change this according to the model/LoRA that you are using. Used in instruct and chat-instruct modes.')595 shared.gradio['name1_instruct'] = gr.Textbox(value='', lines=2, label='User string')596 shared.gradio['name2_instruct'] = gr.Textbox(value='', lines=1, label='Bot string')597 shared.gradio['context_instruct'] = gr.Textbox(value='', lines=4, label='Context')598 shared.gradio['turn_template'] = gr.Textbox(value=shared.settings['turn_template'], lines=1, label='Turn template', info='Used to precisely define the placement of spaces and new line characters in instruction prompts.')599 with gr.Row():600 shared.gradio['chat-instruct_command'] = gr.Textbox(value=shared.settings['chat-instruct_command'], lines=4, label='Command for chat-instruct mode', info='<|character|> gets replaced by the bot name, and <|prompt|> gets replaced by the regular chat prompt.')601 602 with gr.Row():603 with gr.Tab('Chat history'):604 with gr.Row():605 with gr.Column():606 gr.Markdown('## Upload')607 shared.gradio['upload_chat_history'] = gr.File(type='binary', file_types=['.json', '.txt'])608 609 with gr.Column():610 gr.Markdown('## Download')611 shared.gradio['download'] = gr.File()612 shared.gradio['download_button'] = gr.Button(value='Click me')613 614 with gr.Tab('Upload character'):615 gr.Markdown('## JSON format')616 with gr.Row():617 with gr.Column():618 gr.Markdown('1. Select the JSON file')619 shared.gradio['upload_json'] = gr.File(type='binary', file_types=['.json'])620 621 with gr.Column():622 gr.Markdown('2. Select your character\'s profile picture (optional)')623 shared.gradio['upload_img_bot'] = gr.File(type='binary', file_types=['image'])624 625 shared.gradio['Upload character'] = gr.Button(value='Submit')626 gr.Markdown('## TavernAI PNG format')627 shared.gradio['upload_img_tavern'] = gr.File(type='binary', file_types=['image'])628 629 with gr.Tab("Parameters", elem_id="parameters"):630 with gr.Box():631 gr.Markdown("Chat parameters")632 with gr.Row():633 with gr.Column():634 shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], step=1, label='max_new_tokens', value=shared.settings['max_new_tokens'])635 shared.gradio['chat_prompt_size'] = gr.Slider(minimum=shared.settings['chat_prompt_size_min'], maximum=shared.settings['chat_prompt_size_max'], step=1, label='Maximum prompt size in tokens', value=shared.settings['chat_prompt_size'])636 637 with gr.Column():638 shared.gradio['chat_generation_attempts'] = gr.Slider(minimum=shared.settings['chat_generation_attempts_min'], maximum=shared.settings['chat_generation_attempts_max'], value=shared.settings['chat_generation_attempts'], step=1, label='Generation attempts (for longer replies)', info='New generations will be called until either this number is reached or no new content is generated between two iterations')639 shared.gradio['stop_at_newline'] = gr.Checkbox(value=shared.settings['stop_at_newline'], label='Stop generating at new line character')640 641 create_settings_menus(default_preset)642 643 # Create notebook mode interface644 elif shared.args.notebook:645 shared.input_elements = ui.list_interface_input_elements(chat=False)646 shared.gradio['interface_state'] = gr.State({k: None for k in shared.input_elements})647 shared.gradio['last_input'] = gr.State('')648 with gr.Tab("Text generation", elem_id="main"):649 with gr.Row():650 with gr.Column(scale=4):651 with gr.Tab('Raw'):652 shared.gradio['textbox'] = gr.Textbox(value=default_text, elem_classes="textbox", lines=27)653 654 with gr.Tab('Markdown'):655 shared.gradio['markdown'] = gr.Markdown()656 657 with gr.Tab('HTML'):658 shared.gradio['html'] = gr.HTML()659 660 with gr.Row():661 shared.gradio['Generate'] = gr.Button('Generate', variant='primary', elem_classes="small-button")662 shared.gradio['Stop'] = gr.Button('Stop', elem_classes="small-button")663 shared.gradio['Undo'] = gr.Button('Undo', elem_classes="small-button")664 shared.gradio['Regenerate'] = gr.Button('Regenerate', elem_classes="small-button")665 666 with gr.Column(scale=1):667 gr.HTML('<div style="padding-bottom: 13px"></div>')668 shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], step=1, label='max_new_tokens', value=shared.settings['max_new_tokens'])669 with gr.Row():670 shared.gradio['prompt_menu'] = gr.Dropdown(choices=utils.get_available_prompts(), value='None', label='Prompt')671 ui.create_refresh_button(shared.gradio['prompt_menu'], lambda: None, lambda: {'choices': utils.get_available_prompts()}, 'refresh-button')672 673 shared.gradio['open_save_prompt'] = gr.Button('Save prompt')674 shared.gradio['save_prompt'] = gr.Button('Confirm save prompt', visible=False)675 shared.gradio['prompt_to_save'] = gr.Textbox(elem_classes="textbox_default", lines=1, label='Prompt name:', interactive=True, visible=False)676 shared.gradio['count_tokens'] = gr.Button('Count tokens')677 shared.gradio['status'] = gr.Markdown('')678 679 with gr.Tab("Parameters", elem_id="parameters"):680 create_settings_menus(default_preset)681 682 # Create default mode interface683 else:684 shared.input_elements = ui.list_interface_input_elements(chat=False)685 shared.gradio['interface_state'] = gr.State({k: None for k in shared.input_elements})686 shared.gradio['last_input'] = gr.State('')687 with gr.Tab("Text generation", elem_id="main"):688 with gr.Row():689 with gr.Column():690 shared.gradio['textbox'] = gr.Textbox(value=default_text, elem_classes="textbox_default", lines=27, label='Input')691 shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], step=1, label='max_new_tokens', value=shared.settings['max_new_tokens'])692 with gr.Row():693 shared.gradio['Generate'] = gr.Button('Generate', variant='primary', elem_classes="small-button")694 shared.gradio['Stop'] = gr.Button('Stop', elem_classes="small-button")695 shared.gradio['Continue'] = gr.Button('Continue', elem_classes="small-button")696 shared.gradio['open_save_prompt'] = gr.Button('Save prompt', elem_classes="small-button")697 shared.gradio['save_prompt'] = gr.Button('Confirm save prompt', visible=False, elem_classes="small-button")698 shared.gradio['count_tokens'] = gr.Button('Count tokens', elem_classes="small-button")699 700 with gr.Row():701 with gr.Column():702 with gr.Row():703 shared.gradio['prompt_menu'] = gr.Dropdown(choices=utils.get_available_prompts(), value='None', label='Prompt')704 ui.create_refresh_button(shared.gradio['prompt_menu'], lambda: None, lambda: {'choices': utils.get_available_prompts()}, 'refresh-button')705 706 with gr.Column():707 shared.gradio['prompt_to_save'] = gr.Textbox(elem_classes="textbox_default", lines=1, label='Prompt name:', interactive=True, visible=False)708 shared.gradio['status'] = gr.Markdown('')709 710 with gr.Column():711 with gr.Tab('Raw'):712 shared.gradio['output_textbox'] = gr.Textbox(elem_classes="textbox_default_output", lines=27, label='Output')713 714 with gr.Tab('Markdown'):715 shared.gradio['markdown'] = gr.Markdown()716 717 with gr.Tab('HTML'):718 shared.gradio['html'] = gr.HTML()719 720 with gr.Tab("Parameters", elem_id="parameters"):721 create_settings_menus(default_preset)722 723 # Model tab724 with gr.Tab("Model", elem_id="model-tab"):725 create_model_menus()726 727 # Training tab728 with gr.Tab("Training", elem_id="training-tab"):729 training.create_train_interface()730 731 # Interface mode tab732 with gr.Tab("Interface mode", elem_id="interface-mode"):733 modes = ["default", "notebook", "chat"]734 current_mode = "default"735 for mode in modes[1:]:736 if getattr(shared.args, mode):737 current_mode = mode738 break739 740 cmd_list = vars(shared.args)741 bool_list = sorted([k for k in cmd_list if type(cmd_list[k]) is bool and k not in modes + ui.list_model_elements()])742 bool_active = [k for k in bool_list if vars(shared.args)[k]]743 744 shared.gradio['interface_modes_menu'] = gr.Dropdown(choices=modes, value=current_mode, label="Mode")745 shared.gradio['extensions_menu'] = gr.CheckboxGroup(choices=utils.get_available_extensions(), value=shared.args.extensions, label="Available extensions")746 shared.gradio['bool_menu'] = gr.CheckboxGroup(choices=bool_list, value=bool_active, label="Boolean command-line flags")747 shared.gradio['reset_interface'] = gr.Button("Apply and restart the interface")748 749 # Reset interface event750 shared.gradio['reset_interface'].click(751 set_interface_arguments, [shared.gradio[k] for k in ['interface_modes_menu', 'extensions_menu', 'bool_menu']], None).then(752 lambda: None, None, None, _js='() => {document.body.innerHTML=\'<h1 style="font-family:monospace;margin-top:20%;color:lightgray;text-align:center;">Reloading...</h1>\'; setTimeout(function(){location.reload()},2500); return []}')753 754 # chat mode event handlers755 if shared.is_chat():756 shared.input_params = [shared.gradio[k] for k in ['Chat input', 'interface_state']]757 clear_arr = [shared.gradio[k] for k in ['Clear history-confirm', 'Clear history', 'Clear history-cancel']]758 shared.reload_inputs = [shared.gradio[k] for k in ['name1', 'name2', 'mode', 'chat_style']]759 760 gen_events.append(shared.gradio['Generate'].click(761 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(762 lambda x: (x, ''), shared.gradio['textbox'], [shared.gradio['Chat input'], shared.gradio['textbox']], show_progress=False).then(763 chat.generate_chat_reply_wrapper, shared.input_params, shared.gradio['display'], show_progress=False).then(764 chat.save_history, shared.gradio['mode'], None, show_progress=False)765 )766 767 gen_events.append(shared.gradio['textbox'].submit(768 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(769 lambda x: (x, ''), shared.gradio['textbox'], [shared.gradio['Chat input'], shared.gradio['textbox']], show_progress=False).then(770 chat.generate_chat_reply_wrapper, shared.input_params, shared.gradio['display'], show_progress=False).then(771 chat.save_history, shared.gradio['mode'], None, show_progress=False)772 )773 774 gen_events.append(shared.gradio['Regenerate'].click(775 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(776 partial(chat.generate_chat_reply_wrapper, regenerate=True), shared.input_params, shared.gradio['display'], show_progress=False).then(777 chat.save_history, shared.gradio['mode'], None, show_progress=False)778 )779 780 gen_events.append(shared.gradio['Continue'].click(781 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(782 partial(chat.generate_chat_reply_wrapper, _continue=True), shared.input_params, shared.gradio['display'], show_progress=False).then(783 chat.save_history, shared.gradio['mode'], None, show_progress=False)784 )785 786 gen_events.append(shared.gradio['Impersonate'].click(787 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(788 lambda x: x, shared.gradio['textbox'], shared.gradio['Chat input'], show_progress=False).then(789 chat.impersonate_wrapper, shared.input_params, shared.gradio['textbox'], show_progress=False)790 )791 792 shared.gradio['Replace last reply'].click(793 chat.replace_last_reply, shared.gradio['textbox'], None).then(794 lambda: '', None, shared.gradio['textbox'], show_progress=False).then(795 chat.save_history, shared.gradio['mode'], None, show_progress=False).then(796 chat.redraw_html, shared.reload_inputs, shared.gradio['display'])797 798 shared.gradio['Send dummy message'].click(799 chat.send_dummy_message, shared.gradio['textbox'], None).then(800 lambda: '', None, shared.gradio['textbox'], show_progress=False).then(801 chat.save_history, shared.gradio['mode'], None, show_progress=False).then(802 chat.redraw_html, shared.reload_inputs, shared.gradio['display'])803 804 shared.gradio['Send dummy reply'].click(805 chat.send_dummy_reply, shared.gradio['textbox'], None).then(806 lambda: '', None, shared.gradio['textbox'], show_progress=False).then(807 chat.save_history, shared.gradio['mode'], None, show_progress=False).then(808 chat.redraw_html, shared.reload_inputs, shared.gradio['display'])809 810 shared.gradio['Clear history-confirm'].click(811 lambda: [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)], None, clear_arr).then(812 chat.clear_chat_log, [shared.gradio[k] for k in ['greeting', 'mode']], None).then(813 chat.save_history, shared.gradio['mode'], None, show_progress=False).then(814 chat.redraw_html, shared.reload_inputs, shared.gradio['display'])815 816 shared.gradio['Stop'].click(817 stop_everything_event, None, None, queue=False, cancels=gen_events if shared.args.no_stream else None).then(818 chat.redraw_html, shared.reload_inputs, shared.gradio['display'])819 820 shared.gradio['mode'].change(821 lambda x: gr.update(visible=x != 'instruct'), shared.gradio['mode'], shared.gradio['chat_style'], show_progress=False).then(822 chat.redraw_html, shared.reload_inputs, shared.gradio['display'])823 824 825 shared.gradio['chat_style'].change(chat.redraw_html, shared.reload_inputs, shared.gradio['display'])826 shared.gradio['instruction_template'].change(827 partial(chat.load_character, instruct=True), [shared.gradio[k] for k in ['instruction_template', 'name1_instruct', 'name2_instruct']], [shared.gradio[k] for k in ['name1_instruct', 'name2_instruct', 'dummy', 'dummy', 'context_instruct', 'turn_template']])828 829 shared.gradio['upload_chat_history'].upload(830 chat.load_history, [shared.gradio[k] for k in ['upload_chat_history', 'name1', 'name2']], None).then(831 chat.redraw_html, shared.reload_inputs, shared.gradio['display'])832 833 shared.gradio['Copy last reply'].click(chat.send_last_reply_to_input, None, shared.gradio['textbox'], show_progress=False)834 shared.gradio['Clear history'].click(lambda: [gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)], None, clear_arr)835 shared.gradio['Clear history-cancel'].click(lambda: [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)], None, clear_arr)836 shared.gradio['Remove last'].click(837 chat.remove_last_message, None, shared.gradio['textbox'], show_progress=False).then(838 chat.save_history, shared.gradio['mode'], None, show_progress=False).then(839 chat.redraw_html, shared.reload_inputs, shared.gradio['display'])840 841 shared.gradio['download_button'].click(lambda x: chat.save_history(x, timestamp=True), shared.gradio['mode'], shared.gradio['download'])842 shared.gradio['Upload character'].click(chat.upload_character, [shared.gradio['upload_json'], shared.gradio['upload_img_bot']], [shared.gradio['character_menu']])843 shared.gradio['character_menu'].change(844 partial(chat.load_character, instruct=False), [shared.gradio[k] for k in ['character_menu', 'name1', 'name2']], [shared.gradio[k] for k in ['name1', 'name2', 'character_picture', 'greeting', 'context', 'dummy']]).then(845 chat.redraw_html, shared.reload_inputs, shared.gradio['display'])846 847 shared.gradio['upload_img_tavern'].upload(chat.upload_tavern_character, [shared.gradio['upload_img_tavern'], shared.gradio['name1'], shared.gradio['name2']], [shared.gradio['character_menu']])848 shared.gradio['your_picture'].change(849 chat.upload_your_profile_picture, shared.gradio['your_picture'], None).then(850 partial(chat.redraw_html, reset_cache=True), shared.reload_inputs, shared.gradio['display'])851 852 # notebook/default modes event handlers853 else:854 shared.input_params = [shared.gradio[k] for k in ['textbox', 'interface_state']]855 if shared.args.notebook:856 output_params = [shared.gradio[k] for k in ['textbox', 'markdown', 'html']]857 else:858 output_params = [shared.gradio[k] for k in ['output_textbox', 'markdown', 'html']]859 860 gen_events.append(shared.gradio['Generate'].click(861 lambda x: x, shared.gradio['textbox'], shared.gradio['last_input']).then(862 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(863 generate_reply_wrapper, shared.input_params, output_params, show_progress=False) # .then(864 # None, None, None, _js="() => {element = document.getElementsByTagName('textarea')[0]; element.scrollTop = element.scrollHeight}")865 )866 867 gen_events.append(shared.gradio['textbox'].submit(868 lambda x: x, shared.gradio['textbox'], shared.gradio['last_input']).then(869 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(870 generate_reply_wrapper, shared.input_params, output_params, show_progress=False) # .then(871 # None, None, None, _js="() => {element = document.getElementsByTagName('textarea')[0]; element.scrollTop = element.scrollHeight}")872 )873 874 if shared.args.notebook:875 shared.gradio['Undo'].click(lambda x: x, shared.gradio['last_input'], shared.gradio['textbox'], show_progress=False)876 gen_events.append(shared.gradio['Regenerate'].click(877 lambda x: x, shared.gradio['last_input'], shared.gradio['textbox'], show_progress=False).then(878 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(879 generate_reply_wrapper, shared.input_params, output_params, show_progress=False) # .then(880 # None, None, None, _js="() => {element = document.getElementsByTagName('textarea')[0]; element.scrollTop = element.scrollHeight}")881 )882 else:883 gen_events.append(shared.gradio['Continue'].click(884 ui.gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then(885 generate_reply_wrapper, [shared.gradio['output_textbox']] + shared.input_params[1:], output_params, show_progress=False) # .then(886 # None, None, None, _js="() => {element = document.getElementsByTagName('textarea')[1]; element.scrollTop = element.scrollHeight}")887 )888 889 shared.gradio['Stop'].click(stop_everything_event, None, None, queue=False, cancels=gen_events if shared.args.no_stream else None)890 shared.gradio['prompt_menu'].change(load_prompt, shared.gradio['prompt_menu'], shared.gradio['textbox'], show_progress=False)891 shared.gradio['open_save_prompt'].click(open_save_prompt, None, [shared.gradio[k] for k in ['prompt_to_save', 'open_save_prompt', 'save_prompt']], show_progress=False)892 shared.gradio['save_prompt'].click(save_prompt, [shared.gradio[k] for k in ['textbox', 'prompt_to_save']], [shared.gradio[k] for k in ['status', 'prompt_to_save', 'open_save_prompt', 'save_prompt']], show_progress=False)893 shared.gradio['count_tokens'].click(count_tokens, shared.gradio['textbox'], shared.gradio['status'], show_progress=False)894 895 shared.gradio['interface'].load(None, None, None, _js=f"() => {{{js}}}")896 shared.gradio['interface'].load(partial(ui.apply_interface_values, {}, use_persistent=True), None, [shared.gradio[k] for k in ui.list_interface_input_elements(chat=shared.is_chat())], show_progress=False)897 # Extensions tabs898 extensions_module.create_extensions_tabs()899 900 # Extensions block901 extensions_module.create_extensions_block()902 903 # Launch the interface904 shared.gradio['interface'].queue()905 if shared.args.listen:906 shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_name=shared.args.listen_host or '0.0.0.0', server_port=shared.args.listen_port, inbrowser=shared.args.auto_launch, auth=auth)907 else:908 shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_port=shared.args.listen_port, inbrowser=shared.args.auto_launch, auth=auth)909 910 911if __name__ == "__main__":912 # Loading custom settings913 settings_file = None914 if shared.args.settings is not None and Path(shared.args.settings).exists():915 settings_file = Path(shared.args.settings)916 elif Path('settings.json').exists():917 settings_file = Path('settings.json')918 919 if settings_file is not None:920 logging.info(f"Loading settings from {settings_file}...")921 new_settings = json.loads(open(settings_file, 'r').read())922 for item in new_settings:923 shared.settings[item] = new_settings[item]924 925 # Set default model settings based on settings.json926 shared.model_config['.*'] = {927 'wbits': 'None',928 'model_type': 'None',929 'groupsize': 'None',930 'pre_layer': 0,931 'mode': shared.settings['mode'],932 'skip_special_tokens': shared.settings['skip_special_tokens'],933 'custom_stopping_strings': shared.settings['custom_stopping_strings'],934 }935 936 shared.model_config.move_to_end('.*', last=False) # Move to the beginning937 938 # Default extensions939 extensions_module.available_extensions = utils.get_available_extensions()940 if shared.is_chat():941 for extension in shared.settings['chat_default_extensions']:942 shared.args.extensions = shared.args.extensions or []943 if extension not in shared.args.extensions:944 shared.args.extensions.append(extension)945 else:946 for extension in shared.settings['default_extensions']:947 shared.args.extensions = shared.args.extensions or []948 if extension not in shared.args.extensions:949 shared.args.extensions.append(extension)950 951 available_models = utils.get_available_models()952 953 # Model defined through --model954 if shared.args.model is not None:955 shared.model_name = shared.args.model956 957 # Only one model is available958 elif len(available_models) == 1:959 shared.model_name = available_models[0]960 961 # Select the model from a command-line menu962 elif shared.args.model_menu:963 if len(available_models) == 0:964 logging.error('No models are available! Please download at least one.')965 sys.exit(0)966 else:967 print('The following models are available:\n')968 for i, model in enumerate(available_models):969 print(f'{i+1}. {model}')970 971 print(f'\nWhich one do you want to load? 1-{len(available_models)}\n')972 i = int(input()) - 1973 print()974 975 shared.model_name = available_models[i]976 977 # If any model has been selected, load it978 if shared.model_name != 'None':979 model_settings = get_model_specific_settings(shared.model_name)980 shared.settings.update(model_settings) # hijacking the interface defaults981 update_model_parameters(model_settings, initial=True) # hijacking the command-line arguments982 983 # Load the model984 shared.model, shared.tokenizer = load_model(shared.model_name)985 if shared.args.lora:986 add_lora_to_model(shared.args.lora)987 988 # Force a character to be loaded989 if shared.is_chat():990 shared.persistent_interface_state.update({991 'mode': shared.settings['mode'],992 'character_menu': shared.args.character or shared.settings['character'],993 'instruction_template': shared.settings['instruction_template']994 })995 996 # Launch the web UI997 create_interface()998 while True:999 time.sleep(0.5)1000 if shared.need_restart:1001 shared.need_restart = False1002 shared.gradio['interface'].close()1003 time.sleep(0.5)1004 create_interface()1005