forestcalled/text-generation-webui

1import functools2from collections import OrderedDict3 4import gradio as gr5 6from modules import shared7 8loaders_and_params = OrderedDict({9    'Transformers': [10        'cpu_memory',11        'gpu_memory',12        'load_in_8bit',13        'bf16',14        'cpu',15        'disk',16        'auto_devices',17        'load_in_4bit',18        'use_double_quant',19        'quant_type',20        'compute_dtype',21        'trust_remote_code',22        'no_use_fast',23        'use_flash_attention_2',24        'alpha_value',25        'rope_freq_base',26        'compress_pos_emb',27        'disable_exllama',28        'disable_exllamav2',29        'transformers_info'30    ],31    'llama.cpp': [32        'n_ctx',33        'n_gpu_layers',34        'tensor_split',35        'n_batch',36        'threads',37        'threads_batch',38        'no_mmap',39        'mlock',40        'no_mul_mat_q',41        'alpha_value',42        'rope_freq_base',43        'compress_pos_emb',44        'cpu',45        'numa',46        'no_offload_kqv',47        'tensorcores',48    ],49    'llamacpp_HF': [50        'n_ctx',51        'n_gpu_layers',52        'tensor_split',53        'n_batch',54        'threads',55        'threads_batch',56        'no_mmap',57        'mlock',58        'no_mul_mat_q',59        'alpha_value',60        'rope_freq_base',61        'compress_pos_emb',62        'cpu',63        'numa',64        'cfg_cache',65        'trust_remote_code',66        'no_use_fast',67        'logits_all',68        'no_offload_kqv',69        'tensorcores',70        'llamacpp_HF_info',71    ],72    'ExLlamav2_HF': [73        'gpu_split',74        'max_seq_len',75        'cfg_cache',76        'no_flash_attn',77        'num_experts_per_token',78        'cache_8bit',79        'alpha_value',80        'compress_pos_emb',81        'trust_remote_code',82        'no_use_fast',83    ],84    'ExLlama_HF': [85        'gpu_split',86        'max_seq_len',87        'alpha_value',88        'rope_freq_base',89        'compress_pos_emb',90        'cfg_cache',91        'trust_remote_code',92        'no_use_fast',93    ],94    'AutoGPTQ': [95        'triton',96        'no_inject_fused_attention',97        'no_inject_fused_mlp',98        'no_use_cuda_fp16',99        'wbits',100        'groupsize',101        'desc_act',102        'disable_exllama',103        'disable_exllamav2',104        'gpu_memory',105        'cpu_memory',106        'cpu',107        'disk',108        'auto_devices',109        'trust_remote_code',110        'no_use_fast',111        'autogptq_info',112    ],113    'AutoAWQ': [114        'cpu_memory',115        'gpu_memory',116        'auto_devices',117        'max_seq_len',118        'no_inject_fused_attention',119        'trust_remote_code',120        'no_use_fast',121    ],122    'GPTQ-for-LLaMa': [123        'wbits',124        'groupsize',125        'model_type',126        'pre_layer',127        'trust_remote_code',128        'no_use_fast',129        'gptq_for_llama_info',130    ],131    'ExLlamav2': [132        'gpu_split',133        'max_seq_len',134        'no_flash_attn',135        'num_experts_per_token',136        'cache_8bit',137        'alpha_value',138        'compress_pos_emb',139        'exllamav2_info',140    ],141    'ExLlama': [142        'gpu_split',143        'max_seq_len',144        'alpha_value',145        'rope_freq_base',146        'compress_pos_emb',147        'exllama_info',148    ],149    'ctransformers': [150        'n_ctx',151        'n_gpu_layers',152        'n_batch',153        'threads',154        'model_type',155        'no_mmap',156        'mlock'157    ],158    'QuIP#': [159        'trust_remote_code',160        'no_use_fast',161        'no_flash_attn',162        'quipsharp_info',163    ],164    'HQQ': [165        'hqq_backend',166        'trust_remote_code',167        'no_use_fast',168    ]169})170 171 172def transformers_samplers():173    return {174        'temperature',175        'temperature_last',176        'top_p',177        'min_p',178        'top_k',179        'typical_p',180        'epsilon_cutoff',181        'eta_cutoff',182        'tfs',183        'top_a',184        'repetition_penalty',185        'presence_penalty',186        'frequency_penalty',187        'repetition_penalty_range',188        'encoder_repetition_penalty',189        'no_repeat_ngram_size',190        'min_length',191        'seed',192        'do_sample',193        'penalty_alpha',194        'num_beams',195        'length_penalty',196        'early_stopping',197        'mirostat_mode',198        'mirostat_tau',199        'mirostat_eta',200        'grammar_file_row',201        'grammar_string',202        'guidance_scale',203        'negative_prompt',204        'ban_eos_token',205        'custom_token_bans',206        'add_bos_token',207        'skip_special_tokens',208        'auto_max_new_tokens',209    }210 211 212loaders_samplers = {213    'Transformers': transformers_samplers(),214    'AutoGPTQ': transformers_samplers(),215    'GPTQ-for-LLaMa': transformers_samplers(),216    'AutoAWQ': transformers_samplers(),217    'QuIP#': transformers_samplers(),218    'HQQ': transformers_samplers(),219    'ExLlama_HF': {220        'temperature',221        'temperature_last',222        'top_p',223        'min_p',224        'top_k',225        'typical_p',226        'epsilon_cutoff',227        'eta_cutoff',228        'tfs',229        'top_a',230        'repetition_penalty',231        'presence_penalty',232        'frequency_penalty',233        'repetition_penalty_range',234        'encoder_repetition_penalty',235        'no_repeat_ngram_size',236        'min_length',237        'seed',238        'do_sample',239        'mirostat_mode',240        'mirostat_tau',241        'mirostat_eta',242        'grammar_file_row',243        'grammar_string',244        'guidance_scale',245        'negative_prompt',246        'ban_eos_token',247        'custom_token_bans',248        'add_bos_token',249        'skip_special_tokens',250        'auto_max_new_tokens',251    },252    'ExLlama': {253        'temperature',254        'top_p',255        'top_k',256        'typical_p',257        'repetition_penalty',258        'repetition_penalty_range',259        'seed',260        'guidance_scale',261        'negative_prompt',262        'ban_eos_token',263        'add_bos_token',264        'custom_token_bans',265        'auto_max_new_tokens',266    },267    'ExLlamav2': {268        'temperature',269        'top_p',270        'min_p',271        'top_k',272        'typical_p',273        'tfs',274        'repetition_penalty',275        'repetition_penalty_range',276        'seed',277        'mirostat_mode',278        'mirostat_tau',279        'mirostat_eta',280        'ban_eos_token',281        'add_bos_token',282        'custom_token_bans',283        'skip_special_tokens',284        'auto_max_new_tokens',285    },286    'ExLlamav2_HF': {287        'temperature',288        'temperature_last',289        'top_p',290        'min_p',291        'top_k',292        'typical_p',293        'epsilon_cutoff',294        'eta_cutoff',295        'tfs',296        'top_a',297        'repetition_penalty',298        'presence_penalty',299        'frequency_penalty',300        'repetition_penalty_range',301        'encoder_repetition_penalty',302        'no_repeat_ngram_size',303        'min_length',304        'seed',305        'do_sample',306        'mirostat_mode',307        'mirostat_tau',308        'mirostat_eta',309        'grammar_file_row',310        'grammar_string',311        'guidance_scale',312        'negative_prompt',313        'ban_eos_token',314        'custom_token_bans',315        'add_bos_token',316        'skip_special_tokens',317        'auto_max_new_tokens',318    },319    'llama.cpp': {320        'temperature',321        'top_p',322        'min_p',323        'top_k',324        'typical_p',325        'tfs',326        'repetition_penalty',327        'presence_penalty',328        'frequency_penalty',329        'seed',330        'mirostat_mode',331        'mirostat_tau',332        'mirostat_eta',333        'grammar_file_row',334        'grammar_string',335        'ban_eos_token',336        'custom_token_bans',337    },338    'llamacpp_HF': {339        'temperature',340        'temperature_last',341        'top_p',342        'min_p',343        'top_k',344        'typical_p',345        'epsilon_cutoff',346        'eta_cutoff',347        'tfs',348        'top_a',349        'repetition_penalty',350        'presence_penalty',351        'frequency_penalty',352        'repetition_penalty_range',353        'encoder_repetition_penalty',354        'no_repeat_ngram_size',355        'min_length',356        'seed',357        'do_sample',358        'mirostat_mode',359        'mirostat_tau',360        'mirostat_eta',361        'grammar_file_row',362        'grammar_string',363        'guidance_scale',364        'negative_prompt',365        'ban_eos_token',366        'custom_token_bans',367        'add_bos_token',368        'skip_special_tokens',369        'auto_max_new_tokens',370    },371    'ctransformers': {372        'temperature',373        'top_p',374        'top_k',375        'repetition_penalty',376        'repetition_penalty_range',377    },378}379 380loaders_model_types = {381    'GPTQ-for-LLaMa': [382        "None",383        "llama",384        "opt",385        "gptj"386    ],387    'ctransformers': [388        "None",389        "gpt2",390        "gptj",391        "gptneox",392        "llama",393        "mpt",394        "dollyv2",395        "replit",396        "starcoder",397        "gptbigcode",398        "falcon"399    ],400}401 402 403@functools.cache404def list_all_samplers():405    all_samplers = set()406    for k in loaders_samplers:407        for sampler in loaders_samplers[k]:408            all_samplers.add(sampler)409 410    return sorted(all_samplers)411 412 413def blacklist_samplers(loader):414    all_samplers = list_all_samplers()415    if loader == 'All':416        return [gr.update(visible=True) for sampler in all_samplers]417    else:418        return [gr.update(visible=True) if sampler in loaders_samplers[loader] else gr.update(visible=False) for sampler in all_samplers]419 420 421def get_model_types(loader):422    if loader in loaders_model_types:423        return loaders_model_types[loader]424 425    return ["None"]426 427 428def get_gpu_memory_keys():429    return [k for k in shared.gradio if k.startswith('gpu_memory')]430 431 432@functools.cache433def get_all_params():434    all_params = set()435    for k in loaders_and_params:436        for el in loaders_and_params[k]:437            all_params.add(el)438 439    if 'gpu_memory' in all_params:440        all_params.remove('gpu_memory')441        for k in get_gpu_memory_keys():442            all_params.add(k)443 444    return sorted(all_params)445 446 447def make_loader_params_visible(loader):448    params = []449    all_params = get_all_params()450    if loader in loaders_and_params:451        params = loaders_and_params[loader]452 453        if 'gpu_memory' in params:454            params.remove('gpu_memory')455            params += get_gpu_memory_keys()456 457    return [gr.update(visible=True) if k in params else gr.update(visible=False) for k in all_params]458