forestcalled/text-generation-webui

1import ast2import copy3import html4import random5import re6import time7import traceback8 9import numpy as np10import torch11import transformers12from transformers import LogitsProcessorList, is_torch_xpu_available13 14import modules.shared as shared15from modules.callbacks import (16    Iteratorize,17    Stream,18    _StopEverythingStoppingCriteria19)20from modules.extensions import apply_extensions21from modules.grammar.grammar_utils import initialize_grammar22from modules.grammar.logits_process import GrammarConstrainedLogitsProcessor23from modules.html_generator import generate_4chan_html, generate_basic_html24from modules.logging_colors import logger25from modules.models import clear_torch_cache, local_rank26 27 28def generate_reply(*args, **kwargs):29    shared.generation_lock.acquire()30    try:31        for result in _generate_reply(*args, **kwargs):32            yield result33    finally:34        shared.generation_lock.release()35 36 37def _generate_reply(question, state, stopping_strings=None, is_chat=False, escape_html=False, for_ui=False):38 39    # Find the appropriate generation function40    generate_func = apply_extensions('custom_generate_reply')41    if generate_func is None:42        if shared.model_name == 'None' or shared.model is None:43            logger.error("No model is loaded! Select one in the Model tab.")44            yield ''45            return46 47        if shared.model.__class__.__name__ in ['LlamaCppModel', 'RWKVModel', 'ExllamaModel', 'Exllamav2Model', 'CtransformersModel']:48            generate_func = generate_reply_custom49        else:50            generate_func = generate_reply_HF51 52    # Prepare the input53    original_question = question54    if not is_chat:55        state = apply_extensions('state', state)56        question = apply_extensions('input', question, state)57 58    # Find the stopping strings59    all_stop_strings = []60    for st in (stopping_strings, state['custom_stopping_strings']):61        if type(st) is str:62            st = ast.literal_eval(f"[{st}]")63 64        if type(st) is list and len(st) > 0:65            all_stop_strings += st66 67    if shared.args.verbose:68        print(f'\n\n{question}\n--------------------\n')69 70    shared.stop_everything = False71    clear_torch_cache()72    seed = set_manual_seed(state['seed'])73    last_update = -174    reply = ''75    is_stream = state['stream']76    if len(all_stop_strings) > 0 and not state['stream']:77        state = copy.deepcopy(state)78        state['stream'] = True79 80    min_update_interval = 081    if state.get('max_updates_second', 0) > 0:82        min_update_interval = 1 / state['max_updates_second']83 84    # Generate85    for reply in generate_func(question, original_question, seed, state, stopping_strings, is_chat=is_chat):86        reply, stop_found = apply_stopping_strings(reply, all_stop_strings)87        if escape_html:88            reply = html.escape(reply)89        if is_stream:90            cur_time = time.time()91 92            # Maximum number of tokens/second93            if state['max_tokens_second'] > 0:94                diff = 1 / state['max_tokens_second'] - (cur_time - last_update)95                if diff > 0:96                    time.sleep(diff)97 98                last_update = time.time()99                yield reply100 101            # Limit updates to avoid lag in the Gradio UI102            # API updates are not limited103            else:104                if cur_time - last_update > min_update_interval:105                    last_update = cur_time106                    yield reply107 108        if stop_found or (state['max_tokens_second'] > 0 and shared.stop_everything):109            break110 111    if not is_chat:112        reply = apply_extensions('output', reply, state)113 114    yield reply115 116 117def encode(prompt, add_special_tokens=True, add_bos_token=True, truncation_length=None):118    if shared.tokenizer is None:119        raise ValueError('No tokenizer is loaded')120 121    if shared.model.__class__.__name__ in ['LlamaCppModel', 'RWKVModel', 'CtransformersModel', 'Exllamav2Model']:122        input_ids = shared.tokenizer.encode(str(prompt))123        if shared.model.__class__.__name__ not in ['Exllamav2Model']:124            input_ids = np.array(input_ids).reshape(1, len(input_ids))125    else:126        input_ids = shared.tokenizer.encode(str(prompt), return_tensors='pt', add_special_tokens=add_special_tokens)127        if not add_bos_token:128            while len(input_ids[0]) > 0 and input_ids[0][0] == shared.tokenizer.bos_token_id:129                input_ids = input_ids[:, 1:]130 131    # Handling truncation132    if truncation_length is not None:133        input_ids = input_ids[:, -truncation_length:]134 135    if shared.model.__class__.__name__ in ['LlamaCppModel', 'RWKVModel', 'ExllamaModel', 'Exllamav2Model', 'CtransformersModel'] or shared.args.cpu:136        return input_ids137    elif shared.args.deepspeed:138        return input_ids.to(device=local_rank)139    elif torch.backends.mps.is_available():140        device = torch.device('mps')141        return input_ids.to(device)142    elif is_torch_xpu_available():143        return input_ids.to("xpu:0")144    else:145        return input_ids.cuda()146 147 148def decode(output_ids, skip_special_tokens=True):149    if shared.tokenizer is None:150        raise ValueError('No tokenizer is loaded')151 152    return shared.tokenizer.decode(output_ids, skip_special_tokens=skip_special_tokens)153 154 155def get_encoded_length(prompt):156    length_after_extensions = apply_extensions('tokenized_length', prompt)157    if length_after_extensions is not None:158        return length_after_extensions159 160    return len(encode(prompt)[0])161 162 163def get_token_ids(prompt):164    tokens = encode(prompt)[0]165    decoded_tokens = [shared.tokenizer.decode([i]) for i in tokens]166 167    output = ''168    for row in list(zip(tokens, decoded_tokens)):169        output += f"{str(int(row[0])).ljust(5)}  -  {repr(row[1])}\n"170 171    return output172 173 174def get_max_prompt_length(state):175    return state['truncation_length'] - state['max_new_tokens']176 177 178def generate_reply_wrapper(question, state, stopping_strings=None):179    """180    Returns formatted outputs for the UI181    """182    reply = question if not shared.is_seq2seq else ''183    yield formatted_outputs(reply, shared.model_name)184 185    for reply in generate_reply(question, state, stopping_strings, is_chat=False, escape_html=True, for_ui=True):186        if not shared.is_seq2seq:187            reply = question + reply188 189        yield formatted_outputs(reply, shared.model_name)190 191 192def formatted_outputs(reply, model_name):193    if any(s in model_name for s in ['gpt-4chan', 'gpt4chan']):194        reply = fix_gpt4chan(reply)195        return html.unescape(reply), generate_4chan_html(reply)196    else:197        return html.unescape(reply), generate_basic_html(reply)198 199 200def fix_gpt4chan(s):201    """202    Removes empty replies from gpt4chan outputs203    """204    for i in range(10):205        s = re.sub("--- [0-9]*\n>>[0-9]*\n---", "---", s)206        s = re.sub("--- [0-9]*\n *\n---", "---", s)207        s = re.sub("--- [0-9]*\n\n\n---", "---", s)208 209    return s210 211 212def fix_galactica(s):213    """214    Fix the LaTeX equations in GALACTICA215    """216    s = s.replace(r'\[', r'$')217    s = s.replace(r'\]', r'$')218    s = s.replace(r'\(', r'$')219    s = s.replace(r'\)', r'$')220    s = s.replace(r'$$', r'$')221    s = re.sub(r'\n', r'\n\n', s)222    s = re.sub(r"\n{3,}", "\n\n", s)223    return s224 225 226def set_manual_seed(seed):227    seed = int(seed)228    if seed == -1:229        seed = random.randint(1, 2**31)230 231    torch.manual_seed(seed)232    if torch.cuda.is_available():233        torch.cuda.manual_seed_all(seed)234    elif is_torch_xpu_available():235        torch.xpu.manual_seed_all(seed)236 237    return seed238 239 240def stop_everything_event():241    shared.stop_everything = True242 243 244def apply_stopping_strings(reply, all_stop_strings):245    stop_found = False246    for string in all_stop_strings:247        idx = reply.find(string)248        if idx != -1:249            reply = reply[:idx]250            stop_found = True251            break252 253    if not stop_found:254        # If something like "\nYo" is generated just before "\nYou:"255        # is completed, trim it256        for string in all_stop_strings:257            for j in range(len(string) - 1, 0, -1):258                if reply[-j:] == string[:j]:259                    reply = reply[:-j]260                    break261            else:262                continue263 264            break265 266    return reply, stop_found267 268 269def get_reply_from_output_ids(output_ids, state, starting_from=0):270    reply = decode(output_ids[starting_from:], state['skip_special_tokens'])271 272    # Handle tokenizers that do not add the leading space for the first token273    if (hasattr(shared.tokenizer, 'convert_ids_to_tokens') and len(output_ids) > starting_from) and not reply.startswith(' '):274        first_token = shared.tokenizer.convert_ids_to_tokens(int(output_ids[starting_from]))275        if isinstance(first_token, (bytes,)):276            first_token = first_token.decode('utf8')277 278        if first_token.startswith('▁'):279            reply = ' ' + reply280 281    return reply282 283 284def generate_reply_HF(question, original_question, seed, state, stopping_strings=None, is_chat=False):285    generate_params = {}286    for k in ['max_new_tokens', 'do_sample', 'temperature', 'temperature_last', 'top_p', 'min_p', 'typical_p', 'repetition_penalty', 'presence_penalty', 'frequency_penalty', 'repetition_penalty_range', 'encoder_repetition_penalty', 'top_k', 'min_length', 'no_repeat_ngram_size', 'num_beams', 'penalty_alpha', 'length_penalty', 'early_stopping', 'tfs', 'top_a', 'mirostat_mode', 'mirostat_tau', 'mirostat_eta', 'guidance_scale']:287        generate_params[k] = state[k]288 289    if state['negative_prompt'] != '':290        generate_params['negative_prompt_ids'] = encode(state['negative_prompt'])291 292    for k in ['epsilon_cutoff', 'eta_cutoff']:293        if state[k] > 0:294            generate_params[k] = state[k] * 1e-4295 296    if state['ban_eos_token']:297        generate_params['suppress_tokens'] = [shared.tokenizer.eos_token_id]298 299    if state['custom_token_bans']:300        to_ban = [int(x) for x in state['custom_token_bans'].split(',')]301        if len(to_ban) > 0:302            if generate_params.get('suppress_tokens', None):303                generate_params['suppress_tokens'] += to_ban304            else:305                generate_params['suppress_tokens'] = to_ban306 307    generate_params.update({'use_cache': not shared.args.no_cache})308    if shared.args.deepspeed:309        generate_params.update({'synced_gpus': True})310 311    # Encode the input312    input_ids = encode(question, add_bos_token=state['add_bos_token'], truncation_length=get_max_prompt_length(state))313    output = input_ids[0]314    cuda = not any((shared.args.cpu, shared.args.deepspeed))315    if state['auto_max_new_tokens']:316        generate_params['max_new_tokens'] = state['truncation_length'] - input_ids.shape[-1]317 318    # Add the encoded tokens to generate_params319    question, input_ids, inputs_embeds = apply_extensions('tokenizer', state, question, input_ids, None)320    original_input_ids = input_ids321    generate_params.update({'inputs': input_ids})322    if inputs_embeds is not None:323        generate_params.update({'inputs_embeds': inputs_embeds})324 325    # Stopping criteria / eos token326    eos_token_ids = [shared.tokenizer.eos_token_id] if shared.tokenizer.eos_token_id is not None else []327    generate_params['eos_token_id'] = eos_token_ids328    generate_params['stopping_criteria'] = transformers.StoppingCriteriaList()329    generate_params['stopping_criteria'].append(_StopEverythingStoppingCriteria())330 331    # Logits processor332    processor = state.get('logits_processor', LogitsProcessorList([]))333    if not isinstance(processor, LogitsProcessorList):334        processor = LogitsProcessorList([processor])335 336    # Grammar337    if state['grammar_string'].strip() != '':338        grammar = initialize_grammar(state['grammar_string'])339        grammar_processor = GrammarConstrainedLogitsProcessor(grammar)340        processor.append(grammar_processor)341 342    apply_extensions('logits_processor', processor, input_ids)343    generate_params['logits_processor'] = processor344 345    t0 = time.time()346    try:347        if not is_chat and not shared.is_seq2seq:348            yield ''349 350        # Generate the entire reply at once.351        if not state['stream']:352            with torch.no_grad():353                output = shared.model.generate(**generate_params)[0]354                if cuda:355                    output = output.cuda()356 357            starting_from = 0 if shared.is_seq2seq else len(input_ids[0])358            yield get_reply_from_output_ids(output, state, starting_from=starting_from)359 360        # Stream the reply 1 token at a time.361        # This is based on the trick of using 'stopping_criteria' to create an iterator.362        else:363 364            def generate_with_callback(callback=None, *args, **kwargs):365                kwargs['stopping_criteria'].append(Stream(callback_func=callback))366                clear_torch_cache()367                with torch.no_grad():368                    shared.model.generate(**kwargs)369 370            def generate_with_streaming(**kwargs):371                return Iteratorize(generate_with_callback, [], kwargs, callback=None)372 373            with generate_with_streaming(**generate_params) as generator:374                cumulative_reply = ''375                starting_from = 0 if shared.is_seq2seq else len(input_ids[0])376                for output in generator:377                    if output[-1] in eos_token_ids:378                        break379 380                    new_content = get_reply_from_output_ids(output, state, starting_from=starting_from)381                    # check the partial unicode character382                    if chr(0xfffd) in new_content:383                        continue384 385                    cumulative_reply += new_content386                    starting_from = len(output)387                    yield cumulative_reply388 389    except Exception:390        traceback.print_exc()391    finally:392        t1 = time.time()393        original_tokens = len(original_input_ids[0])394        new_tokens = len(output) - (original_tokens if not shared.is_seq2seq else 0)395        print(f'Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens}, seed {seed})')396        return397 398 399def generate_reply_custom(question, original_question, seed, state, stopping_strings=None, is_chat=False):400    """401    For models that do not use the transformers library for sampling402    """403    seed = set_manual_seed(state['seed'])404 405    t0 = time.time()406    reply = ''407    try:408        if not is_chat:409            yield ''410 411        if not state['stream']:412            reply = shared.model.generate(question, state)413            yield reply414        else:415            for reply in shared.model.generate_with_streaming(question, state):416                yield reply417 418    except Exception:419        traceback.print_exc()420    finally:421        t1 = time.time()422        original_tokens = len(encode(original_question)[0])423        new_tokens = len(encode(original_question + reply)[0]) - original_tokens424        print(f'Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens}, seed {seed})')425        return426