forestcalled/text-generation-webui

1import datetime2from pathlib import Path3 4import pandas as pd5import torch6from datasets import load_dataset7from tqdm import tqdm8 9from modules import shared10from modules.logging_colors import logger11from modules.models import clear_torch_cache, load_model, unload_model12from modules.models_settings import get_model_metadata, update_model_parameters13from modules.text_generation import encode14 15 16def load_past_evaluations():17    if Path('logs/evaluations.csv').exists():18        df = pd.read_csv(Path('logs/evaluations.csv'), dtype=str)19        df['Perplexity'] = pd.to_numeric(df['Perplexity'])20        return df21    else:22        return pd.DataFrame(columns=['Model', 'LoRAs', 'Dataset', 'Perplexity', 'stride', 'max_length', 'Date', 'Comment'])23 24 25past_evaluations = load_past_evaluations()26 27 28def save_past_evaluations(df):29    global past_evaluations30    past_evaluations = df31    filepath = Path('logs/evaluations.csv')32    filepath.parent.mkdir(parents=True, exist_ok=True)33    df.to_csv(filepath, index=False)34 35 36def calculate_perplexity(models, input_dataset, stride, _max_length):37    '''38    Based on:39    https://huggingface.co/docs/transformers/perplexity#calculating-ppl-with-fixedlength-models40    '''41 42    if not shared.args.no_use_fast:43        logger.warning("--no_use_fast is not being used. If tokenizing the input dataset takes a long time, consider loading the model with that option checked.")44 45    global past_evaluations46    cumulative_log = ''47    cumulative_log += "Loading the input dataset...\n\n"48    yield cumulative_log49 50    # Copied from https://github.com/qwopqwop200/GPTQ-for-LLaMa/blob/triton/utils/datautils.py51    if input_dataset == 'wikitext':52        data = load_dataset('wikitext', 'wikitext-2-raw-v1', split='test')53        text = "\n\n".join(data['text'])54    elif input_dataset == 'ptb':55        data = load_dataset('ptb_text_only', 'penn_treebank', split='validation')56        text = "\n\n".join(data['sentence'])57    elif input_dataset == 'ptb_new':58        data = load_dataset('ptb_text_only', 'penn_treebank', split='test')59        text = " ".join(data['sentence'])60    else:61        with open(Path(f'training/datasets/{input_dataset}.txt'), 'r', encoding='utf-8') as f:62            text = f.read()63 64    for model in models:65        if is_in_past_evaluations(model, input_dataset, stride, _max_length):66            cumulative_log += f"`{model}` has already been tested. Ignoring.\n\n"67            yield cumulative_log68            continue69 70        if model != 'current model':71            try:72                yield cumulative_log + f"Loading `{model}`...\n\n"73                model_settings = get_model_metadata(model)74                shared.settings.update({k: v for k, v in model_settings.items() if k in shared.settings})  # hijacking the interface defaults75                update_model_parameters(model_settings)  # hijacking the command-line arguments76                unload_model()77                shared.model, shared.tokenizer = load_model(model)78            except:79                cumulative_log += f"Failed to load `{model}`. Moving on.\n\n"80                yield cumulative_log81                continue82 83        cumulative_log += f"Processing `{shared.model_name}`...\n\n"84        yield cumulative_log + "Tokenizing the input dataset...\n\n"85        encodings = encode(text, add_special_tokens=False)86        seq_len = encodings.shape[1]87        if _max_length:88            max_length = _max_length89        elif hasattr(shared.model.config, 'max_position_embeddings'):90            max_length = shared.model.config.max_position_embeddings91        else:92            max_length = 204893 94        nlls = []95        prev_end_loc = 096        for begin_loc in tqdm(range(0, seq_len, stride)):97            yield cumulative_log + f"Evaluating... {100*begin_loc/seq_len:.2f}%"98            end_loc = min(begin_loc + max_length, seq_len)99            trg_len = end_loc - prev_end_loc  # may be different from stride on last loop100            input_ids = encodings[:, begin_loc:end_loc]101            target_ids = input_ids.clone()102            target_ids[:, :-trg_len] = -100103            clear_torch_cache()104            with torch.no_grad():105                outputs = shared.model(input_ids=input_ids, labels=target_ids)106 107                # loss is calculated using CrossEntropyLoss which averages over valid labels108                # N.B. the model only calculates loss over trg_len - 1 labels, because it internally shifts the labels109                # to the left by 1.110                neg_log_likelihood = outputs.loss111 112            nlls.append(neg_log_likelihood)113            prev_end_loc = end_loc114            if end_loc == seq_len:115                break116 117        ppl = torch.exp(torch.stack(nlls).mean())118        add_entry_to_past_evaluations(float(ppl), shared.model_name, input_dataset, stride, _max_length)119        save_past_evaluations(past_evaluations)120        cumulative_log += f"The perplexity for `{shared.model_name}` is: {float(ppl)}\n\n"121        yield cumulative_log122 123 124def add_entry_to_past_evaluations(perplexity, model, dataset, stride, max_length):125    global past_evaluations126    entry = {127        'Model': model,128        'LoRAs': ', '.join(shared.lora_names) or '-',129        'Dataset': dataset,130        'Perplexity': perplexity,131        'stride': str(stride),132        'max_length': str(max_length),133        'Date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),134        'Comment': ''135    }136    past_evaluations = pd.concat([past_evaluations, pd.DataFrame([entry])], ignore_index=True)137 138 139def is_in_past_evaluations(model, dataset, stride, max_length):140    entries = past_evaluations[(past_evaluations['Model'] == model) &141                               (past_evaluations['Dataset'] == dataset) &142                               (past_evaluations['max_length'] == str(max_length)) &143                               (past_evaluations['stride'] == str(stride))]144 145    if entries.shape[0] > 0:146        return True147    else:148        return False149 150 151def generate_markdown_table():152    sorted_df = past_evaluations.sort_values(by=['Dataset', 'stride', 'Perplexity', 'Date'])153    return sorted_df154