oscars47/Thinking_Parrot_Reading_Club

1import command2# need to update pip3res = command.run(['/usr/local/bin/python -m pip install --upgrade pip'])4 5import gradio as gr6import numpy as np7import keras8 9# helper files----10# 1. clean data11# open the textfile; convert all text to lower case for ease of use12# takes in tetxfile path13def clean_data(text):14    # lowercase!!15    text = text.lower()16 17    # print('number of characters in textfile, including newline:', len(text))18    # remove all new line '\n' characters as these don't have any meaning19    # break up into list of characters; if the char is '\n' don't add it20    # then recompile into string21 22    # list of all the bad characters23    forbidden_char = ['\n', '\\', '^', '{', '|', '}', '~', '£', 24    '¥', '§', '©', '«', '¬', '®', '°', '»', '„', '•', '™', '■', '□', '►']25 26    temp = []27    i = 028    while i < (len(text)-3):29        char = text[i]30        char_next = text[i+1]31        char_next_next = text[i+2]32        char_next_next_next = text[i+3]33        34        # if the next character isn't a new line and char isn't '\n', add it35        if not(char in forbidden_char):36            # check if next character is '¬'37            if char_next == '¬':38                if (char_next_next == '\n') or (char_next_next_next=='\n'):39                    temp.append(char)40                    i+=341            42            elif not(char_next in forbidden_char):43                temp.append(char)44    45            # next char is newline46            elif char_next == '\n':47                if char != ' ':48                    temp.append(char)49                    temp.append(' ')50                else:51                    temp.append(char)52            else:53                temp.append(char)54        i+=155 56        # make sure we don't forget to append final character if not illegal!!57        # print(i == len(text)-3)58        if (i == len(text)-3) and not(char_next in forbidden_char):59            temp.append(char_next)60            if not(char_next_next) in forbidden_char:61                temp.append(char_next_next)62                if not(char_next_next_next) in forbidden_char:63                    temp.append(char_next_next_next)64 65    #reset nasrudin string66    text = ''67    for char in temp:68        text += char69 70    # print('number of characters in textfile:', len(text))71 72    # return cleaned data file73    return text74 75 76# get nasrudin text cleaned77with open('sufis_full.txt', 'r') as file:78    text = file.read()79    nasrudin = clean_data(text)80 81# 2. helper function to parse string into alphabet and get mapping dictionaries from char to int and int to char82def parse_text(text):83    # first find all the unique characters; sort them84    alphabet = sorted(list(set(text)))85 86    # create a dictionary for a 1-1 map from character to integer and vice versa so we can seamlessly convert87    char_to_int = dict((c, i) for i, c in enumerate (alphabet))88    int_to_char = dict((i, c) for i, c in enumerate (alphabet))89 90    return alphabet, char_to_int, int_to_char91 92alphabet, char_to_int, int_to_char = parse_text(nasrudin)93 94# set max_Char value; this is length of sentence which we train on -- do not change this95global maxChar96maxChar=4097 98# helper functions from Keras99 100# interpret probabilities101def sample(preds, temperature=1.0):102    # helper function to sample an index from a probability array103    # rescale data104    preds = np.asarray(preds).astype('float64')105    #preds = np.log(preds) / temperature106    exp_preds = np.exp(1/temperature)*preds107    preds = exp_preds / np.sum(exp_preds)108    # create multinomial distribution; run experiment 10 times, select most probable outcome109    probas = np.random.multinomial(10, preds, 1)110    return np.argmax(probas)111 112# helper function that we call to generate text113# takes in an input string, hdf5 trained model, and desired output length of text114model_types = ['Nasrudin', 'Shakespeare', 'Hemingway']115 116# function takes in input string, what text TP was trained on, and the text length as provided by huggingface input117def generate_text(input, text_len):118    # make sure at least 40 characters for training119    if len(input) < maxChar:120        raise gr.Error('Input must have >= %i characters. You have %i.' %(maxChar, len(input)))121 122    # make sure output num characters is integer123    if type(text_len) != int:124        raise gr.Error('Number of generated characters must be an integer!')125 126    # clean input data127    input = clean_data(input)128 129    # load desired model and set maxChar limit -- change these as we generate new models!130    131    model = keras.models.load_model('nasrudin_v1.0.0.hdf5')132 133    # grab last maxChar characters134    sentence = input[-maxChar:]135 136    # initalize generated string137    generated = ''138    #generated += input139        140    # randomly pick diversity parameter141    diversities = [0.2, 0.5, 1.0, 1.2]142    div_index = int(np.random.random()*(len(diversities)))143    diversity = diversities[div_index]144    # print('diversity:', diversity)145    # sys.stdout.write(input)146 147    # generate text_len characters worth of test148    for i in range(text_len):149        # prepare chosen sentence as part of new dataset150        x_pred = np.zeros((1, len(sentence), len(alphabet)))151        for t, char in enumerate(sentence):152            x_pred[0, t, char_to_int[char]] = 1.0153 154        # use the current model to predict what outputs are155        preds = model.predict(x_pred, verbose=0)[0]156        # call the function above to interpret the probabilities and add a degree of freedom157        next_index = sample(preds, diversity)158        #convert predicted number to character159        next_char = int_to_char[next_index]160 161        # append to existing string so as to build it up162        generated += next_char163        # append new character to previous sentence and delete the old one in front; now we train on predictions164        sentence = sentence[1:] + next_char165 166        # print the new character as we create it167        # sys.stdout.write(next_char)168        # sys.stdout.flush()169    print()170 171    return generated172 173# call hugging space interactive interface; use Blocks174 175with gr.Blocks() as think:176    # have intro blurb177    gr.Markdown("Hi! I'm Thinking Parrot, a text generating AI! 🦜" )178    179    # have accordian blurb180    with gr.Accordion("Click for more details!"):181        gr.Markdown("Simply type at least 40 characters into the box labeled 'Your Input Text' below and then select the number of output characters you want (note: try lower values for a faster response). Then click 'Think'! My response will appear in the box labeled 'My Response'.")182    183    # setup user interface184    input = [gr.Textbox(label = 'Your Input Text'), gr.Slider(minimum=40, maximum =500, label='Number of output characters', step=10)]185    output = gr.Textbox(label = 'My Response')186    think_btn = gr.Button('Think!')187    think_btn.click(fn= generate_text, inputs = input, outputs = output)188 189# enable queing if heavy traffic190think.queue(concurrency_count=3)191think.launch()