DanielNonStop/TextProcessing

1import os2import numpy as np3import gradio as gr4import stanza5from simpletransformers.classification import ClassificationModel, ClassificationArgs6import preprocessor as p7 8 9def clean_text(text):10    text = text.replace("#", "")11    return p.clean(text)12 13 14def softmax(x):15    16    return np.exp(x) / np.sum(np.exp(x), axis=0)17 18 19def number_to_sentiment(number):20 21    sentiments = {22        '0': 'Negative',23        '1': 'Neutral',24        '2': 'Positive'25    }26 27    return sentiments[str(number)]28 29 30def number_to_topic(number):31    topics = {32        '0': 'Abortions',33        '1': 'Taiwan',34        '2': 'Afghanistan',35        '3': 'Insurance',36        '4': 'Undefined'37    }38    return topics[str(number)]39 40 41def text_processing(text):42 43    results = nlp(text)44    text = clean_text(text)45    number_of_sentiments = 046    number_of_sentences = 047    for i, sentence in enumerate(results.sentences):48        number_of_sentiments += int(sentence.sentiment)49        number_of_sentences += 150 51    sentiment = int(round(number_of_sentiments/number_of_sentences))52    sentiment = number_to_sentiment(sentiment)53 54    predictions, raw_outputs = model.predict(text)55 56    print(predictions[0], raw_outputs[0])57 58    softmax_pred = softmax(raw_outputs[0])59    if softmax_pred.max() > 0.90:60        topic = number_to_topic(softmax_pred.argmax())61        print(softmax_pred.argmax())62    else:63        print(4)64        topic = number_to_topic(4)65 66    return f'Text topic: {topic}, text sentiment: {sentiment}'67 68 69if __name__ == "__main__":70 71    eval_model_args = ClassificationArgs(max_seq_length=128, use_multiprocessing_for_evaluation=False,72                                         eval_batch_size=1)73 74    model = ClassificationModel(75        "xlnet", "./", use_cuda=False, args=eval_model_args76    )77 78    stanza.download('en')79    nlp = stanza.Pipeline('en', processors='sentiment,tokenize,mwt', tokenize_no_ssplit=True)80 81    with gr.Blocks() as demo:82        with gr.Tab("Get text topic and sentiment"):83            text_input = gr.Textbox(label='Input text', placeholder='Put your text here')84            text_output = gr.Textbox(label='Output', placeholder="Topic and sentiment of the text")85            text_button = gr.Button("Run processing")86 87        text_button.click(text_processing, inputs=text_input, outputs=text_output)88 89    demo.launch()90