rayan2106/NLP-Preprocessing

1import pandas as pd2import gradio as gr3from nltk.stem import PorterStemmer4import spacy5from spacy.cli import download6try:7    nlp = spacy.load("en_core_web_sm")8except OSError:9    download("en_core_web_sm")10    nlp = spacy.load("en_core_web_sm")11 12nlp = spacy.load("en_core_web_sm")13stemmer = PorterStemmer()14 15def tokenize(text):16    doc = nlp(text)17    return [token.text for token in doc]18 19def stop_words(text):20    doc = nlp(text)21    return [token.text for token in doc if token.is_stop and not token.is_punct]22 23def stemming(text):24    doc = nlp(text)25    return [stemmer.stem(token.text) for token in doc]26 27def lemmatization(text):28    doc = nlp(text)29    return [token.lemma_ for token in doc]30 31def pos_tagging(text):32    doc = nlp(text)33    return [(token.text, token.pos_) for token in doc]34 35def ner(text):36    doc = nlp(text)37    return [(ent.text, ent.label_) for ent in doc.ents]38 39def run_all(text, mode):40    result = {41        "tokenize": tokenize(text),42        "stop_words": stop_words(text),43        "pos_tagging": pos_tagging(text),44        "ner": ner(text),45    }46    if (mode == "stemming"):47        result["stemmed"] = stemming(text)48    elif (mode == "lemmatization"):49        result["lemmatized"] = lemmatization(text)50    else:51        result["lemmatized"] = lemmatization(text)52 53    return result54 55def toggle_visibility(nlp_choice):56    if nlp_choice == "Full Pipeline":57        return gr.update(visible=True)58    else:59        return gr.update(visible=False)60 61def get_columns(file):62    if file is None:63        return {"error": "No file uploaded."}64    elif file.name.endswith(".csv"):65        df = pd.read_csv(file.name)66    else:67        df = pd.read_excel(file.name)68    return gr.update(choices=["All Columns"] + df.columns.tolist())69 70def process_file(file, column, nlp_choice, mode="lemmatization"):71    if not file:72        return {"error": "No file uploaded."}73    if not column:74        return {"error": "No column selected."}75    if not nlp_choice:76        return {"error": "No NLP function selected."}77 78    # Load file79    if file.name.endswith(".csv"):80        df = pd.read_csv(file)81    elif file.name.endswith((".xlsx", ".xls")):82        df = pd.read_excel(file)83    else:84        return {"error": "Unsupported file type."}85 86    # Determine columns87    columns = [column] if column != "All Columns" else df.columns.tolist()88 89    # NLP function map90    nlp_map = {91        "Tokenization": tokenize,92        "Stop Words": stop_words,93        "Stemming": stemming,94        "Lemmatization": lemmatization,95        "POS Tagging": pos_tagging,96        "NER": ner,97    }98 99    for col in columns:100        def process_text(text):101            text = str(text)102            try:103                if nlp_choice == "Full Pipeline":104                    return run_all(text, mode)105 106                func = nlp_map.get(nlp_choice)107                return func(text) if func else {}108            except Exception as e:109                return {"error": str(e)}110 111        df[f"{col}_processed"] = df[col].apply(process_text)112 113    return df.to_dict(orient="records")114 115text_tabs = gr.TabbedInterface(116    [117        gr.Interface(fn=lambda text, mode: run_all(text, mode), inputs=[gr.Textbox(label="Text"), gr.Radio(["stemming", "lemmatization"])], outputs="json", title="Full Pipeline"),118        gr.Interface(fn=tokenize, inputs="text", outputs="json", title="Tokenization"),119        gr.Interface(fn=stop_words, inputs="text", outputs="json", title="Stop Words"),120        gr.Interface(fn=stemming, inputs="text", outputs="json", title="Stemming"),121        gr.Interface(fn=lemmatization, inputs="text", outputs="json", title="Lemmatization"),122        gr.Interface(fn=pos_tagging, inputs="text", outputs="json", title="POS_Tagging"),123        gr.Interface(fn=ner, inputs="text", outputs="json", title="NER"),124    ],125    tab_names=["Full Pipeline", "Tokenization", "Stop Words", "Stemming", "Lemmatization", "POS_Tagging", "NER"]126)127 128with gr.Blocks() as dataset_tab:129    gr.Markdown("## Run NLP on Uploaded File")130    file_input = gr.File(label="Upload Your File", type="filepath")131    column_dropdown = gr.Radio(["All Columns"], value="All Columns", label="Select Column to Process")132    nlp_choice = gr.Radio(["Full Pipeline", "Tokenization", "Stop Words", "POS Tagging", "NER", "Stemming", "Lemmatization"], label="Select NLP Function")133    mode = gr.Radio(["stemming", "lemmatization"], visible=False)134    output = gr.JSON(label="Processed Data")135    run_btn = gr.Button("Run NLP")136 137    nlp_choice.change(fn=toggle_visibility, inputs=nlp_choice, outputs=mode)138    file_input.change(fn=get_columns, inputs=file_input, outputs=column_dropdown)139    run_btn.click(fn=process_file, inputs=[file_input, column_dropdown, nlp_choice, mode], outputs=output)140 141app = gr.TabbedInterface([text_tabs, dataset_tab], ["Text", "Dataset"])142 143app.launch()