UmarKhattab09/WebScraping_With_LLM_FineTuning

1import gradio as gr2from webscraping import WebScraping3import pandas as pd4from transformers import pipeline5from newscategory import NewsCategoriesImpact6from newscategory import NewsCategories7import os8from transformers import AutoModelForSequenceClassification, AutoTokenizer9 10 11 12def outputdf(count):13    savepath = f"outputs/{count}.csv"14    try:15        df = pd.read_csv(savepath)16        return df17    except FileNotFoundError:18        web = WebScraping(count)19        df = web.dataframe()20        df.to_csv(savepath,index=False)21        return df22 23def outputdfimpact(count):24    savepath = f"outputs/{count}_iMPACT.csv"25    try:26        df = pd.read_csv(savepath)27        return df28    except FileNotFoundError:29        web = WebScraping(count)30        df = web.slowdataframe()31        df.to_csv(savepath,index=False)32        return df33 34 35def trainingllm(text):36    token=os.environ.get("hf_token")37    model_id = "UmarKhattab09/llmfinetuning"38    model_name = "UmarKhattab09/llmfinetuning"39    model = AutoModelForSequenceClassification.from_pretrained(model_id, use_auth_token=token)40    tokenizer = AutoTokenizer.from_pretrained(model_id, use_auth_token=token)41    classifier = pipeline(42    "text-classification",43        model=model_name,44        tokenizer=model_name,45 46    )    47    result = classifier.predict(text)48    PredictedNews=result[0]['label']49    score=result[0]['score']50    return PredictedNews,score51 52 53def characternetwork(df):54    Network = NewsCategories(df)55    html = Network.newscategories() 56    return html57 58 59def characternetworkimpact(df):60    Network = NewsCategoriesImpact(df)61    html = Network.newscategoriesimpact() 62    return html   63 64 65def load_df_and_graph(count_range):66    df = outputdf(count_range)67    html = characternetwork(df)68    return df,html69 70def load_df_and_graph2(count_range):71    df = outputdfimpact(count_range)72    html = characternetworkimpact(df)73    return df,html74 75def main():76    with gr.Blocks() as demo:77        with gr.Row():78            with gr.Column():79                gr.HTML('<h1> WEB SCRAPING OF www.npr.org. LLM WITH FINE TUNING.')80 81        with gr.Row():82            with gr.Column():83                dataframe = gr.Dataframe(headers=["NewsType","News"],datatype=["str","str"])84            85            with gr.Column():  86                Counts = gr.Textbox(label="Range To Train LLM ON")  87                range = gr.Button("Load DataFrame")88                count = gr.Button("LoadDataFrame with NewsImpact (Slow)")89                count.click(outputdfimpact,inputs=[Counts],outputs=[dataframe])90                range.click(outputdf,inputs=[Counts],outputs=[dataframe])91 92        with gr.Row():93            with gr.Column():94                gr.HTML('<h1> Predicting a News Category.</h1>')95                gr.HTML('<p> This Model is trained on very small dataset, around 5 per category and 5 epochs. THere is a trainingllm folder where you can train it. The class is not wokring however you can train it on trainingllm.ipynb. Training takes a lot of time.</p>')96 97        with gr.Row():98            with gr.Column():99                News = gr.Textbox(label="Enter a News ")100                Getcategory = gr.Button("Get Category")       101            102        with gr.Column():  103                NewsCategory = gr.Textbox()104                Confidence = gr.Textbox()105        Getcategory.click(trainingllm,inputs=[News],outputs=[NewsCategory,Confidence])106 107 108 109 110        with gr.Row():111            with gr.Column():112                gr.HTML('<h1>Categories Implemented</h1>')113                gr.HTML('<p> Also, I am gonna add weights basically the LLM can learn from the urgency/importance of the News and add weights which will be displayed as a graph. Will Work Soon.</p>')114 115         116 117        with gr.Row():118            networkplot=gr.HTML()119        range.click(fn=load_df_and_graph, inputs=[Counts], outputs=[dataframe, networkplot])120        count.click(fn=load_df_and_graph2,inputs=[Counts],outputs=[dataframe,networkplot])121            122          123  124    demo.launch()125 126if __name__ =="__main__":127    main()128 129                130 131 132