CodeFella/summarization_publiq

1# pip install nltk2 3import nltk4nltk.download('stopwords')5import streamlit as st6from nltk.corpus import stopwords7from nltk.cluster.util import cosine_distance8import numpy as np9import networkx as nx10 11def read_article(file_name):12    file = file_name13    filedata = file.splitlines()14    article = filedata[0].split(". ")15    sentences = []16 17    for sentence in article:18        # print(sentence)19        sentences.append(sentence.replace("[^a-zA-Z]", " ").split(" "))20    sentences.pop() 21    22    return sentences23 24def sentence_similarity(sent1, sent2, stopwords=None):25    if stopwords is None:26        stopwords = []27 28    sent1 = [w.lower() for w in sent1]29    sent2 = [w.lower() for w in sent2]30 31    all_words = list(set(sent1 + sent2))32 33    vector1 = [0] * len(all_words)34    vector2 = [0] * len(all_words)35 36    # build the vector for the first sentence37    for w in sent1:38        if w in stopwords:39            continue40        vector1[all_words.index(w)] += 141 42    # build the vector for the second sentence43    for w in sent2:44        if w in stopwords:45            continue46        vector2[all_words.index(w)] += 147 48    return 1 - cosine_distance(vector1, vector2)49 50def build_similarity_matrix(sentences, stop_words):51    # Create an empty similarity matrix52    similarity_matrix = np.zeros((len(sentences), len(sentences)))53 54    for idx1 in range(len(sentences)):55        for idx2 in range(len(sentences)):56            if idx1 == idx2: #ignore if both are same sentences57                continue 58            similarity_matrix[idx1][idx2] = sentence_similarity(sentences[idx1], sentences[idx2], stop_words)59 60    return similarity_matrix61 62def generate_summary(file_name, top_n=5):63    stop_words = stopwords.words('english')64    summarize_text = []65 66    # Step 1 - Read text anc split it67    sentences =  read_article(file_name)68 69    # Step 2 - Generate Similary Martix across sentences70    sentence_similarity_martix = build_similarity_matrix(sentences, stop_words)71 72    # Step 3 - Rank sentences in similarity martix73    sentence_similarity_graph = nx.from_numpy_array(sentence_similarity_martix)74    scores = nx.pagerank(sentence_similarity_graph)75 76    # Step 4 - Sort the rank and pick top sentences77    ranked_sentence = sorted(((scores[i],s) for i,s in enumerate(sentences)), reverse=True)    78    # print("Indexes of top ranked_sentence order are ", ranked_sentence)    79 80    for i in range(top_n):81      summarize_text.append(" ".join(ranked_sentence[i][1]))82        # st.text(ranked_sentence[i][1])83 84    # Step 5 - Offcourse, output the summarize texr85    # print("Summarize Text: \n", ". ".join(summarize_text))86    st.write( "Summarize Text:")87    st.write(".  ".join(summarize_text))88 89 90st.title("Text Summarization: ")91inp = st.text_input("Enter text")92 93if(st.button('Generate')):94    generate_summary(inp)95 96 97 98 99 100 101# let's begin102 103 104 105# generate_summary( 'In an attempt to build an AI-ready workforce, Microsoft announced Intelligent Cloud Hub which has been launched to empower the next generation of students with AI-ready skills. Envisioned as a three-year collaborative program, Intelligent Cloud Hub will support around 100 institutions with AI infrastructure, course content and curriculum, developer support, development tools and give students access to cloud and AI services. As part of the program, the Redmond giant which wants to expand its reach and is planning to build a strong developer ecosystem in India with the program will set up the core AI infrastructure and IoT Hub for the selected campuses. The company will provide AI development tools and Azure AI services such as Microsoft Cognitive Services, Bot Services and Azure Machine Learning.According to Manish Prakash, Country General Manager-PS, Health and Education, Microsoft India, said, "With AI being the defining technology of our time, it is transforming lives and industry and the jobs of tomorrow will require a different skillset. This will require more collaborations and training and working with AI. That’s why it has become more critical than ever for educational institutions to integrate new cloud and AI technologies. The program is an attempt to ramp up the institutional set-up and build capabilities among the educators to educate the workforce of tomorrow." The program aims to build up the cognitive skills and in-depth understanding of developing intelligent cloud connected solutions for applications across industry. Earlier in April this year, the company announced Microsoft Professional Program In AI as a learning track open to the public. The program was developed to provide job ready skills to programmers who wanted to hone their skills in AI and data science with a series of online courses which featured hands-on labs and expert instructors as well. This program also included developer-focused AI school that provided a bunch of assets to help build AI skills.', 2)