bjong/blessed_Text_summarization_and_lingual_model

1import pandas as pd
2import nltk
3from nltk.sentiment import SentimentIntensityAnalyzer
4from transformers import AutoTokenizer, AutoModelForSequenceClassification
5from scipy.special import softmax
6
7# Download necessary NLTK data
8nltk.download('vader_lexicon')
9nltk.download('punkt')
10
11# Load VADER SentimentIntensityAnalyzer
12sia = SentimentIntensityAnalyzer()
13
14# Load RoBERTa model and tokenizer
15MODEL = "cardiffnlp/twitter-roberta-base-sentiment"
16tokenizer = AutoTokenizer.from_pretrained(MODEL)
17model = AutoModelForSequenceClassification.from_pretrained(MODEL)
18
19# Function to get polarity scores from RoBERTa
20def polarity_scores_roberta(text):
21    encoded_text = tokenizer(text, return_tensors='pt')
22    output = model(**encoded_text)
23    scores = output[0][0].detach().numpy()
24    scores = softmax(scores)
25    scores_dict = {
26        'roberta_neg': scores[0],
27        'roberta_neu': scores[1],
28        'roberta_pos': scores[2]
29    }
30    return scores_dict
31
32# Function to perform sentiment analysis
33def perform_sentiment_analysis(text):
34    sentences = nltk.sent_tokenize(text)
35    results = []
36    for sentence in sentences:
37        vader_result = sia.polarity_scores(sentence)
38        vader_result_rename = {f"vader_{k}": v for k, v in vader_result.items()}
39        roberta_result = polarity_scores_roberta(sentence)
40        both = {**vader_result_rename, **roberta_result, 'Text': sentence}
41        results.append(both)
42    results_df = pd.DataFrame(results)
43    
44    # Initialize counter for sentences with negative sentiment
45    total_negative_sentences = 0
46    
47    # Identify sentences with negative sentiment
48    negative_sentences = []
49    negative_texts = results_df[(results_df['vader_neg'] > 0.5) | (results_df['roberta_neg'] > 0.5)]
50    if not negative_texts.empty:
51        for index, row in negative_texts.iterrows():
52            sentence = row['Text']
53            vader_score = sia.polarity_scores(sentence)['compound']
54            roberta_score = polarity_scores_roberta(sentence)['roberta_neg']
55            if vader_score < 0 or roberta_score > 0.5:
56                negative_sentences.append(sentence)
57                total_negative_sentences += 1
58
59    return results_df, negative_sentences, total_negative_sentences
60