krish5932/Customer-FeedBack-System

1import re2import pandas as pd3import numpy as np4from sklearn.feature_extraction.text import TfidfVectorizer5from sklearn.linear_model import LogisticRegression6from sklearn.svm import LinearSVC7from sklearn.calibration import CalibratedClassifierCV8from sklearn.naive_bayes import MultinomialNB9from sklearn.cluster import KMeans10from sklearn.decomposition import PCA11from sklearn.model_selection import train_test_split12from sklearn.metrics import classification_report, confusion_matrix, silhouette_score13import nltk14from nltk.corpus import stopwords15from nltk.stem import WordNetLemmatizer16from nltk.tokenize import word_tokenize17 18# --- Robust NLTK Resource Downloader ---19def download_nltk_resources():20    resources = {21        'corpora/stopwords': 'stopwords',22        'tokenizers/punkt': 'punkt',23        'corpora/wordnet': 'wordnet',24        'corpora/omw-1.4': 'omw-1.4'25    }26    for path, name in resources.items():27        try:28            nltk.data.find(path)29        except LookupError:30            try:31                nltk.download(name, quiet=True)32            except Exception as e:33                print(f"Warning: Could not download NLTK resource '{name}': {e}")34 35download_nltk_resources()36 37# Fallback stopwords in case NLTK downloader fails38FALLBACK_STOPWORDS = set([39    "i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", "your", "yours", 40    "yourself", "yourselves", "he", "him", "his", "himself", "she", "her", "hers", 41    "herself", "it", "its", "itself", "they", "them", "their", "theirs", "themselves", 42    "what", "which", "who", "whom", "this", "that", "these", "those", "am", "is", "are", 43    "was", "were", "be", "been", "being", "have", "has", "had", "having", "do", "does", 44    "did", "doing", "a", "an", "the", "and", "but", "if", "or", "because", "as", "until", 45    "while", "of", "at", "by", "for", "with", "about", "against", "between", "into", 46    "through", "during", "before", "after", "above", "below", "to", "from", "up", "down", 47    "in", "out", "on", "off", "over", "under", "again", "further", "then", "once", "here", 48    "there", "when", "where", "why", "how", "all", "any", "both", "each", "few", "more", 49    "most", "other", "some", "such", "no", "nor", "not", "only", "own", "same", "so", 50    "than", "too", "very", "s", "t", "can", "will", "just", "don", "should", "now"51])52 53# Expansion mapping for common English contractions54CONTRACTION_MAP = {55    "don't": "do not",56    "can't": "cannot",57    "won't": "will not",58    "isn't": "is not",59    "aren't": "are not",60    "wasn't": "was not",61    "weren't": "were not",62    "haven't": "have not",63    "hasn't": "has not",64    "hadn't": "had not",65    "doesn't": "does not",66    "shouldn't": "should not",67    "wouldn't": "would not",68    "couldn't": "could not",69    "mustn't": "must not",70    "didn't": "did not",71    "it's": "it is",72    "he's": "he is",73    "she's": "she is",74    "that's": "that is",75    "what's": "what is",76    "where's": "where is",77    "there's": "there is",78    "i'm": "i am",79    "you're": "you are",80    "we're": "we are",81    "they're": "they are",82    "i've": "i have",83    "you've": "you have",84    "we've": "we have",85    "they've": "they have",86    "i'd": "i would",87    "you'd": "you would",88    "we'd": "we would",89    "they'd": "they would",90    "i'll": "i will",91    "you'll": "you will",92    "we'll": "we will",93    "they'll": "they will",94}95 96# Words indicating negation/contrast that should NOT be removed in sentiment analysis97NEGATION_WORDS = {98    'not', 'no', 'nor', 'neither', 'never', 'none', 'but', 'against', 99    'without', 'hardly', 'scarcely', 'barely', 'few', 'less', 'least'100}101 102try:103    NLTK_STOPWORDS = set(stopwords.words('english'))104except Exception:105    NLTK_STOPWORDS = FALLBACK_STOPWORDS106 107# Filter negation words out of the stopword set108STOP_WORDS = NLTK_STOPWORDS - NEGATION_WORDS109 110try:111    LEMMATIZER = WordNetLemmatizer()112except Exception:113    LEMMATIZER = None114 115# --- Preprocessing Function ---116def clean_text(text):117    if not isinstance(text, str):118        return ""119    120    # 1. Lowercase121    text = text.lower()122    123    # 2. Expand contractions124    for contraction, expansion in CONTRACTION_MAP.items():125        text = text.replace(contraction, expansion)126    127    # 3. Remove punctuation and numbers128    text = re.sub(r'[^a-zA-Z\s]', '', text)129    130    # 4. Tokenize131    try:132        tokens = word_tokenize(text)133    except Exception:134        tokens = text.split()135        136    # 5. Remove non-negation stopwords and lemmatize137    cleaned_tokens = []138    for token in tokens:139        if token not in STOP_WORDS and len(token) > 2:140            if LEMMATIZER:141                try:142                    token = LEMMATIZER.lemmatize(token)143                except Exception:144                    pass145            cleaned_tokens.append(token)146            147    return " ".join(cleaned_tokens)148 149# --- Sentiment Classifier Pipeline ---150class SentimentModel:151    def __init__(self, model_type='Logistic Regression', C=1.0, alpha=0.5):152        self.model_type = model_type153        self.vectorizer = TfidfVectorizer(ngram_range=(1, 2), max_features=1500, sublinear_tf=True, min_df=2)154        155        # Initialize selected model156        if model_type == 'Logistic Regression':157            self.classifier = LogisticRegression(C=C, class_weight='balanced', random_state=42, max_iter=1000)158        elif model_type == 'Linear SVM (SVC)':159            base_svc = LinearSVC(C=C, class_weight='balanced', random_state=42, dual=False, max_iter=2000)160            # Calibrate SVM to output reliable probabilities161            self.classifier = CalibratedClassifierCV(base_svc, cv=3)162        elif model_type == 'Multinomial Naive Bayes':163            self.classifier = MultinomialNB(alpha=alpha)164        else:165            self.classifier = LogisticRegression(C=1.0, class_weight='balanced', random_state=42)166            167        self.is_trained = False168        self.test_accuracy = 0.0169        self.conf_matrix = None170        self.class_report = None171        self.classes = []172 173    def train(self, texts, labels):174        cleaned_texts = [clean_text(t) for t in texts]175        labels = np.array(labels)176        177        # 1. Split into 80% train and 20% test to compute clean, unbiased metrics178        X_train_raw, X_test_raw, y_train, y_test = train_test_split(179            cleaned_texts, labels, test_size=0.2, random_state=42, stratify=labels180        )181        182        # 2. Fit vectorizer on training split183        X_train = self.vectorizer.fit_transform(X_train_raw)184        X_test = self.vectorizer.transform(X_test_raw)185        186        # 3. Fit classifier on training split187        self.classifier.fit(X_train, y_train)188        self.classes = list(self.classifier.classes_)189        190        # 4. Generate evaluation metrics191        test_preds = self.classifier.predict(X_test)192        self.test_accuracy = float(np.mean(test_preds == y_test))193        self.conf_matrix = confusion_matrix(y_test, test_preds, labels=self.classes)194        self.class_report = classification_report(y_test, test_preds, output_dict=True)195        196        # 5. Retrain the model on the full dataset so the prediction engine is as robust as possible197        X_full = self.vectorizer.fit_transform(cleaned_texts)198        self.classifier.fit(X_full, labels)199        self.is_trained = True200        201        return self.test_accuracy202 203    def predict(self, text):204        if not self.is_trained:205            raise ValueError("Model is not trained yet!")206        cleaned = clean_text(text)207        X = self.vectorizer.transform([cleaned])208        return self.classifier.predict(X)[0]209 210    def predict_probs(self, text):211        if not self.is_trained:212            raise ValueError("Model is not trained yet!")213        cleaned = clean_text(text)214        X = self.vectorizer.transform([cleaned])215        probs = self.classifier.predict_proba(X)[0]216        classes = self.classifier.classes_217        return dict(zip(classes, probs))218 219# --- Topic Clustering Pipeline ---220class TopicClustering:221    def __init__(self, n_clusters=4):222        self.n_clusters = n_clusters223        self.vectorizer = TfidfVectorizer(ngram_range=(1, 2), max_features=1000)224        self.kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)225        self.pca = PCA(n_components=2, random_state=42)226        self.is_fitted = False227 228    def fit_transform(self, texts):229        cleaned_texts = [clean_text(t) for t in texts]230        self.cleaned_texts = cleaned_texts231        232        self.tfidf_matrix = self.vectorizer.fit_transform(cleaned_texts)233        self.cluster_labels = self.kmeans.fit_predict(self.tfidf_matrix)234        235        self.pca_coords = self.pca.fit_transform(self.tfidf_matrix.toarray())236        self.is_fitted = True237        238        return self.cluster_labels, self.pca_coords239 240    def get_cluster_keywords(self, top_n=5):241        if not self.is_fitted:242            raise ValueError("Clustering is not fitted yet!")243            244        centroids = self.kmeans.cluster_centers_245        feature_names = np.array(self.vectorizer.get_feature_names_out())246        247        cluster_keywords = {}248        for i in range(self.n_clusters):249            sorted_indices = np.argsort(centroids[i])[::-1]250            top_features = feature_names[sorted_indices[:top_n]]251            cluster_keywords[i] = list(top_features)252            253        return cluster_keywords254 255    def assign_new_text(self, text):256        if not self.is_fitted:257            raise ValueError("Clustering is not fitted yet!")258            259        cleaned = clean_text(text)260        X = self.vectorizer.transform([cleaned])261        262        cluster_label = self.kmeans.predict(X)[0]263        coords = self.pca.transform(X.toarray())[0]264        265        return cluster_label, coords266 267    def compute_silhouette_scores(self, texts, max_k=8):268        """Compute silhouette score for KMeans clustering for K in range [2, max_k]"""269        cleaned_texts = [clean_text(t) for t in texts]270        temp_vectorizer = TfidfVectorizer(ngram_range=(1, 2), max_features=1000)271        tfidf_matrix = temp_vectorizer.fit_transform(cleaned_texts)272        273        scores = {}274        for k in range(2, max_k + 1):275            kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)276            labels = kmeans.fit_predict(tfidf_matrix)277            score = silhouette_score(tfidf_matrix, labels)278            scores[k] = float(score)279            280        return scores281