vashu2425/Data_Analysis_And_Feature_Engineering_Platform
0
1"""2EDA Analysis Module3 4This module handles all dataset processing and analysis, providing structured information5about the dataset that can be used for visualization and LLM prompting.6"""7 8import pandas as pd9import numpy as np10from typing import Dict, List, Tuple, Any, Optional11import matplotlib.pyplot as plt12import seaborn as sns13from sklearn.preprocessing import StandardScaler14from io import BytesIO15import base6416 17class DatasetAnalyzer:18 """Class for analyzing datasets and extracting key information"""19 20 def __init__(self, df: pd.DataFrame = None):21 """Initialize with an optional dataframe"""22 self.df = df23 self.analysis_results = {}24 25 def load_dataframe(self, df: pd.DataFrame) -> None:26 """Load a dataframe for analysis"""27 self.df = df28 # Reset analysis results when loading a new dataframe29 self.analysis_results = {}30 31 def analyze_dataset(self) -> Dict[str, Any]:32 """33 Perform comprehensive analysis on the dataset34 35 Returns:36 Dict: Dictionary containing all analysis results37 """38 if self.df is None:39 raise ValueError("No dataframe loaded. Please load a dataframe first.")40 41 # Basic information42 self.analysis_results["shape"] = self.df.shape43 self.analysis_results["columns"] = list(self.df.columns)44 self.analysis_results["dtypes"] = {col: str(self.df[col].dtype) for col in self.df.columns}45 46 # Missing values47 self.analysis_results["missing_values"] = self._analyze_missing_values()48 49 # Basic statistics50 self.analysis_results["basic_stats"] = self._generate_basic_stats()51 52 # Correlations (for numerical columns)53 self.analysis_results["correlations"] = self._analyze_correlations()54 55 # Sample data56 self.analysis_results["sample_data"] = self.df.head().to_string()57 58 # Additional analyses59 self.analysis_results["categorical_columns"] = self._identify_categorical_columns()60 self.analysis_results["numerical_columns"] = self._identify_numerical_columns()61 self.analysis_results["unique_values"] = self._count_unique_values()62 63 return self.analysis_results64 65 def _analyze_missing_values(self) -> Dict[str, Tuple[int, float]]:66 """67 Analyze missing values in the dataset68 69 Returns:70 Dict: Column names as keys, tuples of (count, percentage) as values71 """72 missing_values = {}73 for col in self.df.columns:74 count = self.df[col].isna().sum()75 percentage = round((count / len(self.df)) * 100, 2)76 missing_values[col] = (count, percentage)77 78 return missing_values79 80 def _generate_basic_stats(self) -> str:81 """82 Generate basic statistics for the dataset83 84 Returns:85 str: String representation of basic statistics86 """87 # For numerical columns88 num_stats = self.df.describe().to_string()89 90 # For categorical columns91 cat_columns = self._identify_categorical_columns()92 cat_stats = ""93 if cat_columns:94 cat_stats = "\n\nCategorical columns statistics:\n"95 for col in cat_columns:96 value_counts = self.df[col].value_counts().head(10)97 cat_stats += f"\n{col} - Top values:\n{value_counts.to_string()}\n"98 99 return num_stats + cat_stats100 101 def _analyze_correlations(self) -> str:102 """103 Analyze correlations between numerical features104 105 Returns:106 str: String representation of top correlations107 """108 num_columns = self._identify_numerical_columns()109 110 if not num_columns or len(num_columns) < 2:111 return "Not enough numerical columns for correlation analysis."112 113 corr_matrix = self.df[num_columns].corr()114 115 # Get top correlations (excluding self-correlations)116 corr_pairs = []117 for i in range(len(num_columns)):118 for j in range(i+1, len(num_columns)):119 col1, col2 = num_columns[i], num_columns[j]120 corr_value = corr_matrix.loc[col1, col2]121 if not np.isnan(corr_value):122 corr_pairs.append((col1, col2, corr_value))123 124 # Sort by absolute correlation value125 corr_pairs.sort(key=lambda x: abs(x[2]), reverse=True)126 127 # Format results128 result = "Top correlations:\n"129 for col1, col2, corr in corr_pairs[:10]: # Top 10 correlations130 result += f"{col1} -- {col2}: {corr:.4f}\n"131 132 return result133 134 def _identify_categorical_columns(self) -> List[str]:135 """136 Identify categorical columns in the dataset137 138 Returns:139 List[str]: List of categorical column names140 """141 cat_columns = []142 for col in self.df.columns:143 # Consider object, category, and boolean types as categorical144 if self.df[col].dtype == 'object' or self.df[col].dtype == 'category' or self.df[col].dtype == 'bool':145 cat_columns.append(col)146 # Also consider int/float columns with few unique values as categorical147 elif (self.df[col].dtype == 'int64' or self.df[col].dtype == 'float64') and \148 self.df[col].nunique() < 10 and self.df[col].nunique() / len(self.df) < 0.05:149 cat_columns.append(col)150 151 return cat_columns152 153 def _identify_numerical_columns(self) -> List[str]:154 """155 Identify numerical columns in the dataset156 157 Returns:158 List[str]: List of numerical column names159 """160 num_columns = []161 cat_columns = self._identify_categorical_columns()162 163 for col in self.df.columns:164 if col not in cat_columns and pd.api.types.is_numeric_dtype(self.df[col].dtype):165 num_columns.append(col)166 167 return num_columns168 169 def _count_unique_values(self) -> Dict[str, int]:170 """171 Count unique values for each column172 173 Returns:174 Dict: Column names as keys, unique count as values175 """176 return {col: self.df[col].nunique() for col in self.df.columns}177 178 def generate_eda_visualizations(self) -> Dict[str, str]:179 """180 Generate common EDA visualizations181 182 Returns:183 Dict: Dictionary of visualization titles and their base64-encoded images184 """185 if self.df is None:186 raise ValueError("No dataframe loaded. Please load a dataframe first.")187 188 visualizations = {}189 190 # 1. Missing values heatmap191 visualizations["missing_values_heatmap"] = self._plot_missing_values()192 193 # 2. Distribution plots for numerical features194 num_columns = self._identify_numerical_columns()195 for i, col in enumerate(num_columns[:5]): # Limit to first 5 numerical columns196 visualizations[f"distribution_{col}"] = self._plot_distribution(col)197 198 # 3. Correlation heatmap199 visualizations["correlation_heatmap"] = self._plot_correlation_heatmap()200 201 # 4. Categorical feature distributions202 cat_columns = self._identify_categorical_columns()203 for i, col in enumerate(cat_columns[:5]): # Limit to first 5 categorical columns204 visualizations[f"categorical_{col}"] = self._plot_categorical_distribution(col)205 206 # 5. Scatter plot of 2 most correlated features207 if len(num_columns) >= 2:208 visualizations["scatter_plot"] = self._plot_scatter_correlation()209 210 return visualizations211 212 def _plot_missing_values(self) -> str:213 """Generate missing values heatmap"""214 plt.figure(figsize=(10, 6))215 sns.heatmap(self.df.isnull(), cmap='viridis', yticklabels=False, cbar=True, cbar_kws={'label': 'Missing Data'})216 plt.tight_layout()217 plt.title('Missing Values Heatmap')218 219 # Convert plot to base64 string220 return self._fig_to_base64(plt.gcf())221 222 def _plot_distribution(self, column: str) -> str:223 """Generate distribution plot for a numerical column"""224 plt.figure(figsize=(10, 6))225 226 # Histogram with KDE227 sns.histplot(data=self.df, x=column, kde=True)228 229 plt.title(f'Distribution of {column}')230 plt.xlabel(column)231 plt.ylabel('Frequency')232 plt.tight_layout()233 234 # Convert plot to base64 string235 return self._fig_to_base64(plt.gcf())236 237 def _plot_correlation_heatmap(self) -> str:238 """Generate correlation heatmap"""239 num_columns = self._identify_numerical_columns()240 241 if not num_columns or len(num_columns) < 2:242 return ""243 244 plt.figure(figsize=(12, 10))245 corr_matrix = self.df[num_columns].corr()246 mask = np.triu(np.ones_like(corr_matrix, dtype=bool))247 248 # Custom diverging palette249 cmap = sns.diverging_palette(230, 20, as_cmap=True)250 251 # Draw heatmap252 sns.heatmap(corr_matrix, mask=mask, cmap=cmap, vmax=1, vmin=-1, center=0,253 square=True, linewidths=.5, annot=True, fmt=".2f")254 255 plt.title('Correlation Heatmap')256 plt.tight_layout()257 258 # Convert plot to base64 string259 return self._fig_to_base64(plt.gcf())260 261 def _plot_categorical_distribution(self, column: str) -> str:262 """Generate bar plot for categorical column"""263 plt.figure(figsize=(10, 6))264 265 # Get value counts and limit to top 10 categories if there are too many266 value_counts = self.df[column].value_counts()267 if len(value_counts) > 10:268 # Keep top 9 categories and group the rest as 'Other'269 top_categories = value_counts.nlargest(9).index270 data = self.df.copy()271 data[column] = data[column].apply(lambda x: x if x in top_categories else 'Other')272 sns.countplot(y=column, data=data, order=data[column].value_counts().index)273 else:274 sns.countplot(y=column, data=self.df, order=value_counts.index)275 276 plt.title(f'Distribution of {column}')277 plt.xlabel('Count')278 plt.ylabel(column)279 plt.tight_layout()280 281 # Convert plot to base64 string282 return self._fig_to_base64(plt.gcf())283 284 def _plot_scatter_correlation(self) -> str:285 """Generate scatter plot of two most correlated features"""286 num_columns = self._identify_numerical_columns()287 288 if not num_columns or len(num_columns) < 2:289 return ""290 291 # Find the two most correlated features292 corr_matrix = self.df[num_columns].corr().abs()293 294 # Get upper triangle mask295 mask = np.triu(np.ones_like(corr_matrix, dtype=bool))296 corr_matrix = corr_matrix.mask(mask)297 298 # Find the max correlation299 max_corr = corr_matrix.max().max()300 max_corr_idx = corr_matrix.stack().idxmax()301 302 if pd.isna(max_corr):303 return ""304 305 # Get the column names306 col1, col2 = max_corr_idx307 308 # Create scatter plot309 plt.figure(figsize=(10, 6))310 311 # Add regression line312 sns.regplot(x=col1, y=col2, data=self.df, scatter_kws={'alpha': 0.5})313 314 plt.title(f'Scatter plot of {col1} vs {col2} (correlation: {corr_matrix.loc[col1, col2]:.2f})')315 plt.tight_layout()316 317 # Convert plot to base64 string318 return self._fig_to_base64(plt.gcf())319 320 def _fig_to_base64(self, fig) -> str:321 """Convert matplotlib figure to base64 string"""322 buf = BytesIO()323 fig.savefig(buf, format='png', bbox_inches='tight')324 buf.seek(0)325 img_str = base64.b64encode(buf.read()).decode('utf-8')326 plt.close(fig)327 return img_str328 329 def suggest_data_preprocessing(self) -> Dict[str, List[str]]:330 """331 Suggest preprocessing steps based on dataset analysis332 333 Returns:334 Dict: Dictionary of preprocessing suggestions for each column type335 """336 if not self.analysis_results:337 self.analyze_dataset()338 339 suggestions = {340 "numerical": [],341 "categorical": [],342 "missing_values": [],343 "outliers": [],344 "general": []345 }346 347 # Missing values suggestions348 missing_cols = [col for col, (count, _) in self.analysis_results["missing_values"].items() if count > 0]349 if missing_cols:350 suggestions["missing_values"].append(f"Found {len(missing_cols)} columns with missing values.")351 if len(missing_cols) > 5:352 suggestions["missing_values"].append(f"Columns with highest missing values: {', '.join(missing_cols[:5])}...")353 else:354 suggestions["missing_values"].append(f"Columns with missing values: {', '.join(missing_cols)}")355 356 suggestions["missing_values"].append("Consider these strategies for handling missing values:")357 suggestions["missing_values"].append("- Imputation (mean/median for numerical, mode for categorical)")358 suggestions["missing_values"].append("- Creating missing value indicators as new features")359 suggestions["missing_values"].append("- Removing rows or columns with too many missing values")360 361 # Numerical column suggestions362 num_cols = self.analysis_results["numerical_columns"]363 if num_cols:364 suggestions["numerical"].append(f"Found {len(num_cols)} numerical columns.")365 suggestions["numerical"].append("Consider these preprocessing steps:")366 suggestions["numerical"].append("- Scaling (StandardScaler or MinMaxScaler)")367 suggestions["numerical"].append("- Check for skewness and apply log or Box-Cox transformation if needed")368 suggestions["numerical"].append("- Create binned versions of continuous variables")369 370 # Check for potential outliers371 for col in num_cols:372 if col in self.df.columns: # Safety check373 q1 = self.df[col].quantile(0.25)374 q3 = self.df[col].quantile(0.75)375 iqr = q3 - q1376 outlier_count = ((self.df[col] < (q1 - 1.5 * iqr)) | (self.df[col] > (q3 + 1.5 * iqr))).sum()377 378 if outlier_count > 0:379 percentage = round((outlier_count / len(self.df)) * 100, 2)380 if percentage > 5: # If more than 5% are outliers381 suggestions["outliers"].append(f"Column '{col}' has {outlier_count} potential outliers ({percentage}%).")382 383 # Categorical column suggestions384 cat_cols = self.analysis_results["categorical_columns"]385 if cat_cols:386 suggestions["categorical"].append(f"Found {len(cat_cols)} categorical columns.")387 388 # Check cardinality (number of unique values)389 high_cardinality = []390 for col in cat_cols:391 unique_count = self.analysis_results["unique_values"].get(col, 0)392 if unique_count > 10:393 high_cardinality.append((col, unique_count))394 395 if high_cardinality:396 suggestions["categorical"].append("High cardinality columns (many unique values):")397 for col, count in sorted(high_cardinality, key=lambda x: x[1], reverse=True)[:5]:398 suggestions["categorical"].append(f"- {col}: {count} unique values")399 400 suggestions["categorical"].append("For high cardinality columns, consider:")401 suggestions["categorical"].append("- Grouping less frequent categories")402 suggestions["categorical"].append("- Target encoding or embedding techniques")403 404 suggestions["categorical"].append("General categorical encoding strategies:")405 suggestions["categorical"].append("- One-hot encoding for low cardinality columns")406 suggestions["categorical"].append("- Label encoding for ordinal variables")407 408 # General suggestions409 suggestions["general"].append("General preprocessing recommendations:")410 suggestions["general"].append("- Check for duplicate rows and remove if necessary")411 suggestions["general"].append("- Normalize text fields (lowercase, remove special characters)")412 suggestions["general"].append("- Create feature interactions for highly correlated features")413 414 return suggestions415 416 def generate_feature_engineering_ideas(self) -> List[str]:417 """418 Generate feature engineering ideas based on dataset analysis419 420 Returns:421 List[str]: List of feature engineering suggestions422 """423 if not self.analysis_results:424 self.analyze_dataset()425 426 ideas = []427 428 # Get column types429 num_cols = self.analysis_results["numerical_columns"]430 cat_cols = self.analysis_results["categorical_columns"]431 432 # Aggregation features433 if len(num_cols) >= 2:434 ideas.append("### Numerical Feature Transformations:")435 ideas.append("1. Create polynomial features for continuous variables")436 ideas.append("2. Apply mathematical transformations (log, sqrt, square) to handle skewed distributions")437 ideas.append("3. Create binned versions of continuous features to capture non-linear relationships")438 439 # Check for date/time related column names440 time_related_cols = [col for col in self.df.columns if any(x in col.lower() for x in ['date', 'time', 'year', 'month', 'day'])]441 if time_related_cols:442 ideas.append("\n### Time-Based Features:")443 ideas.append(f"Detected potential date/time columns: {', '.join(time_related_cols)}")444 ideas.append("1. Extract components like year, month, day, weekday, quarter")445 ideas.append("2. Create cyclical features using sine/cosine transformations for periodic time components")446 ideas.append("3. Calculate time since specific events or time differences between dates")447 448 # Categorical interactions449 if len(cat_cols) >= 2:450 ideas.append("\n### Categorical Feature Engineering:")451 ideas.append("1. Create interaction features by combining categorical variables")452 ideas.append("2. Use target encoding for high cardinality categorical features")453 ideas.append("3. Combine rare categories into an 'Other' category to reduce dimensionality")454 455 # Mixed interactions456 if num_cols and cat_cols:457 ideas.append("\n### Feature Interactions:")458 ideas.append("1. Create group-based statistics (mean, median, min, max) of numerical features grouped by categorical features")459 ideas.append("2. Calculate the difference from group means for numerical features")460 ideas.append("3. Create ratio or difference features between related numerical columns")461 462 # Dimensionality reduction463 if len(num_cols) > 10:464 ideas.append("\n### Dimensionality Reduction:")465 ideas.append("1. Apply PCA to reduce dimensionality and create principal components")466 ideas.append("2. Use feature selection methods (information gain, chi-square, mutual information)")467 ideas.append("3. Try UMAP or t-SNE for non-linear dimensionality reduction")468 469 # Text features470 text_cols = [col for col in self.df.columns if self.df[col].dtype == 'object' and471 self.df[col].apply(lambda x: isinstance(x, str) and len(x.split()) > 3).mean() > 0.5]472 if text_cols:473 ideas.append("\n### Text Feature Engineering:")474 ideas.append(f"Detected potential text columns: {', '.join(text_cols)}")475 ideas.append("1. Create bag-of-words or TF-IDF representations")476 ideas.append("2. Extract text length, word count, and other statistical features")477 ideas.append("3. Consider pretrained word embeddings or sentence transformers")478 479 return ideas480