vashu2425/Data_Analysis_And_Feature_Engineering_Platform
0
1"""2AI-Powered EDA & Feature Engineering Assistant3 4This application enables users to upload a CSV dataset, and utilizes LLMs to analyze5the dataset to provide EDA and feature engineering recommendations.6"""7 8import streamlit as st9import pandas as pd10import os11import base6412from io import BytesIO13from dotenv import load_dotenv14from typing import Dict, List, Any, Optional15import time16import logging17import plotly.express as px18import numpy as np19# Import LangChain memory components20from langchain.memory import ConversationBufferMemory21from langchain_core.messages import AIMessage, HumanMessage22 23# Import local modules24from eda_analysis import DatasetAnalyzer25from llm_inference import LLMInference26 27# Configure logging28logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')29logger = logging.getLogger(__name__)30 31# Load environment variables32load_dotenv()33 34# Set page configuration - must be the first Streamlit command35st.set_page_config(36 page_title="AI-Powered EDA & Feature Engineering Assistant",37 page_icon="📊",38 layout="wide",39 initial_sidebar_state="expanded"40)41 42# Initialize our classes43@st.cache_resource44def get_llm_inference():45 try:46 return LLMInference()47 except Exception as e:48 st.error(f"Error initializing LLM inference: {str(e)}")49 return None50 51llm_inference = get_llm_inference()52 53# Session state initialization54if "dataset_analyzer" not in st.session_state:55 st.session_state.dataset_analyzer = DatasetAnalyzer()56 57if "dataset_loaded" not in st.session_state:58 st.session_state.dataset_loaded = False59 60if "dataset_info" not in st.session_state:61 st.session_state.dataset_info = {}62 63if "visualizations" not in st.session_state:64 st.session_state.visualizations = {}65 66if "eda_insights" not in st.session_state:67 st.session_state.eda_insights = ""68 69if "feature_engineering_recommendations" not in st.session_state:70 st.session_state.feature_engineering_recommendations = ""71 72if "data_quality_insights" not in st.session_state:73 st.session_state.data_quality_insights = ""74 75if "active_tab" not in st.session_state:76 st.session_state.active_tab = "welcome"77 78# Add new functions to support the updated UI79def initialize_session_state():80 """Initialize session state variables needed for the application"""81 # Initialize session variables with appropriate defaults82 if "chat_history" not in st.session_state:83 st.session_state.chat_history = []84 85 # Initialize conversation memory for LangChain86 if "conversation_memory" not in st.session_state:87 st.session_state.conversation_memory = ConversationBufferMemory(88 memory_key="chat_history", 89 return_messages=True90 )91 92 # For dataframe and related variables, ensure proper initialization93 # df should not be in session_state until a proper DataFrame is loaded94 if "descriptive_stats" not in st.session_state:95 st.session_state.descriptive_stats = None96 97 if "selected_columns" not in st.session_state:98 st.session_state.selected_columns = []99 100 if "filtered_df" not in st.session_state:101 st.session_state.filtered_df = None102 103 if "ai_insights" not in st.session_state:104 st.session_state.ai_insights = None105 106 if "loading_insights" not in st.session_state:107 st.session_state.loading_insights = False108 109 if "selected_tab" not in st.session_state:110 st.session_state.selected_tab = 'tab-overview'111 112 if "dataset_name" not in st.session_state:113 st.session_state.dataset_name = ""114 115 # Logging initialization116 logger.info("Session state initialized")117 118def apply_custom_css():119 """Apply additional custom CSS that's not already in the main CSS block"""120 st.markdown("""121 <style>122 /* Base theme variables */123 :root {124 --primary: #4F46E5;125 --secondary: #06B6D4;126 --text-light: #F3F4F6;127 --text-muted: #9CA3AF;128 --bg-card: rgba(31, 41, 55, 0.7);129 --bg-dark: #111827;130 }131 132 /* Global styles */133 .stApp {134 background-color: var(--bg-dark);135 color: var(--text-light);136 }137 138 /* Improve sidebar styling */139 .sidebar-header {140 background: linear-gradient(90deg, var(--primary), var(--secondary));141 color: white;142 padding: 1rem;143 border-radius: 8px;144 margin-bottom: 1.5rem;145 font-size: 1.2rem;146 font-weight: 600;147 text-align: center;148 }149 150 151 /*152 div[data-testid="stBottomBlockContainer"] {153 background-color: #111827 !important;154 }155 156 div[data-testid="stChatInput"]{157 background-color: #111827 !important;158 } */159 160 /* Override the bottom chat input container */161 div.stChatFloatingInputContainer {162 background-color: #111827 !important;163 }164 165 /* Override the inner chat input box */166 div.stChatInputContainer {167 background-color: #111827 !important;168 169 }170 171 /* Optional: Override text area background */172 textarea {173 background-color: #111827 !important;174 color: white !important;175 }176 177 .sidebar-section {178 background: rgba(31, 41, 55, 0.4);179 border-radius: 8px;180 padding: 1rem;181 margin-bottom: 1.5rem;182 border: 1px solid rgba(99, 102, 241, 0.1);183 }184 185 .sidebar-footer {186 text-align: center;187 padding: 1rem;188 font-size: 0.8rem;189 color: var(--text-muted);190 margin-top: 3rem;191 }192 193 /* Feature Engineering Cards */194 .fe-cards-container {195 display: grid;196 grid-template-columns: repeat(2, 1fr);197 gap: 0.8rem;198 margin-top: 1rem;199 }200 201 .fe-card {202 background: rgba(31, 41, 55, 0.6);203 border-radius: 8px;204 padding: 0.8rem;205 text-align: center;206 cursor: pointer;207 transition: all 0.2s ease;208 border: 1px solid rgba(99, 102, 241, 0.1);209 position: relative;210 overflow: hidden;211 }212 213 .fe-card::before {214 content: '';215 position: absolute;216 top: 0;217 left: 0;218 right: 0;219 bottom: 0;220 background: linear-gradient(135deg, var(--primary), var(--secondary));221 opacity: 0;222 transition: opacity 0.3s ease;223 z-index: 0;224 }225 226 .fe-card:hover::before {227 opacity: 0.1;228 }229 230 .fe-card:hover {231 transform: translateY(-2px);232 box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);233 border-color: rgba(99, 102, 241, 0.3);234 }235 236 .fe-card-active {237 border-color: var(--primary);238 background: rgba(79, 70, 229, 0.1);239 }240 241 .fe-card-icon {242 font-size: 1.8rem;243 margin-bottom: 0.3rem;244 position: relative;245 z-index: 1;246 }247 248 .fe-card-title {249 font-size: 0.85rem;250 font-weight: 600;251 color: var(--text-light);252 position: relative;253 z-index: 1;254 }255 256 /* Tab content styling */257 .tab-title {258 font-size: 1.8rem;259 margin-bottom: 1.5rem;260 position: relative;261 display: inline-block;262 color: var(--text-light);263 }264 265 .tab-title:after {266 content: '';267 position: absolute;268 bottom: -10px;269 left: 0;270 width: 100%;271 height: 3px;272 background: linear-gradient(90deg, var(--primary) 0%, var(--secondary) 100%);273 border-radius: 3px;274 }275 276 /* Navigation Tabs */277 .custom-tabs {278 display: flex;279 background: rgba(31, 41, 55, 0.6);280 border-radius: 12px;281 padding: 0.5rem;282 margin-bottom: 2rem;283 justify-content: space-between;284 overflow: hidden;285 border: 1px solid rgba(99, 102, 241, 0.1);286 }287 288 .tab-item {289 flex: 1;290 text-align: center;291 padding: 0.8rem 0.5rem;292 border-radius: 8px;293 cursor: pointer;294 transition: all 0.3s ease;295 position: relative;296 z-index: 1;297 margin: 0 0.2rem;298 }299 300 .tab-item.active {301 background: rgba(79, 70, 229, 0.1);302 }303 304 .tab-item.active::before {305 content: '';306 position: absolute;307 bottom: 0;308 left: 10%;309 right: 10%;310 height: 3px;311 background: linear-gradient(90deg, var(--primary), var(--secondary));312 border-radius: 3px;313 }314 315 .tab-item:hover {316 background: rgba(79, 70, 229, 0.05);317 }318 319 .tab-icon {320 font-size: 1.5rem;321 margin-bottom: 0.3rem;322 }323 324 .tab-label {325 font-size: 0.85rem;326 font-weight: 500;327 color: var(--text-light);328 }329 330 .tab-content-spacer {331 height: 1rem;332 }333 334 /* Card styling */335 .stats-card, .info-card, .chart-card {336 background: rgba(31, 41, 55, 0.3);337 border-radius: 10px;338 padding: 1.2rem;339 margin-bottom: 1.5rem;340 border: 1px solid rgba(99, 102, 241, 0.1);341 transition: all 0.3s ease;342 }343 344 .stats-card:hover, .info-card:hover, .chart-card:hover {345 transform: translateY(-5px);346 box-shadow: 0 8px 15px rgba(0, 0, 0, 0.2);347 border-color: rgba(99, 102, 241, 0.3);348 }349 350 /* Dataset stats styling */351 .dataset-stats {352 display: flex;353 flex-wrap: wrap;354 gap: 0.8rem;355 justify-content: center;356 }357 358 .stat-item {359 text-align: center;360 padding: 0.8rem;361 background: rgba(31, 41, 55, 0.6);362 border-radius: 8px;363 min-width: 80px;364 border: 1px solid rgba(99, 102, 241, 0.2);365 }366 367 .stat-value {368 font-size: 1.5rem;369 font-weight: 700;370 color: var(--primary);371 }372 373 .stat-label {374 font-size: 0.8rem;375 color: var(--text-muted);376 margin-top: 0.3rem;377 }378 379 /* Chart styling */380 .chart-container {381 margin-top: 1.5rem;382 }383 384 .chart-card h3 {385 font-size: 1.2rem;386 margin-bottom: 1rem;387 color: var(--text-light);388 }389 390 .stat-summary {391 display: grid;392 grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));393 gap: 0.5rem;394 margin-top: 1rem;395 }396 397 .stat-pair {398 display: flex;399 justify-content: space-between;400 padding: 0.3rem 0.5rem;401 background: rgba(31, 41, 55, 0.4);402 border-radius: 4px;403 font-size: 0.9rem;404 }405 406 .stat-pair span {407 color: var(--text-muted);408 }409 410 .stat-pair strong {411 color: var(--text-light);412 }413 414 /* Filter container */415 .filter-container {416 background: rgba(31, 41, 55, 0.3);417 border-radius: 10px;418 padding: 1.2rem;419 margin-bottom: 1.5rem;420 border: 1px solid rgba(99, 102, 241, 0.1);421 }422 423 /* AI Insights styling */424 .insights-container {425 margin-top: 1rem;426 }427 428 .insights-category {429 margin-top: 0.5rem;430 }431 432 .insight-card {433 background: rgba(31, 41, 55, 0.3);434 border-radius: 10px;435 padding: 1.2rem;436 margin-bottom: 1rem;437 border: 1px solid rgba(99, 102, 241, 0.1);438 display: flex;439 align-items: flex-start;440 }441 442 .insight-content {443 display: flex;444 align-items: flex-start;445 gap: 1rem;446 }447 448 .insight-icon {449 font-size: 1.5rem;450 margin-top: 0.1rem;451 }452 453 .insight-text {454 flex: 1;455 line-height: 1.5;456 }457 458 .generate-insights-container {459 display: flex;460 justify-content: center;461 align-items: center;462 margin: 3rem 0;463 }464 465 .placeholder-card {466 background: rgba(31, 41, 55, 0.3);467 border-radius: 15px;468 padding: 2rem;469 text-align: center;470 border: 1px solid rgba(99, 102, 241, 0.1);471 max-width: 500px;472 margin: 0 auto;473 }474 475 .placeholder-icon {476 font-size: 3rem;477 margin-bottom: 1rem;478 animation: float 3s ease-in-out infinite;479 }480 481 .placeholder-text {482 color: var(--text-muted);483 line-height: 1.6;484 margin-bottom: 1.5rem;485 }486 487 .loading-container {488 display: flex;489 justify-content: center;490 margin: 2rem 0;491 }492 493 .loading-pulse {494 width: 80px;495 height: 80px;496 border-radius: 50%;497 background: linear-gradient(to right, var(--primary), var(--secondary));498 animation: pulse-animation 1.5s ease infinite;499 }500 501 @keyframes pulse-animation {502 0% {503 transform: scale(0.6);504 opacity: 0.5;505 }506 50% {507 transform: scale(1);508 opacity: 1;509 }510 100% {511 transform: scale(0.6);512 opacity: 0.5;513 }514 }515 516 @keyframes float {517 0% { transform: translateY(0px); }518 50% { transform: translateY(-10px); }519 100% { transform: translateY(0px); }520 }521 522 /* Button styling */523 button[kind="primary"] {524 background: linear-gradient(90deg, var(--primary), var(--secondary)) !important;525 color: white !important;526 border: none !important;527 border-radius: 8px !important;528 padding: 0.6rem 1.2rem !important;529 font-weight: 600 !important;530 transition: all 0.3s ease !important;531 box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1) !important;532 }533 534 button[kind="primary"]:hover {535 transform: translateY(-2px) !important;536 box-shadow: 0 6px 10px rgba(0, 0, 0, 0.15) !important;537 }538 539 button[kind="secondary"] {540 background: rgba(79, 70, 229, 0.1) !important;541 color: var(--text-light) !important;542 border: 1px solid rgba(79, 70, 229, 0.3) !important;543 border-radius: 8px !important;544 padding: 0.6rem 1.2rem !important;545 font-weight: 600 !important;546 transition: all 0.3s ease !important;547 }548 549 button[kind="secondary"]:hover {550 background: rgba(79, 70, 229, 0.2) !important;551 transform: translateY(-2px) !important;552 }553 554 /* Override Streamlit default button styles */555 .stButton>button {556 background: linear-gradient(90deg, var(--primary), var(--secondary)) !important;557 color: white !important;558 border: none !important;559 border-radius: 8px !important;560 padding: 0.6rem 1.2rem !important;561 font-weight: 600 !important;562 transition: all 0.3s ease !important;563 box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1) !important;564 width: 100%;565 }566 567 .stButton>button:hover {568 transform: translateY(-2px) !important;569 box-shadow: 0 6px 10px rgba(0, 0, 0, 0.15) !important;570 }571 572 /* Chat interface styling */573 .chat-interface-container {574 padding: 1rem 0;575 margin-bottom: 100px;576 position: relative;577 }578 579 .chat-messages {580 display: flex;581 flex-direction: column;582 gap: 15px;583 margin-bottom: 20px;584 }585 586 .chat-message-user, .chat-message-ai {587 padding: 12px 16px;588 border-radius: 12px;589 max-width: 80%;590 box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);591 }592 593 .chat-message-user {594 align-self: flex-end;595 background: linear-gradient(135deg, var(--primary) 0%, var(--secondary) 100%);596 color: white;597 border-bottom-right-radius: 0;598 margin-left: auto;599 }600 601 .chat-message-ai {602 align-self: flex-start;603 background: var(--bg-card);604 color: var(--text-light);605 border-bottom-left-radius: 0;606 margin-right: auto;607 }608 609 .chat-input-container {610 display: flex;611 align-items: center;612 gap: 10px;613 margin-top: 1.5rem;614 }615 616 .chat-suggestions {617 display: flex;618 flex-wrap: wrap;619 gap: 10px;620 margin: 1.5rem 0;621 }622 623 .chat-suggestion {624 background: rgba(99, 102, 241, 0.1);625 border: 1px solid rgba(99, 102, 241, 0.3);626 border-radius: 30px;627 padding: 8px 15px;628 font-size: 0.9rem;629 color: var(--text-light);630 cursor: pointer;631 transition: all 0.3s ease;632 display: inline-block;633 margin-bottom: 8px;634 }635 636 .chat-suggestion:hover {637 background: rgba(99, 102, 241, 0.2);638 transform: translateY(-2px);639 }640 641 /* Expander styling */642 .st-expander {643 background: rgba(31, 41, 55, 0.2) !important;644 border-radius: 8px !important;645 margin-bottom: 1rem !important;646 border: 1px solid rgba(99, 102, 241, 0.1) !important;647 }648 649 /* Streamlit widget styling */650 div[data-testid="stForm"] {651 background: rgba(31, 41, 55, 0.2) !important;652 border-radius: 10px !important;653 padding: 1rem !important;654 border: 1px solid rgba(99, 102, 241, 0.1) !important;655 }656 657 .stSelectbox>div>div {658 background: rgba(31, 41, 55, 0.4) !important;659 border: 1px solid rgba(99, 102, 241, 0.2) !important;660 border-radius: 8px !important;661 }662 663 .stTextInput>div>div>input {664 background: rgba(31, 41, 55, 0.4) !important;665 border: 1px solid rgba(99, 102, 241, 0.2) !important;666 border-radius: 8px !important;667 color: var(--text-light) !important;668 padding: 1rem !important;669 }670 671 /* Streamlit multiselect dropdown styling */672 div[data-baseweb="popover"] {673 background: var(--bg-dark) !important;674 border: 1px solid rgba(99, 102, 241, 0.2) !important;675 border-radius: 8px !important;676 }677 678 div[data-baseweb="menu"] {679 background: var(--bg-dark) !important;680 }681 682 div[role="listbox"] {683 background: var(--bg-dark) !important;684 }685 686 /* Fix for the upload button */687 .stFileUploader > div {688 display: flex;689 flex-direction: column;690 align-items: center;691 }692 693 .stFileUploader > div > button {694 background: linear-gradient(90deg, var(--primary), var(--secondary)) !important;695 color: white !important;696 border: none !important;697 width: 100%;698 margin-top: 1rem;699 }700 701 /* Fix for tab content spacing */702 .tab-content {703 margin-top: 2rem;704 padding: 1rem;705 background: rgba(31, 41, 55, 0.2);706 border-radius: 10px;707 border: 1px solid rgba(99, 102, 241, 0.1);708 }709 </style>710 """, unsafe_allow_html=True)711 712def generate_ai_insights():713 """Generate AI-powered insights about the dataset"""714 # Make sure we have a dataframe to analyze715 if 'df' not in st.session_state:716 logger.warning("Cannot generate AI insights: No dataframe in session state")717 return {}718 719 df = st.session_state.df720 insights = {}721 722 # Try to use the LLM for insights generation first723 try:724 if llm_inference is not None:725 # Create dataset_info dictionary for LLM726 num_rows, num_cols = df.shape727 num_numerical = len(df.select_dtypes(include=['number']).columns)728 num_categorical = len(df.select_dtypes(include=['object', 'category']).columns)729 num_missing = df.isnull().sum().sum()730 731 # Format missing values for better readability732 missing_cols = df.isnull().sum()[df.isnull().sum() > 0]733 missing_values = {}734 for col in missing_cols.index:735 count = missing_cols[col]736 percent = round(count / len(df) * 100, 2)737 missing_values[col] = (count, percent)738 739 # Get numerical columns and their correlations if applicable740 num_cols = df.select_dtypes(include=['number']).columns741 correlations = "No numerical columns to calculate correlations."742 if len(num_cols) > 1:743 # Calculate correlations744 corr_matrix = df[num_cols].corr()745 # Get top correlations (absolute values)746 corr_pairs = []747 for i in range(len(num_cols)):748 for j in range(i):749 val = corr_matrix.iloc[i, j]750 if abs(val) > 0.5: # Only show strong correlations751 corr_pairs.append((num_cols[i], num_cols[j], val))752 753 # Sort by absolute correlation and format754 if corr_pairs:755 corr_pairs.sort(key=lambda x: abs(x[2]), reverse=True)756 formatted_corrs = []757 for col1, col2, val in corr_pairs[:5]: # Top 5758 formatted_corrs.append(f"{col1} and {col2}: {val:.3f}")759 correlations = "\n".join(formatted_corrs)760 761 dataset_info = {762 "shape": f"{num_rows} rows, {num_cols} columns",763 "columns": df.columns.tolist(),764 "dtypes": {col: str(dtype) for col, dtype in df.dtypes.items()},765 "missing_values": missing_values,766 "basic_stats": df.describe().to_string(),767 "correlations": correlations,768 "sample_data": df.head(5).to_string()769 }770 771 # Generate EDA insights with better error handling772 logger.info("Requesting EDA insights from LLM")773 try:774 eda_insights = llm_inference.generate_eda_insights(dataset_info)775 776 if eda_insights and isinstance(eda_insights, str) and len(eda_insights) > 50:777 # Clean and format the response778 eda_insights = eda_insights.strip()779 insights["EDA Insights"] = [eda_insights]780 logger.info("Successfully generated EDA insights")781 else:782 logger.warning(f"EDA insights response was invalid: {type(eda_insights)}, length: {len(eda_insights) if isinstance(eda_insights, str) else 'N/A'}")783 except Exception as e:784 logger.error(f"Error generating EDA insights: {str(e)}")785 786 # Generate feature engineering recommendations787 if "EDA Insights" in insights: # Only proceed if EDA worked788 logger.info("Requesting feature engineering recommendations from LLM")789 try:790 fe_insights = llm_inference.generate_feature_engineering_recommendations(dataset_info)791 792 if fe_insights and isinstance(fe_insights, str) and len(fe_insights) > 50:793 fe_insights = fe_insights.strip()794 insights["Feature Engineering Recommendations"] = [fe_insights]795 logger.info("Successfully generated feature engineering recommendations")796 else:797 logger.warning(f"Feature engineering response was invalid: {type(fe_insights)}, length: {len(fe_insights) if isinstance(fe_insights, str) else 'N/A'}")798 except Exception as e:799 logger.error(f"Error generating feature engineering recommendations: {str(e)}")800 801 # Generate data quality insights802 logger.info("Requesting data quality insights from LLM")803 try:804 dq_insights = llm_inference.generate_data_quality_insights(dataset_info)805 806 if dq_insights and isinstance(dq_insights, str) and len(dq_insights) > 50:807 dq_insights = dq_insights.strip()808 insights["Data Quality Insights"] = [dq_insights]809 logger.info("Successfully generated data quality insights")810 else:811 logger.warning(f"Data quality response was invalid: {type(dq_insights)}, length: {len(dq_insights) if isinstance(dq_insights, str) else 'N/A'}")812 except Exception as e:813 logger.error(f"Error generating data quality insights: {str(e)}")814 815 # If we have at least one type of insights, consider it a success816 if insights:817 # Mark that the insights are loaded818 st.session_state['loading_insights'] = False819 logger.info("Successfully generated AI insights using LLM")820 return insights821 822 logger.warning("All LLM generated insights failed or were too short. Falling back to template insights.")823 else:824 logger.warning("LLM inference is not available. Falling back to template insights.")825 except Exception as e:826 logger.error(f"Error in generate_ai_insights(): {str(e)}. Falling back to template insights.")827 828 # If LLM fails or is not available, generate template-based insights829 logger.info("Falling back to template-based insights generation")830 831 # Add missing values insights832 missing_data = df.isnull().sum()833 missing_percent = (missing_data / len(df)) * 100834 missing_cols = missing_data[missing_data > 0]835 836 missing_insights = []837 if len(missing_cols) > 0:838 missing_insights.append(f"Found {len(missing_cols)} columns with missing values.")839 for col in missing_cols.index[:3]: # Show details for top 3840 missing_insights.append(f"Column '{col}' has {missing_data[col]} missing values ({missing_percent[col]:.2f}%).")841 842 if len(missing_cols) > 3:843 missing_insights.append(f"And {len(missing_cols) - 3} more columns have missing values.")844 845 # Add recommendation846 if any(missing_percent > 50):847 high_missing = missing_percent[missing_percent > 50].index.tolist()848 missing_insights.append(f"Consider dropping columns with >50% missing values: {', '.join(high_missing[:3])}.")849 else:850 missing_insights.append("Consider using imputation techniques for columns with missing values.")851 else:852 missing_insights.append("No missing values found in the dataset. Great job!")853 854 insights["Missing Values Analysis"] = missing_insights855 856 # Add distribution insights857 num_cols = df.select_dtypes(include=['number']).columns858 dist_insights = []859 860 if len(num_cols) > 0:861 for col in num_cols[:3]: # Analyze top 3 numeric columns862 # Check for skewness863 skew = df[col].skew()864 if abs(skew) > 1:865 direction = "right" if skew > 0 else "left"866 dist_insights.append(f"Column '{col}' is {direction}-skewed (skewness: {skew:.2f}). Consider log transformation.")867 868 # Check for outliers using IQR869 Q1 = df[col].quantile(0.25)870 Q3 = df[col].quantile(0.75)871 IQR = Q3 - Q1872 outliers = df[(df[col] < (Q1 - 1.5 * IQR)) | (df[col] > (Q3 + 1.5 * IQR))][col].count()873 874 if outliers > 0:875 pct = (outliers / len(df)) * 100876 dist_insights.append(f"Column '{col}' has {outliers} outliers ({pct:.2f}%). Consider outlier treatment.")877 878 if len(num_cols) > 3:879 dist_insights.append(f"Additional {len(num_cols) - 3} numerical columns not analyzed here.")880 else:881 dist_insights.append("No numerical columns found for distribution analysis.")882 883 insights["Distribution Insights"] = dist_insights884 885 # Add correlation insights886 corr_insights = []887 if len(num_cols) > 1:888 # Calculate correlation889 corr_matrix = df[num_cols].corr()890 high_corr = []891 892 # Find high correlations893 for i in range(len(corr_matrix.columns)):894 for j in range(i):895 if abs(corr_matrix.iloc[i, j]) > 0.7:896 high_corr.append((corr_matrix.columns[i], corr_matrix.columns[j], corr_matrix.iloc[i, j]))897 898 if high_corr:899 corr_insights.append(f"Found {len(high_corr)} pairs of highly correlated features.")900 for col1, col2, corr_val in high_corr[:3]: # Show top 3901 corr_direction = "positively" if corr_val > 0 else "negatively"902 corr_insights.append(f"'{col1}' and '{col2}' are strongly {corr_direction} correlated (r={corr_val:.2f}).")903 904 if len(high_corr) > 3:905 corr_insights.append(f"And {len(high_corr) - 3} more highly correlated pairs found.")906 907 corr_insights.append("Consider removing some highly correlated features to reduce dimensionality.")908 else:909 corr_insights.append("No strong correlations found between features.")910 else:911 corr_insights.append("Need at least 2 numerical columns to analyze correlations.")912 913 insights["Correlation Analysis"] = corr_insights914 915 # Add feature engineering recommendations916 fe_insights = []917 918 # Check for date columns919 date_cols = []920 for col in df.columns:921 if df[col].dtype == 'object':922 try:923 pd.to_datetime(df[col])924 date_cols.append(col)925 except:926 pass927 928 if date_cols:929 fe_insights.append(f"Found {len(date_cols)} potential date columns: {', '.join(date_cols[:3])}.")930 fe_insights.append("Consider extracting year, month, day, weekday from these columns.")931 932 # Check for categorical columns933 cat_cols = df.select_dtypes(include=['object']).columns934 if len(cat_cols) > 0:935 fe_insights.append(f"Found {len(cat_cols)} categorical columns.")936 fe_insights.append("Consider one-hot encoding or label encoding for categorical features.")937 938 # Check for high cardinality939 high_card_cols = []940 for col in cat_cols:941 if df[col].nunique() > 10:942 high_card_cols.append((col, df[col].nunique()))943 944 if high_card_cols:945 fe_insights.append(f"Some categorical columns have high cardinality:")946 for col, card in high_card_cols[:2]:947 fe_insights.append(f"Column '{col}' has {card} unique values. Consider grouping less common categories.")948 949 # Suggest polynomial features if few numeric features950 if 1 < len(num_cols) < 5:951 fe_insights.append("Consider creating polynomial features or interaction terms between numerical features.")952 953 insights["Feature Engineering Recommendations"] = fe_insights954 955 # Add a slight delay to simulate processing956 time.sleep(1)957 958 # Mark that the insights are loaded959 st.session_state['loading_insights'] = False960 logger.info("Template-based insights generation completed")961 962 return insights963 964def display_chat_interface():965 """Display a chat interface for interacting with the data"""966 st.markdown('<div class="tab-content">', unsafe_allow_html=True)967 st.markdown('<h2 class="tab-title">💬 Chat with Your Data</h2>', unsafe_allow_html=True)968 969 # Initialize chat history if not present970 if "chat_history" not in st.session_state:971 st.session_state.chat_history = []972 973 # Make sure we have data to chat about974 if 'df' not in st.session_state or st.session_state.df is None:975 st.error("No dataset loaded. Please upload a CSV file to chat with your data.")976 977 # Show a preview of chat capabilities978 st.markdown("""979 <div style="margin-top: 2rem;">980 <h3>What can I help you with?</h3>981 <p>Once you upload a dataset, you can ask questions like:</p>982 <ul>983 <li>What patterns do you see in my data?</li>984 <li>How many missing values are there?</li>985 <li>What feature engineering would you recommend?</li>986 <li>Show me the distribution of a specific column</li>987 <li>What are the correlations between features?</li>988 </ul>989 </div>990 """, unsafe_allow_html=True)991 992 st.markdown('</div>', unsafe_allow_html=True)993 return994 995 # Add a button to clear chat history996 col1, col2 = st.columns([4, 1])997 with col2:998 if st.button("Clear Chat", key="clear_chat"):999 st.session_state.chat_history = []1000 # Reset conversation memory1001 if "conversation_memory" in st.session_state:1002 st.session_state.conversation_memory = ConversationBufferMemory(1003 memory_key="chat_history", 1004 return_messages=True1005 )1006 logger.info("Chat history and memory cleared")1007 st.rerun()1008 1009 # Display chat history1010 for message in st.session_state.chat_history:1011 if message["role"] == "user":1012 st.chat_message("user").write(message["content"])1013 else:1014 st.chat_message("assistant").write(message["content"])1015 1016 # If no chat history, show some example questions1017 if not st.session_state.chat_history:1018 st.info("Ask me anything about your dataset! I can help you understand patterns, identify issues, and suggest improvements.")1019 1020 st.markdown("### Example questions you can ask:")1021 1022 # Create a grid of example questions using columns1023 col1, col2 = st.columns(2)1024 1025 with col1:1026 example_questions = [1027 "What are the key patterns in this dataset?",1028 "Which columns have missing values?",1029 "What kind of feature engineering would help?"1030 ]1031 1032 for i, question in enumerate(example_questions):1033 if st.button(question, key=f"example_q_{i}"):1034 process_chat_message(question)1035 st.rerun()1036 1037 with col2:1038 more_questions = [1039 "How are the numerical variables distributed?",1040 "What are the strongest correlations?",1041 "How can I prepare this data for modeling?"1042 ]1043 1044 for i, question in enumerate(more_questions):1045 if st.button(question, key=f"example_q_{i+3}"):1046 process_chat_message(question)1047 st.rerun()1048 1049 # Input area for new messages1050 user_input = st.chat_input("Ask a question about your data...", key="chat_input")1051 1052 if user_input:1053 # Add user message to chat history1054 process_chat_message(user_input)1055 st.rerun()1056 1057 st.markdown('</div>', unsafe_allow_html=True)1058 1059def display_descriptive_tab():1060 st.markdown('<div class="tab-content">', unsafe_allow_html=True)1061 st.markdown('<h2 class="tab-title">📊 Descriptive Statistics</h2>', unsafe_allow_html=True)1062 1063 # Make sure we access the data from session state1064 if 'df' not in st.session_state or 'descriptive_stats' not in st.session_state:1065 st.error("No dataset loaded. Please upload a CSV file.")1066 st.markdown('</div>', unsafe_allow_html=True)1067 return1068 1069 df = st.session_state.df1070 descriptive_stats = st.session_state.descriptive_stats1071 1072 # Display descriptive statistics in a more visually appealing way1073 col1, col2 = st.columns([3, 1])1074 1075 with col1:1076 # Style the dataframe1077 st.markdown('<div class="stats-card">', unsafe_allow_html=True)1078 st.subheader("Numerical Summary")1079 st.dataframe(descriptive_stats.style.background_gradient(cmap='Blues', axis=0)1080 .format(precision=2, na_rep="Missing"), use_container_width=True)1081 st.markdown('</div>', unsafe_allow_html=True)1082 1083 with col2:1084 st.markdown('<div class="info-card">', unsafe_allow_html=True)1085 st.subheader("Dataset Overview")1086 1087 # Display dataset information in a cleaner format1088 total_rows = df.shape[0]1089 total_cols = df.shape[1]1090 numeric_cols = len(df.select_dtypes(include=['number']).columns)1091 cat_cols = len(df.select_dtypes(include=['object', 'category']).columns)1092 date_cols = len(df.select_dtypes(include=['datetime']).columns)1093 1094 st.markdown(f"""1095 <div class="dataset-stats">1096 <div class="stat-item">1097 <div class="stat-value">{total_rows:,}</div>1098 <div class="stat-label">Rows</div>1099 </div>1100 <div class="stat-item">1101 <div class="stat-value">{total_cols}</div>1102 <div class="stat-label">Columns</div>1103 </div>1104 <div class="stat-item">1105 <div class="stat-value">{numeric_cols}</div>1106 <div class="stat-label">Numerical</div>1107 </div>1108 <div class="stat-item">1109 <div class="stat-value">{cat_cols}</div>1110 <div class="stat-label">Categorical</div>1111 </div>1112 <div class="stat-item">1113 <div class="stat-value">{date_cols}</div>1114 <div class="stat-label">Date/Time</div>1115 </div>1116 </div>1117 """, unsafe_allow_html=True)1118 st.markdown('</div>', unsafe_allow_html=True)1119 1120 # Add missing values information with visualization1121 st.markdown('<div class="stats-card">', unsafe_allow_html=True)1122 st.subheader("Missing Values")1123 col1, col2 = st.columns([2, 3])1124 1125 with col1:1126 # Calculate missing values1127 missing_data = df.isnull().sum()1128 missing_percent = (missing_data / len(df)) * 1001129 missing_data = pd.DataFrame({1130 'Missing Values': missing_data,1131 'Percentage (%)': missing_percent.round(2)1132 })1133 missing_data = missing_data[missing_data['Missing Values'] > 0].sort_values('Missing Values', ascending=False)1134 1135 if not missing_data.empty:1136 st.dataframe(missing_data.style.background_gradient(cmap='Reds', subset=['Percentage (%)'])1137 .format({'Percentage (%)': '{:.2f}%'}), use_container_width=True)1138 else:1139 st.success("No missing values found in the dataset! 🎉")1140 1141 with col2:1142 if not missing_data.empty:1143 # Create a horizontal bar chart for missing values1144 fig = px.bar(missing_data, 1145 x='Percentage (%)', 1146 y=missing_data.index, 1147 orientation='h',1148 color='Percentage (%)',1149 color_continuous_scale='Reds',1150 title='Missing Values by Column')1151 1152 fig.update_layout(1153 height=max(350, len(missing_data) * 30),1154 xaxis_title='Missing (%)',1155 yaxis_title='',1156 coloraxis_showscale=False,1157 margin=dict(l=0, r=10, t=30, b=0)1158 )1159 1160 st.plotly_chart(fig, use_container_width=True)1161 1162 st.markdown('</div>', unsafe_allow_html=True)1163 st.markdown('</div>', unsafe_allow_html=True)1164 1165def display_distribution_tab():1166 st.markdown('<div class="tab-content">', unsafe_allow_html=True)1167 st.markdown('<h2 class="tab-title">📈 Data Distribution</h2>', unsafe_allow_html=True)1168 1169 # Make sure we access the data from session state1170 if 'df' not in st.session_state:1171 st.error("No dataset loaded. Please upload a CSV file.")1172 st.markdown('</div>', unsafe_allow_html=True)1173 return1174 1175 df = st.session_state.df1176 1177 # Add filters for better UX1178 st.markdown('<div class="filter-container">', unsafe_allow_html=True)1179 col1, col2 = st.columns([1, 1])1180 1181 with col1:1182 chart_type = st.selectbox(1183 "Select Chart Type",1184 ["Histogram", "Box Plot", "Violin Plot", "Distribution Plot"],1185 key="chart_type_select"1186 )1187 1188 with col2:1189 if chart_type != "Distribution Plot":1190 column_type = "Numerical" if chart_type in ["Histogram", "Box Plot", "Violin Plot"] else "Categorical"1191 columns_to_show = list(df.select_dtypes(include=['number']).columns) if column_type == "Numerical" else list(df.select_dtypes(include=['object', 'category']).columns)1192 1193 selected_columns = st.multiselect(1194 f"Select {column_type} Columns to Visualize",1195 options=columns_to_show,1196 default=list(columns_to_show[:min(3, len(columns_to_show))]), # Convert to list ✅1197 key="column_select"1198 )1199 else:1200 num_cols = list(df.select_dtypes(include=['number']).columns) # Convert to list ✅