pranit144/Real_Eatate_ChatBot
0
1from flask import Flask, render_template, request, jsonify
2import google.generativeai as genai
3import pandas as pd
4import numpy as np
5import joblib
6import os
7import json
8from datetime import datetime
9import re
10from typing import Dict, List, Any, Optional
11import logging
12from sklearn.model_selection import train_test_split
13from sklearn.ensemble import RandomForestRegressor
14from sklearn.preprocessing import StandardScaler
15from sklearn.pipeline import Pipeline
16from sklearn.metrics import mean_absolute_error, r2_score
17import sklearn # Import sklearn to get version
18
19app = Flask(__name__)
20
21# Configure logging
22logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
23logger = logging.getLogger(__name__)
24
25# Configure Gemini API
26GEMINI_API_KEY = os.getenv('GEMINI_API_KEY', 'AIzaSyAU8NFxxscrLcPcYqp5bnnyBPTVH0v1aZY') # IMPORTANT: Replace with a secure way to manage API keys
27genai.configure(api_key=GEMINI_API_KEY)
28# Using gemini-1.5-flash for potentially larger context window and better performance
29model = genai.GenerativeModel('gemini-2.0-flash')
30
31
32class EnhancedRealEstateSalesAssistant:
33 def __init__(self):
34 self.pipeline = None # Initialize pipeline to None
35 self.training_feature_names = [] # To store feature names from training
36 self.model_version = 'N/A'
37 self.load_model_and_data()
38 self.conversation_history = []
39 self.client_preferences = {}
40 self.last_search_results = None
41 self.context_memory = []
42
43 def _clean_gemini_text(self, text: str) -> str:
44 """Removes common Markdown formatting characters (like asterisks, hashes) from Gemini's output."""
45 if not isinstance(text, str):
46 return text # Return as is if not a string
47
48 # Remove bold/italic markers (**)
49 cleaned_text = re.sub(r'\*\*', '', text)
50 # Remove italic markers (*)
51 cleaned_text = re.sub(r'\*', '', cleaned_text)
52 # Remove header markers (#) at the start of a line, and any following whitespace
53 cleaned_text = re.sub(r'^\s*#+\s*', '', cleaned_text, flags=re.MULTILINE)
54 # Remove typical list bullet points like '- ' or '+ ' or '* ' at the start of a line
55 # (Note: '*' removal above might catch list items, this is for redundancy/clarity)
56 cleaned_text = re.sub(r'^\s*[-+]\s+', '', cleaned_text, flags=re.MULTILINE)
57 # Replace multiple newlines with a single one (or two, depending on desired paragraph spacing)
58 cleaned_text = re.sub(r'\n\s*\n', '\n\n', cleaned_text)
59 # Remove excessive leading/trailing whitespace from each line
60 cleaned_text = '\n'.join([line.strip() for line in cleaned_text.splitlines()])
61 # Final strip for the whole block
62 cleaned_text = cleaned_text.strip()
63 return cleaned_text
64
65 def load_model_and_data(self):
66 """Load the trained model and dataset with version compatibility"""
67 try:
68 # Load the dataset first
69 self.df = pd.read_csv('Pune_House_Data.csv')
70
71 # Clean and process data
72 self.df = self.clean_dataset()
73
74 # Extract unique values for filtering
75 self.locations = sorted(self.df['site_location'].unique().tolist())
76 self.area_types = sorted(self.df['area_type'].unique().tolist())
77 self.availability_types = sorted(self.df['availability'].unique().tolist())
78 self.bhk_options = sorted(self.df['bhk'].unique().tolist())
79
80 # Try to load model with version compatibility
81 model_loaded = self.load_model_with_version_check()
82
83 if not model_loaded:
84 logger.warning("Model loading failed, prediction will use fallback method.")
85 self.pipeline = None # Ensure pipeline is None if loading failed
86
87 logger.info("✅ Model and data loading process completed.")
88 logger.info(f"Dataset shape: {self.df.shape}")
89 logger.info(f"Available locations: {len(self.locations)}")
90
91 except Exception as e:
92 logger.error(f"❌ Critical error loading model/data: {e}")
93 # Create fallback data
94 self.df = self.create_fallback_data()
95 self.locations = ['Baner', 'Hadapsar', 'Kothrud', 'Viman Nagar', 'Wakad', 'Hinjewadi']
96 self.area_types = ['Built-up Area', 'Super built-up Area', 'Plot Area', 'Carpet Area']
97 self.availability_types = ['Ready To Move', 'Not Ready', 'Under Construction']
98 self.bhk_options = [1, 2, 3, 4, 5, 6]
99 self.pipeline = None # Ensure pipeline is None if data loading failed
100
101 def load_model_with_version_check(self):
102 """Load model with version compatibility check, or retrain if necessary."""
103 try:
104 # First try to load the new version-compatible model
105 try:
106 model_data = joblib.load('house_price_prediction_v2.pkl')
107 self.pipeline = model_data['pipeline']
108 self.training_feature_names = model_data.get('feature_names', [])
109 self.model_version = model_data.get('version', '2.0')
110 logger.info(f"✅ Loaded model version {self.model_version} with {len(self.training_feature_names)} features.")
111 return True
112 except FileNotFoundError:
113 logger.info("New model version (house_price_prediction_v2.pkl) not found, attempting to retrain.")
114 return self.retrain_model_with_current_version()
115 except Exception as v2_load_error:
116 logger.error(f"Failed to load new model version (v2): {v2_load_error}. Attempting retraining.")
117 return self.retrain_model_with_current_version()
118
119 except Exception as e:
120 logger.error(f"Error in model loading process: {e}. Retraining will be attempted as a last resort.")
121 return self.retrain_model_with_current_version() # Final attempt to retrain if other loading fails
122
123 def retrain_model_with_current_version(self):
124 """Retrain the model with current scikit-learn version"""
125 try:
126 logger.info(f"🔄 Retraining model with scikit-learn version {sklearn.__version__}")
127
128 # Prepare features (this method also sets self.training_feature_names)
129 X, y = self.create_training_data_with_proper_features()
130
131 if X is None or y is None:
132 logger.error("Failed to create training data, cannot retrain model.")
133 self.pipeline = None
134 return False
135
136 # Split data
137 X_train, X_test, y_train, y_test = train_test_split(
138 X, y, test_size=0.2, random_state=42
139 )
140
141 # Create pipeline
142 pipeline = Pipeline([
143 ('scaler', StandardScaler()),
144 ('model', RandomForestRegressor(
145 n_estimators=100,
146 random_state=42,
147 n_jobs=-1 # Use all available cores
148 ))
149 ])
150
151 # Train model
152 logger.info("Training model...")
153 pipeline.fit(X_train, y_train)
154
155 # Test model
156 y_pred = pipeline.predict(X_test)
157 mae = mean_absolute_error(y_test, y_pred)
158 r2 = r2_score(y_test, y_pred)
159
160 logger.info(f"✅ Model trained successfully:")
161 logger.info(f" MAE: {mae:.2f} lakhs")
162 logger.info(f" R² Score: {r2:.3f}")
163
164 # Save the new model
165 self.pipeline = pipeline
166 # self.training_feature_names is already set by create_training_data_with_proper_features
167
168 model_data = {
169 'pipeline': self.pipeline,
170 'feature_names': self.training_feature_names,
171 'version': '2.0',
172 'sklearn_version': sklearn.__version__,
173 'training_info': {
174 'mae': mae,
175 'r2_score': r2,
176 'training_samples': len(X_train),
177 'test_samples': len(X_test),
178 'feature_count': len(self.training_feature_names)
179 },
180 'created_at': datetime.now().isoformat()
181 }
182
183 # Save with version info
184 joblib.dump(model_data, 'house_price_prediction_v2.pkl')
185 logger.info("✅ Model saved as house_price_prediction_v2.pkl")
186
187 return True
188
189 except Exception as e:
190 logger.error(f"Error retraining model: {e}")
191 self.pipeline = None
192 return False
193
194 def create_training_data_with_proper_features(self):
195 """Create training data with proper feature engineering for consistent predictions"""
196 try:
197 # Prepare features for training
198 df_features = self.df.copy()
199
200 # Select relevant columns for training
201 feature_columns = ['site_location', 'area_type', 'availability', 'bhk', 'bath', 'balcony', 'total_sqft']
202
203 # Check if all required columns exist
204 missing_columns = [col for col in feature_columns + ['price'] if col not in df_features.columns]
205 if missing_columns:
206 logger.error(f"Missing essential columns for training: {missing_columns}")
207 return None, None
208
209 df_features = df_features[feature_columns + ['price']]
210
211 # Clean data (ensure numeric columns are correct, fillna where appropriate)
212 df_features['bhk'] = pd.to_numeric(df_features['bhk'], errors='coerce')
213 df_features['bath'] = pd.to_numeric(df_features['bath'], errors='coerce').fillna(df_features['bath'].median())
214 df_features['balcony'] = pd.to_numeric(df_features['balcony'], errors='coerce').fillna(df_features['balcony'].median())
215 df_features['total_sqft'] = pd.to_numeric(df_features['total_sqft'], errors='coerce')
216 df_features['price'] = pd.to_numeric(df_features['price'], errors='coerce')
217
218 # Remove rows with critical missing values for training
219 df_features = df_features.dropna(subset=['price', 'total_sqft', 'bhk', 'site_location'])
220
221 # Remove outliers before one-hot encoding for more robust training
222 df_features = self.remove_outliers(df_features)
223
224 # One-hot encode categorical variables
225 categorical_cols = ['site_location', 'area_type', 'availability']
226 df_encoded = pd.get_dummies(df_features, columns=categorical_cols, prefix=categorical_cols, dummy_na=False)
227
228 # Separate features and target
229 X = df_encoded.drop('price', axis=1)
230 y = df_encoded['price']
231
232 # Store feature names for later use in prediction
233 self.training_feature_names = X.columns.tolist()
234
235 logger.info(f"Training data prepared: {X.shape[0]} samples, {X.shape[1]} features")
236
237 return X, y
238
239 except Exception as e:
240 logger.error(f"Error creating training data: {e}")
241 return None, None
242
243 def remove_outliers(self, df):
244 """Remove outliers from the dataset based on price and area."""
245 try:
246 # Calculate price_per_sqft for outlier removal, then drop it if not needed as a feature
247 df['price_per_sqft'] = (df['price'] * 100000) / df['total_sqft']
248
249 # Remove price outliers (e.g., beyond 3 standard deviations from mean)
250 price_mean = df['price'].mean()
251 price_std = df['price'].std()
252 df = df[abs(df['price'] - price_mean) <= 3 * price_std]
253
254 # Remove area outliers (e.g., beyond 3 standard deviations from mean)
255 area_mean = df['total_sqft'].mean()
256 area_std = df['total_sqft'].std()
257 df = df[abs(df['total_sqft'] - area_mean) <= 3 * area_std]
258
259 # Remove properties with unrealistic price per sqft (e.g., <1000 or >20000 INR/sqft for Pune)
260 df = df[(df['price_per_sqft'] >= 1000) & (df['price_per_sqft'] <= 20000)]
261
262 # Drop temporary price_per_sqft column
263 df = df.drop(columns=['price_per_sqft'])
264
265 logger.info(f"Dataset after outlier removal: {len(df)} rows")
266 return df
267
268 except Exception as e:
269 logger.error(f"Error removing outliers: {e}")
270 return df # Return original df if error occurs
271
272 def predict_single_price(self, location, bhk, bath, balcony, sqft, area_type, availability):
273 """Predict price for a single property using ML model, with fallback to heuristic."""
274 try:
275 # Check if ML pipeline is loaded
276 if not hasattr(self, 'pipeline') or self.pipeline is None or not self.training_feature_names:
277 logger.warning("ML pipeline or training features not available. Using fallback prediction.")
278 return self.predict_single_price_fallback(location, bhk, bath, balcony, sqft, area_type, availability)
279
280 try:
281 # Create a feature dictionary initialized with 0s for all expected features
282 feature_dict = {feature: 0 for feature in self.training_feature_names}
283
284 # Set numerical features
285 numerical_mapping = {
286 'bhk': bhk,
287 'bath': bath,
288 'balcony': balcony,
289 'total_sqft': sqft
290 }
291 for feature, value in numerical_mapping.items():
292 if feature in feature_dict:
293 feature_dict[feature] = value
294 else:
295 logger.warning(f"Numerical feature '{feature}' not found in training features. Skipping.")
296
297 # Set categorical features (one-hot encoded)
298 # Location
299 location_key = f"site_location_{location}"
300 if location_key in feature_dict:
301 feature_dict[location_key] = 1
302 else:
303 found_loc = False
304 for feature in self.training_feature_names:
305 if feature.startswith('site_location_') and location.lower() in feature.lower():
306 feature_dict[feature] = 1
307 found_loc = True
308 break
309 if not found_loc:
310 logger.warning(f"Location '{location}' not found as exact or fuzzy match in training features. This location will not contribute to the ML prediction specific to its one-hot encoding.")
311
312 # Area Type
313 area_type_key = f"area_type_{area_type}"
314 if area_type_key in feature_dict:
315 feature_dict[area_type_key] = 1
316 else:
317 found_area_type = False
318 for feature in self.training_feature_names:
319 if feature.startswith('area_type_') and area_type.lower() in feature.lower():
320 feature_dict[feature] = 1
321 found_area_type = True
322 break
323 if not found_area_type:
324 logger.warning(f"Area type '{area_type}' not found in training features. This area type will not contribute to the ML prediction specific to its one-hot encoding.")
325
326 # Availability
327 availability_key = f"availability_{availability}"
328 if availability_key in feature_dict:
329 feature_dict[availability_key] = 1
330 else:
331 found_availability = False
332 for feature in self.training_feature_names:
333 if feature.startswith('availability_') and availability.lower() in feature.lower():
334 feature_dict[feature] = 1
335 found_availability = True
336 break
337 if not found_availability:
338 logger.warning(f"Availability '{availability}' not found in training features. This availability will not contribute to the ML prediction specific to its one-hot encoding.")
339
340 # Create DataFrame from the feature dictionary and ensure column order
341 input_df = pd.DataFrame([feature_dict])[self.training_feature_names]
342
343 predicted_price = self.pipeline.predict(input_df)[0]
344 return round(predicted_price, 2)
345
346 except Exception as ml_prediction_error:
347 logger.warning(f"ML prediction failed for input {location, bhk, sqft}: {ml_prediction_error}. Falling back to heuristic model.")
348 return self.predict_single_price_fallback(location, bhk, bath, balcony, sqft, area_type, availability)
349
350 except Exception as general_error:
351 logger.error(f"General error in predict_single_price wrapper: {general_error}. Falling back to heuristic.")
352 return self.predict_single_price_fallback(location, bhk, bath, balcony, sqft, area_type, availability)
353
354
355 def predict_single_price_fallback(self, location, bhk, bath, balcony, sqft, area_type, availability):
356 """Enhanced fallback prediction method when ML model is unavailable or fails."""
357 try:
358 # Base price per sqft for different areas in Pune (extensive map for better fallback)
359 location_price_map = {
360 'Koregaon Park': 12000, 'Kalyani Nagar': 11000, 'Boat Club Road': 10500,
361 'Aundh': 9500, 'Baner': 8500, 'Balewadi': 8000, 'Kothrud': 8500,
362 'Viman Nagar': 7500, 'Wakad': 7000, 'Hinjewadi': 7500, 'Pune': 7000,
363 'Hadapsar': 6500, 'Kharadi': 7200, 'Yerawada': 7000, 'Lohegaon': 6000,
364 'Wagholi': 5500, 'Mundhwa': 7000, 'Undri': 6500, 'Kondhwa': 6000,
365 'Katraj': 5800, 'Dhankawadi': 6000, 'Warje': 6500, 'Karve Nagar': 7500,
366 'Bavdhan': 8000, 'Pashan': 7500, 'Sus': 7200, 'Pimpri': 6000,
367 'Chinchwad': 6200, 'Akurdi': 5800, 'Nigdi': 6500, 'Bhosari': 5500,
368 'Chakan': 4800, 'Talegaon': 4500, 'Alandi': 4200, 'Dehu': 4000,
369 'Lonavala': 5000, 'Kamshet': 4500, 'Sinhagad Road': 6800,
370 'Balaji Nagar': 5800, 'Parvati': 5500, 'Gultekdi': 5200, 'Wanowrie': 6200,
371 'Shivajinagar': 9000, 'Deccan': 8500, 'Camp': 7500, 'Koregaon': 8000,
372 'Sadashiv Peth': 8200, 'Shukrawar Peth': 7800, 'Kasba Peth': 7500,
373 'Narayan Peth': 7200, 'Rasta Peth': 7000, 'Ganj Peth': 6800,
374 'Magarpatta': 7500, 'Fursungi': 5000, 'Handewadi': 4800,
375 'Mahatma Phule Peth': 6200, 'Bhavani Peth': 6000, 'Bibwewadi': 7000
376 }
377
378 # Get base price per sqft, try fuzzy matching if exact match not found
379 base_price_per_sqft = location_price_map.get(location, 6500) # Default if no match
380 if location not in location_price_map:
381 for loc_key, price in location_price_map.items():
382 if location.lower() in loc_key.lower() or loc_key.lower() in location.lower():
383 base_price_per_sqft = price
384 break
385
386 # Area type multiplier
387 area_type_multiplier = {
388 'Super built-up Area': 1.0,
389 'Built-up Area': 0.92, # Typically slightly less than super built-up for same listed price
390 'Plot Area': 0.85, # Plot area might translate to lower built-up
391 'Carpet Area': 1.08 # Carpet area is typically more expensive per sqft
392 }.get(area_type, 1.0)
393
394 # Availability multiplier (Under Construction/Not Ready typically cheaper)
395 availability_multiplier = {
396 'Ready To Move': 1.0,
397 'Under Construction': 0.88,
398 'Not Ready': 0.82
399 }.get(availability, 0.95)
400
401 # BHK-based pricing adjustments (larger BHKs might have slightly lower price/sqft due to bulk)
402 bhk_multiplier = {
403 1: 1.15, # Premium for 1BHK
404 2: 1.0, # Base
405 3: 0.95, # Slight economy
406 4: 0.92,
407 5: 0.90,
408 6: 0.88
409 }.get(bhk, 1.0)
410
411 # Bathroom and Balcony multipliers (simple linear adjustment)
412 bath_multiplier = 1.0 + (bath - 2) * 0.03 # Each extra bath adds 3% to price
413 balcony_multiplier = 1.0 + (balcony - 1) * 0.02 # Each extra balcony adds 2% to price
414
415 # Ensure multipliers don't go negative or too low
416 bath_multiplier = max(0.9, bath_multiplier)
417 balcony_multiplier = max(0.95, balcony_multiplier)
418
419 # Calculate final estimated price
420 estimated_price_raw = (sqft * base_price_per_sqft *
421 area_type_multiplier * availability_multiplier *
422 bhk_multiplier * bath_multiplier * balcony_multiplier)
423
424 estimated_price_lakhs = estimated_price_raw / 100000
425
426 return round(estimated_price_lakhs, 2)
427
428 except Exception as e:
429 logger.error(f"Error in fallback prediction: {e}")
430 return None # Indicate failure
431
432 def clean_dataset(self):
433 """Clean and process the dataset"""
434 df = self.df.copy()
435
436 # Rename 'size' to 'bhk' based on previous logic (if 'size' exists)
437 if 'size' in df.columns and 'bhk' not in df.columns:
438 df['bhk'] = df['size'].str.extract(r'(\d+)').astype(float)
439 elif 'bhk' not in df.columns:
440 df['bhk'] = np.nan # Ensure bhk column exists even if 'size' is missing
441
442 # Clean total_sqft column
443 df['total_sqft'] = pd.to_numeric(df['total_sqft'], errors='coerce')
444
445 # Clean price column
446 df['price'] = pd.to_numeric(df['price'], errors='coerce')
447
448 # Fill missing values for numerical columns (using median for robustness)
449 df['bath'] = pd.to_numeric(df['bath'], errors='coerce').fillna(df['bath'].median())
450 df['balcony'] = pd.to_numeric(df['balcony'], errors='coerce').fillna(df['balcony'].median())
451
452 # Fill missing values for categorical columns
453 df['society'] = df['society'].fillna('Not Specified')
454 df['area_type'] = df['area_type'].fillna('Super built-up Area') # Common default
455 df['availability'] = df['availability'].fillna('Ready To Move') # Common default
456 df['site_location'] = df['site_location'].fillna('Pune') # General default
457
458 # Remove rows with critical missing values after initial cleaning
459 df = df.dropna(subset=['price', 'total_sqft', 'bhk', 'site_location'])
460
461 # Ensure 'bhk' is an integer (or float if decimals are possible)
462 df['bhk'] = df['bhk'].astype(int)
463 df['bath'] = df['bath'].astype(int)
464 df['balcony'] = df['balcony'].astype(int)
465
466 logger.info(f"Dataset cleaned. Original rows: {self.df.shape[0]}, Cleaned rows: {df.shape[0]}")
467 return df
468
469 def create_fallback_data(self):
470 """Create fallback data if CSV loading fails"""
471 logger.warning("Creating fallback dataset due to CSV loading failure.")
472 fallback_data = {
473 'area_type': ['Super built-up Area', 'Built-up Area', 'Carpet Area'] * 10,
474 'availability': ['Ready To Move', 'Under Construction', 'Ready To Move'] * 10,
475 'size': ['2 BHK', '3 BHK', '4 BHK'] * 10, # Keep 'size' for cleaning process if needed
476 'society': ['Sample Society'] * 30,
477 'total_sqft': [1000, 1200, 1500] * 10,
478 'bath': [2, 3, 4] * 10,
479 'balcony': [1, 2, 3] * 10,
480 'price': [50, 75, 100] * 10,
481 'site_location': ['Baner', 'Hadapsar', 'Kothrud'] * 10,
482 }
483 df_fallback = pd.DataFrame(fallback_data)
484 # Ensure 'bhk' is generated if 'size' is used, and other derived columns
485 if 'size' in df_fallback.columns:
486 df_fallback['bhk'] = df_fallback['size'].str.extract(r'(\d+)').astype(float)
487 else:
488 df_fallback['bhk'] = [2, 3, 4] * 10 # Direct bhk if 'size' is not used
489 # Add a placeholder for price_per_sqft as it's used in analysis/outlier removal
490 df_fallback['price_per_sqft'] = (df_fallback['price'] * 100000) / df_fallback['total_sqft']
491 return df_fallback
492
493
494 def filter_properties(self, filters: Dict[str, Any]) -> pd.DataFrame:
495 """Filter properties based on user criteria"""
496 filtered_df = self.df.copy()
497
498 # Apply filters
499 if filters.get('location'):
500 # Use a more robust search for location, allowing partial or case-insensitive matches
501 search_location = filters['location'].lower()
502 filtered_df = filtered_df[filtered_df['site_location'].str.lower().str.contains(search_location, na=False)]
503
504 if filters.get('bhk') is not None: # Check for None explicitly as bhk can be 0
505 filtered_df = filtered_df[filtered_df['bhk'] == filters['bhk']]
506
507 if filters.get('min_price'):
508 filtered_df = filtered_df[filtered_df['price'] >= filters['min_price']]
509
510 if filters.get('max_price'):
511 filtered_df = filtered_df[filtered_df['price'] <= filters['max_price']]
512
513 if filters.get('min_area'):
514 filtered_df = filtered_df[filtered_df['total_sqft'] >= filters['min_area']]
515
516 if filters.get('max_area'):
517 filtered_df = filtered_df[filtered_df['total_sqft'] <= filters['max_area']]
518
519 if filters.get('area_type'):
520 search_area_type = filters['area_type'].lower()
521 filtered_df = filtered_df[filtered_df['area_type'].str.lower().str.contains(search_area_type, na=False)]
522
523 if filters.get('availability'):
524 search_availability = filters['availability'].lower()
525 filtered_df = filtered_df[filtered_df['availability'].str.lower().str.contains(search_availability, na=False)]
526
527 return filtered_df.head(20) # Limit results for display
528
529 def get_price_range_analysis(self, filtered_df: pd.DataFrame) -> Dict[str, Any]:
530 """Analyze price range and provide insights"""
531 if filtered_df.empty:
532 return {
533 "total_properties": 0,
534 "price_range": {"min": 0.0, "max": 0.0, "avg": 0.0, "median": 0.0},
535 "area_range": {"min": 0.0, "max": 0.0, "avg": 0.0},
536 "price_per_sqft": {"min": 0.0, "max": 0.0, "avg": 0.0},
537 "location_distribution": {},
538 "area_type_distribution": {},
539 "availability_distribution": {}
540 }
541
542 analysis = {
543 "total_properties": len(filtered_df),
544 "price_range": {
545 "min": float(filtered_df['price'].min()),
546 "max": float(filtered_df['price'].max()),
547 "avg": float(filtered_df['price'].mean()),
548 "median": float(filtered_df['price'].median())
549 },
550 "area_range": {
551 "min": float(filtered_df['total_sqft'].min()),
552 "max": float(filtered_df['total_sqft'].max()),
553 "avg": float(filtered_df['total_sqft'].mean())
554 },
555 # Recalculate price_per_sqft for analysis as it might be dropped after outlier removal
556 "price_per_sqft": {
557 "min": float(((filtered_df['price'] * 100000) / filtered_df['total_sqft']).min()),
558 "max": float(((filtered_df['price'] * 100000) / filtered_df['total_sqft']).max()),
559 "avg": float(((filtered_df['price'] * 100000) / filtered_df['total_sqft']).mean())
560 },
561 "location_distribution": filtered_df['site_location'].value_counts().to_dict(),
562 "area_type_distribution": filtered_df['area_type'].value_counts().to_dict(),
563 "availability_distribution": filtered_df['availability'].value_counts().to_dict()
564 }
565
566 return analysis
567
568 def predict_prices_for_filtered_results(self, filtered_df: pd.DataFrame) -> pd.DataFrame:
569 """Predict prices for all filtered properties and add to DataFrame."""
570 if filtered_df.empty:
571 return filtered_df
572
573 predictions = []
574 for index, row in filtered_df.iterrows():
575 predicted_price = self.predict_single_price(
576 location=row['site_location'],
577 bhk=row['bhk'],
578 bath=row['bath'],
579 balcony=row['balcony'],
580 sqft=row['total_sqft'],
581 area_type=row['area_type'],
582 availability=row['availability']
583 )
584 # Use original price if prediction fails (returns None)
585 predictions.append(predicted_price if predicted_price is not None else row['price'])
586
587 filtered_df['predicted_price'] = predictions
588 filtered_df['price_difference'] = filtered_df['predicted_price'] - filtered_df['price']
589 # Avoid division by zero if original price is 0
590 filtered_df['price_difference_pct'] = (
591 filtered_df['price_difference'] / filtered_df['price'].replace(0, np.nan) * 100
592 ).fillna(0) # Fill NaN with 0 if original price was 0 or prediction failed
593
594 return filtered_df
595
596 def generate_deep_insights(self, filtered_df: pd.DataFrame, analysis: Dict[str, Any]) -> str:
597 """Generate deep insights using Gemini AI"""
598 try:
599 # Prepare context for Gemini
600 properties_sample_str = filtered_df[['site_location', 'bhk', 'total_sqft', 'price', 'predicted_price', 'price_difference_pct']].head().to_string(index=False)
601 if properties_sample_str:
602 properties_sample_str = f"Top 5 Properties Sample:\n{properties_sample_str}\n"
603 else:
604 properties_sample_str = "No specific property samples available to list."
605
606
607 context = f"""
608 You are a highly experienced real estate market expert specializing in Pune properties.
609 Your task is to analyze the provided property data and offer comprehensive, actionable insights for a potential property buyer.
610
611 Here's a summary of the current search results:
612 - Total Properties Found: {analysis['total_properties']}
613 - Price Range: ₹{analysis['price_range']['min']:.2f}L to ₹{analysis['price_range']['max']:.2f}L
614 - Average Price: ₹{analysis['price_range']['avg']:.2f}L
615 - Median Price: ₹{analysis['price_range']['median']:.2f}L
616 - Average Area: {analysis['area_range']['avg']:.0f} sq ft
617 - Average Price per sq ft: ₹{analysis['price_per_sqft']['avg']:.0f}
618
619 Location Distribution: {json.dumps(analysis['location_distribution'], indent=2)}
620 Area Type Distribution: {json.dumps(analysis['area_type_distribution'], indent=2)}
621 Availability Distribution: {json.dumps(analysis['availability_distribution'], indent=2)}
622
623 {properties_sample_str}
624
625 Please provide detailed market insights including:
626 1. **Current Market Trends**: What are the prevailing trends in Pune's real estate market based on this data? (e.g., buyer's/seller's market, growth areas, demand patterns).
627 2. **Price Competitiveness**: How do the prices of these properties compare to the overall Pune market? Are they underpriced, overpriced, or fair value? Highlight any anomalies based on the 'price_difference_pct' if available.
628 3. **Best Value Propositions**: Identify what types of properties (BHK, area, location) offer the best value for money right now based on average price per sqft and price ranges.
629 4. **Location-Specific Insights**: For the top 2-3 locations, provide specific observations on amenities, connectivity, future development potential, and why they might be good/bad for investment or living.
630 5. **Investment Recommendations**: Based on the data, what would be your general investment advice for someone looking into Pune real estate?
631 6. **Future Market Outlook**: Briefly discuss the short-to-medium term outlook (next 1-2 years) for the Pune real estate market.
632
633 Ensure your response is conversational, professional, and directly actionable for a property buyer.
634 """
635
636 response = model.generate_content(context)
637 return self._clean_gemini_text(response.text) # Apply cleaning here
638
639 except Exception as e:
640 logger.error(f"Error generating insights using Gemini: {e}")
641 return "I'm analyzing the market data to provide you with detailed insights. Based on the current search results, I can see various opportunities in your preferred areas. Please bear with me while I gather more information."
642
643 def get_nearby_properties_gemini(self, location: str, requirements: Dict[str, Any]) -> str:
644 """Get nearby properties and recent market data using Gemini"""
645 try:
646 # Refine requirements to be more human-readable for Gemini
647 req_str = ""
648 for k, v in requirements.items():
649 if k == 'min_price': req_str += f"Minimum Budget: {v} Lakhs. "
650 elif k == 'max_price': req_str += f"Maximum Budget: {v} Lakhs. "
651 elif k == 'min_area': req_str += f"Minimum Area: {v} sq ft. "
652 elif k == 'max_area': req_str += f"Maximum Area: {v} sq ft. "
653 elif k == 'bhk': req_str += f"BHK: {v}. "
654 elif k == 'area_type': req_str += f"Area Type: {v}. "
655 elif k == 'availability': req_str += f"Availability: {v}. "
656 elif k == 'location': req_str += f"Preferred Location: {v}. "
657 if not req_str: req_str = "No specific requirements provided beyond location."
658
659
660 context = f"""
661 You are a knowledgeable real estate advisor with up-to-date market information for Pune.
662
663 A client is interested in properties near **{location}**.
664 Their current general requirements are: {req_str.strip()}
665
666 Please provide a concise but informative overview focusing on:
667 1. **Recent Property Launches**: Mention any significant residential projects launched in {location} or very close vicinity in the last 6-12 months.
668 2. **Upcoming Projects**: Are there any major upcoming residential or commercial developments (e.g., IT parks, malls, new infrastructure) that could impact property values in {location} or adjacent areas?
669 3. **Infrastructure Developments**: Discuss any planned or ongoing infrastructure improvements (roads, metro, civic amenities) that might affect property values and lifestyle in {location}.
670 4. **Market Trends in {location}**: What are the current buying/rental trends specific to {location}? Is it a high-demand area, or is supply outpacing demand?
671 5. **Comparative Analysis**: Briefly compare {location} with one or two similar, nearby localities in terms of property values, amenities, and lifestyle.
672 6. **Investment Potential**: What is the general investment outlook and potential rental yield for properties in {location}?
673
674 Focus on specific details relevant to the Pune real estate market in 2024-2025.
675 """
676
677 response = model.generate_content(context)
678 return self._clean_gemini_text(response.text) # Apply cleaning here
679
680 except Exception as e:
681 logger.error(f"Error getting nearby properties from Gemini: {e}")
682 return f"I'm gathering information about recent properties and market trends near {location}. This area has shown consistent growth in property values, and I'll get you more specific details shortly."
683
684 def update_context_memory(self, user_query: str, results: Dict[str, Any]):
685 """Update context memory to maintain conversation continuity"""
686 self.context_memory.append({
687 'timestamp': datetime.now().isoformat(),
688 'user_query': user_query,
689 'results_summary': {
690 'total_properties': results.get('total_properties', 0),
691 'price_range': results.get('price_range', {}),
692 'locations': list(results.get('location_distribution', {}).keys())
693 }
694 })
695
696 # Keep only last 5 interactions
697 if len(self.context_memory) > 5:
698 self.context_memory.pop(0)
699
700 def extract_requirements(self, user_message: str) -> Dict[str, Any]:
701 """Extract property requirements from user message"""
702 requirements = {}
703 message_lower = user_message.lower()
704
705 # Extract BHK
706 bhk_match = re.search(r'(\d+)\s*bhk', message_lower)
707 if bhk_match:
708 try:
709 requirements['bhk'] = int(bhk_match.group(1))
710 except ValueError:
711 pass # Ignore if not a valid number
712
713 # Extract budget range (flexible with "lakh", "crore", "million")
714 budget_match = re.search(
715 r'(?:budget|price|cost)(?:.*?(?:of|between|from)\s*)?'
716 r'(\d+(?:\.\d+)?)\s*(?:lakhs?|crores?|million)?(?:(?:\s*to|\s*-\s*)\s*(\d+(?:\.\d+)?)\s*(?:lakhs?|crores?|million)?)?',
717 message_lower
718 )
719 if budget_match:
720 try:
721 min_amount = float(budget_match.group(1))
722 max_amount = float(budget_match.group(2)) if budget_match.group(2) else None
723
724 # Detect units for both min and max if provided
725 if 'crore' in budget_match.group(0): # Check original matched string for "crore"
726 requirements['min_price'] = min_amount * 100
727 if max_amount:
728 requirements['max_price'] = max_amount * 100
729 elif 'million' in budget_match.group(0):
730 requirements['min_price'] = min_amount * 10
731 if max_amount:
732 requirements['max_price'] = max_amount * 10
733 else: # Default to lakhs
734 requirements['min_price'] = min_amount
735 if max_amount:
736 requirements['max_price'] = max_amount
737 except ValueError:
738 pass
739
740 # Extract location
741 found_location = False
742 for location in self.locations:
743 if location.lower() in message_lower:
744 requirements['location'] = location
745 found_location = True
746 break
747 # Broader location matching if specific location not found
748 if not found_location:
749 general_locations = ['pune', 'pcmp'] # Add more general terms if needed
750 for gen_loc in general_locations:
751 if gen_loc in message_lower:
752 requirements['location'] = gen_loc.capitalize() # Or a default location like 'Pune'
753 break
754
755
756 # Extract area requirements
757 area_match = re.search(r'(\d+(?:\.\d+)?)\s*(?:to|-)?\s*(\d+(?:\.\d+)?)?\s*(?:sq\s*ft|sqft|square\s*feet|sft)',
758 message_lower)
759 if area_match:
760 try:
761 requirements['min_area'] = float(area_match.group(1))
762 if area_match.group(2):
763 requirements['max_area'] = float(area_match.group(2))
764 except ValueError:
765 pass
766
767 # Extract area type
768 for area_type in self.area_types:
769 if area_type.lower() in message_lower:
770 requirements['area_type'] = area_type
771 break
772
773 # Extract availability
774 for availability in self.availability_types:
775 if availability.lower() in message_lower:
776 requirements['availability'] = availability
777 break
778
779 return requirements
780
781 def handle_query(self, user_message: str) -> Dict[str, Any]:
782 """Main function to handle user queries with context awareness"""
783 try:
784 # Extract requirements from current message
785 requirements = self.extract_requirements(user_message)
786
787 # Merge with previous context if building on last query
788 if self.is_follow_up_query(user_message):
789 requirements = self.merge_with_context(requirements)
790
791 # Update client preferences (store all extracted preferences)
792 self.client_preferences.update(requirements)
793
794 # Filter properties based on requirements
795 filtered_df = self.filter_properties(requirements)
796
797 # If no properties found, provide suggestions
798 if filtered_df.empty:
799 return self.handle_no_results(requirements)
800
801 # Predict prices for filtered results
802 filtered_df = self.predict_prices_for_filtered_results(filtered_df)
803
804 # Get price range analysis
805 analysis = self.get_price_range_analysis(filtered_df)
806
807 # Generate deep insights
808 insights = self.generate_deep_insights(filtered_df, analysis)
809
810 # Get nearby properties information
811 nearby_info = ""
812 if requirements.get('location'):
813 nearby_info = self.get_nearby_properties_gemini(requirements['location'], requirements)
814
815 # Prepare response
816 response_data = {
817 'filtered_properties': filtered_df.to_dict('records'),
818 'analysis': analysis,
819 'insights': insights,
820 'nearby_properties': nearby_info,
821 'requirements': requirements,
822 'context_aware': self.is_follow_up_query(user_message)
823 }
824
825 # Update context memory
826 self.update_context_memory(user_message, analysis)
827 self.last_search_results = response_data # Store the full response
828
829 return response_data
830
831 except Exception as e:
832 logger.error(f"Error handling query: {e}")
833 return {'error': f'An error occurred while processing your query: {str(e)}'}
834
835 def is_follow_up_query(self, user_message: str) -> bool:
836 """Check if this is a follow-up query building on previous results"""
837 follow_up_indicators = ['also', 'and', 'additionally', 'what about', 'show me more', 'similar', 'nearby',
838 'around', 'close to', 'next', 'other', 'different', 'change', 'update']
839 return any(indicator in user_message.lower() for indicator in follow_up_indicators) and len(
840 self.context_memory) > 0
841
842 def merge_with_context(self, new_requirements: Dict[str, Any]) -> Dict[str, Any]:
843 """Merge new requirements with context from previous queries"""
844 # Start with the overall client preferences
845 merged = self.client_preferences.copy()
846
847 # Overwrite with any new requirements from the current message
848 merged.update(new_requirements)
849
850 # Apply default or previously established values if not specified in current query
851 # Example: If a new query doesn't specify location, use the last known location
852 if 'location' not in new_requirements and 'location' in self.client_preferences:
853 merged['location'] = self.client_preferences['location']
854
855 # You can add more sophisticated merging logic here if needed
856 # e.g., if a new budget is provided, replace the old one entirely.
857 # The current update() method already handles this for direct key conflicts.
858
859 return merged
860
861 def handle_no_results(self, requirements: Dict[str, Any]) -> Dict[str, Any]:
862 """Handle cases where no properties match the criteria"""
863 suggestions = ["Try relaxing some of your criteria."]
864
865 if requirements.get('max_price'):
866 suggestions.append(f"Consider increasing your budget above ₹{requirements['max_price']:.2f}L.")
867 # Suggest a slightly higher range
868 suggestions.append(f"Perhaps a range of ₹{requirements['max_price']:.2f}L to ₹{(requirements['max_price'] * 1.2):.2f}L?")
869
870 if requirements.get('location'):
871 # Find nearby popular locations in the dataset, excluding the one searched
872 nearby_locs = [loc for loc in self.locations if loc.lower() != requirements['location'].lower()]
873 if nearby_locs:
874 # Sort by count if possible, or just pick top 3
875 top_locations = self.df['site_location'].value_counts().index.tolist()
876 suggested_nearby = [loc for loc in top_locations if loc.lower() != requirements['location'].lower()][:3]
877 if suggested_nearby:
878 suggestions.append(f"Consider nearby popular areas like: {', '.join(suggested_nearby)}.")
879
880 if requirements.get('bhk') is not None:
881 # Suggest +/- 1 BHK if not already at min/max reasonable bhk
882 if requirements['bhk'] > 1:
883 suggestions.append(f"Consider looking for {requirements['bhk'] - 1} BHK options.")
884 if requirements['bhk'] < 6: # Assuming 6 is a practical upper limit for most searches
885 suggestions.append(f"Consider {requirements['bhk'] + 1} BHK options.")
886
887 if requirements.get('max_area'):
888 suggestions.append(f"You might find more options if you consider properties up to ~{(requirements['max_area'] * 1.1):.0f} sq ft.")
889
890 # Default message if specific suggestions are not generated
891 if len(suggestions) == 1 and "relaxing" in suggestions[0]:
892 final_suggestion_text = "No properties found matching your exact criteria. Please try broadening your search or clarifying your preferences."
893 else:
894 final_suggestion_text = f"No properties found matching your exact criteria. Here are some suggestions: {'; '.join(suggestions)}."
895
896 return {
897 'filtered_properties': [],
898 'analysis': self.get_price_range_analysis(pd.DataFrame()), # Empty analysis
899 'insights': final_suggestion_text,
900 'nearby_properties': '',
901 'requirements': requirements,
902 'suggestions': suggestions
903 }
904
905
906# Initialize the enhanced sales assistant (MUST be after class definition)
907sales_assistant = EnhancedRealEstateSalesAssistant()
908
909
910@app.route('/')
911def index():
912 return render_template('index2.html')
913
914
915@app.route('/chat', methods=['POST'])
916def chat():
917 """Enhanced chat endpoint with comprehensive property search"""
918 try:
919 data = request.get_json()
920 user_message = data.get('message', '').strip()
921
922 if not user_message:
923 return jsonify({'error': 'Please provide your query'}), 400
924
925 # Get comprehensive response from sales assistant
926 response_data = sales_assistant.handle_query(user_message)
927
928 # Ensure response_data always includes necessary keys even on error
929 if 'error' in response_data:
930 return jsonify({
931 'response': response_data,
932 'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
933 'conversation_context': len(sales_assistant.context_memory)
934 }), 500 # Return 500 for internal errors
935 else:
936 return jsonify({
937 'response': response_data,
938 'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
939 'conversation_context': len(sales_assistant.context_memory)
940 })
941
942 except Exception as e:
943 logger.error(f"Chat endpoint error: {e}")
944 return jsonify({'error': f'An unexpected error occurred in chat endpoint: {str(e)}'}), 500
945
946
947# Flask routes must be defined at the module level, not inside a class
948@app.route('/model_status')
949def model_status():
950 """Get current model status and version info"""
951 try:
952 status = {
953 'model_loaded': hasattr(sales_assistant, 'pipeline') and sales_assistant.pipeline is not None,
954 'model_version': getattr(sales_assistant, 'model_version', 'unknown'),
955 'feature_count': len(getattr(sales_assistant, 'training_feature_names', [])),
956 'prediction_method': 'ML' if hasattr(sales_assistant, 'pipeline') and sales_assistant.pipeline else 'Fallback',
957 'sklearn_version': sklearn.__version__
958 }
959 return jsonify(status)
960 except Exception as e:
961 logger.error(f"Error in model_status endpoint: {e}")
962 return jsonify({'error': str(e)}), 500
963
964@app.route('/retrain_model', methods=['POST'])
965def retrain_model():
966 """Retrain the model with current version"""
967 try:
968 logger.info("Starting model retraining via endpoint...")
969 success = sales_assistant.retrain_model_with_current_version()
970
971 if success:
972 return jsonify({
973 'message': 'Model retrained successfully',
974 'version': sales_assistant.model_version,
975 'feature_count': len(sales_assistant.training_feature_names),
976 'sample_features': sales_assistant.training_feature_names[:10] if sales_assistant.training_feature_names else []
977 })
978 else:
979 return jsonify({'error': 'Failed to retrain model. Check logs for details.'}), 500
980
981 except Exception as e:
982 logger.error(f"Retrain endpoint error: {e}")
983 return jsonify({'error': f'An unexpected error occurred during retraining: {str(e)}'}), 500
984
985
986@app.route('/filter_properties', methods=['POST'])
987def filter_properties_endpoint(): # Renamed to avoid conflict with class method
988 """Dedicated endpoint for property filtering"""
989 try:
990 filters = request.get_json()
991
992 filtered_df = sales_assistant.filter_properties(filters)
993 filtered_df = sales_assistant.predict_prices_for_filtered_results(filtered_df)
994 analysis = sales_assistant.get_price_range_analysis(filtered_df)
995
996 return jsonify({
997 'properties': filtered_df.to_dict('records'),
998 'analysis': analysis,
999 'total_count': len(filtered_df)
1000 })
1001
1002 except Exception as e:
1003 logger.error(f"Filter endpoint error: {e}")
1004 return jsonify({'error': f'Filtering error: {str(e)}'}), 500
1005
1006
1007@app.route('/get_insights', methods=['POST'])
1008def get_insights():
1009 """Get AI-powered insights for current search results"""
1010 try:
1011 data = request.get_json()
1012 location_from_request = data.get('location')
1013 requirements_from_request = data.get('requirements', {})
1014
1015 if sales_assistant.last_search_results:
1016 filtered_df = pd.DataFrame(sales_assistant.last_search_results['filtered_properties'])
1017 analysis = sales_assistant.last_search_results['analysis']
1018 insights = sales_assistant.generate_deep_insights(filtered_df, analysis)
1019
1020 nearby_info = ""
1021 # Use location from request if provided, otherwise from last search results
1022 effective_location = location_from_request or sales_assistant.last_search_results['requirements'].get('location')
1023 if effective_location:
1024 nearby_info = sales_assistant.get_nearby_properties_gemini(effective_location, requirements_from_request or sales_assistant.last_search_results['requirements'])
1025
1026 return jsonify({
1027 'insights': insights,
1028 'nearby_properties': nearby_info
1029 })
1030 else:
1031 return jsonify({'error': 'No previous search results available for insights. Please perform a search first.'}), 400
1032
1033 except Exception as e:
1034 logger.error(f"Insights endpoint error: {e}")
1035 return jsonify({'error': f'Insights error: {str(e)}'}), 500
1036
1037
1038@app.route('/get_market_data')
1039def get_market_data():
1040 """Get comprehensive market data"""
1041 try:
1042 # Ensure df is loaded before accessing its properties
1043 if not hasattr(sales_assistant, 'df') or sales_assistant.df.empty:
1044 return jsonify({'error': 'Market data not available, dataset not loaded.'}), 500
1045
1046 market_data = {
1047 'locations': sales_assistant.locations,
1048 'area_types': sales_assistant.area_types,
1049 'availability_types': sales_assistant.availability_types,
1050 'bhk_options': sales_assistant.bhk_options,
1051 'price_statistics': {
1052 'min_price': float(sales_assistant.df['price'].min()),
1053 'max_price': float(sales_assistant.df['price'].max()),
1054 'avg_price': float(sales_assistant.df['price'].mean()),
1055 'median_price': float(sales_assistant.df['price'].median())
1056 },
1057 'area_statistics': {
1058 'min_area': float(sales_assistant.df['total_sqft'].min()),
1059 'max_area': float(sales_assistant.df['total_sqft'].max()),
1060 'avg_area': float(sales_assistant.df['total_sqft'].mean())
1061 }
1062 }
1063
1064 return jsonify(market_data)
1065
1066 except Exception as e:
1067 logger.error(f"Market data endpoint error: {e}")
1068 return jsonify({'error': f'Market data error: {str(e)}'}), 500
1069
1070
1071@app.route('/client_preferences')
1072def get_client_preferences():
1073 """Get current client preferences and conversation history"""
1074 return jsonify({
1075 'preferences': sales_assistant.client_preferences,
1076 'conversation_history': sales_assistant.context_memory,
1077 'last_search_available': sales_assistant.last_search_results is not None
1078 })
1079
1080
1081@app.route('/reset_session', methods=['POST'])
1082def reset_session():
1083 """Reset entire session including preferences and context"""
1084 try:
1085 sales_assistant.client_preferences = {}
1086 sales_assistant.context_memory = []
1087 sales_assistant.last_search_results = None
1088 return jsonify({'message': 'Session reset successfully'})
1089 except Exception as e:
1090 logger.error(f"Reset session error: {e}")
1091 return jsonify({'error': f'Failed to reset session: {str(e)}'}), 500
1092
1093
1094@app.route('/health')
1095def health():
1096 """Health check endpoint"""
1097 try:
1098 return jsonify({
1099 'status': 'healthy',
1100 'model_loaded': hasattr(sales_assistant, 'pipeline') and sales_assistant.pipeline is not None,
1101 'model_version': sales_assistant.model_version,
1102 'data_loaded': hasattr(sales_assistant, 'df') and not sales_assistant.df.empty,
1103 'dataset_size': len(sales_assistant.df) if hasattr(sales_assistant, 'df') else 0,
1104 'locations_available': len(sales_assistant.locations),
1105 'context_memory_size': len(sales_assistant.context_memory),
1106 'timestamp': datetime.now().isoformat()
1107 })
1108 except Exception as e:
1109 logger.error(f"Health check error: {e}")
1110 return jsonify({'status': 'unhealthy', 'error': str(e)}), 500
1111
1112
1113if __name__ == '__main__':
1114 # It's better to explicitly set the host for deployment environments.
1115 # For development, debug=True is useful but should be False in production.
1116 app.run(debug=True, host='0.0.0.0', port=5000)