AIBotsForYou/Ensemble_Fraud_Detection

1# app.py
2
3import streamlit as st
4import pandas as pd
5import numpy as np
6import matplotlib.pyplot as plt
7
8from models.ensemble_model import load_data, train_ensemble  # (existing module for training)
9from utils.evaluation import plot_roc, plot_confusion_matrix
10from utils.data_generator import simulate_transaction_stream
11from utils.preprocessing import preprocess_data
12
13# Set page config
14st.set_page_config(page_title="Ensemble-Based Credit Fraud Detector", layout="wide")
15
16# Sidebar Navigation with Data Preprocessing Overview at the top
17st.sidebar.title("Navigation")
18app_mode = st.sidebar.radio("Choose the app mode", 
19                            ["Data Preprocessing Overview", "Model Training & Evaluation", "Real-Time Simulation"])
20
21# --- Data Preprocessing Overview Section ---
22if app_mode == "Data Preprocessing Overview":
23    st.title("Data Preprocessing Overview")
24    st.markdown("""
25    **Why Preprocess?**  
26    Raw data can contain inconsistencies such as missing values, varying scales, and non-numeric types. Preprocessing ensures that the data fed to machine learning models is clean, standardized, and in a format they can understand.
27
28    **Steps in Our Preprocessing Pipeline:**
29
30    1. **Missing Value Handling:**  
31       - Rows with missing target values are dropped to ensure data quality.
32    
33    2. **Feature Separation:**  
34       - The target column (`is_fraud`) is separated from the features.
35    
36    3. **Feature Type Identification:**  
37       - **Numerical Features:** Automatically identified based on data types (e.g., int64, float64).  
38       - **Categorical Features:** Identified as object or category types.
39    
40    4. **Transformation Pipelines:**  
41       - **Numeric Transformer:** Uses StandardScaler to normalize numerical features. This ensures that features with larger scales do not dominate the model training.
42       - **Categorical Transformer:** Uses OneHotEncoder to convert categorical variables into a numerical format that the model can use.
43    
44    5. **Column Transformer:**  
45       - Combines both pipelines so that each feature is processed appropriately.
46    
47    **Interactive Learning:**
48    - Experiment by uploading your own dataset.
49    - Observe how the data is transformed (e.g., check the shape of the processed data).
50    - Learn how each step improves model performance and robustness in fraud detection.
51    
52    **Interview Insights:**
53    - Understand how scaling and encoding work.
54    - Be ready to explain the benefits of using pipelines and transformers.
55    - Know how raw data is transformed into a format that machine learning models can use effectively.
56    """)
57    
58    st.info("Try uploading different datasets and observe the preprocessing steps to deepen your understanding!")
59
60# --- Model Training & Evaluation Section ---
61elif app_mode == "Model Training & Evaluation":
62    st.title("Ensemble-Based Credit Fraud Detector")
63    st.markdown("""
64    **Overview:**  
65    In this section, you will learn how to convert raw transaction data into numerical features and train an ensemble classifier for fraud detection.  
66    The process includes:
67    
68    1. **Data Preprocessing:** Converting raw data (which may contain text, missing values, etc.) into a numerical matrix using scaling and encoding.
69    2. **Model Training:** Using Logistic Regression, Decision Tree, and Random Forest combined via a soft voting ensemble.
70    3. **Model Evaluation:** Generating ROC curves and confusion matrices to assess model performance.
71    """)
72    
73    # File uploader to allow custom dataset upload
74    uploaded_file = st.file_uploader("Upload your CSV file (must include 'is_fraud' column)", type="csv")
75
76    if uploaded_file is not None:
77        # Load the raw data from CSV
78        df = pd.read_csv(uploaded_file)
79        st.write("### Raw Data Preview", df.head())
80        
81        if "is_fraud" not in df.columns:
82            st.error("Uploaded CSV must include an 'is_fraud' column as the target.")
83        else:
84            # Preprocess the data: raw to numerical conversion
85            st.info("Preprocessing data: converting raw inputs into numerical features...")
86            X_processed, y, preprocessor = preprocess_data(df, target_column="is_fraud")
87            st.write("**Processed Data Shape:**", X_processed.shape)
88            
89            # Split the data for training and evaluation
90            from sklearn.model_selection import train_test_split
91            X_train, X_test, y_train, y_test = train_test_split(X_processed, y, test_size=0.2, random_state=42)
92
93            # Import classifiers and VotingClassifier
94            from sklearn.linear_model import LogisticRegression
95            from sklearn.tree import DecisionTreeClassifier
96            from sklearn.ensemble import RandomForestClassifier, VotingClassifier
97
98            # Define individual models
99            clf1 = LogisticRegression(max_iter=1000, solver='lbfgs')
100            clf2 = DecisionTreeClassifier(max_depth=5, random_state=42)
101            clf3 = RandomForestClassifier(n_estimators=100, random_state=42)
102
103            # Build the ensemble using soft voting (to support predict_proba)
104            ensemble = VotingClassifier(estimators=[
105                ('lr', clf1), ('dt', clf2), ('rf', clf3)
106            ], voting='soft')
107
108            # Train the ensemble classifier
109            ensemble.fit(X_train, y_train)
110            st.success("Model trained successfully!")
111
112            # Evaluation Plots
113            st.subheader("Model Evaluation")
114            col1, col2 = st.columns(2)
115
116            with col1:
117                st.write("**ROC Curve**")
118                try:
119                    roc_plot = plot_roc(ensemble, X_test, y_test)
120                    st.pyplot(roc_plot)
121                except Exception as e:
122                    st.error("ROC curve could not be generated: " + str(e))
123            
124            with col2:
125                st.write("**Confusion Matrix**")
126                try:
127                    cm_plot = plot_confusion_matrix(ensemble, X_test, y_test)
128                    st.pyplot(cm_plot)
129                except Exception as e:
130                    st.error("Confusion Matrix could not be generated: " + str(e))
131    else:
132        st.info("Please upload a CSV file to proceed with training.")
133
134# --- Real-Time Simulation Section ---
135elif app_mode == "Real-Time Simulation":
136    st.title("Real-Time Fraud Detection Simulation")
137    st.markdown("""
138    **Overview:**  
139    In this simulation, synthetic raw transaction data is preprocessed and then passed to the pre-trained ensemble model for fraud detection in real time.
140    
141    **Process:**
142    1. **Synthetic Data Creation:** Raw data is generated with a mix of numerical and (if needed) categorical features.
143    2. **Preprocessing:** The same pipeline converts raw transaction data into a numerical format.
144    3. **Prediction:** Each transaction is evaluated by the ensemble model (which uses soft voting to obtain probability estimates).
145    """)
146    
147    st.info("Training a demo model on synthetic data for simulation...")
148
149    # Create a synthetic dataset
150    n_samples = 1000
151    df_demo = pd.DataFrame({
152        "feature1": np.random.rand(n_samples),
153        "feature2": np.random.randint(0, 100, n_samples),
154        "feature3": np.random.rand(n_samples) * 50,
155        "feature4": np.random.choice([0, 1], n_samples),
156    })
157    # Create a synthetic target with imbalance
158    df_demo["is_fraud"] = np.where((df_demo["feature1"] + df_demo["feature4"]) > 1.2, 1, 0)
159
160    # Preprocess synthetic data
161    X_processed, y_demo, preprocessor = preprocess_data(df_demo, target_column="is_fraud")
162
163    from sklearn.model_selection import train_test_split
164    X_train, X_test, y_train, y_test = train_test_split(X_processed, y_demo, test_size=0.2, random_state=42)
165
166    from sklearn.linear_model import LogisticRegression
167    from sklearn.tree import DecisionTreeClassifier
168    from sklearn.ensemble import RandomForestClassifier, VotingClassifier
169
170    clf1 = LogisticRegression(max_iter=1000, solver='lbfgs')
171    clf2 = DecisionTreeClassifier(max_depth=5, random_state=42)
172    clf3 = RandomForestClassifier(n_estimators=100, random_state=42)
173
174    ensemble_demo = VotingClassifier(estimators=[
175        ('lr', clf1), ('dt', clf2), ('rf', clf3)
176    ], voting='soft')
177    ensemble_demo.fit(X_train, y_train)
178    st.success("Demo model trained successfully!")
179
180    st.markdown("### Starting Transaction Stream Simulation")
181    simulate = st.button("Start Simulation")
182
183    if simulate:
184        st.markdown("#### Real-Time Transaction Predictions")
185        placeholder = st.empty()
186        # Simulate stream in real time
187        for transaction in simulate_transaction_stream(n_transactions=10, delay=1):
188            # Convert transaction dict to DataFrame
189            transaction_df = pd.DataFrame([transaction])
190            # Preprocess new transaction using the fitted preprocessor
191            X_new = preprocessor.transform(transaction_df)
192            prediction = ensemble_demo.predict(X_new)[0]
193            prediction_text = "Fraudulent" if prediction == 1 else "Legitimate"
194            placeholder.write(f"Transaction: {transaction} -> Prediction: **{prediction_text}**")
195