ehsanulhaque92/multimodal-prescriptive-pdm
0
1# =========================================================================2# === THIS IS THE NEW, MEMORY-EFFICIENT app/utils.py FILE ===3# =========================================================================4import shap5import matplotlib6matplotlib.use('Agg')7import matplotlib.pyplot as plt8import io9import base6410import joblib11import pandas as pd12import numpy as np13import app_config as config14import traceback15import logging16import json17from prescriptive_rag.chains import create_rag_chain18 19# --- Setup Logging ---20logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')21 22# --- Global Placeholders for Lazily-Loaded Models ---23# We initialize them to None. They will be loaded into these global24# variables the first time they are requested.25RUL_MODEL = None26RUL_SCALER = None27FAULT_CLASSIFIER = None28CLASSIFICATION_PREPROCESSOR = None29RAG_CHAIN = None30 31# --- "Getter" Functions for Lazy Loading ---32 33def get_rag_chain_instance():34 """35 Loads the RAG chain on first call and caches it in a global variable.36 """37 global RAG_CHAIN38 if RAG_CHAIN is None:39 logging.info("RAG chain not loaded. Initializing now...")40 try:41 RAG_CHAIN = create_rag_chain()42 logging.info("RAG chain successfully initialized and cached.")43 except Exception as e:44 logging.error(f"Failed to initialize RAG chain: {e}", exc_info=True)45 RAG_CHAIN = None # Ensure it remains None on failure46 return RAG_CHAIN47 48def get_rul_model_and_scaler():49 """50 Loads the RUL model and scaler on first call and caches them.51 """52 global RUL_MODEL, RUL_SCALER53 if RUL_MODEL is None or RUL_SCALER is None:54 logging.info("RUL model/scaler not loaded. Initializing now...")55 try:56 RUL_MODEL = joblib.load(config.RUL_MODEL_PATH)57 RUL_SCALER = joblib.load(config.RUL_SCALER_PATH)58 logging.info("RUL model and scaler successfully loaded and cached.")59 except FileNotFoundError:60 logging.error(f"Could not load RUL model/scaler from {config.RUL_MODEL_PATH}")61 RUL_MODEL, RUL_SCALER = None, None62 return RUL_MODEL, RUL_SCALER63 64def get_classification_model_and_preprocessor():65 """66 Loads the classification model and preprocessor on first call and caches them.67 """68 global FAULT_CLASSIFIER, CLASSIFICATION_PREPROCESSOR69 if FAULT_CLASSIFIER is None or CLASSIFICATION_PREPROCESSOR is None:70 logging.info("Classification model/preprocessor not loaded. Initializing now...")71 try:72 FAULT_CLASSIFIER = joblib.load(config.CLASSIFICATION_MODEL_PATH)73 CLASSIFICATION_PREPROCESSOR = joblib.load(config.CLASSIFICATION_PREPROCESSOR_PATH)74 logging.info("Classification model and preprocessor successfully loaded and cached.")75 except FileNotFoundError:76 logging.error(f"Could not load classification model from {config.CLASSIFICATION_MODEL_PATH}")77 FAULT_CLASSIFIER, CLASSIFICATION_PREPROCESSOR = None, None78 return FAULT_CLASSIFIER, CLASSIFICATION_PREPROCESSOR79 80 81# --- Prediction Functions (Now using the "getter" functions) ---82 83def get_rul_prediction(data: pd.DataFrame, with_shap: bool = False) -> dict:84 rul_model, rul_scaler = get_rul_model_and_scaler()85 if rul_model is None or rul_scaler is None:86 return {"error": "RUL model is not available."}87 # ... (rest of the function is identical to before)88 df = data.copy()89 sensor_cols = [col for col in df.columns if 'sensor' in col and '_rolling' not in col]90 for col in sensor_cols:91 df[f'{col}_rolling_mean'] = df[col].rolling(window=config.RUL_WINDOW_SIZE, min_periods=1).mean()92 df[f'{col}_rolling_std'] = df[col].rolling(window=config.RUL_WINDOW_SIZE, min_periods=1).std()93 df.fillna(0, inplace=True)94 last_row = df.iloc[[-1]]95 required_features = rul_model.feature_names_in_96 features_for_model = last_row[required_features]97 X_scaled = rul_scaler.transform(features_for_model)98 predicted_rul = min(float(rul_model.predict(X_scaled)[0]), config.RUL_CAP)99 shap_plot_base64 = None100 if with_shap:101 try:102 explainer = shap.TreeExplainer(rul_model)103 shap_plot_base64 = _generate_shap_plot(rul_model, explainer, X_scaled, required_features)104 except Exception as e:105 logging.error(f"Error generating RUL SHAP plot: {e}")106 return {"predicted_rul": round(predicted_rul, 2), "shap_plot": shap_plot_base64}107 108def get_fault_prediction(data: pd.DataFrame) -> dict:109 fault_classifier, classification_preprocessor = get_classification_model_and_preprocessor()110 if fault_classifier is None or classification_preprocessor is None:111 return {"error": "Classification model is not available."}112 # ... (rest of the function is identical to before)113 X_processed = classification_preprocessor.transform(data)114 prediction = fault_classifier.predict(X_processed)[0]115 probabilities = fault_classifier.predict_proba(X_processed)[0]116 class_index = np.where(fault_classifier.classes_ == prediction)[0][0]117 confidence = probabilities[class_index]118 shap_plot_base64 = None119 try:120 explainer = shap.TreeExplainer(fault_classifier)121 feature_names = classification_preprocessor.get_feature_names_out()122 shap_plot_base64 = _generate_shap_plot(fault_classifier, explainer, X_processed, feature_names)123 except Exception as e:124 logging.error(f"Error generating Classification SHAP plot: {e}")125 return {"predicted_fault": str(prediction), "confidence": round(float(confidence), 2), "shap_plot": shap_plot_base64}126 127 128# --- Other Functions (SHAP plot, monitoring, etc. remain the same) ---129# --- NOTE: For brevity, I am omitting the other functions. Copy them from your existing file ---130def _generate_shap_plot(model, explainer, processed_data, feature_names):131 """A generic helper to generate a SHAP force plot."""132 # ... (This function remains IDENTICAL)133 shap_values = explainer(processed_data)134 is_classification = hasattr(model, 'classes_')135 if is_classification:136 prediction = model.predict(processed_data)[0]137 class_list = model.classes_.tolist()138 prediction_index = class_list.index(prediction)139 base_value = shap_values.base_values[0, prediction_index]140 shap_values_for_plot = shap_values.values[0, :, prediction_index]141 feature_names_clean = [name.split('__')[1] for name in feature_names]142 else: # Regression143 base_value = explainer.expected_value144 shap_values_for_plot = shap_values.values[0]145 feature_names_clean = feature_names146 force_plot = shap.force_plot(147 base_value=base_value, shap_values=shap_values_for_plot, features=processed_data[0],148 feature_names=feature_names_clean, matplotlib=True, show=False, figsize=(20, 5), text_rotation=15149 )150 buf = io.BytesIO()151 plt.savefig(buf, format='png', bbox_inches='tight', dpi=150)152 plt.close(force_plot)153 buf.seek(0)154 image_base64 = base64.b64encode(buf.read()).decode('utf-8')155 return f"data:image/png;base64,{image_base64}"156 157def simulate_drift_detection():158 """ Simulates monitoring a data stream for concept drift. """159 # ... (This function remains IDENTICAL)160 try:161 df = pd.read_csv(config.PROCESSED_DATA_DIR / "drift_simulation_data.csv")162 except FileNotFoundError:163 return {"error": "Drift simulation data not found."}164 # ... (rest of the logic)165 initial_train_size=200; model=SGDClassifier(loss='log_loss',random_state=42); X_initial=df.iloc[:initial_train_size][['feature1','feature2']]; y_initial=df.iloc[:initial_train_size]['target']; model.fit(X_initial, y_initial); stream_data=df.iloc[initial_train_size:]; chunk_size=50; time_steps,accuracies,drift_points=[],[],[]; drift_threshold,is_drift_detected=0.70,False166 for i in range(0,len(stream_data),chunk_size):167 chunk=stream_data.iloc[i:i+chunk_size];168 if chunk.empty:continue169 X_chunk,y_chunk_true=chunk[['feature1','feature2']],chunk['target']; y_chunk_pred=model.predict(X_chunk); acc=accuracy_score(y_chunk_true,y_chunk_pred); time_steps.append(initial_train_size+i+chunk_size/2); accuracies.append(acc)170 if acc<drift_threshold and not is_drift_detected:drift_points.append({"time":initial_train_size+i+chunk_size/2,"label":"Concept Drift Detected"}); is_drift_detected=True171 return {"time_steps":time_steps,"accuracies":accuracies,"drift_points":drift_points,"drift_threshold":drift_threshold}172 173def get_fleet_topics():174 """ Loads the discovered topics from the JSON file. """175 # ... (This function remains IDENTICAL)176 try:177 with open(config.PROCESSED_DATA_DIR / "dashboard_topics.json", 'r') as f:178 topics = json.load(f)179 return topics180 except FileNotFoundError:181 return []182 except json.JSONDecodeError:183 return []184 185def get_dashboard_data(rul_df_processed: pd.DataFrame) -> dict:186 """ Generates the dynamic data for the dashboard, including assets and topics. """187 # ... (This function remains IDENTICAL)188 all_dashboard_assets=[]189 if rul_df_processed is not None:190 # ... (rest of logic)191 pass # Placeholder for your existing RUL logic192 classification_samples = {193 "Milling Machine #XYZ": pd.DataFrame([{"type":"L","air_temperature":302.5,"process_temperature":311.8,"rotational_speed":1390,"torque":55.3,"tool_wear":208}]),194 "Conveyor Belt #A-3": pd.DataFrame([{"type":"M","air_temperature":300.1,"process_temperature":309.7,"rotational_speed":1550,"torque":41.2,"tool_wear":20}])195 }196 for asset_id,sample_df in classification_samples.items():197 asset_info={"id":asset_id,"type":"classification"}; prediction_result=get_fault_prediction(sample_df); asset_info["prediction"]=prediction_result198 fault=prediction_result.get("predicted_fault","No Failure")199 if fault!="No Failure":asset_info["status_class"]="border-red"; asset_info["status_text"]="Fault Predicted"200 else:asset_info["status_class"]="border-green"; asset_info["status_text"]="Healthy"201 all_dashboard_assets.append(asset_info)202 status_order={"border-red":0,"border-orange":1,"border-green":2}; final_sorted_assets=sorted(all_dashboard_assets,key=lambda x:status_order.get(x.get('status_class'),99))203 fleet_topics=get_fleet_topics()204 return {"assets":final_sorted_assets,"topics":fleet_topics}