rodts28/Inverse_3DCP_Evolutionary_Algorithm
0
1# -*- coding: utf-8 -*-2"""3Created on Tue Dec 9 18:32:07 20254 5@author: rod-t6"""7 8"""93DCP Evolutionary Inverse-Design Dashboard (Hugging Face Space entrypoint)10 11Tabs:121. Inverse Evolutionary (targets → mix) [currently nearest-neighbor fallback]132. Quality & Sensitivity143. kNN Analogs & Clusters154. Engineering Reality-Gap16"""17 18import os19import time20import numpy as np21import pandas as pd22import gradio as gr23 24from core_inverse import CFG, load_data25from analysis_quality import (26 analyze_inverse_and_sensitivity,27 local_sensitivity_with_constraints,28)29from analysis_neighbors_clusters import (30 nearest_neighbor_analogs_and_manifold_check,31 gmm_cluster_on_outputs,32 cluster_explainability_outputs,33)34from analysis_reality_gap import engineering_reality_gap_analysis35 36# ---------------------------------------------------------37# Configure save directory for evolutionary inverse results38# ---------------------------------------------------------39CFG["save_dir"] = "./inverse_results"40os.makedirs(CFG["save_dir"], exist_ok=True)41 42 43# ---------------------------------------------------------44# Evolutionary wrapper (currently nearest-neighbor in output space)45# ---------------------------------------------------------46 47def run_inverse_evo_inverse_design(targets_dict: dict) -> dict:48 """49 Inverse design wrapper for the evolutionary pipeline.50 51 Currently implemented as a nearest-neighbor fallback in OUTPUT space,52 using the dataset and forward model loaded via load_data().53 54 This function:55 - Loads df, model, mappings, ranges via load_data()56 - Finds the dataset row whose outputs best match the requested targets57 in a range-normalized Euclidean sense58 - Builds REPORT and ELITES CSV files with the standard schema:59 * Target::OutputName60 * Pred::OutputName61 * Mix::IngredientName62 """63 (64 df,65 model,66 mix_map,67 out_map,68 lb,69 ub,70 y_min,71 y_max,72 y_rng,73 nn,74 nn_scale,75 ) = load_data()76 77 out_cols_actual = list(out_map.values())78 out_actual_to_canon = {v: k for k, v in out_map.items()}79 80 # Canonical mix order and actual columns for X81 mix_canon_order = [c for c in CFG["canon_mix_cols"] if c in mix_map]82 mix_cols_actual = [mix_map[c] for c in mix_canon_order]83 84 X = df[mix_cols_actual].values85 Y = df[out_cols_actual].values86 87 # Build target vector in actual-output order88 target_vec = np.zeros(len(out_cols_actual), dtype=float)89 mask = np.zeros(len(out_cols_actual), dtype=bool)90 for i, col in enumerate(out_cols_actual):91 canon = out_actual_to_canon[col]92 if canon in targets_dict and pd.notna(targets_dict[canon]):93 target_vec[i] = float(targets_dict[canon])94 mask[i] = True95 96 if not mask.any():97 raise ValueError("No valid targets provided; please specify at least one.")98 99 Y_sub = Y[:, mask]100 t_sub = target_vec[mask]101 102 # Range-normalize errors to balance outputs103 y_rng_vec = y_rng[out_cols_actual].values104 y_rng_sub = y_rng_vec[mask]105 y_rng_sub = np.where(y_rng_sub > 0, y_rng_sub, 1.0)106 107 Y_norm = (Y_sub - t_sub) / y_rng_sub108 dists = np.linalg.norm(Y_norm, axis=1)109 best_idx = int(np.argmin(dists))110 111 x_best = X[best_idx, :]112 y_best = Y[best_idx, :]113 114 # Build canonical mix dict115 mix_canon = {116 canon: float(x_best[mix_canon_order.index(canon)]) for canon in mix_canon_order117 }118 # Canonical predicted outputs119 preds_canon = {120 out_actual_to_canon[col]: float(y_best[i])121 for i, col in enumerate(out_cols_actual)122 }123 124 # REPORT row125 report_row = {}126 for canon in out_actual_to_canon.values():127 report_row[f"Target::{canon}"] = float(targets_dict.get(canon, np.nan))128 for canon, val in preds_canon.items():129 report_row[f"Pred::{canon}"] = val130 for canon in mix_canon_order:131 report_row[f"Mix::{canon}"] = mix_canon[canon]132 133 report_df = pd.DataFrame([report_row])134 135 # ELITES row (simple compatibility table)136 elites_row = {"Score": 0.0}137 elites_row.update(138 {k: v for k, v in report_row.items() if not k.startswith("Target::")}139 )140 elites_df = pd.DataFrame([elites_row])141 142 save_dir = CFG.get("save_dir", "./inverse_results")143 os.makedirs(save_dir, exist_ok=True)144 145 ts = time.strftime("%Y%m%d_%H%M%S")146 base = f"inverseEVO_ui_{ts}"147 148 report_csv_path = os.path.join(save_dir, base + "_REPORT.csv")149 elites_csv_path = os.path.join(save_dir, base + "_ELITES.csv")150 151 report_df.to_csv(report_csv_path, index=False)152 elites_df.to_csv(elites_csv_path, index=False)153 154 return {155 "best_mix": mix_canon,156 "best_preds": preds_canon,157 "report_csv_path": report_csv_path,158 "elites_csv_path": elites_csv_path,159 "best_idx": best_idx,160 "dist_norm": float(dists[best_idx]),161 }162 163 164# ---------------------------------------------------------165# UI helpers166# ---------------------------------------------------------167 168def _targets_from_ui(ps, fer, sys, vm, cs):169 targets = {}170 if ps is not None:171 targets["Pump-Speed"] = float(ps)172 if fer is not None:173 targets["Flow Extrusion Rate"] = float(fer)174 if sys is not None:175 targets["Static Yield Stress"] = float(sys)176 if vm is not None:177 targets["Viscosity Mixture"] = float(vm)178 if cs is not None:179 targets["CS"] = float(cs)180 return targets181 182 183# ---------------- Tab 1: Inverse Evolutionary -----------------184 185def ui_run_inverse(ps, fer, sys, vm, cs, iterations, exploration, save_tag, state_dict):186 targets = _targets_from_ui(ps, fer, sys, vm, cs)187 if not targets:188 return (189 pd.DataFrame(),190 pd.DataFrame(),191 "No outputs selected. Provide at least one target.",192 "",193 "",194 state_dict,195 )196 197 try:198 res = run_inverse_evo_inverse_design(targets_dict=targets)199 except Exception as e:200 msg = f"[ERROR] Evolutionary inverse-design failed: {e}"201 return pd.DataFrame(), pd.DataFrame(), msg, "", "", state_dict202 203 report_path = res.get("report_csv_path", "")204 elites_path = res.get("elites_csv_path", "")205 best_mix = res.get("best_mix", {})206 best_preds = res.get("best_preds", {})207 208 state_dict = dict(state_dict or {})209 state_dict["report_path"] = report_path210 state_dict["elites_path"] = elites_path211 state_dict["targets"] = targets212 state_dict["best_mix"] = best_mix213 state_dict["best_preds"] = best_preds214 215 mix_df = (216 pd.DataFrame(217 [{"Ingredient": k, "Value": v} for k, v in best_mix.items()]218 )219 if best_mix220 else pd.DataFrame()221 )222 preds_df = (223 pd.DataFrame(224 [225 {226 "Output": k,227 "Predicted": v,228 "Target": targets.get(k, np.nan),229 }230 for k, v in best_preds.items()231 ]232 )233 if best_preds234 else pd.DataFrame()235 )236 237 msg = (238 "[OK] Inverse design (nearest-neighbor baseline) complete.\n"239 f"REPORT: {report_path}\n"240 f"ELITES: {elites_path}"241 )242 243 return mix_df, preds_df, msg, report_path, elites_path, state_dict244 245 246# ---------------- Tab 2: Quality & Sensitivity ----------------247 248def ui_quality_and_sensitivity(state_dict, do_local_sens, delta_frac):249 if not state_dict or "report_path" not in state_dict:250 return (251 "[ERROR] No inverse run in this session. Run Tab 1 first.",252 pd.DataFrame(),253 pd.DataFrame(),254 [],255 pd.DataFrame(),256 )257 258 report_path = state_dict["report_path"]259 elites_path = state_dict["elites_path"]260 261 try:262 results = analyze_inverse_and_sensitivity(263 report_csv_path=report_path,264 elites_csv_path=elites_path,265 perturb_frac=0.05,266 )267 except Exception as e:268 return (269 f"[ERROR] analyze_inverse_and_sensitivity failed: {e}",270 pd.DataFrame(),271 pd.DataFrame(),272 [],273 pd.DataFrame(),274 )275 276 per_out = results.get("per_output_table", pd.DataFrame())277 sens_rank = results.get("sensitivity_rank", pd.Series(dtype=float))278 279 sens_table = pd.DataFrame()280 msg_local = ""281 if do_local_sens:282 try:283 sens_res = local_sensitivity_with_constraints(284 report_csv_path=report_path,285 elites_csv_path=elites_path,286 delta_frac_of_sum=float(delta_frac),287 outputs_to_match=None,288 save_excel_path=None,289 )290 sens_table = sens_res.get("table", pd.DataFrame())291 msg_local = "[OK] Local constraint-aware sensitivity computed."292 except Exception as e:293 msg_local = f"[WARN] local_sensitivity_with_constraints failed: {e}"294 295 msg = "[OK] Global quality analysis complete.\n" + msg_local296 297 if isinstance(sens_rank, pd.Series) and not sens_rank.empty:298 sens_rank_df = sens_rank.reset_index()299 sens_rank_df.columns = ["Mix_Var", "MeanAbsInfluence"]300 else:301 sens_rank_df = pd.DataFrame()302 303 return msg, per_out, sens_rank_df, [], sens_table304 305 306# ---------------- Tab 3: kNN + GMM & explainability -----------307 308def ui_neighbors_and_clusters(state_dict, k_neighbors, auto_k, manual_k, tag):309 """310 Run:311 - kNN analogs & manifold distance312 - GMM clustering on outputs313 - Cluster explainability (PCA + logit drivers)314 315 All errors are caught and reported in the Status textbox.316 """317 empty_df = pd.DataFrame()318 none_img = None # safest "no image" for type="filepath"319 320 # Need a previous inverse run (Tab 1)321 if not state_dict or "report_path" not in state_dict or "elites_path" not in state_dict:322 msg = "[ERROR] No inverse run in this session. Please run Tab 1 first."323 return (324 msg,325 empty_df,326 empty_df,327 none_img,328 empty_df,329 none_img,330 none_img,331 none_img,332 )333 334 report_path = state_dict["report_path"]335 elites_path = state_dict["elites_path"]336 337 # Unified save directory338 save_dir = os.path.abspath(CFG.get("save_dir", "./inverse_results"))339 os.makedirs(save_dir, exist_ok=True)340 341 # k value342 try:343 k = int(k_neighbors) if k_neighbors is not None else 10344 if k <= 0:345 k = 10346 except Exception:347 k = 10348 349 save_tag = tag or "nn_gmm"350 351 # Initialize outputs352 nn_neighbors_df = empty_df353 nn_summary_df = empty_df354 nn_plot_img = none_img355 gmm_summary_df = empty_df356 gmm_plot_img = none_img357 expl_pca_img = none_img358 expl_drivers_img = none_img359 360 messages = []361 362 # --------- 1) kNN analogs ----------363 try:364 res_nn = nearest_neighbor_analogs_and_manifold_check(365 report_csv_path=report_path,366 elites_csv_path=elites_path,367 k=k,368 save_dir=save_dir,369 save_tag=save_tag,370 include_neighbor_mixes=False,371 )372 nn_neighbors_df = res_nn.get("neighbors_table", empty_df)373 nn_summary_df = res_nn.get("summary", empty_df)374 nn_plot_img = res_nn.get("paths", {}).get("plot_png", None)375 d_scaled = res_nn.get("base_manifold_scaled_distance", None)376 if d_scaled is not None:377 messages.append(378 f"[OK] kNN analogs computed. Base manifold scaled distance = {d_scaled:.3f}"379 )380 else:381 messages.append("[OK] kNN analogs computed.")382 except Exception as e:383 messages.append(f"[ERROR] nearest_neighbor_analogs_and_manifold_check failed: {e}")384 385 # --------- 2) GMM clustering ----------386 xlsx_path = None387 try:388 if auto_k:389 n_components = None390 components_grid = list(range(2, 8))391 else:392 try:393 n_components = int(manual_k) if manual_k is not None else 3394 if n_components <= 1:395 n_components = 3396 except Exception:397 n_components = 3398 components_grid = None399 400 res_gmm = gmm_cluster_on_outputs(401 report_csv_path=report_path,402 elites_csv_path=elites_path,403 n_components=n_components,404 components_grid=components_grid,405 save_tag=save_tag,406 save_dir=save_dir,407 )408 gmm_summary_df = res_gmm.get("cluster_summary", empty_df)409 gmm_plot_img = res_gmm.get("paths", {}).get("plot_png", None)410 xlsx_path = res_gmm.get("paths", {}).get("workbook_xlsx", None)411 messages.append("[OK] GMM clustering on outputs computed.")412 messages.append(f"[DEBUG] GMM workbook: {xlsx_path}")413 except Exception as e:414 messages.append(f"[ERROR] gmm_cluster_on_outputs failed: {e}")415 xlsx_path = None416 417 # --------- 3) Cluster explainability ----------418 try:419 if xlsx_path:420 res_expl = cluster_explainability_outputs(421 gmm_workbook_xlsx=xlsx_path,422 save_dir=save_dir,423 save_tag=save_tag,424 )425 expl_paths = res_expl.get("paths", {})426 expl_pca_img = expl_paths.get("pca_png", None)427 expl_drivers_img = expl_paths.get("drivers_png", None)428 429 # Debug info + existence checks430 if expl_pca_img:431 exists_pca = os.path.exists(expl_pca_img)432 messages.append(433 f"[OK] Cluster explainability PCA path: {expl_pca_img} (exists={exists_pca})"434 )435 if not exists_pca:436 expl_pca_img = None437 else:438 messages.append("[WARN] No PCA image path returned from explainability.")439 440 if expl_drivers_img:441 exists_drv = os.path.exists(expl_drivers_img)442 messages.append(443 f"[OK] Cluster explainability drivers path: {expl_drivers_img} (exists={exists_drv})"444 )445 if not exists_drv:446 expl_drivers_img = None447 else:448 messages.append(449 "[WARN] No drivers image path returned from explainability (logit may have failed)."450 )451 452 messages.append("[OK] Cluster explainability computed.")453 else:454 messages.append(455 "[WARN] No workbook path returned from GMM; skipping cluster explainability."456 )457 except Exception as e:458 messages.append(f"[WARN] cluster_explainability_outputs failed: {e}")459 460 if not messages:461 messages.append("[WARN] Nothing executed (unexpected).")462 463 msg = "\n".join(messages)464 465 return (466 msg,467 nn_neighbors_df,468 nn_summary_df,469 nn_plot_img,470 gmm_summary_df,471 gmm_plot_img,472 expl_pca_img,473 expl_drivers_img,474 )475 476 477# ---------------- Tab 4: Engineering Reality-Gap --------------478 479def ui_reality_gap(state_dict, tag):480 if not state_dict or "report_path" not in state_dict:481 return (482 "[ERROR] No inverse run in this session. Run Tab 1 first.",483 pd.DataFrame(),484 pd.DataFrame(),485 pd.DataFrame(),486 "",487 )488 489 report_path = state_dict["report_path"]490 save_tag = tag or "gap"491 492 try:493 res_gap = engineering_reality_gap_analysis(494 report_csv_path=report_path,495 save_dir=CFG["save_dir"],496 save_tag=save_tag,497 )498 except Exception as e:499 return (500 f"[ERROR] engineering_reality_gap_analysis failed: {e}",501 pd.DataFrame(),502 pd.DataFrame(),503 pd.DataFrame(),504 "",505 )506 507 raw_mix_df = res_gap.get("raw_mix_reality", pd.DataFrame())508 kpi_df = res_gap.get("kpi_reality", pd.DataFrame())509 summary_df = res_gap.get("summary_table", pd.DataFrame())510 xlsx_path = res_gap.get("paths", {}).get("xlsx", "")511 512 msg = "[OK] Engineering reality-gap computed.\n" f"Workbook: {xlsx_path}"513 return msg, raw_mix_df, kpi_df, summary_df, xlsx_path514 515 516 517# ---------------------------------------------------------518# Build Gradio UI519# ---------------------------------------------------------520 521with gr.Blocks(title="3DCP Evolutionary Inverse Design & Analysis") as demo:522 gr.Markdown(523 "## 🧱 3DCP Evolutionary Inverse-Design Dashboard\n"524 "Inverse mix design (currently nearest-neighbor baseline) + analysis:\n"525 "- errors & sensitivities\n"526 "- kNN analogs & clusters\n"527 "- engineering reality check (dataset vs inverse)."528 )529 530 state = gr.State({})531 532 # Tab 1533 with gr.Tab("1️⃣ Inverse Evolutionary (Targets → Mix)"):534 gr.Markdown("### Set target performance and run inverse design")535 536 with gr.Row():537 with gr.Column():538 ps = gr.Number(label="Target Pump-Speed", value=75.0)539 fer = gr.Number(label="Target Flow Extrusion Rate", value=8.0)540 sys = gr.Number(label="Target Static Yield Stress", value=1800.0)541 vm = gr.Number(label="Target Viscosity Mixture", value=15.0)542 cs = gr.Number(label="Target CS", value=60.0)543 with gr.Column():544 iterations = gr.Number(545 label="Search iterations (unused for now)", value=5000, precision=0546 )547 exploration = gr.Number(548 label="Exploration parameter (unused for now)", value=0.2549 )550 save_tag = gr.Textbox(label="Save tag", value="all5_ui")551 run_btn = gr.Button("🚀 Run Inverse (baseline)", variant="primary")552 553 msg_inv = gr.Textbox(label="Status", interactive=False)554 report_path_out = gr.Textbox(label="REPORT path", interactive=False)555 elites_path_out = gr.Textbox(label="ELITES path", interactive=False)556 557 gr.Markdown("#### Inverse-designed mix (ingredients)")558 mix_df_out = gr.Dataframe(559 headers=["Ingredient", "Value"],560 datatype=["str", "number"],561 interactive=False,562 label="Inverse Mix",563 )564 565 gr.Markdown("#### Predicted vs Target outputs")566 preds_df_out = gr.Dataframe(567 headers=["Output", "Predicted", "Target"],568 datatype=["str", "number", "number"],569 interactive=False,570 label="Predicted vs Target",571 )572 573 run_btn.click(574 fn=ui_run_inverse,575 inputs=[ps, fer, sys, vm, cs, iterations, exploration, save_tag, state],576 outputs=[577 mix_df_out,578 preds_df_out,579 msg_inv,580 report_path_out,581 elites_path_out,582 state,583 ],584 )585 586 # Tab 2587 with gr.Tab("2️⃣ Quality & Sensitivity"):588 gr.Markdown("### Error analysis and local robustness around the inverse mix")589 590 do_local_sens = gr.Checkbox(591 label="Compute constraint-aware local sensitivity", value=True592 )593 delta_frac = gr.Number(594 label="Local sensitivity δ (fraction of total mix per variable)",595 value=0.05,596 )597 run_qs_btn = gr.Button(598 "📊 Run Quality & Sensitivity Analysis", variant="primary"599 )600 601 msg_qs = gr.Textbox(label="Status", interactive=False)602 per_out_df = gr.Dataframe(label="Per-output error table")603 sens_rank_df = gr.Dataframe(label="Local sensitivity rank (central diff)")604 sens_loc_df = gr.Dataframe(label="Constraint-aware local sensitivity table")605 606 run_qs_btn.click(607 fn=ui_quality_and_sensitivity,608 inputs=[state, do_local_sens, delta_frac],609 outputs=[msg_qs, per_out_df, sens_rank_df, gr.State(), sens_loc_df],610 )611 612 # Tab 3613 with gr.Tab("3️⃣ kNN Analogs & Clusters"):614 gr.Markdown("### Nearest-neighbor analogs, GMM clusters, and explainability")615 616 with gr.Row():617 k_neighbors = gr.Number(618 label="k (nearest neighbors)", value=10, precision=0619 )620 auto_k = gr.Checkbox(621 label="Auto-select GMM clusters via BIC", value=True622 )623 manual_k = gr.Number(624 label="Manual clusters (if auto_k=False)", value=3, precision=0625 )626 tag_nc = gr.Textbox(label="Save tag", value="demo")627 628 run_nc_btn = gr.Button(629 "🔎 Run kNN + GMM + Explainability", variant="primary"630 )631 632 msg_nc = gr.Textbox(label="Status", interactive=False)633 634 gr.Markdown("#### Nearest neighbors in dataset")635 nn_neighbors_df = gr.Dataframe(636 label="Neighbor samples (true outputs, distances)"637 )638 nn_summary_df = gr.Dataframe(639 label="Output summary (Target vs NN mean true)"640 )641 nn_plot_img = gr.Image(642 label="Targets vs Neighbor Mean (True)", type="filepath"643 )644 645 gr.Markdown("#### GMM cluster summary")646 gmm_summary_df = gr.Dataframe(label="Cluster output stats (mean/std)")647 gmm_plot_img = gr.Image(648 label="GMM: Dataset mean vs cluster mean vs inverse", type="filepath"649 )650 651 gr.Markdown("#### Cluster explainability")652 expl_pca_img = gr.Image(653 label="PCA clusters (outputs) with centroids", type="filepath"654 )655 expl_drivers_img = gr.Image(656 label="Global cluster drivers (|logit coef|)", type="filepath"657 )658 659 run_nc_btn.click(660 fn=ui_neighbors_and_clusters,661 inputs=[state, k_neighbors, auto_k, manual_k, tag_nc],662 outputs=[663 msg_nc,664 nn_neighbors_df,665 nn_summary_df,666 nn_plot_img,667 gmm_summary_df,668 gmm_plot_img,669 expl_pca_img,670 expl_drivers_img,671 ],672 )673 674 # Tab 4675 with gr.Tab("4️⃣ Engineering Reality-Gap"):676 gr.Markdown(677 "### Compare inverse mix to dataset & KPIs (engineering reality check)"678 )679 680 tag_gap = gr.Textbox(label="Save tag", value="gap")681 run_gap_btn = gr.Button("🧪 Run Reality-Gap Analysis", variant="primary")682 683 msg_gap = gr.Textbox(label="Status", interactive=False)684 raw_mix_df = gr.Dataframe(685 label="Raw variables: inverse vs dataset (z-scores, percentiles)"686 )687 kpi_df = gr.Dataframe(688 label="Engineering KPIs (w/c, binder fraction, SP/binder, etc.)"689 )690 summary_gap_df = gr.Dataframe(691 label="Summary (out-of-distribution counts)"692 )693 gap_xlsx_path = gr.Textbox(label="Workbook path", interactive=False)694 695 run_gap_btn.click(696 fn=ui_reality_gap,697 inputs=[state, tag_gap],698 outputs=[msg_gap, raw_mix_df, kpi_df, summary_gap_df, gap_xlsx_path],699 )700 701 702if __name__ == "__main__":703 demo.launch()704