ThinkingBit/Efficient_Deep_IAD
0
1# Import required libraries2import time3from pathlib import Path4import numpy as np5import gradio as gr6import matplotlib.pyplot as plt7import seaborn as sns8import pandas as pd9import openvino as ov10from anomalib.deploy import OpenVINOInferencer11 12# Instantiate the OpenVINO Core13core = ov.Core()14 15# Retrieve available inference devices16device_list = core.get_available_devices()17 18# Set global variables19## Inferencers20ov_inferencer = None21 22## User Selections23object_category = None24prev_object_category = None25prev_model_name = None26prev_selected_device = None27 28example_list = [["bottle/examples/broken_large_004.png","segmentations","bottle",["stfpm", "padim"], "CPU",120],29 ["cable/examples/missing_wire_003.png","heat map","cable",["patchcore","cflow"], "CPU",90],30 ["grid/examples/broken_001.png","predicted mask","grid",["padim","patchcore"], "CPU",60],31 ["hazelnut/examples/print_000.png","anomaly map","hazelnut",["efficient_ad","stfpm"], "CPU",45],32 ["metal_nut/examples/bent_024.png","segmentations","metal_nut",["cflow","padim"], "CPU",30]]33 34# Compile OpenVINO inferencer35def compile_OV_model(object_category: str, model_name: str, selected_device):36 """Compiles relevant OpenVINO model for inference based on user selections."""37 global ov_inferencer38 39 model_path = Path.cwd() / object_category / "models" / model_name / "weights" / "openvino" / "model.bin"40 metadata_path = Path.cwd() / object_category / "models" / model_name / "weights" / "openvino" / "metadata.json"41 42 ov_inferencer = OpenVINOInferencer(43 path=model_path,44 metadata=metadata_path,45 device=selected_device46 )47 return ov_inferencer48 49# Run inference50def run_OV_inference(input_image, visual_output_choice: str):51 """Runs inference on given input image."""52 # Start the timer53 start_time = time.perf_counter()54 55 # Run inference56 prediction_results = ov_inferencer.predict(image=input_image)57 58 # End the timer59 end_time = time.perf_counter()60 time_to_inference = end_time - start_time61 62 # Extract predictions63 confidence_score = prediction_results.pred_score64 65 if visual_output_choice == "segmentations":66 output_image = prediction_results.segmentations67 elif visual_output_choice == "anomaly map":68 output_image = prediction_results.anomaly_map69 elif visual_output_choice == "heat map":70 output_image = prediction_results.heat_map71 elif visual_output_choice == "predicted mask":72 output_image = prediction_results.pred_mask73 else:74 output_image = prediction_results.image75 76 return output_image, round(confidence_score*100, 2), round(time_to_inference*1000)77 78# Compile/Re-compile and run inference79def compile_plus_run_OV_model(object_category: str, model_name: str, selected_device, input_image, visual_output_choice):80 """Compiles or Re-compiles the model inferencer if any model-related user selections are modified then81 runs inference.82 """83 global prev_object_category84 global prev_model_name85 global prev_selected_device86 87 # Compile/re-compile OpenVINO Inferencer if user selection changes (optional)88 if selected_device != prev_selected_device or object_category != prev_object_category or model_name != prev_model_name:89 compile_OV_model(object_category, model_name, selected_device)90 prev_object_category = object_category91 prev_model_name = model_name92 prev_selected_device = selected_device93 94 # Run model inference95 output_image, output_conf_score, output_inf_time = run_OV_inference(input_image, visual_output_choice)96 return output_image, output_conf_score, output_inf_time97 98# Extract then merge Pixel-AUROC data & model outputs99def load_transform_data(object_category, selected_models, inf_values):100 """Extracts & merges the model Pixel-AUROC data from a CSV file and model inference latencies101 into a single dataframe."""102 # Load the pixel AUROC data103 pixel_auroc_data = pd.read_csv('pixel_auroc_data.csv')104 105 # Load the latency data106 latency_data = {"model": selected_models,107 "latency": inf_values}108 109 # Filter data for the selected object category110 selected_pixel_auroc_data = pixel_auroc_data[['model', object_category]].copy()111 112 # Convert latency data into dataframe113 selected_latency_data = pd.DataFrame(latency_data)114 115 # Merge the two dataframes on the 'Model' column116 merged_data = pd.merge(selected_pixel_auroc_data, selected_latency_data, on='model')117 118 # Rename columns for clarity119 merged_data.columns = ['Model', 'Pixel_AUROC', 'Latency']120 121 # Display the resulting dataframe122 123 return merged_data124 125# Convert FPS throughput to latency126def latency_calc(fps_threshold):127 """Converts frames-per-second (fps_threshold) throughput threshold to latency in milliseconds(ms)"""128 # Latency calculation129 min_latency_threshold = 1000 / fps_threshold130 return min_latency_threshold131 132# Plot model comparison chart133def plot_grouped_bar(dataframe, object_category, fps_threshold):134 """Plots grouped bar chart of Pixel-AUROC and Latency Values of selected models."""135 # Extracting data from the DataFrame136 models = dataframe['Model'].tolist()137 latencies = dataframe['Latency'].tolist()138 auroc_scores = dataframe['Pixel_AUROC'].tolist()139 140 min_latency_threshold = latency_calc(fps_threshold)141 142 # Set a seaborn color palette for colorblind-friendliness143 colors = sns.color_palette("colorblind", n_colors=4)144 145 # Plotting146 bar_width = 0.35147 index = np.arange(len(models))148 149 fig, ax1 = plt.subplots(figsize=(15, 6))150 151 # Inference Latencies152 ax1.bar(index, latencies, bar_width, label=f'Inference Latency ({object_category})', color=colors[0])153 154 # Adding data labels155 for bar in ax1.patches:156 yval = bar.get_height()157 ax1.text(bar.get_x() + bar.get_width()/2, yval, round(yval, 2), ha='center', va='bottom')158 159 ax1.set_xlabel('Models')160 ax1.set_ylabel('Latency (ms)', color=colors[0])161 ax1.tick_params(axis='y', labelcolor=colors[0])162 163 # Create a secondary y-axis for Pixel AUROC164 ax2 = ax1.twinx()165 ax2.bar(index + bar_width, auroc_scores, bar_width, label=f'Pixel-AUROC ({object_category})', color=colors[2])166 167 # Adding data labels for Pixel AUROC168 for bar in ax2.patches:169 yval = bar.get_height()170 ax2.text(bar.get_x() + bar.get_width()/2, yval, round(yval, 2), ha='center', va='bottom')171 172 ax2.set_ylabel('Pixel AUROC', color=colors[2])173 ax2.tick_params(axis='y', labelcolor=colors[2])174 175 # Minimum Latency Threshold Line176 ax1.axhline(y=min_latency_threshold, color=colors[3], linestyle='--', label=f'Min. Latency Threshold (60fps)')177 178 # Adding labels and title179 plt.title(f'CPU Inference Latencies & Pixel-AUROC (mean) Scores for Selected Models ({object_category})')180 plt.xticks(index + bar_width / 2, models)181 fig.tight_layout()182 183 # Place legend above the chart area184 lines, labels = ax1.get_legend_handles_labels()185 lines2, labels2 = ax2.get_legend_handles_labels()186 lines.extend(lines2)187 labels.extend(labels2)188 plt.legend(lines, labels, loc='lower center', bbox_to_anchor=(0.5, 1.15), ncol=3)189 190 return fig191 192# Extract user inputs and return dynamic outputs193def run_model_selection(object_category: str, model_selection: list,194 selected_device: str, fps_threshold, input_image: np.array,195 visual_output_choice: str):196 """Extracts user inputs and returns the relevant model outputs"""197 198 selected_models = []199 inf_values = []200 201 model_outputs = {}202 203 # Run inference for each selected model204 for model in model_selection:205 img, conf_score, inf_time = compile_plus_run_OV_model(object_category, model, selected_device,206 input_image,visual_output_choice)207 208 # Update model outputs (image, confidence score and inference time)209 model_outputs.update({model_to_ui_output[model][0]: img})210 model_outputs.update({model_to_ui_output[model][1]: conf_score})211 model_outputs.update({model_to_ui_output[model][2]: inf_time})212 213 # Save model names and inference values214 selected_models.append(model)215 inf_values.append(inf_time)216 217 model_plot = plot_grouped_bar(load_transform_data(object_category, selected_models, inf_values),218 object_category, fps_threshold)219 model_outputs.update({model_comparison_plot: model_plot})220 221 return model_outputs222 223 224# Gradio UI menu variables225object_list = ["bottle", "cable", "grid", "hazelnut", "metal_nut"]226model_list = ["cflow", "efficient_ad", "padim", "patchcore", "stfpm"]227visual_output_list = ["anomaly map", "heat map",228 "predicted mask", "segmentations"]229 230# Gradio UI231with gr.Blocks() as demo:232 # Header233 gr.Markdown("""234 <img align="left" width="150" src= "https://github.com/openvinotoolkit/anomalib/assets/10940214/7e61a627-d1b0-4ad4-b602-da9b348c0cbe"> 235 <img align="right" width="150" src= "https://github.com/openvinotoolkit/anomalib/assets/10940214/5d6dd038-b40c-441f-ad38-1cf526137de2">236 <h1 align="center"> Benchmarking Deep Anomaly Detection Models </h1>""")237 238 with gr.Row():239 with gr.Column():240 gr.Markdown(241 """242 Benchmark the performance of multiple state-of-the-art anomaly detection models implemented using the Anomalib-OpenVINO toolkit.243 All models were trained on the different objects of MVTecAD visual anomaly dataset.244 245 This demo app allows you to compare and contrast the varying image outputs, average pixel-AUROC scores over the test set alongside inference latency performance for a set throughput threshold.246 """247 ) 248 249 with gr.Column():250 gr.Markdown(251 """252 <img src="https://github.com/openvinotoolkit/openvino_notebooks/assets/10940214/45dfb61f-c6d1-4098-88d1-8498f0a42e11" alt="drawing" width="500"/>253 """254 ) 255 256 # Select Object Category257 gr.Markdown("## Step 1: Select an object category to detect.")258 object_category = gr.Dropdown(object_list, label="Choose the object type")259 260 # Select Model261 gr.Markdown("## Step 2: Select the model(s) you want to benchmark.")262 model_checkbox = gr.CheckboxGroup(model_list, label="Choose anomaly detection models to compare")263 264 # Select Visual Output265 gr.Markdown("## Step 3: Select the type of model output you want.")266 visual_output_choice = gr.Radio(visual_output_list, label="Select model output")267 268 # Select Inference Device269 gr.Markdown("## Step 4: Choose your inference device.")270 selected_device = gr.Dropdown(device_list, label="Select device")271 272 # Input Throughput Threshold273 gr.Markdown("## Step 5: Enter the throughput threshold for your application (in frames-per-second (FPS)).")274 fps_threshold = gr.Number(60, label="Throughput Threshold in FPS")275 276 # Input Model Image277 gr.Markdown("## Step 6: Upload your image and run inference.")278 input_image = gr.Image(type="numpy", label="Input Image")279 280 # Run Inference281 run_inference_btn = gr.Button(value="Run Inference")282 283 # Comparison Plot284 model_comparison_plot = gr.Plot(label="Pixel AUROC and OpenVINO Inference Latencies of S.O.T.A Models")285 286 # Cflow287 with gr.Column(visible=False) as cflow:288 cflow_img_output = gr.Image(type="numpy", label=f"Cflow Model Output", )289 with gr.Row():290 cflow_conf_score = gr.Textbox(label="Confidence Score (%)")291 cflow_time = gr.Textbox(label="Inference Time (ms)")292 293 # EfficientAD294 with gr.Column(visible=False) as efficient_ad:295 efficient_ad_img_output = gr.Image(type="numpy", label=f"EfficientAD Model Output")296 with gr.Row():297 efficient_ad_conf_score = gr.Textbox(label="Confidence Score (%)")298 efficient_ad_time = gr.Textbox(label="Inference Time (ms)")299 300 # PADIM301 with gr.Column(visible=False) as padim:302 padim_img_output = gr.Image(type="numpy", label=f"PADIM Model Output")303 with gr.Row():304 padim_conf_score = gr.Textbox(label="Confidence Score (%)")305 padim_time = gr.Textbox(label="Inference Time (ms)")306 307 # Patchcore308 with gr.Column(visible=False) as patchcore:309 patchcore_img_output = gr.Image(type="numpy", label=f"Patchcore Model Output")310 with gr.Row():311 patchcore_conf_score = gr.Textbox(label="Confidence Score (%)")312 patchcore_time = gr.Textbox(label="Inference Time (ms)")313 314 # STFPM315 with gr.Column(visible=False) as stfpm:316 stfpm_img_output = gr.Image(type="numpy", label=f"STFPM Model Output")317 with gr.Row():318 stfpm_conf_score = gr.Textbox(label="Confidence Score (%)")319 stfpm_time = gr.Textbox(label="Inference Time (ms)")320 321 gr.Markdown("## OR use any of these examples for a quick start")322 gr.Examples(323 examples=example_list,324 inputs=[input_image, visual_output_choice, object_category, model_checkbox,325 selected_device, fps_threshold])326 327 328 # Map model names to respective UI components329 model_ui_components = {330 "cflow": cflow,331 "efficient_ad": efficient_ad,332 "padim": padim,333 "patchcore": patchcore,334 "stfpm": stfpm335 }336 337 # Map model names to respective outputs338 model_to_ui_output = {339 "cflow": [cflow_img_output, cflow_conf_score, cflow_time],340 "efficient_ad": [efficient_ad_img_output, efficient_ad_conf_score, efficient_ad_time],341 "padim": [padim_img_output, padim_conf_score, padim_time],342 "patchcore": [patchcore_img_output, patchcore_conf_score, patchcore_time],343 "stfpm": [stfpm_img_output, stfpm_conf_score, stfpm_time]344 }345 346 # Display UI component blocks for each model dynamically347 def variable_outputs(model_selection):348 """Toggles relevant model display outputs on/off based on current user selection."""349 global model_list350 global model_ui_components 351 352 # Initialize the output dictionary353 current_model_selection = {}354 355 # Initialize selected and unselected model lists356 selected_models = [model for model in model_selection if model in model_list]357 unselected_models = [model for model in model_list if model not in model_selection]358 359 # Modify the selection to reflect in the UI360 selected_dict = {model_ui_components [model]: gr.Column(visible=True) for model in selected_models}361 unselected_dict = {model_ui_components [model]: gr.Column(visible=False) for model in unselected_models}362 363 # Update the user selection state364 current_model_selection.update(selected_dict)365 current_model_selection.update(unselected_dict)366 367 return current_model_selection 368 369 # Event Handlers370 ## Check if checkbox input has changed371 model_checkbox.change(variable_outputs, model_checkbox,372 [cflow, efficient_ad, padim, patchcore, stfpm])373 374 ## Run inference on button click event375 run_inference_btn.click(run_model_selection,376 inputs=[object_category, model_checkbox, selected_device,377 fps_threshold, input_image, visual_output_choice],378 outputs = [model_comparison_plot,379 cflow_img_output, cflow_conf_score, cflow_time, # cflow outputs380 efficient_ad_img_output, efficient_ad_conf_score, efficient_ad_time, # efficient_ad outputs381 padim_img_output, padim_conf_score, padim_time, # padim outputs382 patchcore_img_output, patchcore_conf_score, patchcore_time, # patchcore outputs383 stfpm_img_output, stfpm_conf_score, stfpm_time]384 )385 386if __name__=="__main__":387 demo.launch()