iasjkk/bbox_detection
0
1import os2import sys3import random4import itertools5import colorsys6 7import numpy as np8from skimage.measure import find_contours9import matplotlib.pyplot as plt10from matplotlib import patches, lines11from matplotlib.patches import Polygon12import IPython.display13 14# Root directory of the project15ROOT_DIR = os.path.abspath("../")16 17 18sys.path.append(ROOT_DIR) # To find local version of the library19from bboxcnn import utils20 21 22############################################################23# Visualization24############################################################25 26def display_images(images, titles=None, cols=4, cmap=None, norm=None,27 interpolation=None):28 """Display the given set of images, optionally with titles.29 images: list or array of image tensors in HWC format.30 titles: optional. A list of titles to display with each image.31 cols: number of images per row32 cmap: Optional. Color map to use. For example, "Blues".33 norm: Optional. A Normalize instance to map values to colors.34 interpolation: Optional. Image interpolation to use for display.35 """36 titles = titles if titles is not None else [""] * len(images)37 rows = len(images) // cols + 138 plt.figure(figsize=(14, 14 * rows // cols))39 i = 140 for image, title in zip(images, titles):41 plt.subplot(rows, cols, i)42 plt.title(title, fontsize=9)43 plt.axis('off')44 plt.imshow(image.astype(np.uint8), cmap=cmap,45 norm=norm, interpolation=interpolation)46 i += 147 plt.show()48 49 50def random_colors(N, bright=True):51 """52 Generate random colors.53 To get visually distinct colors, generate them in HSV space then54 convert to RGB.55 """56 brightness = 1.0 if bright else 0.757 hsv = [(i / N, 1, brightness) for i in range(N)]58 colors = list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv))59 random.shuffle(colors)60 return colors61 62 63def apply_mask(image, mask, color, alpha=0.5):64 """Apply the given mask to the image.65 """66 for c in range(3):67 image[:, :, c] = np.where(mask == 1,68 image[:, :, c] *69 (1 - alpha) + alpha * color[c] * 255,70 image[:, :, c])71 return image72 73 74def display_instances(image, boxes, masks, class_ids, class_names,75 scores=None, title="",76 figsize=(16, 16), ax=None,77 show_mask=True, show_bbox=True,78 colors=None, captions=None):79 """80 boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.81 masks: [height, width, num_instances]82 class_ids: [num_instances]83 class_names: list of class names of the dataset84 scores: (optional) confidence scores for each box85 title: (optional) Figure title86 show_mask, show_bbox: To show masks and bounding boxes or not87 figsize: (optional) the size of the image88 colors: (optional) An array or colors to use with each object89 captions: (optional) A list of strings to use as captions for each object90 """91 # Number of instances92 N = boxes.shape[0]93 if not N:94 print("\n*** No instances to display *** \n")95 else:96 assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]97 98 # If no axis is passed, create one and automatically call show()99 auto_show = False100 if not ax:101 _, ax = plt.subplots(1, figsize=figsize)102 auto_show = True103 104 # Generate random colors105 colors = colors or random_colors(N)106 107 # Show area outside image boundaries.108 height, width = image.shape[:2]109 ax.set_ylim(height + 10, -10)110 ax.set_xlim(-10, width + 10)111 ax.axis('off')112 ax.set_title(title)113 114 masked_image = image.astype(np.uint32).copy()115 for i in range(N):116 color = colors[i]117 118 # Bounding box119 if not np.any(boxes[i]):120 # Skip this instance. Has no bbox. Likely lost in image cropping.121 continue122 y1, x1, y2, x2 = boxes[i]123 if show_bbox:124 p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,125 alpha=0.7, linestyle="dashed",126 edgecolor=color, facecolor='none')127 ax.add_patch(p)128 129 # Label130 if not captions:131 class_id = class_ids[i]132 score = scores[i] if scores is not None else None133 label = class_names[class_id]134 caption = "{} {:.3f}".format(label, score) if score else label135 else:136 caption = captions[i]137 ax.text(x1, y1 + 8, caption,138 color='w', size=11, backgroundcolor="none")139 140 # Mask141 mask = masks[:, :, i]142 if show_mask:143 masked_image = apply_mask(masked_image, mask, color)144 145 # Mask Polygon146 # Pad to ensure proper polygons for masks that touch image edges.147 padded_mask = np.zeros(148 (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)149 padded_mask[1:-1, 1:-1] = mask150 contours = find_contours(padded_mask, 0.5)151 for verts in contours:152 # Subtract the padding and flip (y, x) to (x, y)153 verts = np.fliplr(verts) - 1154 p = Polygon(verts, facecolor="none", edgecolor=color)155 ax.add_patch(p)156 ax.imshow(masked_image.astype(np.uint8))157 if auto_show:158 plt.show()159 160 161def display_differences(image,162 gt_box, gt_class_id, gt_mask,163 pred_box, pred_class_id, pred_score, pred_mask,164 class_names, title="", ax=None,165 show_mask=True, show_box=True,166 iou_threshold=0.5, score_threshold=0.5):167 """Display ground truth and prediction instances on the same image."""168 # Match predictions to ground truth169 gt_match, pred_match, overlaps = utils.compute_matches(170 gt_box, gt_class_id, gt_mask,171 pred_box, pred_class_id, pred_score, pred_mask,172 iou_threshold=iou_threshold, score_threshold=score_threshold)173 # Ground truth = green. Predictions = red174 colors = [(0, 1, 0, .8)] * len(gt_match)\175 + [(1, 0, 0, 1)] * len(pred_match)176 # Concatenate GT and predictions177 class_ids = np.concatenate([gt_class_id, pred_class_id])178 scores = np.concatenate([np.zeros([len(gt_match)]), pred_score])179 boxes = np.concatenate([gt_box, pred_box])180 masks = np.concatenate([gt_mask, pred_mask], axis=-1)181 # Captions per instance show score/IoU182 captions = ["" for m in gt_match] + ["{:.2f} / {:.2f}".format(183 pred_score[i],184 (overlaps[i, int(pred_match[i])]185 if pred_match[i] > -1 else overlaps[i].max()))186 for i in range(len(pred_match))]187 # Set title if not provided188 title = title or "Ground Truth and Detections\n GT=green, pred=red, captions: score/IoU"189 # Display190 display_instances(191 image,192 boxes, masks, class_ids,193 class_names, scores, ax=ax,194 show_bbox=show_box, show_mask=show_mask,195 colors=colors, captions=captions,196 title=title)197 198 199def draw_rois(image, rois, refined_rois, mask, class_ids, class_names, limit=10):200 """201 anchors: [n, (y1, x1, y2, x2)] list of anchors in image coordinates.202 proposals: [n, 4] the same anchors but refined to fit objects better.203 """204 masked_image = image.copy()205 206 # Pick random anchors in case there are too many.207 ids = np.arange(rois.shape[0], dtype=np.int32)208 ids = np.random.choice(209 ids, limit, replace=False) if ids.shape[0] > limit else ids210 211 fig, ax = plt.subplots(1, figsize=(12, 12))212 if rois.shape[0] > limit:213 plt.title("Showing {} random ROIs out of {}".format(214 len(ids), rois.shape[0]))215 else:216 plt.title("{} ROIs".format(len(ids)))217 218 # Show area outside image boundaries.219 ax.set_ylim(image.shape[0] + 20, -20)220 ax.set_xlim(-50, image.shape[1] + 20)221 ax.axis('off')222 223 for i, id in enumerate(ids):224 color = np.random.rand(3)225 class_id = class_ids[id]226 # ROI227 y1, x1, y2, x2 = rois[id]228 p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,229 edgecolor=color if class_id else "gray",230 facecolor='none', linestyle="dashed")231 ax.add_patch(p)232 # Refined ROI233 if class_id:234 ry1, rx1, ry2, rx2 = refined_rois[id]235 p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2,236 edgecolor=color, facecolor='none')237 ax.add_patch(p)238 # Connect the top-left corners of the anchor and proposal for easy visualization239 ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))240 241 # Label242 label = class_names[class_id]243 ax.text(rx1, ry1 + 8, "{}".format(label),244 color='w', size=11, backgroundcolor="none")245 246 # Mask247 m = utils.unmold_mask(mask[id], rois[id]248 [:4].astype(np.int32), image.shape)249 masked_image = apply_mask(masked_image, m, color)250 251 ax.imshow(masked_image)252 253 # Print stats254 print("Positive ROIs: ", class_ids[class_ids > 0].shape[0])255 print("Negative ROIs: ", class_ids[class_ids == 0].shape[0])256 print("Positive Ratio: {:.2f}".format(257 class_ids[class_ids > 0].shape[0] / class_ids.shape[0]))258 259 260# TODO: Replace with matplotlib equivalent?261def draw_box(image, box, color):262 """Draw 3-pixel width bounding boxes on the given image array.263 color: list of 3 int values for RGB.264 """265 y1, x1, y2, x2 = box266 image[y1:y1 + 2, x1:x2] = color267 image[y2:y2 + 2, x1:x2] = color268 image[y1:y2, x1:x1 + 2] = color269 image[y1:y2, x2:x2 + 2] = color270 return image271 272 273def display_top_masks(image, mask, class_ids, class_names, limit=4):274 """Display the given image and the top few class masks."""275 to_display = []276 titles = []277 to_display.append(image)278 titles.append("H x W={}x{}".format(image.shape[0], image.shape[1]))279 # Pick top prominent classes in this image280 unique_class_ids = np.unique(class_ids)281 mask_area = [np.sum(mask[:, :, np.where(class_ids == i)[0]])282 for i in unique_class_ids]283 top_ids = [v[0] for v in sorted(zip(unique_class_ids, mask_area),284 key=lambda r: r[1], reverse=True) if v[1] > 0]285 # Generate images and titles286 for i in range(limit):287 class_id = top_ids[i] if i < len(top_ids) else -1288 # Pull masks of instances belonging to the same class.289 m = mask[:, :, np.where(class_ids == class_id)[0]]290 m = np.sum(m * np.arange(1, m.shape[-1] + 1), -1)291 to_display.append(m)292 titles.append(class_names[class_id] if class_id != -1 else "-")293 display_images(to_display, titles=titles, cols=limit + 1, cmap="Blues_r")294 295 296def plot_precision_recall(AP, precisions, recalls):297 """Draw the precision-recall curve.298 299 AP: Average precision at IoU >= 0.5300 precisions: list of precision values301 recalls: list of recall values302 """303 # Plot the Precision-Recall curve304 _, ax = plt.subplots(1)305 ax.set_title("Precision-Recall Curve. AP@50 = {:.3f}".format(AP))306 ax.set_ylim(0, 1.1)307 ax.set_xlim(0, 1.1)308 _ = ax.plot(recalls, precisions)309 310 311def plot_overlaps(gt_class_ids, pred_class_ids, pred_scores,312 overlaps, class_names, threshold=0.5):313 """Draw a grid showing how ground truth objects are classified.314 gt_class_ids: [N] int. Ground truth class IDs315 pred_class_id: [N] int. Predicted class IDs316 pred_scores: [N] float. The probability scores of predicted classes317 overlaps: [pred_boxes, gt_boxes] IoU overlaps of predictions and GT boxes.318 class_names: list of all class names in the dataset319 threshold: Float. The prediction probability required to predict a class320 """321 gt_class_ids = gt_class_ids[gt_class_ids != 0]322 pred_class_ids = pred_class_ids[pred_class_ids != 0]323 324 plt.figure(figsize=(12, 10))325 plt.imshow(overlaps, interpolation='nearest', cmap=plt.cm.Blues)326 plt.yticks(np.arange(len(pred_class_ids)),327 ["{} ({:.2f})".format(class_names[int(id)], pred_scores[i])328 for i, id in enumerate(pred_class_ids)])329 plt.xticks(np.arange(len(gt_class_ids)),330 [class_names[int(id)] for id in gt_class_ids], rotation=90)331 332 thresh = overlaps.max() / 2.333 for i, j in itertools.product(range(overlaps.shape[0]),334 range(overlaps.shape[1])):335 text = ""336 if overlaps[i, j] > threshold:337 text = "match" if gt_class_ids[j] == pred_class_ids[i] else "wrong"338 color = ("white" if overlaps[i, j] > thresh339 else "black" if overlaps[i, j] > 0340 else "grey")341 plt.text(j, i, "{:.3f}\n{}".format(overlaps[i, j], text),342 horizontalalignment="center", verticalalignment="center",343 fontsize=9, color=color)344 345 plt.tight_layout()346 plt.xlabel("Ground Truth")347 plt.ylabel("Predictions")348 349 350def draw_boxes(image, boxes=None, refined_boxes=None,351 masks=None, captions=None, visibilities=None,352 title="", ax=None):353 """Draw bounding boxes and segmentation masks with different354 customizations.355 356 boxes: [N, (y1, x1, y2, x2, class_id)] in image coordinates.357 refined_boxes: Like boxes, but draw with solid lines to show358 that they're the result of refining 'boxes'.359 masks: [N, height, width]360 captions: List of N titles to display on each box361 visibilities: (optional) List of values of 0, 1, or 2. Determine how362 prominent each bounding box should be.363 title: An optional title to show over the image364 ax: (optional) Matplotlib axis to draw on.365 """366 # Number of boxes367 assert boxes is not None or refined_boxes is not None368 N = boxes.shape[0] if boxes is not None else refined_boxes.shape[0]369 370 # Matplotlib Axis371 if not ax:372 _, ax = plt.subplots(1, figsize=(12, 12))373 374 # Generate random colors375 colors = random_colors(N)376 377 # Show area outside image boundaries.378 margin = image.shape[0] // 10379 ax.set_ylim(image.shape[0] + margin, -margin)380 ax.set_xlim(-margin, image.shape[1] + margin)381 ax.axis('off')382 383 ax.set_title(title)384 385 masked_image = image.astype(np.uint32).copy()386 for i in range(N):387 # Box visibility388 visibility = visibilities[i] if visibilities is not None else 1389 if visibility == 0:390 color = "gray"391 style = "dotted"392 alpha = 0.5393 elif visibility == 1:394 color = colors[i]395 style = "dotted"396 alpha = 1397 elif visibility == 2:398 color = colors[i]399 style = "solid"400 alpha = 1401 402 # Boxes403 if boxes is not None:404 if not np.any(boxes[i]):405 # Skip this instance. Has no bbox. Likely lost in cropping.406 continue407 y1, x1, y2, x2 = boxes[i]408 p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,409 alpha=alpha, linestyle=style,410 edgecolor=color, facecolor='none')411 ax.add_patch(p)412 413 # Refined boxes414 if refined_boxes is not None and visibility > 0:415 ry1, rx1, ry2, rx2 = refined_boxes[i].astype(np.int32)416 p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2,417 edgecolor=color, facecolor='none')418 ax.add_patch(p)419 # Connect the top-left corners of the anchor and proposal420 if boxes is not None:421 ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))422 423 # Captions424 if captions is not None:425 caption = captions[i]426 # If there are refined boxes, display captions on them427 if refined_boxes is not None:428 y1, x1, y2, x2 = ry1, rx1, ry2, rx2429 ax.text(x1, y1, caption, size=11, verticalalignment='top',430 color='w', backgroundcolor="none",431 bbox={'facecolor': color, 'alpha': 0.5,432 'pad': 2, 'edgecolor': 'none'})433 434 # Masks435 if masks is not None:436 mask = masks[:, :, i]437 masked_image = apply_mask(masked_image, mask, color)438 # Mask Polygon439 # Pad to ensure proper polygons for masks that touch image edges.440 padded_mask = np.zeros(441 (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)442 padded_mask[1:-1, 1:-1] = mask443 contours = find_contours(padded_mask, 0.5)444 for verts in contours:445 # Subtract the padding and flip (y, x) to (x, y)446 verts = np.fliplr(verts) - 1447 p = Polygon(verts, facecolor="none", edgecolor=color)448 ax.add_patch(p)449 ax.imshow(masked_image.astype(np.uint8))450 451 452def display_table(table):453 """Display values in a table format.454 table: an iterable of rows, and each row is an iterable of values.455 """456 html = ""457 for row in table:458 row_html = ""459 for col in row:460 row_html += "<td>{:40}</td>".format(str(col))461 html += "<tr>" + row_html + "</tr>"462 html = "<table>" + html + "</table>"463 IPython.display.display(IPython.display.HTML(html))464 465 466def display_weight_stats(model):467 """Scans all the weights in the model and returns a list of tuples468 that contain stats about each weight.469 """470 layers = model.get_trainable_layers()471 table = [["WEIGHT NAME", "SHAPE", "MIN", "MAX", "STD"]]472 for l in layers:473 weight_values = l.get_weights() # list of Numpy arrays474 weight_tensors = l.weights # list of TF tensors475 for i, w in enumerate(weight_values):476 weight_name = weight_tensors[i].name477 # Detect problematic layers. Exclude biases of conv layers.478 alert = ""479 if w.min() == w.max() and not (l.__class__.__name__ == "Conv2D" and i == 1):480 alert += "<span style='color:red'>*** dead?</span>"481 if np.abs(w.min()) > 1000 or np.abs(w.max()) > 1000:482 alert += "<span style='color:red'>*** Overflow?</span>"483 # Add row484 table.append([485 weight_name + alert,486 str(w.shape),487 "{:+9.4f}".format(w.min()),488 "{:+10.4f}".format(w.max()),489 "{:+9.4f}".format(w.std()),490 ])491 display_table(table)