coreml-community/ControlNet-v1-1-Annotators-cpu
15
1# ------------------------------------------------------------------------------2# Reference: https://github.com/facebookresearch/detectron2/blob/main/detectron2/evaluation/coco_evaluation.py3# Modified by Jitesh Jain (https://github.com/praeclarumjj3)4# ------------------------------------------------------------------------------5 6import contextlib7import copy8import io9import itertools10import json11import logging12import numpy as np13import os14import pickle15from collections import OrderedDict16import pycocotools.mask as mask_util17import torch18from pycocotools.coco import COCO19from pycocotools.cocoeval import COCOeval20from tabulate import tabulate21 22import annotator.oneformer.detectron2.utils.comm as comm23from annotator.oneformer.detectron2.config import CfgNode24from annotator.oneformer.detectron2.data import MetadataCatalog25from annotator.oneformer.detectron2.data.datasets.coco import convert_to_coco_json26from annotator.oneformer.detectron2.structures import Boxes, BoxMode, pairwise_iou27from annotator.oneformer.detectron2.utils.file_io import PathManager28from annotator.oneformer.detectron2.utils.logger import create_small_table29 30from .evaluator import DatasetEvaluator31 32try:33 from annotator.oneformer.detectron2.evaluation.fast_eval_api import COCOeval_opt34except ImportError:35 COCOeval_opt = COCOeval36 37 38class DetectionCOCOEvaluator(DatasetEvaluator):39 """40 Evaluate AR for object proposals, AP for instance detection/segmentation, AP41 for keypoint detection outputs using COCO's metrics.42 See http://cocodataset.org/#detection-eval and43 http://cocodataset.org/#keypoints-eval to understand its metrics.44 The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means45 the metric cannot be computed (e.g. due to no predictions made).46 47 In addition to COCO, this evaluator is able to support any bounding box detection,48 instance segmentation, or keypoint detection dataset.49 """50 51 def __init__(52 self,53 dataset_name,54 tasks=None,55 distributed=True,56 output_dir=None,57 *,58 max_dets_per_image=None,59 use_fast_impl=True,60 kpt_oks_sigmas=(),61 allow_cached_coco=True,62 ):63 """64 Args:65 dataset_name (str): name of the dataset to be evaluated.66 It must have either the following corresponding metadata:67 68 "json_file": the path to the COCO format annotation69 70 Or it must be in detectron2's standard dataset format71 so it can be converted to COCO format automatically.72 tasks (tuple[str]): tasks that can be evaluated under the given73 configuration. A task is one of "bbox", "segm", "keypoints".74 By default, will infer this automatically from predictions.75 distributed (True): if True, will collect results from all ranks and run evaluation76 in the main process.77 Otherwise, will only evaluate the results in the current process.78 output_dir (str): optional, an output directory to dump all79 results predicted on the dataset. The dump contains two files:80 81 1. "instances_predictions.pth" a file that can be loaded with `torch.load` and82 contains all the results in the format they are produced by the model.83 2. "coco_instances_results.json" a json file in COCO's result format.84 max_dets_per_image (int): limit on the maximum number of detections per image.85 By default in COCO, this limit is to 100, but this can be customized86 to be greater, as is needed in evaluation metrics AP fixed and AP pool87 (see https://arxiv.org/pdf/2102.01066.pdf)88 This doesn't affect keypoint evaluation.89 use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP.90 Although the results should be very close to the official implementation in COCO91 API, it is still recommended to compute results with the official API for use in92 papers. The faster implementation also uses more RAM.93 kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS.94 See http://cocodataset.org/#keypoints-eval95 When empty, it will use the defaults in COCO.96 Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS.97 allow_cached_coco (bool): Whether to use cached coco json from previous validation98 runs. You should set this to False if you need to use different validation data.99 Defaults to True.100 """101 self._logger = logging.getLogger(__name__)102 self._distributed = distributed103 self._output_dir = output_dir104 105 if use_fast_impl and (COCOeval_opt is COCOeval):106 self._logger.info("Fast COCO eval is not built. Falling back to official COCO eval.")107 use_fast_impl = False108 self._use_fast_impl = use_fast_impl109 110 # COCOeval requires the limit on the number of detections per image (maxDets) to be a list111 # with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the112 # 3rd element (100) is used as the limit on the number of detections per image when113 # evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval,114 # we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults.115 if max_dets_per_image is None:116 max_dets_per_image = [1, 10, 100]117 else:118 max_dets_per_image = [1, 10, max_dets_per_image]119 self._max_dets_per_image = max_dets_per_image120 121 if tasks is not None and isinstance(tasks, CfgNode):122 kpt_oks_sigmas = (123 tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas124 )125 self._logger.warn(126 "COCO Evaluator instantiated using config, this is deprecated behavior."127 " Please pass in explicit arguments instead."128 )129 self._tasks = None # Infering it from predictions should be better130 else:131 self._tasks = tasks132 133 self._cpu_device = torch.device("cpu")134 135 self._metadata = MetadataCatalog.get(dataset_name)136 if not hasattr(self._metadata, "json_file"):137 if output_dir is None:138 raise ValueError(139 "output_dir must be provided to COCOEvaluator "140 "for datasets not in COCO format."141 )142 self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...")143 144 cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json")145 self._metadata.json_file = cache_path146 convert_to_coco_json(dataset_name, cache_path, allow_cached=allow_cached_coco)147 148 json_file = PathManager.get_local_path(self._metadata.json_file)149 with contextlib.redirect_stdout(io.StringIO()):150 self._coco_api = COCO(json_file)151 152 # Test set json files do not contain annotations (evaluation must be153 # performed using the COCO evaluation server).154 self._do_evaluation = "annotations" in self._coco_api.dataset155 if self._do_evaluation:156 self._kpt_oks_sigmas = kpt_oks_sigmas157 158 def reset(self):159 self._predictions = []160 161 def process(self, inputs, outputs):162 """163 Args:164 inputs: the inputs to a COCO model (e.g., GeneralizedRCNN).165 It is a list of dict. Each dict corresponds to an image and166 contains keys like "height", "width", "file_name", "image_id".167 outputs: the outputs of a COCO model. It is a list of dicts with key168 "box_instances" that contains :class:`Instances`.169 """170 for input, output in zip(inputs, outputs):171 prediction = {"image_id": input["image_id"]}172 173 if "box_instances" in output:174 instances = output["box_instances"].to(self._cpu_device)175 prediction["box_instances"] = instances_to_coco_json(instances, input["image_id"])176 if "proposals" in output:177 prediction["proposals"] = output["proposals"].to(self._cpu_device)178 if len(prediction) > 1:179 self._predictions.append(prediction)180 181 def evaluate(self, img_ids=None):182 """183 Args:184 img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset185 """186 if self._distributed:187 comm.synchronize()188 predictions = comm.gather(self._predictions, dst=0)189 predictions = list(itertools.chain(*predictions))190 191 if not comm.is_main_process():192 return {}193 else:194 predictions = self._predictions195 196 if len(predictions) == 0:197 self._logger.warning("[COCOEvaluator] Did not receive valid predictions.")198 return {}199 200 if self._output_dir:201 PathManager.mkdirs(self._output_dir)202 file_path = os.path.join(self._output_dir, "instances_predictions.pth")203 with PathManager.open(file_path, "wb") as f:204 torch.save(predictions, f)205 206 self._results = OrderedDict()207 if "proposals" in predictions[0]:208 self._eval_box_proposals(predictions)209 if "box_instances" in predictions[0]:210 self._eval_predictions(predictions, img_ids=img_ids)211 # Copy so the caller can do whatever with results212 return copy.deepcopy(self._results)213 214 def _tasks_from_predictions(self, predictions):215 """216 Get COCO API "tasks" (i.e. iou_type) from COCO-format predictions.217 """218 tasks = {"bbox"}219 for pred in predictions:220 if "keypoints" in pred:221 tasks.add("keypoints")222 return sorted(tasks)223 224 def _eval_predictions(self, predictions, img_ids=None):225 """226 Evaluate predictions. Fill self._results with the metrics of the tasks.227 """228 self._logger.info("Preparing results for COCO format ...")229 coco_results = list(itertools.chain(*[x["box_instances"] for x in predictions]))230 tasks = self._tasks or self._tasks_from_predictions(coco_results)231 232 # unmap the category ids for COCO233 if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"):234 dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id235 all_contiguous_ids = list(dataset_id_to_contiguous_id.values())236 num_classes = len(all_contiguous_ids)237 assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1238 239 reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()}240 for result in coco_results:241 category_id = result["category_id"]242 assert category_id < num_classes, (243 f"A prediction has class={category_id}, "244 f"but the dataset only has {num_classes} classes and "245 f"predicted class id should be in [0, {num_classes - 1}]."246 )247 result["category_id"] = reverse_id_mapping[category_id]248 249 if self._output_dir:250 file_path = os.path.join(self._output_dir, "coco_instances_results.json")251 self._logger.info("Saving results to {}".format(file_path))252 with PathManager.open(file_path, "w") as f:253 f.write(json.dumps(coco_results))254 f.flush()255 256 if not self._do_evaluation:257 self._logger.info("Annotations are not available for evaluation.")258 return259 260 self._logger.info(261 "Evaluating predictions with {} COCO API...".format(262 "unofficial" if self._use_fast_impl else "official"263 )264 )265 for task in sorted(tasks):266 assert task in {"bbox", "keypoints"}, f"Got unknown task: {task}!"267 coco_eval = (268 _evaluate_predictions_on_coco(269 self._coco_api,270 coco_results,271 task,272 kpt_oks_sigmas=self._kpt_oks_sigmas,273 use_fast_impl=self._use_fast_impl,274 img_ids=img_ids,275 max_dets_per_image=self._max_dets_per_image,276 )277 if len(coco_results) > 0278 else None # cocoapi does not handle empty results very well279 )280 281 res = self._derive_coco_results(282 coco_eval, task, class_names=self._metadata.get("thing_classes")283 )284 self._results[task] = res285 286 def _eval_box_proposals(self, predictions):287 """288 Evaluate the box proposals in predictions.289 Fill self._results with the metrics for "box_proposals" task.290 """291 if self._output_dir:292 # Saving generated box proposals to file.293 # Predicted box_proposals are in XYXY_ABS mode.294 bbox_mode = BoxMode.XYXY_ABS.value295 ids, boxes, objectness_logits = [], [], []296 for prediction in predictions:297 ids.append(prediction["image_id"])298 boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy())299 objectness_logits.append(prediction["proposals"].objectness_logits.numpy())300 301 proposal_data = {302 "boxes": boxes,303 "objectness_logits": objectness_logits,304 "ids": ids,305 "bbox_mode": bbox_mode,306 }307 with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f:308 pickle.dump(proposal_data, f)309 310 if not self._do_evaluation:311 self._logger.info("Annotations are not available for evaluation.")312 return313 314 self._logger.info("Evaluating bbox proposals ...")315 res = {}316 areas = {"all": "", "small": "s", "medium": "m", "large": "l"}317 for limit in [100, 1000]:318 for area, suffix in areas.items():319 stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit)320 key = "AR{}@{:d}".format(suffix, limit)321 res[key] = float(stats["ar"].item() * 100)322 self._logger.info("Proposal metrics: \n" + create_small_table(res))323 self._results["box_proposals"] = res324 325 def _derive_coco_results(self, coco_eval, iou_type, class_names=None):326 """327 Derive the desired score numbers from summarized COCOeval.328 329 Args:330 coco_eval (None or COCOEval): None represents no predictions from model.331 iou_type (str):332 class_names (None or list[str]): if provided, will use it to predict333 per-category AP.334 335 Returns:336 a dict of {metric name: score}337 """338 339 metrics = {340 "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl"],341 "keypoints": ["AP", "AP50", "AP75", "APm", "APl"],342 }[iou_type]343 344 if coco_eval is None:345 self._logger.warn("No predictions from the model!")346 return {metric: float("nan") for metric in metrics}347 348 # the standard metrics349 results = {350 metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else "nan")351 for idx, metric in enumerate(metrics)352 }353 self._logger.info(354 "Evaluation results for {}: \n".format(iou_type) + create_small_table(results)355 )356 if not np.isfinite(sum(results.values())):357 self._logger.info("Some metrics cannot be computed and is shown as NaN.")358 359 if class_names is None or len(class_names) <= 1:360 return results361 # Compute per-category AP362 # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa363 precisions = coco_eval.eval["precision"]364 # precision has dims (iou, recall, cls, area range, max dets)365 assert len(class_names) == precisions.shape[2]366 367 results_per_category = []368 for idx, name in enumerate(class_names):369 # area range index 0: all area ranges370 # max dets index -1: typically 100 per image371 precision = precisions[:, :, idx, 0, -1]372 precision = precision[precision > -1]373 ap = np.mean(precision) if precision.size else float("nan")374 results_per_category.append(("{}".format(name), float(ap * 100)))375 376 # tabulate it377 N_COLS = min(6, len(results_per_category) * 2)378 results_flatten = list(itertools.chain(*results_per_category))379 results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)])380 table = tabulate(381 results_2d,382 tablefmt="pipe",383 floatfmt=".3f",384 headers=["category", "AP"] * (N_COLS // 2),385 numalign="left",386 )387 self._logger.info("Per-category {} AP: \n".format(iou_type) + table)388 389 results.update({"AP-" + name: ap for name, ap in results_per_category})390 return results391 392 393def instances_to_coco_json(instances, img_id):394 """395 Dump an "Instances" object to a COCO-format json that's used for evaluation.396 397 Args:398 instances (Instances):399 img_id (int): the image id400 401 Returns:402 list[dict]: list of json annotations in COCO format.403 """404 num_instance = len(instances)405 if num_instance == 0:406 return []407 408 boxes = instances.pred_boxes.tensor.numpy()409 boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)410 boxes = boxes.tolist()411 scores = instances.scores.tolist()412 classes = instances.pred_classes.tolist()413 414 has_mask = instances.has("pred_masks")415 if has_mask:416 # use RLE to encode the masks, because they are too large and takes memory417 # since this evaluator stores outputs of the entire dataset418 rles = [419 mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0]420 for mask in instances.pred_masks421 ]422 for rle in rles:423 # "counts" is an array encoded by mask_util as a byte-stream. Python3's424 # json writer which always produces strings cannot serialize a bytestream425 # unless you decode it. Thankfully, utf-8 works out (which is also what426 # the pycocotools/_mask.pyx does).427 rle["counts"] = rle["counts"].decode("utf-8")428 429 has_keypoints = instances.has("pred_keypoints")430 if has_keypoints:431 keypoints = instances.pred_keypoints432 433 results = []434 for k in range(num_instance):435 result = {436 "image_id": img_id,437 "category_id": classes[k],438 "bbox": boxes[k],439 "score": scores[k],440 }441 if has_mask:442 result["segmentation"] = rles[k]443 if has_keypoints:444 # In COCO annotations,445 # keypoints coordinates are pixel indices.446 # However our predictions are floating point coordinates.447 # Therefore we subtract 0.5 to be consistent with the annotation format.448 # This is the inverse of data loading logic in `datasets/coco.py`.449 keypoints[k][:, :2] -= 0.5450 result["keypoints"] = keypoints[k].flatten().tolist()451 results.append(result)452 return results453 454 455# inspired from Detectron:456# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa457def _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area="all", limit=None):458 """459 Evaluate detection proposal recall metrics. This function is a much460 faster alternative to the official COCO API recall evaluation code. However,461 it produces slightly different results.462 """463 # Record max overlap value for each gt box464 # Return vector of overlap values465 areas = {466 "all": 0,467 "small": 1,468 "medium": 2,469 "large": 3,470 "96-128": 4,471 "128-256": 5,472 "256-512": 6,473 "512-inf": 7,474 }475 area_ranges = [476 [0**2, 1e5**2], # all477 [0**2, 32**2], # small478 [32**2, 96**2], # medium479 [96**2, 1e5**2], # large480 [96**2, 128**2], # 96-128481 [128**2, 256**2], # 128-256482 [256**2, 512**2], # 256-512483 [512**2, 1e5**2],484 ] # 512-inf485 assert area in areas, "Unknown area range: {}".format(area)486 area_range = area_ranges[areas[area]]487 gt_overlaps = []488 num_pos = 0489 490 for prediction_dict in dataset_predictions:491 predictions = prediction_dict["proposals"]492 493 # sort predictions in descending order494 # TODO maybe remove this and make it explicit in the documentation495 inds = predictions.objectness_logits.sort(descending=True)[1]496 predictions = predictions[inds]497 498 ann_ids = coco_api.getAnnIds(imgIds=prediction_dict["image_id"])499 anno = coco_api.loadAnns(ann_ids)500 gt_boxes = [501 BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS)502 for obj in anno503 if obj["iscrowd"] == 0504 ]505 gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes506 gt_boxes = Boxes(gt_boxes)507 gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0])508 509 if len(gt_boxes) == 0 or len(predictions) == 0:510 continue511 512 valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1])513 gt_boxes = gt_boxes[valid_gt_inds]514 515 num_pos += len(gt_boxes)516 517 if len(gt_boxes) == 0:518 continue519 520 if limit is not None and len(predictions) > limit:521 predictions = predictions[:limit]522 523 overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes)524 525 _gt_overlaps = torch.zeros(len(gt_boxes))526 for j in range(min(len(predictions), len(gt_boxes))):527 # find which proposal box maximally covers each gt box528 # and get the iou amount of coverage for each gt box529 max_overlaps, argmax_overlaps = overlaps.max(dim=0)530 531 # find which gt box is 'best' covered (i.e. 'best' = most iou)532 gt_ovr, gt_ind = max_overlaps.max(dim=0)533 assert gt_ovr >= 0534 # find the proposal box that covers the best covered gt box535 box_ind = argmax_overlaps[gt_ind]536 # record the iou coverage of this gt box537 _gt_overlaps[j] = overlaps[box_ind, gt_ind]538 assert _gt_overlaps[j] == gt_ovr539 # mark the proposal box and the gt box as used540 overlaps[box_ind, :] = -1541 overlaps[:, gt_ind] = -1542 543 # append recorded iou coverage level544 gt_overlaps.append(_gt_overlaps)545 gt_overlaps = (546 torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32)547 )548 gt_overlaps, _ = torch.sort(gt_overlaps)549 550 if thresholds is None:551 step = 0.05552 thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32)553 recalls = torch.zeros_like(thresholds)554 # compute recall for each iou threshold555 for i, t in enumerate(thresholds):556 recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos)557 # ar = 2 * np.trapz(recalls, thresholds)558 ar = recalls.mean()559 return {560 "ar": ar,561 "recalls": recalls,562 "thresholds": thresholds,563 "gt_overlaps": gt_overlaps,564 "num_pos": num_pos,565 }566 567 568def _evaluate_predictions_on_coco(569 coco_gt,570 coco_results,571 iou_type,572 kpt_oks_sigmas=None,573 use_fast_impl=True,574 img_ids=None,575 max_dets_per_image=None,576):577 """578 Evaluate the coco results using COCOEval API.579 """580 assert len(coco_results) > 0581 582 if iou_type == "segm":583 coco_results = copy.deepcopy(coco_results)584 # When evaluating mask AP, if the results contain bbox, cocoapi will585 # use the box area as the area of the instance, instead of the mask area.586 # This leads to a different definition of small/medium/large.587 # We remove the bbox field to let mask AP use mask area.588 for c in coco_results:589 c.pop("bbox", None)590 591 coco_dt = coco_gt.loadRes(coco_results)592 coco_eval = (COCOeval_opt if use_fast_impl else COCOeval)(coco_gt, coco_dt, iou_type)593 # For COCO, the default max_dets_per_image is [1, 10, 100].594 if max_dets_per_image is None:595 max_dets_per_image = [1, 10, 100] # Default from COCOEval596 else:597 assert (598 len(max_dets_per_image) >= 3599 ), "COCOeval requires maxDets (and max_dets_per_image) to have length at least 3"600 # In the case that user supplies a custom input for max_dets_per_image,601 # apply COCOevalMaxDets to evaluate AP with the custom input.602 if max_dets_per_image[2] != 100:603 coco_eval = COCOevalMaxDets(coco_gt, coco_dt, iou_type)604 if iou_type != "keypoints":605 coco_eval.params.maxDets = max_dets_per_image606 607 if img_ids is not None:608 coco_eval.params.imgIds = img_ids609 610 if iou_type == "keypoints":611 # Use the COCO default keypoint OKS sigmas unless overrides are specified612 if kpt_oks_sigmas:613 assert hasattr(coco_eval.params, "kpt_oks_sigmas"), "pycocotools is too old!"614 coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas)615 # COCOAPI requires every detection and every gt to have keypoints, so616 # we just take the first entry from both617 num_keypoints_dt = len(coco_results[0]["keypoints"]) // 3618 num_keypoints_gt = len(next(iter(coco_gt.anns.values()))["keypoints"]) // 3619 num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas)620 assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, (621 f"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. "622 f"Ground truth contains {num_keypoints_gt} keypoints. "623 f"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. "624 "They have to agree with each other. For meaning of OKS, please refer to "625 "http://cocodataset.org/#keypoints-eval."626 )627 628 coco_eval.evaluate()629 coco_eval.accumulate()630 coco_eval.summarize()631 632 return coco_eval633 634 635class COCOevalMaxDets(COCOeval):636 """637 Modified version of COCOeval for evaluating AP with a custom638 maxDets (by default for COCO, maxDets is 100)639 """640 641 def summarize(self):642 """643 Compute and display summary metrics for evaluation results given644 a custom value for max_dets_per_image645 """646 647 def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100):648 p = self.params649 iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}"650 titleStr = "Average Precision" if ap == 1 else "Average Recall"651 typeStr = "(AP)" if ap == 1 else "(AR)"652 iouStr = (653 "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1])654 if iouThr is None655 else "{:0.2f}".format(iouThr)656 )657 658 aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]659 mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]660 if ap == 1:661 # dimension of precision: [TxRxKxAxM]662 s = self.eval["precision"]663 # IoU664 if iouThr is not None:665 t = np.where(iouThr == p.iouThrs)[0]666 s = s[t]667 s = s[:, :, :, aind, mind]668 else:669 # dimension of recall: [TxKxAxM]670 s = self.eval["recall"]671 if iouThr is not None:672 t = np.where(iouThr == p.iouThrs)[0]673 s = s[t]674 s = s[:, :, aind, mind]675 if len(s[s > -1]) == 0:676 mean_s = -1677 else:678 mean_s = np.mean(s[s > -1])679 print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s))680 return mean_s681 682 def _summarizeDets():683 stats = np.zeros((12,))684 # Evaluate AP using the custom limit on maximum detections per image685 stats[0] = _summarize(1, maxDets=self.params.maxDets[2])686 stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2])687 stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2])688 stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2])689 stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2])690 stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2])691 stats[6] = _summarize(0, maxDets=self.params.maxDets[0])692 stats[7] = _summarize(0, maxDets=self.params.maxDets[1])693 stats[8] = _summarize(0, maxDets=self.params.maxDets[2])694 stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2])695 stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2])696 stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2])697 return stats698 699 def _summarizeKps():700 stats = np.zeros((10,))701 stats[0] = _summarize(1, maxDets=20)702 stats[1] = _summarize(1, maxDets=20, iouThr=0.5)703 stats[2] = _summarize(1, maxDets=20, iouThr=0.75)704 stats[3] = _summarize(1, maxDets=20, areaRng="medium")705 stats[4] = _summarize(1, maxDets=20, areaRng="large")706 stats[5] = _summarize(0, maxDets=20)707 stats[6] = _summarize(0, maxDets=20, iouThr=0.5)708 stats[7] = _summarize(0, maxDets=20, iouThr=0.75)709 stats[8] = _summarize(0, maxDets=20, areaRng="medium")710 stats[9] = _summarize(0, maxDets=20, areaRng="large")711 return stats712 713 if not self.eval:714 raise Exception("Please run accumulate() first")715 iouType = self.params.iouType716 if iouType == "segm" or iouType == "bbox":717 summarize = _summarizeDets718 elif iouType == "keypoints":719 summarize = _summarizeKps720 self.stats = summarize()721 722 def __str__(self):723 self.summarize()