coreml-community/ControlNet-v1-1-Annotators-cpu
15
1# Copyright (c) Facebook, Inc. and its affiliates.2import contextlib3import copy4import io5import itertools6import json7import logging8import numpy as np9import os10import pickle11from collections import OrderedDict12import pycocotools.mask as mask_util13import torch14from pycocotools.coco import COCO15from pycocotools.cocoeval import COCOeval16from tabulate import tabulate17 18import annotator.oneformer.detectron2.utils.comm as comm19from annotator.oneformer.detectron2.config import CfgNode20from annotator.oneformer.detectron2.data import MetadataCatalog21from annotator.oneformer.detectron2.data.datasets.coco import convert_to_coco_json22from annotator.oneformer.detectron2.structures import Boxes, BoxMode, pairwise_iou23from annotator.oneformer.detectron2.utils.file_io import PathManager24from annotator.oneformer.detectron2.utils.logger import create_small_table25 26from .evaluator import DatasetEvaluator27 28try:29 from annotator.oneformer.detectron2.evaluation.fast_eval_api import COCOeval_opt30except ImportError:31 COCOeval_opt = COCOeval32 33 34class COCOEvaluator(DatasetEvaluator):35 """36 Evaluate AR for object proposals, AP for instance detection/segmentation, AP37 for keypoint detection outputs using COCO's metrics.38 See http://cocodataset.org/#detection-eval and39 http://cocodataset.org/#keypoints-eval to understand its metrics.40 The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means41 the metric cannot be computed (e.g. due to no predictions made).42 43 In addition to COCO, this evaluator is able to support any bounding box detection,44 instance segmentation, or keypoint detection dataset.45 """46 47 def __init__(48 self,49 dataset_name,50 tasks=None,51 distributed=True,52 output_dir=None,53 *,54 max_dets_per_image=None,55 use_fast_impl=True,56 kpt_oks_sigmas=(),57 allow_cached_coco=True,58 ):59 """60 Args:61 dataset_name (str): name of the dataset to be evaluated.62 It must have either the following corresponding metadata:63 64 "json_file": the path to the COCO format annotation65 66 Or it must be in detectron2's standard dataset format67 so it can be converted to COCO format automatically.68 tasks (tuple[str]): tasks that can be evaluated under the given69 configuration. A task is one of "bbox", "segm", "keypoints".70 By default, will infer this automatically from predictions.71 distributed (True): if True, will collect results from all ranks and run evaluation72 in the main process.73 Otherwise, will only evaluate the results in the current process.74 output_dir (str): optional, an output directory to dump all75 results predicted on the dataset. The dump contains two files:76 77 1. "instances_predictions.pth" a file that can be loaded with `torch.load` and78 contains all the results in the format they are produced by the model.79 2. "coco_instances_results.json" a json file in COCO's result format.80 max_dets_per_image (int): limit on the maximum number of detections per image.81 By default in COCO, this limit is to 100, but this can be customized82 to be greater, as is needed in evaluation metrics AP fixed and AP pool83 (see https://arxiv.org/pdf/2102.01066.pdf)84 This doesn't affect keypoint evaluation.85 use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP.86 Although the results should be very close to the official implementation in COCO87 API, it is still recommended to compute results with the official API for use in88 papers. The faster implementation also uses more RAM.89 kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS.90 See http://cocodataset.org/#keypoints-eval91 When empty, it will use the defaults in COCO.92 Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS.93 allow_cached_coco (bool): Whether to use cached coco json from previous validation94 runs. You should set this to False if you need to use different validation data.95 Defaults to True.96 """97 self._logger = logging.getLogger(__name__)98 self._distributed = distributed99 self._output_dir = output_dir100 101 if use_fast_impl and (COCOeval_opt is COCOeval):102 self._logger.info("Fast COCO eval is not built. Falling back to official COCO eval.")103 use_fast_impl = False104 self._use_fast_impl = use_fast_impl105 106 # COCOeval requires the limit on the number of detections per image (maxDets) to be a list107 # with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the108 # 3rd element (100) is used as the limit on the number of detections per image when109 # evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval,110 # we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults.111 if max_dets_per_image is None:112 max_dets_per_image = [1, 10, 100]113 else:114 max_dets_per_image = [1, 10, max_dets_per_image]115 self._max_dets_per_image = max_dets_per_image116 117 if tasks is not None and isinstance(tasks, CfgNode):118 kpt_oks_sigmas = (119 tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas120 )121 self._logger.warn(122 "COCO Evaluator instantiated using config, this is deprecated behavior."123 " Please pass in explicit arguments instead."124 )125 self._tasks = None # Infering it from predictions should be better126 else:127 self._tasks = tasks128 129 self._cpu_device = torch.device("cpu")130 131 self._metadata = MetadataCatalog.get(dataset_name)132 if not hasattr(self._metadata, "json_file"):133 if output_dir is None:134 raise ValueError(135 "output_dir must be provided to COCOEvaluator "136 "for datasets not in COCO format."137 )138 self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...")139 140 cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json")141 self._metadata.json_file = cache_path142 convert_to_coco_json(dataset_name, cache_path, allow_cached=allow_cached_coco)143 144 json_file = PathManager.get_local_path(self._metadata.json_file)145 with contextlib.redirect_stdout(io.StringIO()):146 self._coco_api = COCO(json_file)147 148 # Test set json files do not contain annotations (evaluation must be149 # performed using the COCO evaluation server).150 self._do_evaluation = "annotations" in self._coco_api.dataset151 if self._do_evaluation:152 self._kpt_oks_sigmas = kpt_oks_sigmas153 154 def reset(self):155 self._predictions = []156 157 def process(self, inputs, outputs):158 """159 Args:160 inputs: the inputs to a COCO model (e.g., GeneralizedRCNN).161 It is a list of dict. Each dict corresponds to an image and162 contains keys like "height", "width", "file_name", "image_id".163 outputs: the outputs of a COCO model. It is a list of dicts with key164 "instances" that contains :class:`Instances`.165 """166 for input, output in zip(inputs, outputs):167 prediction = {"image_id": input["image_id"]}168 169 if "instances" in output:170 instances = output["instances"].to(self._cpu_device)171 prediction["instances"] = instances_to_coco_json(instances, input["image_id"])172 if "proposals" in output:173 prediction["proposals"] = output["proposals"].to(self._cpu_device)174 if len(prediction) > 1:175 self._predictions.append(prediction)176 177 def evaluate(self, img_ids=None):178 """179 Args:180 img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset181 """182 if self._distributed:183 comm.synchronize()184 predictions = comm.gather(self._predictions, dst=0)185 predictions = list(itertools.chain(*predictions))186 187 if not comm.is_main_process():188 return {}189 else:190 predictions = self._predictions191 192 if len(predictions) == 0:193 self._logger.warning("[COCOEvaluator] Did not receive valid predictions.")194 return {}195 196 if self._output_dir:197 PathManager.mkdirs(self._output_dir)198 file_path = os.path.join(self._output_dir, "instances_predictions.pth")199 with PathManager.open(file_path, "wb") as f:200 torch.save(predictions, f)201 202 self._results = OrderedDict()203 if "proposals" in predictions[0]:204 self._eval_box_proposals(predictions)205 if "instances" in predictions[0]:206 self._eval_predictions(predictions, img_ids=img_ids)207 # Copy so the caller can do whatever with results208 return copy.deepcopy(self._results)209 210 def _tasks_from_predictions(self, predictions):211 """212 Get COCO API "tasks" (i.e. iou_type) from COCO-format predictions.213 """214 tasks = {"bbox"}215 for pred in predictions:216 if "segmentation" in pred:217 tasks.add("segm")218 if "keypoints" in pred:219 tasks.add("keypoints")220 return sorted(tasks)221 222 def _eval_predictions(self, predictions, img_ids=None):223 """224 Evaluate predictions. Fill self._results with the metrics of the tasks.225 """226 self._logger.info("Preparing results for COCO format ...")227 coco_results = list(itertools.chain(*[x["instances"] for x in predictions]))228 tasks = self._tasks or self._tasks_from_predictions(coco_results)229 230 # unmap the category ids for COCO231 if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"):232 dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id233 all_contiguous_ids = list(dataset_id_to_contiguous_id.values())234 num_classes = len(all_contiguous_ids)235 assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1236 237 reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()}238 for result in coco_results:239 category_id = result["category_id"]240 assert category_id < num_classes, (241 f"A prediction has class={category_id}, "242 f"but the dataset only has {num_classes} classes and "243 f"predicted class id should be in [0, {num_classes - 1}]."244 )245 result["category_id"] = reverse_id_mapping[category_id]246 247 if self._output_dir:248 file_path = os.path.join(self._output_dir, "coco_instances_results.json")249 self._logger.info("Saving results to {}".format(file_path))250 with PathManager.open(file_path, "w") as f:251 f.write(json.dumps(coco_results))252 f.flush()253 254 if not self._do_evaluation:255 self._logger.info("Annotations are not available for evaluation.")256 return257 258 self._logger.info(259 "Evaluating predictions with {} COCO API...".format(260 "unofficial" if self._use_fast_impl else "official"261 )262 )263 for task in sorted(tasks):264 assert task in {"bbox", "segm", "keypoints"}, f"Got unknown task: {task}!"265 coco_eval = (266 _evaluate_predictions_on_coco(267 self._coco_api,268 coco_results,269 task,270 kpt_oks_sigmas=self._kpt_oks_sigmas,271 cocoeval_fn=COCOeval_opt if self._use_fast_impl else COCOeval,272 img_ids=img_ids,273 max_dets_per_image=self._max_dets_per_image,274 )275 if len(coco_results) > 0276 else None # cocoapi does not handle empty results very well277 )278 279 res = self._derive_coco_results(280 coco_eval, task, class_names=self._metadata.get("thing_classes")281 )282 self._results[task] = res283 284 def _eval_box_proposals(self, predictions):285 """286 Evaluate the box proposals in predictions.287 Fill self._results with the metrics for "box_proposals" task.288 """289 if self._output_dir:290 # Saving generated box proposals to file.291 # Predicted box_proposals are in XYXY_ABS mode.292 bbox_mode = BoxMode.XYXY_ABS.value293 ids, boxes, objectness_logits = [], [], []294 for prediction in predictions:295 ids.append(prediction["image_id"])296 boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy())297 objectness_logits.append(prediction["proposals"].objectness_logits.numpy())298 299 proposal_data = {300 "boxes": boxes,301 "objectness_logits": objectness_logits,302 "ids": ids,303 "bbox_mode": bbox_mode,304 }305 with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f:306 pickle.dump(proposal_data, f)307 308 if not self._do_evaluation:309 self._logger.info("Annotations are not available for evaluation.")310 return311 312 self._logger.info("Evaluating bbox proposals ...")313 res = {}314 areas = {"all": "", "small": "s", "medium": "m", "large": "l"}315 for limit in [100, 1000]:316 for area, suffix in areas.items():317 stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit)318 key = "AR{}@{:d}".format(suffix, limit)319 res[key] = float(stats["ar"].item() * 100)320 self._logger.info("Proposal metrics: \n" + create_small_table(res))321 self._results["box_proposals"] = res322 323 def _derive_coco_results(self, coco_eval, iou_type, class_names=None):324 """325 Derive the desired score numbers from summarized COCOeval.326 327 Args:328 coco_eval (None or COCOEval): None represents no predictions from model.329 iou_type (str):330 class_names (None or list[str]): if provided, will use it to predict331 per-category AP.332 333 Returns:334 a dict of {metric name: score}335 """336 337 metrics = {338 "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl"],339 "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl"],340 "keypoints": ["AP", "AP50", "AP75", "APm", "APl"],341 }[iou_type]342 343 if coco_eval is None:344 self._logger.warn("No predictions from the model!")345 return {metric: float("nan") for metric in metrics}346 347 # the standard metrics348 results = {349 metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else "nan")350 for idx, metric in enumerate(metrics)351 }352 self._logger.info(353 "Evaluation results for {}: \n".format(iou_type) + create_small_table(results)354 )355 if not np.isfinite(sum(results.values())):356 self._logger.info("Some metrics cannot be computed and is shown as NaN.")357 358 if class_names is None or len(class_names) <= 1:359 return results360 # Compute per-category AP361 # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa362 precisions = coco_eval.eval["precision"]363 # precision has dims (iou, recall, cls, area range, max dets)364 assert len(class_names) == precisions.shape[2]365 366 results_per_category = []367 for idx, name in enumerate(class_names):368 # area range index 0: all area ranges369 # max dets index -1: typically 100 per image370 precision = precisions[:, :, idx, 0, -1]371 precision = precision[precision > -1]372 ap = np.mean(precision) if precision.size else float("nan")373 results_per_category.append(("{}".format(name), float(ap * 100)))374 375 # tabulate it376 N_COLS = min(6, len(results_per_category) * 2)377 results_flatten = list(itertools.chain(*results_per_category))378 results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)])379 table = tabulate(380 results_2d,381 tablefmt="pipe",382 floatfmt=".3f",383 headers=["category", "AP"] * (N_COLS // 2),384 numalign="left",385 )386 self._logger.info("Per-category {} AP: \n".format(iou_type) + table)387 388 results.update({"AP-" + name: ap for name, ap in results_per_category})389 return results390 391 392def instances_to_coco_json(instances, img_id):393 """394 Dump an "Instances" object to a COCO-format json that's used for evaluation.395 396 Args:397 instances (Instances):398 img_id (int): the image id399 400 Returns:401 list[dict]: list of json annotations in COCO format.402 """403 num_instance = len(instances)404 if num_instance == 0:405 return []406 407 boxes = instances.pred_boxes.tensor.numpy()408 boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)409 boxes = boxes.tolist()410 scores = instances.scores.tolist()411 classes = instances.pred_classes.tolist()412 413 has_mask = instances.has("pred_masks")414 if has_mask:415 # use RLE to encode the masks, because they are too large and takes memory416 # since this evaluator stores outputs of the entire dataset417 rles = [418 mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0]419 for mask in instances.pred_masks420 ]421 for rle in rles:422 # "counts" is an array encoded by mask_util as a byte-stream. Python3's423 # json writer which always produces strings cannot serialize a bytestream424 # unless you decode it. Thankfully, utf-8 works out (which is also what425 # the pycocotools/_mask.pyx does).426 rle["counts"] = rle["counts"].decode("utf-8")427 428 has_keypoints = instances.has("pred_keypoints")429 if has_keypoints:430 keypoints = instances.pred_keypoints431 432 results = []433 for k in range(num_instance):434 result = {435 "image_id": img_id,436 "category_id": classes[k],437 "bbox": boxes[k],438 "score": scores[k],439 }440 if has_mask:441 result["segmentation"] = rles[k]442 if has_keypoints:443 # In COCO annotations,444 # keypoints coordinates are pixel indices.445 # However our predictions are floating point coordinates.446 # Therefore we subtract 0.5 to be consistent with the annotation format.447 # This is the inverse of data loading logic in `datasets/coco.py`.448 keypoints[k][:, :2] -= 0.5449 result["keypoints"] = keypoints[k].flatten().tolist()450 results.append(result)451 return results452 453 454# inspired from Detectron:455# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa456def _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area="all", limit=None):457 """458 Evaluate detection proposal recall metrics. This function is a much459 faster alternative to the official COCO API recall evaluation code. However,460 it produces slightly different results.461 """462 # Record max overlap value for each gt box463 # Return vector of overlap values464 areas = {465 "all": 0,466 "small": 1,467 "medium": 2,468 "large": 3,469 "96-128": 4,470 "128-256": 5,471 "256-512": 6,472 "512-inf": 7,473 }474 area_ranges = [475 [0**2, 1e5**2], # all476 [0**2, 32**2], # small477 [32**2, 96**2], # medium478 [96**2, 1e5**2], # large479 [96**2, 128**2], # 96-128480 [128**2, 256**2], # 128-256481 [256**2, 512**2], # 256-512482 [512**2, 1e5**2],483 ] # 512-inf484 assert area in areas, "Unknown area range: {}".format(area)485 area_range = area_ranges[areas[area]]486 gt_overlaps = []487 num_pos = 0488 489 for prediction_dict in dataset_predictions:490 predictions = prediction_dict["proposals"]491 492 # sort predictions in descending order493 # TODO maybe remove this and make it explicit in the documentation494 inds = predictions.objectness_logits.sort(descending=True)[1]495 predictions = predictions[inds]496 497 ann_ids = coco_api.getAnnIds(imgIds=prediction_dict["image_id"])498 anno = coco_api.loadAnns(ann_ids)499 gt_boxes = [500 BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS)501 for obj in anno502 if obj["iscrowd"] == 0503 ]504 gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes505 gt_boxes = Boxes(gt_boxes)506 gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0])507 508 if len(gt_boxes) == 0 or len(predictions) == 0:509 continue510 511 valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1])512 gt_boxes = gt_boxes[valid_gt_inds]513 514 num_pos += len(gt_boxes)515 516 if len(gt_boxes) == 0:517 continue518 519 if limit is not None and len(predictions) > limit:520 predictions = predictions[:limit]521 522 overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes)523 524 _gt_overlaps = torch.zeros(len(gt_boxes))525 for j in range(min(len(predictions), len(gt_boxes))):526 # find which proposal box maximally covers each gt box527 # and get the iou amount of coverage for each gt box528 max_overlaps, argmax_overlaps = overlaps.max(dim=0)529 530 # find which gt box is 'best' covered (i.e. 'best' = most iou)531 gt_ovr, gt_ind = max_overlaps.max(dim=0)532 assert gt_ovr >= 0533 # find the proposal box that covers the best covered gt box534 box_ind = argmax_overlaps[gt_ind]535 # record the iou coverage of this gt box536 _gt_overlaps[j] = overlaps[box_ind, gt_ind]537 assert _gt_overlaps[j] == gt_ovr538 # mark the proposal box and the gt box as used539 overlaps[box_ind, :] = -1540 overlaps[:, gt_ind] = -1541 542 # append recorded iou coverage level543 gt_overlaps.append(_gt_overlaps)544 gt_overlaps = (545 torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32)546 )547 gt_overlaps, _ = torch.sort(gt_overlaps)548 549 if thresholds is None:550 step = 0.05551 thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32)552 recalls = torch.zeros_like(thresholds)553 # compute recall for each iou threshold554 for i, t in enumerate(thresholds):555 recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos)556 # ar = 2 * np.trapz(recalls, thresholds)557 ar = recalls.mean()558 return {559 "ar": ar,560 "recalls": recalls,561 "thresholds": thresholds,562 "gt_overlaps": gt_overlaps,563 "num_pos": num_pos,564 }565 566 567def _evaluate_predictions_on_coco(568 coco_gt,569 coco_results,570 iou_type,571 kpt_oks_sigmas=None,572 cocoeval_fn=COCOeval_opt,573 img_ids=None,574 max_dets_per_image=None,575):576 """577 Evaluate the coco results using COCOEval API.578 """579 assert len(coco_results) > 0580 581 if iou_type == "segm":582 coco_results = copy.deepcopy(coco_results)583 # When evaluating mask AP, if the results contain bbox, cocoapi will584 # use the box area as the area of the instance, instead of the mask area.585 # This leads to a different definition of small/medium/large.586 # We remove the bbox field to let mask AP use mask area.587 for c in coco_results:588 c.pop("bbox", None)589 590 coco_dt = coco_gt.loadRes(coco_results)591 coco_eval = cocoeval_fn(coco_gt, coco_dt, iou_type)592 # For COCO, the default max_dets_per_image is [1, 10, 100].593 if max_dets_per_image is None:594 max_dets_per_image = [1, 10, 100] # Default from COCOEval595 else:596 assert (597 len(max_dets_per_image) >= 3598 ), "COCOeval requires maxDets (and max_dets_per_image) to have length at least 3"599 # In the case that user supplies a custom input for max_dets_per_image,600 # apply COCOevalMaxDets to evaluate AP with the custom input.601 if max_dets_per_image[2] != 100:602 coco_eval = COCOevalMaxDets(coco_gt, coco_dt, iou_type)603 if iou_type != "keypoints":604 coco_eval.params.maxDets = max_dets_per_image605 606 if img_ids is not None:607 coco_eval.params.imgIds = img_ids608 609 if iou_type == "keypoints":610 # Use the COCO default keypoint OKS sigmas unless overrides are specified611 if kpt_oks_sigmas:612 assert hasattr(coco_eval.params, "kpt_oks_sigmas"), "pycocotools is too old!"613 coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas)614 # COCOAPI requires every detection and every gt to have keypoints, so615 # we just take the first entry from both616 num_keypoints_dt = len(coco_results[0]["keypoints"]) // 3617 num_keypoints_gt = len(next(iter(coco_gt.anns.values()))["keypoints"]) // 3618 num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas)619 assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, (620 f"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. "621 f"Ground truth contains {num_keypoints_gt} keypoints. "622 f"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. "623 "They have to agree with each other. For meaning of OKS, please refer to "624 "http://cocodataset.org/#keypoints-eval."625 )626 627 coco_eval.evaluate()628 coco_eval.accumulate()629 coco_eval.summarize()630 631 return coco_eval632 633 634class COCOevalMaxDets(COCOeval):635 """636 Modified version of COCOeval for evaluating AP with a custom637 maxDets (by default for COCO, maxDets is 100)638 """639 640 def summarize(self):641 """642 Compute and display summary metrics for evaluation results given643 a custom value for max_dets_per_image644 """645 646 def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100):647 p = self.params648 iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}"649 titleStr = "Average Precision" if ap == 1 else "Average Recall"650 typeStr = "(AP)" if ap == 1 else "(AR)"651 iouStr = (652 "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1])653 if iouThr is None654 else "{:0.2f}".format(iouThr)655 )656 657 aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]658 mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]659 if ap == 1:660 # dimension of precision: [TxRxKxAxM]661 s = self.eval["precision"]662 # IoU663 if iouThr is not None:664 t = np.where(iouThr == p.iouThrs)[0]665 s = s[t]666 s = s[:, :, :, aind, mind]667 else:668 # dimension of recall: [TxKxAxM]669 s = self.eval["recall"]670 if iouThr is not None:671 t = np.where(iouThr == p.iouThrs)[0]672 s = s[t]673 s = s[:, :, aind, mind]674 if len(s[s > -1]) == 0:675 mean_s = -1676 else:677 mean_s = np.mean(s[s > -1])678 print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s))679 return mean_s680 681 def _summarizeDets():682 stats = np.zeros((12,))683 # Evaluate AP using the custom limit on maximum detections per image684 stats[0] = _summarize(1, maxDets=self.params.maxDets[2])685 stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2])686 stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2])687 stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2])688 stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2])689 stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2])690 stats[6] = _summarize(0, maxDets=self.params.maxDets[0])691 stats[7] = _summarize(0, maxDets=self.params.maxDets[1])692 stats[8] = _summarize(0, maxDets=self.params.maxDets[2])693 stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2])694 stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2])695 stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2])696 return stats697 698 def _summarizeKps():699 stats = np.zeros((10,))700 stats[0] = _summarize(1, maxDets=20)701 stats[1] = _summarize(1, maxDets=20, iouThr=0.5)702 stats[2] = _summarize(1, maxDets=20, iouThr=0.75)703 stats[3] = _summarize(1, maxDets=20, areaRng="medium")704 stats[4] = _summarize(1, maxDets=20, areaRng="large")705 stats[5] = _summarize(0, maxDets=20)706 stats[6] = _summarize(0, maxDets=20, iouThr=0.5)707 stats[7] = _summarize(0, maxDets=20, iouThr=0.75)708 stats[8] = _summarize(0, maxDets=20, areaRng="medium")709 stats[9] = _summarize(0, maxDets=20, areaRng="large")710 return stats711 712 if not self.eval:713 raise Exception("Please run accumulate() first")714 iouType = self.params.iouType715 if iouType == "segm" or iouType == "bbox":716 summarize = _summarizeDets717 elif iouType == "keypoints":718 summarize = _summarizeKps719 self.stats = summarize()720 721 def __str__(self):722 self.summarize()723