Aluode/PerceptionLabPortable

1from typing import TYPE_CHECKING, Optional, Union2 3import torch4 5from transformers.models.detr.image_processing_detr_fast import DetrImageProcessorFast6 7from ...image_transforms import center_to_corners_format8from ...utils import (9    TensorType,10    logging,11)12 13 14if TYPE_CHECKING:15    from .modeling_grounding_dino import GroundingDinoObjectDetectionOutput16 17 18logger = logging.get_logger(__name__)19 20 21def _scale_boxes(boxes, target_sizes):22    """23    Scale batch of bounding boxes to the target sizes.24 25    Args:26        boxes (`torch.Tensor` of shape `(batch_size, num_boxes, 4)`):27            Bounding boxes to scale. Each box is expected to be in (x1, y1, x2, y2) format.28        target_sizes (`list[tuple[int, int]]` or `torch.Tensor` of shape `(batch_size, 2)`):29            Target sizes to scale the boxes to. Each target size is expected to be in (height, width) format.30 31    Returns:32        `torch.Tensor` of shape `(batch_size, num_boxes, 4)`: Scaled bounding boxes.33    """34 35    if isinstance(target_sizes, (list, tuple)):36        image_height = torch.tensor([i[0] for i in target_sizes])37        image_width = torch.tensor([i[1] for i in target_sizes])38    elif isinstance(target_sizes, torch.Tensor):39        image_height, image_width = target_sizes.unbind(1)40    else:41        raise TypeError("`target_sizes` must be a list, tuple or torch.Tensor")42 43    scale_factor = torch.stack([image_width, image_height, image_width, image_height], dim=1)44    scale_factor = scale_factor.unsqueeze(1).to(boxes.device)45    boxes = boxes * scale_factor46    return boxes47 48 49class GroundingDinoImageProcessorFast(DetrImageProcessorFast):50    def post_process_object_detection(51        self,52        outputs: "GroundingDinoObjectDetectionOutput",53        threshold: float = 0.1,54        target_sizes: Optional[Union[TensorType, list[tuple]]] = None,55    ):56        """57        Converts the raw output of [`GroundingDinoForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y,58        bottom_right_x, bottom_right_y) format.59 60        Args:61            outputs ([`GroundingDinoObjectDetectionOutput`]):62                Raw outputs of the model.63            threshold (`float`, *optional*, defaults to 0.1):64                Score threshold to keep object detection predictions.65            target_sizes (`torch.Tensor` or `list[tuple[int, int]]`, *optional*):66                Tensor of shape `(batch_size, 2)` or list of tuples (`tuple[int, int]`) containing the target size67                `(height, width)` of each image in the batch. If unset, predictions will not be resized.68 69        Returns:70            `list[Dict]`: A list of dictionaries, each dictionary containing the following keys:71            - "scores": The confidence scores for each predicted box on the image.72            - "labels": Indexes of the classes predicted by the model on the image.73            - "boxes": Image bounding boxes in (top_left_x, top_left_y, bottom_right_x, bottom_right_y) format.74        """75        batch_logits, batch_boxes = outputs.logits, outputs.pred_boxes76        batch_size = len(batch_logits)77 78        if target_sizes is not None and len(target_sizes) != batch_size:79            raise ValueError("Make sure that you pass in as many target sizes as images")80 81        # batch_logits of shape (batch_size, num_queries, num_classes)82        batch_class_logits = torch.max(batch_logits, dim=-1)83        batch_scores = torch.sigmoid(batch_class_logits.values)84        batch_labels = batch_class_logits.indices85 86        # Convert to [x0, y0, x1, y1] format87        batch_boxes = center_to_corners_format(batch_boxes)88 89        # Convert from relative [0, 1] to absolute [0, height] coordinates90        if target_sizes is not None:91            batch_boxes = _scale_boxes(batch_boxes, target_sizes)92 93        results = []94        for scores, labels, boxes in zip(batch_scores, batch_labels, batch_boxes):95            keep = scores > threshold96            scores = scores[keep]97            labels = labels[keep]98            boxes = boxes[keep]99            results.append({"scores": scores, "labels": labels, "boxes": boxes})100 101        return results102 103    def post_process(self):104        raise NotImplementedError("Post-processing is not implemented for Grounding-Dino yet.")105 106    def post_process_segmentation(self):107        raise NotImplementedError("Segmentation post-processing is not implemented for Grounding-Dino yet.")108 109    def post_process_instance(self):110        raise NotImplementedError("Instance post-processing is not implemented for Grounding-Dino yet.")111 112    def post_process_panoptic(self):113        raise NotImplementedError("Panoptic post-processing is not implemented for Grounding-Dino yet.")114 115    def post_process_instance_segmentation(self):116        raise NotImplementedError("Segmentation post-processing is not implemented for Grounding-Dino yet.")117 118    def post_process_semantic_segmentation(self):119        raise NotImplementedError("Semantic segmentation post-processing is not implemented for Grounding-Dino yet.")120 121    def post_process_panoptic_segmentation(self):122        raise NotImplementedError("Panoptic segmentation post-processing is not implemented for Grounding-Dino yet.")123 124 125__all__ = ["GroundingDinoImageProcessorFast"]126