Aluode/PerceptionLabPortable

1#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨2#           This file was automatically generated from src/transformers/models/owlv2/modular_owlv2.py.3#               Do NOT edit this file manually as any edits will be overwritten by the generation of4#             the file from the modular. If any change should be done, please apply the change to the5#                          modular_owlv2.py file directly. One of our CI enforces this.6#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨7# coding=utf-88# Copyright 2025 The HuggingFace Inc. team. All rights reserved.9#10# Licensed under the Apache License, Version 2.0 (the "License");11# you may not use this file except in compliance with the License.12# You may obtain a copy of the License at13#14#     http://www.apache.org/licenses/LICENSE-2.015#16# Unless required by applicable law or agreed to in writing, software17# distributed under the License is distributed on an "AS IS" BASIS,18# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.19# See the License for the specific language governing permissions and20# limitations under the License.21 22import warnings23from typing import TYPE_CHECKING, Optional, Union24 25import torch26from torchvision.transforms.v2 import functional as F27 28from ...image_processing_utils_fast import BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs29from ...image_transforms import center_to_corners_format, group_images_by_shape, reorder_images30from ...image_utils import (31    OPENAI_CLIP_MEAN,32    OPENAI_CLIP_STD,33    ChannelDimension,34    ImageInput,35    PILImageResampling,36    SizeDict,37)38from ...processing_utils import Unpack39from ...utils import TensorType, auto_docstring40from .image_processing_owlv2 import _scale_boxes, box_iou41 42 43if TYPE_CHECKING:44    from .modeling_owlv2 import Owlv2ObjectDetectionOutput45 46 47class Owlv2FastImageProcessorKwargs(DefaultFastImageProcessorKwargs): ...48 49 50@auto_docstring51class Owlv2ImageProcessorFast(BaseImageProcessorFast):52    resample = PILImageResampling.BILINEAR53    image_mean = OPENAI_CLIP_MEAN54    image_std = OPENAI_CLIP_STD55    size = {"height": 960, "width": 960}56    default_to_square = True57    crop_size = None58    do_resize = True59    do_center_crop = None60    do_rescale = True61    do_normalize = True62    do_convert_rgb = None63    model_input_names = ["pixel_values"]64    rescale_factor = 1 / 25565    do_pad = True66    valid_kwargs = Owlv2FastImageProcessorKwargs67 68    def post_process(self, outputs, target_sizes):69        """70        Converts the raw output of [`Owlv2ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y,71        bottom_right_x, bottom_right_y) format.72 73        Args:74            outputs ([`Owlv2ObjectDetectionOutput`]):75                Raw outputs of the model.76            target_sizes (`torch.Tensor` of shape `(batch_size, 2)`):77                Tensor containing the size (h, w) of each image of the batch. For evaluation, this must be the original78                image size (before any data augmentation). For visualization, this should be the image size after data79                augment, but before padding.80        Returns:81            `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image82            in the batch as predicted by the model.83        """84        # TODO: (amy) add support for other frameworks85        warnings.warn(86            "`post_process` is deprecated and will be removed in v5 of Transformers, please use"87            " `post_process_object_detection` instead, with `threshold=0.` for equivalent results.",88            FutureWarning,89        )90 91        logits, boxes = outputs.logits, outputs.pred_boxes92 93        if len(logits) != len(target_sizes):94            raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits")95        if target_sizes.shape[1] != 2:96            raise ValueError("Each element of target_sizes must contain the size (h, w) of each image of the batch")97 98        probs = torch.max(logits, dim=-1)99        scores = torch.sigmoid(probs.values)100        labels = probs.indices101 102        # Convert to [x0, y0, x1, y1] format103        boxes = center_to_corners_format(boxes)104 105        # Convert from relative [0, 1] to absolute [0, height] coordinates106        img_h, img_w = target_sizes.unbind(1)107        scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device)108        boxes = boxes * scale_fct[:, None, :]109 110        results = [{"scores": s, "labels": l, "boxes": b} for s, l, b in zip(scores, labels, boxes)]111 112        return results113 114    def post_process_object_detection(115        self,116        outputs: "Owlv2ObjectDetectionOutput",117        threshold: float = 0.1,118        target_sizes: Optional[Union[TensorType, list[tuple]]] = None,119    ):120        """121        Converts the raw output of [`Owlv2ForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y,122        bottom_right_x, bottom_right_y) format.123 124        Args:125            outputs ([`Owlv2ObjectDetectionOutput`]):126                Raw outputs of the model.127            threshold (`float`, *optional*, defaults to 0.1):128                Score threshold to keep object detection predictions.129            target_sizes (`torch.Tensor` or `list[tuple[int, int]]`, *optional*):130                Tensor of shape `(batch_size, 2)` or list of tuples (`tuple[int, int]`) containing the target size131                `(height, width)` of each image in the batch. If unset, predictions will not be resized.132 133        Returns:134            `list[Dict]`: A list of dictionaries, each dictionary containing the following keys:135            - "scores": The confidence scores for each predicted box on the image.136            - "labels": Indexes of the classes predicted by the model on the image.137            - "boxes": Image bounding boxes in (top_left_x, top_left_y, bottom_right_x, bottom_right_y) format.138        """139        batch_logits, batch_boxes = outputs.logits, outputs.pred_boxes140        batch_size = len(batch_logits)141 142        if target_sizes is not None and len(target_sizes) != batch_size:143            raise ValueError("Make sure that you pass in as many target sizes as images")144 145        # batch_logits of shape (batch_size, num_queries, num_classes)146        batch_class_logits = torch.max(batch_logits, dim=-1)147        batch_scores = torch.sigmoid(batch_class_logits.values)148        batch_labels = batch_class_logits.indices149 150        # Convert to [x0, y0, x1, y1] format151        batch_boxes = center_to_corners_format(batch_boxes)152 153        # Convert from relative [0, 1] to absolute [0, height] coordinates154        if target_sizes is not None:155            batch_boxes = _scale_boxes(batch_boxes, target_sizes)156 157        results = []158        for scores, labels, boxes in zip(batch_scores, batch_labels, batch_boxes):159            keep = scores > threshold160            scores = scores[keep]161            labels = labels[keep]162            boxes = boxes[keep]163            results.append({"scores": scores, "labels": labels, "boxes": boxes})164 165        return results166 167    def post_process_image_guided_detection(self, outputs, threshold=0.0, nms_threshold=0.3, target_sizes=None):168        """169        Converts the output of [`Owlv2ForObjectDetection.image_guided_detection`] into the format expected by the COCO170        api.171 172        Args:173            outputs ([`Owlv2ImageGuidedObjectDetectionOutput`]):174                Raw outputs of the model.175            threshold (`float`, *optional*, defaults to 0.0):176                Minimum confidence threshold to use to filter out predicted boxes.177            nms_threshold (`float`, *optional*, defaults to 0.3):178                IoU threshold for non-maximum suppression of overlapping boxes.179            target_sizes (`torch.Tensor`, *optional*):180                Tensor of shape (batch_size, 2) where each entry is the (height, width) of the corresponding image in181                the batch. If set, predicted normalized bounding boxes are rescaled to the target sizes. If left to182                None, predictions will not be unnormalized.183 184        Returns:185            `list[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image186            in the batch as predicted by the model. All labels are set to None as187            `Owlv2ForObjectDetection.image_guided_detection` perform one-shot object detection.188        """189        logits, target_boxes = outputs.logits, outputs.target_pred_boxes190 191        if target_sizes is not None and len(logits) != len(target_sizes):192            raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the logits")193        if target_sizes is not None and target_sizes.shape[1] != 2:194            raise ValueError("Each element of target_sizes must contain the size (h, w) of each image of the batch")195 196        probs = torch.max(logits, dim=-1)197        scores = torch.sigmoid(probs.values)198 199        # Convert to [x0, y0, x1, y1] format200        target_boxes = center_to_corners_format(target_boxes)201 202        # Apply non-maximum suppression (NMS)203        if nms_threshold < 1.0:204            for idx in range(target_boxes.shape[0]):205                for i in torch.argsort(-scores[idx]):206                    if not scores[idx][i]:207                        continue208 209                    ious = box_iou(target_boxes[idx][i, :].unsqueeze(0), target_boxes[idx])[0][0]210                    ious[i] = -1.0  # Mask self-IoU.211                    scores[idx][ious > nms_threshold] = 0.0212 213        # Convert from relative [0, 1] to absolute [0, height] coordinates214        if target_sizes is not None:215            target_boxes = _scale_boxes(target_boxes, target_sizes)216 217        # Compute box display alphas based on prediction scores218        results = []219        alphas = torch.zeros_like(scores)220 221        for idx in range(target_boxes.shape[0]):222            # Select scores for boxes matching the current query:223            query_scores = scores[idx]224            if not query_scores.nonzero().numel():225                continue226 227            # Apply threshold on scores before scaling228            query_scores[query_scores < threshold] = 0.0229 230            # Scale box alpha such that the best box for each query has alpha 1.0 and the worst box has alpha 0.1.231            # All other boxes will either belong to a different query, or will not be shown.232            max_score = torch.max(query_scores) + 1e-6233            query_alphas = (query_scores - (max_score * 0.1)) / (max_score * 0.9)234            query_alphas = torch.clip(query_alphas, 0.0, 1.0)235            alphas[idx] = query_alphas236 237            mask = alphas[idx] > 0238            box_scores = alphas[idx][mask]239            boxes = target_boxes[idx][mask]240            results.append({"scores": box_scores, "labels": None, "boxes": boxes})241 242        return results243 244    def __init__(self, **kwargs: Unpack[Owlv2FastImageProcessorKwargs]):245        super().__init__(**kwargs)246 247    @auto_docstring248    def preprocess(self, images: ImageInput, **kwargs: Unpack[Owlv2FastImageProcessorKwargs]):249        return super().preprocess(images, **kwargs)250 251    def _pad_images(self, images: "torch.Tensor", constant_value: float = 0.5) -> "torch.Tensor":252        """253        Pad an image with zeros to the given size.254        """255        height, width = images.shape[-2:]256        size = max(height, width)257        pad_bottom = size - height258        pad_right = size - width259 260        padding = (0, 0, pad_right, pad_bottom)261        padded_image = F.pad(images, padding, fill=constant_value)262        return padded_image263 264    def pad(265        self,266        images: list["torch.Tensor"],267        disable_grouping: Optional[bool],268        constant_value: float = 0.5,269        **kwargs,270    ) -> list["torch.Tensor"]:271        """272        Unlike the Base class `self.pad` where all images are padded to the maximum image size,273        Owlv2 pads an image to square.274        """275        grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)276        processed_images_grouped = {}277        for shape, stacked_images in grouped_images.items():278            stacked_images = self._pad_images(279                stacked_images,280                constant_value=constant_value,281            )282            processed_images_grouped[shape] = stacked_images283 284        processed_images = reorder_images(processed_images_grouped, grouped_images_index)285 286        return processed_images287 288    def resize(289        self,290        image: "torch.Tensor",291        size: SizeDict,292        anti_aliasing: bool = True,293        anti_aliasing_sigma=None,294        **kwargs,295    ) -> "torch.Tensor":296        """297        Resize an image as per the original implementation.298 299        Args:300            image (`Tensor`):301                Image to resize.302            size (`dict[str, int]`):303                Dictionary containing the height and width to resize the image to.304            anti_aliasing (`bool`, *optional*, defaults to `True`):305                Whether to apply anti-aliasing when downsampling the image.306            anti_aliasing_sigma (`float`, *optional*, defaults to `None`):307                Standard deviation for Gaussian kernel when downsampling the image. If `None`, it will be calculated308                automatically.309        """310        output_shape = (size.height, size.width)311 312        input_shape = image.shape313 314        # select height and width from input tensor315        factors = torch.tensor(input_shape[2:]).to(image.device) / torch.tensor(output_shape).to(image.device)316 317        if anti_aliasing:318            if anti_aliasing_sigma is None:319                anti_aliasing_sigma = ((factors - 1) / 2).clamp(min=0)320            else:321                anti_aliasing_sigma = torch.atleast_1d(anti_aliasing_sigma) * torch.ones_like(factors)322                if torch.any(anti_aliasing_sigma < 0):323                    raise ValueError("Anti-aliasing standard deviation must be greater than or equal to zero")324                elif torch.any((anti_aliasing_sigma > 0) & (factors <= 1)):325                    warnings.warn(326                        "Anti-aliasing standard deviation greater than zero but not down-sampling along all axes"327                    )328            if torch.any(anti_aliasing_sigma == 0):329                filtered = image330            else:331                kernel_sizes = 2 * torch.ceil(3 * anti_aliasing_sigma).int() + 1332 333                filtered = F.gaussian_blur(334                    image, (kernel_sizes[0], kernel_sizes[1]), sigma=anti_aliasing_sigma.tolist()335                )336 337        else:338            filtered = image339 340        out = F.resize(filtered, size=(size.height, size.width), antialias=False)341 342        return out343 344    def _preprocess(345        self,346        images: list["torch.Tensor"],347        do_resize: bool,348        size: SizeDict,349        interpolation: Optional["F.InterpolationMode"],350        do_pad: bool,351        do_rescale: bool,352        rescale_factor: float,353        do_normalize: bool,354        image_mean: Optional[Union[float, list[float]]],355        image_std: Optional[Union[float, list[float]]],356        disable_grouping: Optional[bool],357        return_tensors: Optional[Union[str, TensorType]],358        **kwargs,359    ) -> BatchFeature:360        # Group images by size for batched resizing361        grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)362        processed_images_grouped = {}363 364        for shape, stacked_images in grouped_images.items():365            # Rescale images before other operations as done in original implementation366            stacked_images = self.rescale_and_normalize(367                stacked_images, do_rescale, rescale_factor, False, image_mean, image_std368            )369            processed_images_grouped[shape] = stacked_images370 371        processed_images = reorder_images(processed_images_grouped, grouped_images_index)372 373        if do_pad:374            processed_images = self.pad(processed_images, constant_value=0.5, disable_grouping=disable_grouping)375 376        grouped_images, grouped_images_index = group_images_by_shape(377            processed_images, disable_grouping=disable_grouping378        )379        resized_images_grouped = {}380        for shape, stacked_images in grouped_images.items():381            if do_resize:382                resized_stack = self.resize(383                    image=stacked_images,384                    size=size,385                    interpolation=interpolation,386                    input_data_format=ChannelDimension.FIRST,387                )388                resized_images_grouped[shape] = resized_stack389        resized_images = reorder_images(resized_images_grouped, grouped_images_index)390 391        # Group images by size for further processing392        # Needed in case do_resize is False, or resize returns images with different sizes393        grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)394        processed_images_grouped = {}395        for shape, stacked_images in grouped_images.items():396            # Fused rescale and normalize397            stacked_images = self.rescale_and_normalize(398                stacked_images, False, rescale_factor, do_normalize, image_mean, image_std399            )400            processed_images_grouped[shape] = stacked_images401 402        processed_images = reorder_images(processed_images_grouped, grouped_images_index)403 404        processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images405 406        return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors)407 408 409__all__ = ["Owlv2ImageProcessorFast"]410