Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2025 The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Image processor class for Gemma3."""16 17import itertools18import math19from typing import Optional, Union20 21import numpy as np22 23from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict24from ...image_transforms import (25 convert_to_rgb,26 resize,27 to_channel_dimension_format,28)29from ...image_utils import (30 IMAGENET_STANDARD_MEAN,31 IMAGENET_STANDARD_STD,32 ChannelDimension,33 ImageInput,34 PILImageResampling,35 get_image_size,36 infer_channel_dimension_format,37 is_scaled_image,38 make_flat_list_of_images,39 to_numpy_array,40 valid_images,41 validate_preprocess_arguments,42)43from ...utils import TensorType, filter_out_non_signature_kwargs, is_vision_available, logging44 45 46logger = logging.get_logger(__name__)47 48 49if is_vision_available():50 import PIL51 52 53class Gemma3ImageProcessor(BaseImageProcessor):54 r"""55 Constructs a SigLIP image processor.56 57 Args:58 do_resize (`bool`, *optional*, defaults to `True`):59 Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by60 `do_resize` in the `preprocess` method.61 size (`dict[str, int]` *optional*, defaults to `{"height": 224, "width": 224}`):62 Size of the image after resizing. Can be overridden by `size` in the `preprocess` method.63 resample (`PILImageResampling`, *optional*, defaults to `Resampling.BILINEAR`):64 Resampling filter to use if resizing the image. Can be overridden by `resample` in the `preprocess` method.65 do_rescale (`bool`, *optional*, defaults to `True`):66 Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by `do_rescale` in67 the `preprocess` method.68 rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):69 Scale factor to use if rescaling the image. Can be overridden by `rescale_factor` in the `preprocess`70 method.71 do_normalize (`bool`, *optional*, defaults to `True`):72 Whether to normalize the image by the specified mean and standard deviation. Can be overridden by73 `do_normalize` in the `preprocess` method.74 image_mean (`float` or `list[float]`, *optional*, defaults to `[0.5, 0.5, 0.5]`):75 Mean to use if normalizing the image. This is a float or list of floats the length of the number of76 channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method.77 image_std (`float` or `list[float]`, *optional*, defaults to `[0.5, 0.5, 0.5]`):78 Standard deviation to use if normalizing the image. This is a float or list of floats the length of the79 number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.80 Can be overridden by the `image_std` parameter in the `preprocess` method.81 do_convert_rgb (`bool`, *optional*, defaults to `True`):82 Whether to convert the image to RGB.83 do_pan_and_scan (`bool`, *optional*):84 Whether to apply `pan_and_scan` to images.85 pan_and_scan_min_crop_size (`int`, *optional*):86 Minimum size of each crop in pan and scan.87 pan_and_scan_max_num_crops (`int`, *optional*):88 Maximum number of crops per image in pan and scan.89 pan_and_scan_min_ratio_to_activate (`float`, *optional*):90 Minimum aspect ratio to activate pan and scan.91 """92 93 model_input_names = ["pixel_values", "num_crops"]94 95 def __init__(96 self,97 do_resize: bool = True,98 size: Optional[dict[str, int]] = None,99 resample: PILImageResampling = PILImageResampling.BILINEAR,100 do_rescale: bool = True,101 rescale_factor: Union[int, float] = 1 / 255,102 do_normalize: bool = True,103 image_mean: Optional[Union[float, list[float]]] = None,104 image_std: Optional[Union[float, list[float]]] = None,105 do_convert_rgb: Optional[bool] = True,106 do_pan_and_scan: Optional[bool] = None,107 pan_and_scan_min_crop_size: Optional[int] = None,108 pan_and_scan_max_num_crops: Optional[int] = None,109 pan_and_scan_min_ratio_to_activate: Optional[float] = None,110 **kwargs,111 ) -> None:112 super().__init__(**kwargs)113 size = size if size is not None else {"height": 224, "width": 224}114 size = get_size_dict(size, default_to_square=True)115 image_mean = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN116 image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD117 118 self.do_resize = do_resize119 self.size = size120 self.resample = resample121 self.do_rescale = do_rescale122 self.rescale_factor = rescale_factor123 self.do_normalize = do_normalize124 self.image_mean = image_mean125 self.image_std = image_std126 self.do_convert_rgb = do_convert_rgb127 self.do_pan_and_scan = do_pan_and_scan128 self.pan_and_scan_min_crop_size = pan_and_scan_min_crop_size129 self.pan_and_scan_max_num_crops = pan_and_scan_max_num_crops130 self.pan_and_scan_min_ratio_to_activate = pan_and_scan_min_ratio_to_activate131 132 def pan_and_scan(133 self,134 image: np.ndarray,135 pan_and_scan_min_crop_size: int,136 pan_and_scan_max_num_crops: int,137 pan_and_scan_min_ratio_to_activate: float,138 data_format: Optional[Union[str, ChannelDimension]] = None,139 input_data_format: Optional[Union[str, ChannelDimension]] = None,140 ):141 """142 Pan and Scan and image, by cropping into smaller images when the aspect ratio exceeds143 minimum allowed ratio.144 145 Args:146 image (`np.ndarray`):147 Image to resize.148 pan_and_scan_min_crop_size (`int`, *optional*):149 Minimum size of each crop in pan and scan.150 pan_and_scan_max_num_crops (`int`, *optional*):151 Maximum number of crops per image in pan and scan.152 pan_and_scan_min_ratio_to_activate (`float`, *optional*):153 Minimum aspect ratio to activate pan and scan.154 data_format (`str` or `ChannelDimension`, *optional*):155 The channel dimension format of the image. If not provided, it will be the same as the input image.156 input_data_format (`ChannelDimension` or `str`, *optional*):157 The channel dimension format of the input image. If not provided, it will be inferred.158 """159 height, width = get_image_size(image)160 161 # Square or landscape image.162 if width >= height:163 # Only apply PaS if the image is sufficiently exaggerated164 if width / height < pan_and_scan_min_ratio_to_activate:165 return []166 167 # Select ideal number of crops close to the image aspect ratio and such that crop_size > min_crop_size.168 num_crops_w = int(math.floor(width / height + 0.5)) # Half round up rounding.169 num_crops_w = min(int(math.floor(width / pan_and_scan_min_crop_size)), num_crops_w)170 171 # Make sure the number of crops is in range [2, pan_and_scan_max_num_crops].172 num_crops_w = max(2, num_crops_w)173 num_crops_w = min(pan_and_scan_max_num_crops, num_crops_w)174 num_crops_h = 1175 176 # Portrait image.177 else:178 # Only apply PaS if the image is sufficiently exaggerated179 if height / width < pan_and_scan_min_ratio_to_activate:180 return []181 182 # Select ideal number of crops close to the image aspect ratio and such that crop_size > min_crop_size.183 num_crops_h = int(math.floor(height / width + 0.5))184 num_crops_h = min(int(math.floor(height / pan_and_scan_min_crop_size)), num_crops_h)185 186 # Make sure the number of crops is in range [2, pan_and_scan_max_num_crops].187 num_crops_h = max(2, num_crops_h)188 num_crops_h = min(pan_and_scan_max_num_crops, num_crops_h)189 num_crops_w = 1190 191 crop_size_w = int(math.ceil(width / num_crops_w))192 crop_size_h = int(math.ceil(height / num_crops_h))193 194 # Don't apply PaS if crop size is too small.195 if min(crop_size_w, crop_size_h) < pan_and_scan_min_crop_size:196 return []197 198 crop_positions_w = [crop_size_w * i for i in range(num_crops_w)]199 crop_positions_h = [crop_size_h * i for i in range(num_crops_h)]200 201 if input_data_format == ChannelDimension.LAST:202 image_crops = [203 image[pos_h : pos_h + crop_size_h, pos_w : pos_w + crop_size_w]204 for pos_h, pos_w in itertools.product(crop_positions_h, crop_positions_w)205 ]206 else:207 image_crops = [208 image[:, pos_h : pos_h + crop_size_h, pos_w : pos_w + crop_size_w]209 for pos_h, pos_w in itertools.product(crop_positions_h, crop_positions_w)210 ]211 212 return image_crops213 214 def _process_images_for_pan_and_scan(215 self,216 images: list[np.ndarray],217 do_pan_and_scan: bool,218 pan_and_scan_min_crop_size: int,219 pan_and_scan_max_num_crops: int,220 pan_and_scan_min_ratio_to_activate: float,221 data_format: Optional[Union[str, ChannelDimension]] = None,222 input_data_format: Optional[Union[str, ChannelDimension]] = None,223 ):224 pas_images_list = []225 num_crops = []226 for image in images:227 pas_images = self.pan_and_scan(228 image=image,229 pan_and_scan_min_crop_size=pan_and_scan_min_crop_size,230 pan_and_scan_max_num_crops=pan_and_scan_max_num_crops,231 pan_and_scan_min_ratio_to_activate=pan_and_scan_min_ratio_to_activate,232 data_format=data_format,233 input_data_format=input_data_format,234 )235 pas_images_list.extend([image] + pas_images)236 num_crops.append(len(pas_images))237 return pas_images_list, num_crops238 239 @filter_out_non_signature_kwargs()240 def preprocess(241 self,242 images: ImageInput,243 do_resize: Optional[bool] = None,244 size: Optional[dict[str, int]] = None,245 resample: Optional[PILImageResampling] = None,246 do_rescale: Optional[bool] = None,247 rescale_factor: Optional[float] = None,248 do_normalize: Optional[bool] = None,249 image_mean: Optional[Union[float, list[float]]] = None,250 image_std: Optional[Union[float, list[float]]] = None,251 return_tensors: Optional[Union[str, TensorType]] = None,252 data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,253 input_data_format: Optional[Union[str, ChannelDimension]] = None,254 do_convert_rgb: Optional[bool] = None,255 do_pan_and_scan: Optional[bool] = None,256 pan_and_scan_min_crop_size: Optional[int] = None,257 pan_and_scan_max_num_crops: Optional[int] = None,258 pan_and_scan_min_ratio_to_activate: Optional[float] = None,259 ) -> PIL.Image.Image:260 """261 Preprocess an image or batch of images.262 263 Args:264 images (`ImageInput`):265 Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If266 passing in images with pixel values between 0 and 1, set `do_rescale=False`.267 do_resize (`bool`, *optional*, defaults to `self.do_resize`):268 Whether to resize the image.269 size (`dict[str, int]`, *optional*, defaults to `self.size`):270 Size of the image after resizing.271 resample (`int`, *optional*, defaults to `self.resample`):272 Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only273 has an effect if `do_resize` is set to `True`.274 do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):275 Whether to rescale the image.276 rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):277 Rescale factor to rescale the image by if `do_rescale` is set to `True`.278 do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):279 Whether to normalize the image.280 image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`):281 Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.282 image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`):283 Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to284 `True`.285 return_tensors (`str` or `TensorType`, *optional*):286 The type of tensors to return. Can be one of:287 - Unset: Return a list of `np.ndarray`.288 - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.289 - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.290 - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.291 - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.292 data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):293 The channel dimension format for the output image. Can be one of:294 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.295 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.296 - Unset: Use the channel dimension format of the input image.297 input_data_format (`ChannelDimension` or `str`, *optional*):298 The channel dimension format for the input image. If unset, the channel dimension format is inferred299 from the input image. Can be one of:300 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.301 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.302 - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.303 do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):304 Whether to convert the image to RGB.305 do_pan_and_scan (`bool`, *optional*, defaults to `self.do_pan_and_scan`):306 Whether to apply `pan_and_scan` to images.307 pan_and_scan_min_crop_size (`int`, *optional*, defaults to `self.pan_and_scan_min_crop_size`):308 Minimum size of each crop in pan and scan.309 pan_and_scan_max_num_crops (`int`, *optional*, defaults to `self.pan_and_scan_max_num_crops`):310 Maximum number of crops per image in pan and scan.311 pan_and_scan_min_ratio_to_activate (`float`, *optional*, defaults to `self.pan_and_scan_min_ratio_to_activate`):312 Minimum aspect ratio to activate pan and scan.313 """314 do_resize = do_resize if do_resize is not None else self.do_resize315 size = size if size is not None else self.size316 size = get_size_dict(size, param_name="size", default_to_square=False)317 resample = resample if resample is not None else self.resample318 do_rescale = do_rescale if do_rescale is not None else self.do_rescale319 rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor320 do_normalize = do_normalize if do_normalize is not None else self.do_normalize321 image_mean = image_mean if image_mean is not None else self.image_mean322 image_std = image_std if image_std is not None else self.image_std323 do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb324 do_pan_and_scan = do_pan_and_scan if do_pan_and_scan is not None else self.do_pan_and_scan325 pan_and_scan_min_crop_size = (326 pan_and_scan_min_crop_size if pan_and_scan_min_crop_size is not None else self.pan_and_scan_min_crop_size327 )328 pan_and_scan_max_num_crops = (329 pan_and_scan_max_num_crops if pan_and_scan_max_num_crops is not None else self.pan_and_scan_max_num_crops330 )331 pan_and_scan_min_ratio_to_activate = (332 pan_and_scan_min_ratio_to_activate333 if pan_and_scan_min_ratio_to_activate is not None334 else self.pan_and_scan_min_ratio_to_activate335 )336 337 images = self.fetch_images(images)338 images = make_flat_list_of_images(images)339 340 if not valid_images(images):341 raise ValueError(342 "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "343 "torch.Tensor, tf.Tensor or jax.ndarray."344 )345 346 validate_preprocess_arguments(347 do_rescale=do_rescale,348 rescale_factor=rescale_factor,349 do_normalize=do_normalize,350 image_mean=image_mean,351 image_std=image_std,352 do_resize=do_resize,353 size=size,354 resample=resample,355 )356 if do_convert_rgb:357 images = [convert_to_rgb(image) for image in images]358 359 # All transformations expect numpy arrays.360 images = [to_numpy_array(image) for image in images]361 362 if do_rescale and is_scaled_image(images[0]):363 logger.warning_once(364 "It looks like you are trying to rescale already rescaled images. If the input"365 " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."366 )367 368 if input_data_format is None:369 # We assume that all images have the same channel dimension format.370 input_data_format = infer_channel_dimension_format(images[0])371 372 if do_pan_and_scan:373 images, num_crops = self._process_images_for_pan_and_scan(374 images=images,375 do_pan_and_scan=do_pan_and_scan,376 pan_and_scan_min_crop_size=pan_and_scan_min_crop_size,377 pan_and_scan_max_num_crops=pan_and_scan_max_num_crops,378 pan_and_scan_min_ratio_to_activate=pan_and_scan_min_ratio_to_activate,379 data_format=data_format,380 input_data_format=input_data_format,381 )382 383 else:384 num_crops = [0 for _ in images]385 386 processed_images = []387 for image in images:388 if do_resize:389 height, width = size["height"], size["width"]390 image = resize(391 image=image, size=(height, width), resample=resample, input_data_format=input_data_format392 )393 394 if do_rescale:395 image = self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format)396 397 if do_normalize:398 image = self.normalize(399 image=image, mean=image_mean, std=image_std, input_data_format=input_data_format400 )401 402 image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format)403 processed_images.append(image)404 405 data = {"pixel_values": processed_images, "num_crops": num_crops}406 return BatchFeature(data=data, tensor_type=return_tensors)407 408 409__all__ = ["Gemma3ImageProcessor"]410