Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2022 The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Image processor class for Perceiver."""16 17from typing import Optional, Union18 19import numpy as np20 21from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict22from ...image_transforms import center_crop, resize, to_channel_dimension_format23from ...image_utils import (24 IMAGENET_DEFAULT_MEAN,25 IMAGENET_DEFAULT_STD,26 ChannelDimension,27 ImageInput,28 PILImageResampling,29 get_image_size,30 infer_channel_dimension_format,31 is_scaled_image,32 make_flat_list_of_images,33 to_numpy_array,34 valid_images,35 validate_preprocess_arguments,36)37from ...utils import TensorType, filter_out_non_signature_kwargs, is_vision_available, logging38from ...utils.import_utils import requires39 40 41if is_vision_available():42 import PIL43 44 45logger = logging.get_logger(__name__)46 47 48@requires(backends=("vision",))49class PerceiverImageProcessor(BaseImageProcessor):50 r"""51 Constructs a Perceiver image processor.52 53 Args:54 do_center_crop (`bool`, `optional`, defaults to `True`):55 Whether or not to center crop the image. If the input size if smaller than `crop_size` along any edge, the56 image will be padded with zeros and then center cropped. Can be overridden by the `do_center_crop`57 parameter in the `preprocess` method.58 crop_size (`dict[str, int]`, *optional*, defaults to `{"height": 256, "width": 256}`):59 Desired output size when applying center-cropping. Can be overridden by the `crop_size` parameter in the60 `preprocess` method.61 do_resize (`bool`, *optional*, defaults to `True`):62 Whether to resize the image to `(size["height"], size["width"])`. Can be overridden by the `do_resize`63 parameter in the `preprocess` method.64 size (`dict[str, int]` *optional*, defaults to `{"height": 224, "width": 224}`):65 Size of the image after resizing. Can be overridden by the `size` parameter in the `preprocess` method.66 resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`):67 Defines the resampling filter to use if resizing the image. Can be overridden by the `resample` parameter68 in the `preprocess` method.69 do_rescale (`bool`, *optional*, defaults to `True`):70 Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the `do_rescale`71 parameter in the `preprocess` method.72 rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):73 Defines the scale factor to use if rescaling the image. Can be overridden by the `rescale_factor` parameter74 in the `preprocess` method.75 do_normalize:76 Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess`77 method.78 image_mean (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`):79 Mean to use if normalizing the image. This is a float or list of floats the length of the number of80 channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method.81 image_std (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`):82 Standard deviation to use if normalizing the image. This is a float or list of floats the length of the83 number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.84 """85 86 model_input_names = ["pixel_values"]87 88 def __init__(89 self,90 do_center_crop: bool = True,91 crop_size: Optional[dict[str, int]] = None,92 do_resize: bool = True,93 size: Optional[dict[str, int]] = None,94 resample: PILImageResampling = PILImageResampling.BICUBIC,95 do_rescale: bool = True,96 rescale_factor: Union[int, float] = 1 / 255,97 do_normalize: bool = True,98 image_mean: Optional[Union[float, list[float]]] = None,99 image_std: Optional[Union[float, list[float]]] = None,100 **kwargs,101 ) -> None:102 super().__init__(**kwargs)103 crop_size = crop_size if crop_size is not None else {"height": 256, "width": 256}104 crop_size = get_size_dict(crop_size, param_name="crop_size")105 size = size if size is not None else {"height": 224, "width": 224}106 size = get_size_dict(size)107 108 self.do_center_crop = do_center_crop109 self.crop_size = crop_size110 self.do_resize = do_resize111 self.size = size112 self.resample = resample113 self.do_rescale = do_rescale114 self.rescale_factor = rescale_factor115 self.do_normalize = do_normalize116 self.image_mean = image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN117 self.image_std = image_std if image_std is not None else IMAGENET_DEFAULT_STD118 119 def center_crop(120 self,121 image: np.ndarray,122 crop_size: dict[str, int],123 size: Optional[int] = None,124 data_format: Optional[Union[str, ChannelDimension]] = None,125 input_data_format: Optional[Union[str, ChannelDimension]] = None,126 **kwargs,127 ) -> np.ndarray:128 """129 Center crop an image to `(size["height"] / crop_size["height"] * min_dim, size["width"] / crop_size["width"] *130 min_dim)`. Where `min_dim = min(size["height"], size["width"])`.131 132 If the input size is smaller than `crop_size` along any edge, the image will be padded with zeros and then133 center cropped.134 135 Args:136 image (`np.ndarray`):137 Image to center crop.138 crop_size (`dict[str, int]`):139 Desired output size after applying the center crop.140 size (`dict[str, int]`, *optional*):141 Size of the image after resizing. If not provided, the self.size attribute will be used.142 data_format (`str` or `ChannelDimension`, *optional*):143 The channel dimension format of the image. If not provided, it will be the same as the input image.144 input_data_format (`str` or `ChannelDimension`, *optional*):145 The channel dimension format of the input image. If not provided, it will be inferred.146 """147 size = self.size if size is None else size148 size = get_size_dict(size)149 crop_size = get_size_dict(crop_size, param_name="crop_size")150 151 height, width = get_image_size(image, channel_dim=input_data_format)152 min_dim = min(height, width)153 cropped_height = (size["height"] / crop_size["height"]) * min_dim154 cropped_width = (size["width"] / crop_size["width"]) * min_dim155 return center_crop(156 image,157 size=(cropped_height, cropped_width),158 data_format=data_format,159 input_data_format=input_data_format,160 **kwargs,161 )162 163 # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.resize with PILImageResampling.BILINEAR->PILImageResampling.BICUBIC164 def resize(165 self,166 image: np.ndarray,167 size: dict[str, int],168 resample: PILImageResampling = PILImageResampling.BICUBIC,169 data_format: Optional[Union[str, ChannelDimension]] = None,170 input_data_format: Optional[Union[str, ChannelDimension]] = None,171 **kwargs,172 ) -> np.ndarray:173 """174 Resize an image to `(size["height"], size["width"])`.175 176 Args:177 image (`np.ndarray`):178 Image to resize.179 size (`dict[str, int]`):180 Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image.181 resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`):182 `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`.183 data_format (`ChannelDimension` or `str`, *optional*):184 The channel dimension format for the output image. If unset, the channel dimension format of the input185 image is used. Can be one of:186 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.187 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.188 - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.189 input_data_format (`ChannelDimension` or `str`, *optional*):190 The channel dimension format for the input image. If unset, the channel dimension format is inferred191 from the input image. Can be one of:192 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.193 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.194 - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.195 196 Returns:197 `np.ndarray`: The resized image.198 """199 size = get_size_dict(size)200 if "height" not in size or "width" not in size:201 raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}")202 output_size = (size["height"], size["width"])203 return resize(204 image,205 size=output_size,206 resample=resample,207 data_format=data_format,208 input_data_format=input_data_format,209 **kwargs,210 )211 212 @filter_out_non_signature_kwargs()213 def preprocess(214 self,215 images: ImageInput,216 do_center_crop: Optional[bool] = None,217 crop_size: Optional[dict[str, int]] = None,218 do_resize: Optional[bool] = None,219 size: Optional[dict[str, int]] = None,220 resample: Optional[PILImageResampling] = None,221 do_rescale: Optional[bool] = None,222 rescale_factor: Optional[float] = None,223 do_normalize: Optional[bool] = None,224 image_mean: Optional[Union[float, list[float]]] = None,225 image_std: Optional[Union[float, list[float]]] = None,226 return_tensors: Optional[Union[str, TensorType]] = None,227 data_format: ChannelDimension = ChannelDimension.FIRST,228 input_data_format: Optional[Union[str, ChannelDimension]] = None,229 ) -> PIL.Image.Image:230 """231 Preprocess an image or batch of images.232 233 Args:234 images (`ImageInput`):235 Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If236 passing in images with pixel values between 0 and 1, set `do_rescale=False`.237 do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`):238 Whether to center crop the image to `crop_size`.239 crop_size (`dict[str, int]`, *optional*, defaults to `self.crop_size`):240 Desired output size after applying the center crop.241 do_resize (`bool`, *optional*, defaults to `self.do_resize`):242 Whether to resize the image.243 size (`dict[str, int]`, *optional*, defaults to `self.size`):244 Size of the image after resizing.245 resample (`int`, *optional*, defaults to `self.resample`):246 Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`, Only247 has an effect if `do_resize` is set to `True`.248 do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):249 Whether to rescale the image.250 rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):251 Rescale factor to rescale the image by if `do_rescale` is set to `True`.252 do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):253 Whether to normalize the image.254 image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`):255 Image mean.256 image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`):257 Image standard deviation.258 return_tensors (`str` or `TensorType`, *optional*):259 The type of tensors to return. Can be one of:260 - Unset: Return a list of `np.ndarray`.261 - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.262 - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.263 - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.264 - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.265 data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):266 The channel dimension format for the output image. Can be one of:267 - `ChannelDimension.FIRST`: image in (num_channels, height, width) format.268 - `ChannelDimension.LAST`: image in (height, width, num_channels) format.269 input_data_format (`ChannelDimension` or `str`, *optional*):270 The channel dimension format for the input image. If unset, the channel dimension format is inferred271 from the input image. Can be one of:272 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.273 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.274 - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.275 """276 do_center_crop = do_center_crop if do_center_crop is not None else self.do_center_crop277 crop_size = crop_size if crop_size is not None else self.crop_size278 crop_size = get_size_dict(crop_size, param_name="crop_size")279 do_resize = do_resize if do_resize is not None else self.do_resize280 size = size if size is not None else self.size281 size = get_size_dict(size)282 resample = resample if resample is not None else self.resample283 do_rescale = do_rescale if do_rescale is not None else self.do_rescale284 rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor285 do_normalize = do_normalize if do_normalize is not None else self.do_normalize286 image_mean = image_mean if image_mean is not None else self.image_mean287 image_std = image_std if image_std is not None else self.image_std288 289 images = make_flat_list_of_images(images)290 291 if not valid_images(images):292 raise ValueError(293 "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "294 "torch.Tensor, tf.Tensor or jax.ndarray."295 )296 validate_preprocess_arguments(297 do_rescale=do_rescale,298 rescale_factor=rescale_factor,299 do_normalize=do_normalize,300 image_mean=image_mean,301 image_std=image_std,302 do_center_crop=do_center_crop,303 crop_size=crop_size,304 do_resize=do_resize,305 size=size,306 resample=resample,307 )308 309 # All transformations expect numpy arrays.310 images = [to_numpy_array(image) for image in images]311 312 if do_rescale and is_scaled_image(images[0]):313 logger.warning_once(314 "It looks like you are trying to rescale already rescaled images. If the input"315 " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."316 )317 318 if input_data_format is None:319 # We assume that all images have the same channel dimension format.320 input_data_format = infer_channel_dimension_format(images[0])321 322 if do_center_crop:323 images = [324 self.center_crop(image, crop_size, size=size, input_data_format=input_data_format) for image in images325 ]326 327 if do_resize:328 images = [329 self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format)330 for image in images331 ]332 333 if do_rescale:334 images = [335 self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format)336 for image in images337 ]338 339 if do_normalize:340 images = [341 self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format)342 for image in images343 ]344 345 images = [346 to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images347 ]348 349 data = {"pixel_values": images}350 return BatchFeature(data=data, tensor_type=return_tensors)351 352 353__all__ = ["PerceiverImageProcessor"]354