Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2024 The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Image processor class for Video-LLaVA."""16 17from typing import Optional, Union18 19import numpy as np20 21from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict22from ...image_transforms import (23 convert_to_rgb,24 get_resize_output_image_size,25 resize,26 to_channel_dimension_format,27)28from ...image_utils import (29 OPENAI_CLIP_MEAN,30 OPENAI_CLIP_STD,31 ChannelDimension,32 ImageInput,33 PILImageResampling,34 infer_channel_dimension_format,35 is_scaled_image,36 make_flat_list_of_images,37 to_numpy_array,38 valid_images,39 validate_preprocess_arguments,40)41from ...utils import TensorType, filter_out_non_signature_kwargs, logging42from ...video_utils import VideoInput, make_batched_videos43 44 45logger = logging.get_logger(__name__)46 47 48class VideoLlavaImageProcessor(BaseImageProcessor):49 r"""50 Constructs a CLIP image processor.51 52 Args:53 do_resize (`bool`, *optional*, defaults to `True`):54 Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by55 `do_resize` in the `preprocess` method.56 size (`dict[str, int]` *optional*, defaults to `{"shortest_edge": 224}`):57 Size of the image after resizing. The shortest edge of the image is resized to size["shortest_edge"], with58 the longest edge resized to keep the input aspect ratio. Can be overridden by `size` in the `preprocess`59 method.60 resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):61 Resampling filter to use if resizing the image. Can be overridden by `resample` in the `preprocess` method.62 do_center_crop (`bool`, *optional*, defaults to `True`):63 Whether to center crop the image to the specified `crop_size`. Can be overridden by `do_center_crop` in the64 `preprocess` method.65 crop_size (`dict[str, int]` *optional*, defaults to 224):66 Size of the output image after applying `center_crop`. Can be overridden by `crop_size` in the `preprocess`67 method.68 do_rescale (`bool`, *optional*, defaults to `True`):69 Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by `do_rescale` in70 the `preprocess` method.71 rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):72 Scale factor to use if rescaling the image. Can be overridden by `rescale_factor` in the `preprocess`73 method.74 do_normalize (`bool`, *optional*, defaults to `True`):75 Whether to normalize the image. Can be overridden by `do_normalize` in the `preprocess` method.76 image_mean (`float` or `list[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):77 Mean to use if normalizing the image. This is a float or list of floats the length of the number of78 channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method.79 image_std (`float` or `list[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):80 Standard deviation to use if normalizing the image. This is a float or list of floats the length of the81 number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.82 Can be overridden by the `image_std` parameter in the `preprocess` method.83 do_convert_rgb (`bool`, *optional*, defaults to `True`):84 Whether to convert the image to RGB.85 """86 87 model_input_names = ["pixel_values"]88 89 def __init__(90 self,91 do_resize: bool = True,92 size: Optional[dict[str, int]] = None,93 resample: PILImageResampling = PILImageResampling.BICUBIC,94 do_center_crop: bool = True,95 crop_size: Optional[dict[str, int]] = None,96 do_rescale: bool = True,97 rescale_factor: Union[int, float] = 1 / 255,98 do_normalize: bool = True,99 image_mean: Optional[Union[float, list[float]]] = None,100 image_std: Optional[Union[float, list[float]]] = None,101 do_convert_rgb: bool = True,102 **kwargs,103 ) -> None:104 super().__init__(**kwargs)105 size = size if size is not None else {"shortest_edge": 224}106 size = get_size_dict(size, default_to_square=False)107 crop_size = crop_size if crop_size is not None else {"height": 224, "width": 224}108 crop_size = get_size_dict(crop_size, default_to_square=True, param_name="crop_size")109 110 self.do_resize = do_resize111 self.size = size112 self.resample = resample113 self.do_center_crop = do_center_crop114 self.crop_size = crop_size115 self.do_rescale = do_rescale116 self.rescale_factor = rescale_factor117 self.do_normalize = do_normalize118 self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN119 self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD120 self.do_convert_rgb = do_convert_rgb121 122 def resize(123 self,124 image: np.ndarray,125 size: dict[str, int],126 resample: PILImageResampling = PILImageResampling.BICUBIC,127 data_format: Optional[Union[str, ChannelDimension]] = None,128 input_data_format: Optional[Union[str, ChannelDimension]] = None,129 **kwargs,130 ) -> np.ndarray:131 """132 Resize an image. The shortest edge of the image is resized to size["shortest_edge"], with the longest edge133 resized to keep the input aspect ratio.134 135 Args:136 image (`np.ndarray`):137 Image to resize.138 size (`dict[str, int]`):139 Size of the output image.140 resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`):141 Resampling filter to use when resiizing the image.142 data_format (`str` or `ChannelDimension`, *optional*):143 The channel dimension format of the image. If not provided, it will be the same as the input image.144 input_data_format (`ChannelDimension` or `str`, *optional*):145 The channel dimension format of the input image. If not provided, it will be inferred.146 """147 default_to_square = True148 if "shortest_edge" in size:149 size = size["shortest_edge"]150 default_to_square = False151 elif "height" in size and "width" in size:152 size = (size["height"], size["width"])153 else:154 raise ValueError("Size must contain either 'shortest_edge' or 'height' and 'width'.")155 156 output_size = get_resize_output_image_size(157 image,158 size=size,159 default_to_square=default_to_square,160 input_data_format=input_data_format,161 )162 return resize(163 image,164 size=output_size,165 resample=resample,166 data_format=data_format,167 input_data_format=input_data_format,168 **kwargs,169 )170 171 @filter_out_non_signature_kwargs()172 def preprocess(173 self,174 images: Optional[list[ImageInput]] = None,175 videos: Optional[list[VideoInput]] = None,176 do_resize: Optional[bool] = None,177 size: Optional[dict[str, int]] = None,178 resample: Optional[PILImageResampling] = None,179 do_center_crop: Optional[bool] = None,180 crop_size: Optional[int] = None,181 do_rescale: Optional[bool] = None,182 rescale_factor: Optional[float] = None,183 do_normalize: Optional[bool] = None,184 image_mean: Optional[Union[float, list[float]]] = None,185 image_std: Optional[Union[float, list[float]]] = None,186 do_convert_rgb: Optional[bool] = None,187 return_tensors: Optional[Union[str, TensorType]] = None,188 data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,189 input_data_format: Optional[Union[str, ChannelDimension]] = None,190 ) -> BatchFeature:191 """192 Preprocess an image or batch of images.193 194 Args:195 images (`ImageInput`, *optional*):196 List of images to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If197 passing in images with pixel values between 0 and 1, set `do_rescale=False`.198 videos (`VideoInput`, *optional*):199 List of videos to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If200 passing in videos with pixel values between 0 and 1, set `do_rescale=False`.201 do_resize (`bool`, *optional*, defaults to `self.do_resize`):202 Whether to resize the image.203 size (`dict[str, int]`, *optional*, defaults to `self.size`):204 Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with205 the longest edge resized to keep the input aspect ratio.206 resample (`int`, *optional*, defaults to `self.resample`):207 Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only208 has an effect if `do_resize` is set to `True`.209 do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`):210 Whether to center crop the image.211 crop_size (`dict[str, int]`, *optional*, defaults to `self.crop_size`):212 Size of the center crop. Only has an effect if `do_center_crop` is set to `True`.213 do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):214 Whether to rescale the image.215 rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):216 Rescale factor to rescale the image by if `do_rescale` is set to `True`.217 do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):218 Whether to normalize the image.219 image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`):220 Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.221 image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`):222 Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to223 `True`.224 do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):225 Whether to convert the image to RGB.226 return_tensors (`str` or `TensorType`, *optional*):227 The type of tensors to return. Can be one of:228 - Unset: Return a list of `np.ndarray`.229 - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.230 - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.231 - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.232 - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.233 data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):234 The channel dimension format for the output image. Can be one of:235 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.236 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.237 - Unset: Use the channel dimension format of the input image.238 input_data_format (`ChannelDimension` or `str`, *optional*):239 The channel dimension format for the input image. If unset, the channel dimension format is inferred240 from the input image. Can be one of:241 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.242 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.243 - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.244 """245 do_resize = do_resize if do_resize is not None else self.do_resize246 size = size if size is not None else self.size247 size = get_size_dict(size, param_name="size", default_to_square=False)248 resample = resample if resample is not None else self.resample249 do_center_crop = do_center_crop if do_center_crop is not None else self.do_center_crop250 crop_size = crop_size if crop_size is not None else self.crop_size251 crop_size = get_size_dict(crop_size, param_name="crop_size", default_to_square=True)252 do_rescale = do_rescale if do_rescale is not None else self.do_rescale253 rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor254 do_normalize = do_normalize if do_normalize is not None else self.do_normalize255 image_mean = image_mean if image_mean is not None else self.image_mean256 image_std = image_std if image_std is not None else self.image_std257 do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb258 259 if images is not None:260 images = self.fetch_images(images)261 images = make_flat_list_of_images(images)262 263 if images is not None and not valid_images(images):264 raise ValueError(265 "Invalid input type. Must be of type PIL.Image.Image, numpy.ndarray, "266 "torch.Tensor, tf.Tensor or jax.ndarray."267 )268 269 data = {}270 if videos is not None:271 logger.warning(272 "`VideoLlavaImageProcessor` works only with image inputs and doesn't process videos anymore. "273 "This is a deprecated behavior and will be removed in v5.0. "274 "Your videos should be forwarded to `VideoLlavaVideoProcessor`. "275 )276 videos = make_batched_videos(videos)277 pixel_values_videos = [278 [279 self._preprocess_image(280 image=frame,281 do_resize=do_resize,282 size=size,283 resample=resample,284 do_rescale=do_rescale,285 rescale_factor=rescale_factor,286 do_normalize=do_normalize,287 image_mean=image_mean,288 image_std=image_std,289 do_center_crop=do_center_crop,290 crop_size=crop_size,291 do_convert_rgb=do_convert_rgb,292 data_format=data_format,293 input_data_format=input_data_format,294 )295 for frame in video296 ]297 for video in videos298 ]299 data["pixel_values_videos"] = pixel_values_videos300 301 if images is not None:302 pixel_values_images = [303 self._preprocess_image(304 image=image,305 do_resize=do_resize,306 size=size,307 resample=resample,308 do_rescale=do_rescale,309 rescale_factor=rescale_factor,310 do_normalize=do_normalize,311 image_mean=image_mean,312 image_std=image_std,313 do_center_crop=do_center_crop,314 crop_size=crop_size,315 do_convert_rgb=do_convert_rgb,316 data_format=data_format,317 input_data_format=input_data_format,318 )319 for image in images320 ]321 data["pixel_values_images"] = pixel_values_images322 323 encoded_outputs = BatchFeature(data, tensor_type=return_tensors)324 325 return encoded_outputs326 327 def _preprocess_image(328 self,329 image: Optional[ImageInput] = None,330 do_resize: Optional[bool] = None,331 size: Optional[dict[str, int]] = None,332 resample: Optional[PILImageResampling] = None,333 do_rescale: Optional[bool] = None,334 rescale_factor: Optional[float] = None,335 do_normalize: Optional[bool] = None,336 image_mean: Optional[Union[float, list[float]]] = None,337 image_std: Optional[Union[float, list[float]]] = None,338 do_center_crop: Optional[bool] = None,339 crop_size: Optional[int] = None,340 do_convert_rgb: Optional[bool] = None,341 data_format: ChannelDimension = ChannelDimension.FIRST,342 input_data_format: Optional[Union[str, ChannelDimension]] = None,343 ) -> np.ndarray:344 validate_preprocess_arguments(345 do_rescale=do_rescale,346 rescale_factor=rescale_factor,347 do_normalize=do_normalize,348 image_mean=image_mean,349 image_std=image_std,350 do_center_crop=do_center_crop,351 crop_size=crop_size,352 do_resize=do_resize,353 size=size,354 resample=resample,355 )356 357 # PIL RGBA images are converted to RGB358 if do_convert_rgb:359 image = convert_to_rgb(image)360 361 # All transformations expect numpy arrays.362 image = to_numpy_array(image)363 364 if do_rescale and is_scaled_image(image):365 logger.warning_once(366 "It looks like you are trying to rescale already rescaled images/video frames. If the input"367 " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."368 )369 370 if input_data_format is None:371 # We assume that all images have the same channel dimension format.372 input_data_format = infer_channel_dimension_format(image)373 374 if do_resize:375 image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format)376 377 if do_center_crop:378 image = self.center_crop(image=image, size=crop_size, input_data_format=input_data_format)379 380 if do_rescale:381 image = self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format)382 383 if do_normalize:384 image = self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format)385 386 image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format)387 388 return image389 390 391__all__ = ["VideoLlavaImageProcessor"]392