Flare77/HuLuLLM
013
1# Adopted from https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py.2# Below is the original copyright:3# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.4#5# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX6# and OPT implementations in this library. It has been modified from its7# original forms to accommodate minor architectural differences compared8# to GPT-NeoX and OPT used by the Meta AI team that trained the model.9#10# Licensed under the Apache License, Version 2.0 (the "License");11# you may not use this file except in compliance with the License.12# You may obtain a copy of the License at13#14# http://www.apache.org/licenses/LICENSE-2.015#16# Unless required by applicable law or agreed to in writing, software17# distributed under the License is distributed on an "AS IS" BASIS,18# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.19# See the License for the specific language governing permissions and20# limitations under the License.21"""Image processor class for HuluMed."""22 23import math24from typing import Dict, List, Optional, Union25 26import numpy as np27 28import torch29from transformers.image_processing_utils import BaseImageProcessor, BatchFeature30from transformers.image_utils import ImageInput31from transformers.image_transforms import (32 convert_to_rgb,33 resize,34 to_channel_dimension_format,35)36from transformers.image_utils import (37 OPENAI_CLIP_MEAN,38 OPENAI_CLIP_STD,39 ChannelDimension,40 ImageInput,41 PILImageResampling,42 get_image_size,43 infer_channel_dimension_format,44 is_scaled_image,45 is_valid_image,46 make_list_of_images,47 to_numpy_array,48)49try:50 from transformers.video_utils import VideoInput51except:52 from transformers.image_utils import VideoInput53 54from transformers.utils import TensorType, is_vision_available, logging55 56 57logger = logging.get_logger(__name__)58 59 60if is_vision_available():61 from PIL import Image62 63 64def is_valid_video(video) -> bool:65 if isinstance(video, (list, tuple)):66 return all(is_valid_image(frame) for frame in video)67 elif isinstance(video, np.ndarray):68 return video.ndim == 469 elif isinstance(video, torch.Tensor):70 return video.ndim == 471 return False72 73 74def make_batched_images(images) -> List[List[ImageInput]]:75 """76 Accepts images in list or nested list format, and makes a list of images for preprocessing.77 78 Args:79 images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`):80 The input image.81 82 Returns:83 list: A list of images.84 """85 if isinstance(images, (list, tuple)):86 # list of images/videos87 if not all(is_valid_video(image) or is_valid_image(image) for image in images):88 raise ValueError(f"Could not make batched images from {images}")89 return images90 elif is_valid_video(images) or is_valid_image(images):91 # single image/video92 return [images]93 94 raise ValueError(f"Could not make batched images from {images}")95 96 97def simple_batched_resize(98 images, factor: int = 28, min_tokens: int = 4 * 4, max_tokens: int = 16384, input_data_format: str = None99):100 min_pixels = min_tokens * factor * factor101 max_pixels = max_tokens * factor * factor102 103 num_images = 0104 for image in images:105 if is_valid_video(image):106 num_images += len(image)107 else:108 num_images += 1109 110 image_sizes = []111 for image in images:112 if is_valid_video(image):113 image = image[0]114 if isinstance(image, Image.Image):115 height, width = image.size116 else:117 height, width = get_image_size(image, channel_dim=input_data_format)118 image_sizes.append([height, width])119 120 tmp_image_sizes = []121 for height, width in image_sizes:122 h_bar = round(height / factor) * factor123 w_bar = round(width / factor) * factor124 if h_bar * w_bar > (max_pixels // num_images):125 beta = math.sqrt((height * width) / (max_pixels // num_images))126 h_bar = math.floor(height / beta / factor) * factor127 w_bar = math.floor(width / beta / factor) * factor128 # per image min_pixels129 if h_bar * w_bar < min_pixels:130 beta = math.sqrt(min_pixels / (height * width))131 h_bar = math.ceil(height * beta / factor) * factor132 w_bar = math.ceil(width * beta / factor) * factor133 tmp_image_sizes.append((h_bar, w_bar))134 image_sizes = tmp_image_sizes135 return image_sizes136 137 138def batched_resize(139 images, factors: List[int], min_tokens: int = 4 * 4, max_tokens: int = 16384, input_data_format: str = None140):141 image_sizes = []142 for image in images:143 if is_valid_video(image):144 num_frame = len(image)145 image = image[0]146 else:147 num_frame = 1148 if isinstance(image, Image.Image):149 height, width = image.size150 else:151 height, width = get_image_size(image, channel_dim=input_data_format)152 image_sizes.append([num_frame, height, width])153 154 # global max_pixels155 smart_scale_factors = 1.0156 total_tokens = 0157 for (num_frame, height, width), factor in zip(image_sizes, factors):158 total_tokens += num_frame * math.ceil(height / factor) * math.ceil(width / factor)159 160 # TODO: add min_pixels161 if total_tokens > max_tokens:162 beta = math.sqrt(total_tokens / max_tokens)163 tmp_image_sizes = []164 for (_, height, width), factor in zip(image_sizes, factors):165 h_bar = math.floor(height / beta / factor) * factor166 w_bar = math.floor(width / beta / factor) * factor167 tmp_image_sizes.append((h_bar, w_bar))168 image_sizes = tmp_image_sizes169 else:170 tmp_image_sizes = []171 for (_, height, width), factor in zip(image_sizes, factors):172 height = round(height / factor) * factor173 width = round(width / factor) * factor174 tmp_image_sizes.append((height, width))175 image_sizes = tmp_image_sizes176 177 return image_sizes178 179 180class HulumedImageProcessor(BaseImageProcessor):181 r"""182 Constructs a HuluMed image processor that dynamically resizes images based on the original images.183 184 Args:185 do_resize (`bool`, *optional*, defaults to `True`):186 Whether to resize the image's (height, width) dimensions.187 resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):188 Resampling filter to use when resizing the image.189 do_rescale (`bool`, *optional*, defaults to `True`):190 Whether to rescale the image by the specified scale `rescale_factor`.191 rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):192 Scale factor to use if rescaling the image.193 do_normalize (`bool`, *optional*, defaults to `True`):194 Whether to normalize the image.195 image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):196 Mean to use if normalizing the image. This is a float or list of floats for each channel in the image.197 image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):198 Standard deviation to use if normalizing the image. This is a float or list of floats for each channel in the image.199 do_convert_rgb (`bool`, *optional*, defaults to `True`):200 Whether to convert the image to RGB.201 min_pixels (`int`, *optional*, defaults to `56 * 56`):202 The min pixels of the image to resize the image.203 max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`):204 The max pixels of the image to resize the image.205 patch_size (`int`, *optional*, defaults to 14):206 The spacial patch size of the vision encoder.207 merge_size (`int`, *optional*, defaults to `None`):208 The default merge size for processing. If None, no default merge size is applied.209 """210 211 model_input_names = ["pixel_values", "grid_sizes", "merge_sizes"]212 213 def __init__(214 self,215 do_resize: bool = True,216 resample: PILImageResampling = PILImageResampling.BICUBIC,217 do_rescale: bool = True,218 rescale_factor: Union[int, float] = 1 / 255,219 do_normalize: bool = True,220 image_mean: Optional[Union[float, List[float]]] = None,221 image_std: Optional[Union[float, List[float]]] = None,222 do_convert_rgb: bool = True,223 min_tokens: int = 4 * 4,224 max_tokens: int = 16384,225 patch_size: int = 14,226 merge_size: Optional[int] = None,227 **kwargs,228 ) -> None:229 super().__init__(**kwargs)230 self.do_resize = do_resize231 self.resample = resample232 self.do_rescale = do_rescale233 self.rescale_factor = rescale_factor234 self.do_normalize = do_normalize235 self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN236 self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD237 self.min_tokens = min_tokens238 self.max_tokens = max_tokens239 self.patch_size = patch_size240 self.do_convert_rgb = do_convert_rgb241 self.merge_size = merge_size 242 243 def _preprocess(244 self,245 images: Union[ImageInput, VideoInput],246 target_size: List[int],247 merge_size: int = 1,248 do_resize: bool = None,249 resample: PILImageResampling = None,250 do_rescale: bool = None,251 rescale_factor: float = None,252 do_normalize: bool = None,253 image_mean: Optional[Union[float, List[float]]] = None,254 image_std: Optional[Union[float, List[float]]] = None,255 do_convert_rgb: bool = None,256 data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,257 input_data_format: Optional[Union[str, ChannelDimension]] = None,258 ):259 """260 Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.261 262 Args:263 images (`ImageInput`):264 Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`.265 target_size (`List[int]`):266 The target size to resize the image to. Should be a list of two integers: [target_height, target_width].267 merge_size (`int`, *optional*, defaults to `1`):268 The merge size after the vision encoder.269 do_resize (`bool`, *optional*, defaults to `self.do_resize`):270 Whether to resize the image.271 resample (`PILImageResampling`, *optional*, defaults to `self.resample`):272 Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.273 do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):274 Whether to rescale the image.275 rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):276 Scale factor to use if rescaling the image.277 do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):278 Whether to normalize the image.279 image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):280 Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.281 image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):282 Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.283 do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):284 Whether to convert the image to RGB.285 data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):286 The channel dimension format for the output image. Can be one of:287 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.288 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.289 - Unset: Use the channel dimension format of the input image.290 input_data_format (`ChannelDimension` or `str`, *optional*):291 The channel dimension format for the input image. Can be one of:292 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.293 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.294 - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.295 """296 images = make_list_of_images(images)297 298 if do_convert_rgb:299 images = [convert_to_rgb(image) for image in images]300 301 # All transformations expect numpy arrays.302 images = [to_numpy_array(image) for image in images]303 304 if is_scaled_image(images[0]) and do_rescale:305 logger.warning_once(306 "It looks like you are trying to rescale already rescaled images. If the input"307 " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."308 )309 if input_data_format is None:310 # We assume that all images have the same channel dimension format.311 input_data_format = infer_channel_dimension_format(images[0])312 313 height, width = get_image_size(images[0], channel_dim=input_data_format)314 resized_height, resized_width = height, width315 processed_images = []316 for image in images:317 if do_resize:318 resized_height, resized_width = target_size319 image = resize(320 image, size=(resized_height, resized_width), resample=resample, input_data_format=input_data_format321 )322 323 if do_rescale:324 image = self.rescale(image, scale=rescale_factor, input_data_format=input_data_format)325 326 if do_normalize:327 image = self.normalize(328 image=image, mean=image_mean, std=image_std, input_data_format=input_data_format329 )330 331 image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format)332 processed_images.append(image)333 334 patches = np.array(processed_images)335 if data_format == ChannelDimension.LAST:336 patches = patches.transpose(0, 3, 1, 2)337 t = patches.shape[0]338 channel = patches.shape[1]339 grid_h, grid_w = resized_height // self.patch_size, resized_width // self.patch_size340 patches = patches.reshape(341 t,342 channel,343 grid_h // merge_size,344 merge_size,345 self.patch_size,346 grid_w // merge_size,347 merge_size,348 self.patch_size,349 )350 patches = patches.transpose(0, 2, 5, 3, 6, 1, 4, 7)351 flatten_patches = patches.reshape(352 t * grid_h * grid_w, channel * self.patch_size * self.patch_size353 )354 355 return flatten_patches, (t, grid_h, grid_w)356 357 def preprocess(358 self,359 images: ImageInput,360 do_resize: bool = None,361 resample: PILImageResampling = None,362 do_rescale: bool = None,363 rescale_factor: float = None,364 do_normalize: bool = None,365 image_mean: Optional[Union[float, List[float]]] = None,366 image_std: Optional[Union[float, List[float]]] = None,367 do_convert_rgb: bool = None,368 merge_size: Optional[Union[int, List[int]]] = None,369 return_tensors: Optional[Union[str, TensorType]] = None,370 data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,371 input_data_format: Optional[Union[str, ChannelDimension]] = None,372 ):373 """374 Args:375 images (`ImageInput`):376 Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If377 passing in images with pixel values between 0 and 1, set `do_rescale=False`.378 do_resize (`bool`, *optional*, defaults to `self.do_resize`):379 Whether to resize the image.380 resample (`int`, *optional*, defaults to `self.resample`):381 Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only382 has an effect if `do_resize` is set to `True`.383 do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):384 Whether to rescale the image.385 rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):386 Rescale factor to rescale the image by if `do_rescale` is set to `True`.387 do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):388 Whether to normalize the image.389 image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):390 Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.391 image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):392 Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to393 `True`.394 do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):395 Whether to convert the image to RGB.396 merge_size (`int` or `List[int]`, *optional*, defaults to `self.merge_size`):397 The merge size for processing. Can be a single value or a list of values (one per image).398 return_tensors (`str` or `TensorType`, *optional*):399 The type of tensors to return. Can be one of:400 - Unset: Return a list of `np.ndarray`.401 - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.402 - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.403 - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.404 - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.405 data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):406 The channel dimension format for the output image. Can be one of:407 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.408 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.409 - Unset: Use the channel dimension format of the input image.410 input_data_format (`ChannelDimension` or `str`, *optional*):411 The channel dimension format for the input image. If unset, the channel dimension format is inferred412 from the input image. Can be one of:413 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.414 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.415 - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.416 417 """418 do_resize = do_resize if do_resize is not None else self.do_resize419 resample = resample if resample is not None else self.resample420 do_rescale = do_rescale if do_rescale is not None else self.do_rescale421 rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor422 do_normalize = do_normalize if do_normalize is not None else self.do_normalize423 image_mean = image_mean if image_mean is not None else self.image_mean424 image_std = image_std if image_std is not None else self.image_std425 do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb426 427 # Handle merge_size: use provided value, or fall back to instance default, or use 1428 if merge_size is None:429 merge_size = self.merge_size if self.merge_size is not None else 1430 431 images = make_batched_images(images)432 433 if isinstance(merge_size, (list, tuple)):434 assert len(merge_size) == len(images), "Merge size must be the same length as images."435 merge_sizes = merge_size436 else:437 merge_sizes = [merge_size for _ in images]438 if all(merge_size == merge_sizes[0] for merge_size in merge_sizes):439 target_sizes = simple_batched_resize(440 images,441 factor=self.patch_size * merge_sizes[0],442 min_tokens=self.min_tokens,443 max_tokens=self.max_tokens,444 input_data_format=input_data_format,445 )446 else:447 target_sizes = batched_resize(448 images,449 factors=[self.patch_size * merge_size for merge_size in merge_sizes],450 min_tokens=self.min_tokens,451 max_tokens=self.max_tokens,452 input_data_format=input_data_format,453 )454 455 pixel_values, grid_sizes = [], []456 for image, merge_size, target_size in zip(images, merge_sizes, target_sizes):457 patches, grid_size = self._preprocess(458 image,459 target_size=target_size,460 merge_size=merge_size,461 do_resize=do_resize,462 resample=resample,463 do_rescale=do_rescale,464 rescale_factor=rescale_factor,465 do_normalize=do_normalize,466 image_mean=image_mean,467 image_std=image_std,468 data_format=data_format,469 do_convert_rgb=do_convert_rgb,470 input_data_format=input_data_format,471 )472 pixel_values.append(patches)473 grid_sizes.append(grid_size)474 475 pixel_values = np.concatenate(pixel_values, axis=0)476 grid_sizes = np.array(grid_sizes)477 merge_sizes = np.array(merge_sizes)478 479 data = {480 "pixel_values": pixel_values,481 "grid_sizes": grid_sizes,482 "merge_sizes": merge_sizes,483 }484 485 return BatchFeature(data=data, tensor_type=return_tensors)