optimum-intel-internal-testing/tiny-random-minicpmv-2_6
129k
1import math2from typing import Any, Dict, List, Optional, Union3 4import numpy as np5import PIL6import PIL.Image7import PIL.ImageSequence8import torch9from PIL import Image10from transformers import AutoImageProcessor11from transformers.image_processing_utils import BaseImageProcessor, BatchFeature12from transformers.image_transforms import to_channel_dimension_format13from transformers.image_utils import (14 ChannelDimension,15 infer_channel_dimension_format,16 is_torch_tensor,17 to_numpy_array,18 valid_images,19)20from transformers.utils import TensorType, is_torch_device, is_torch_dtype, requires_backends21 22 23def recursive_converter(converter, value):24 if isinstance(value, list):25 new_value = []26 for v in value:27 new_value += [recursive_converter(converter, v)]28 return new_value29 else:30 return converter(value)31 32 33class MiniCPMVBatchFeature(BatchFeature):34 r"""35 Extend from BatchFeature for supporting various image size36 """37 38 def __init__(self, data: Optional[Dict[str, Any]] = None, tensor_type: Union[None, str, TensorType] = None):39 super().__init__(data)40 self.convert_to_tensors(tensor_type=tensor_type)41 42 def convert_to_tensors(self, tensor_type: Optional[Union[str, TensorType]] = None):43 if tensor_type is None:44 return self45 46 is_tensor, as_tensor = self._get_is_as_tensor_fns(tensor_type)47 48 def converter(value):49 try:50 if not is_tensor(value):51 tensor = as_tensor(value)52 return tensor53 except: # noqa E72254 if key == "overflowing_values":55 raise ValueError("Unable to create tensor returning overflowing values of different lengths. ")56 raise ValueError(57 "Unable to create tensor, you should probably activate padding "58 "with 'padding=True' to have batched tensors with the same length."59 )60 61 for key, value in self.items():62 self[key] = recursive_converter(converter, value)63 return self64 65 def to(self, *args, **kwargs) -> "MiniCPMVBatchFeature":66 requires_backends(self, ["torch"])67 import torch68 69 def cast_tensor(v):70 # check if v is a floating point71 if torch.is_floating_point(v):72 # cast and send to device73 return v.to(*args, **kwargs)74 elif device is not None:75 return v.to(device=device)76 else:77 return v78 79 new_data = {}80 device = kwargs.get("device")81 # Check if the args are a device or a dtype82 if device is None and len(args) > 0:83 # device should be always the first argument84 arg = args[0]85 if is_torch_dtype(arg):86 # The first argument is a dtype87 pass88 elif isinstance(arg, str) or is_torch_device(arg) or isinstance(arg, int):89 device = arg90 else:91 # it's something else92 raise ValueError(f"Attempting to cast a BatchFeature to type {str(arg)}. This is not supported.")93 # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor`94 for k, v in self.items():95 new_data[k] = recursive_converter(cast_tensor, v)96 self.data = new_data97 return self98 99 100class MiniCPMVImageProcessor(BaseImageProcessor):101 model_input_names = ["pixel_values"]102 103 def __init__(self, max_slice_nums=9, scale_resolution=448, patch_size=14, **kwargs):104 super().__init__(**kwargs)105 self.max_slice_nums = max_slice_nums106 self.scale_resolution = scale_resolution107 self.patch_size = patch_size108 self.use_image_id = kwargs.pop("use_image_id", False)109 self.image_feature_size = kwargs.pop("image_feature_size", 64)110 self.im_start_token = kwargs.pop("im_start", "<image>")111 self.im_end_token = kwargs.pop("im_end", "</image>")112 self.slice_start_token = kwargs.pop("slice_start", "<slice>")113 self.slice_end_token = kwargs.pop("slice_end", "</slice>")114 self.unk_token = kwargs.pop("unk", "<unk>")115 self.im_id_start = kwargs.pop("im_id_start", "<image_id>")116 self.im_id_end = kwargs.pop("im_id_end", "</image_id>")117 self.slice_mode = kwargs.pop("slice_mode", True)118 self.mean = kwargs.pop("norm_mean", 0.5)119 self.std = kwargs.pop("norm_std", 0.5)120 self.version = kwargs.pop("version", 2.0)121 122 def ensure_divide(self, length, patch_size):123 return max(round(length / patch_size) * patch_size, patch_size)124 125 def find_best_resize(self, original_size, scale_resolution, patch_size, allow_upscale=False):126 width, height = original_size127 if (width * height > scale_resolution * scale_resolution) or allow_upscale:128 r = width / height129 height = int(scale_resolution / math.sqrt(r))130 width = int(height * r)131 best_width = self.ensure_divide(width, patch_size)132 best_height = self.ensure_divide(height, patch_size)133 return (best_width, best_height)134 135 def get_refine_size(self, original_size, grid, scale_resolution, patch_size, allow_upscale=False):136 width, height = original_size137 grid_x, grid_y = grid138 139 refine_width = self.ensure_divide(width, grid_x)140 refine_height = self.ensure_divide(height, grid_y)141 142 grid_width = refine_width / grid_x143 grid_height = refine_height / grid_y144 145 best_grid_size = self.find_best_resize(146 (grid_width, grid_height), scale_resolution, patch_size, allow_upscale=allow_upscale147 )148 refine_size = (best_grid_size[0] * grid_x, best_grid_size[1] * grid_y)149 return refine_size150 151 def split_to_patches(self, image, grid):152 patches = []153 width, height = image.size154 grid_x = int(width / grid[0])155 grid_y = int(height / grid[1])156 for i in range(0, height, grid_y):157 images = []158 for j in range(0, width, grid_x):159 box = (j, i, j + grid_x, i + grid_y)160 patch = image.crop(box)161 images.append(patch)162 patches.append(images)163 return patches164 165 def slice_image(self, image, max_slice_nums=9, scale_resolution=448, patch_size=14, never_split=False):166 original_size = image.size167 source_image = None168 best_grid = self.get_sliced_grid(original_size, max_slice_nums, never_split)169 patches = []170 171 if best_grid is None:172 # dont need to slice, upsample173 best_size = self.find_best_resize(original_size, scale_resolution, patch_size, allow_upscale=True)174 source_image = image.resize(best_size, resample=Image.Resampling.BICUBIC)175 else:176 # source image, down-sampling and ensure divided by patch_size177 best_resize = self.find_best_resize(original_size, scale_resolution, patch_size)178 source_image = image.copy().resize(best_resize, resample=Image.Resampling.BICUBIC)179 refine_size = self.get_refine_size(180 original_size, best_grid, scale_resolution, patch_size, allow_upscale=True181 )182 refine_image = image.resize(refine_size, resample=Image.Resampling.BICUBIC)183 patches = self.split_to_patches(refine_image, best_grid)184 185 return source_image, patches, best_grid186 187 def get_grid_placeholder(self, grid):188 if grid is None:189 return ""190 slice_image_placeholder = (191 self.slice_start_token + self.unk_token * self.image_feature_size + self.slice_end_token192 )193 194 cols = grid[0]195 rows = grid[1]196 slices = []197 for i in range(rows):198 lines = []199 for j in range(cols):200 lines.append(slice_image_placeholder)201 slices.append("".join(lines))202 203 slice_placeholder = "\n".join(slices)204 return slice_placeholder205 206 def get_image_id_placeholder(self, idx=0):207 return f"{self.im_id_start}{idx}{self.im_id_end}"208 209 def get_sliced_images(self, image, max_slice_nums=None):210 slice_images = []211 212 if not self.slice_mode:213 return [image]214 215 max_slice_nums = self.max_slice_nums if max_slice_nums is None else int(max_slice_nums)216 assert max_slice_nums > 0217 source_image, patches, sliced_grid = self.slice_image(218 image, max_slice_nums, self.scale_resolution, self.patch_size # default: 9 # default: 448 # default: 14219 )220 221 slice_images.append(source_image)222 if len(patches) > 0:223 for i in range(len(patches)):224 for j in range(len(patches[0])):225 slice_images.append(patches[i][j])226 return slice_images227 228 def get_sliced_grid(self, image_size, max_slice_nums, nerver_split=False):229 original_width, original_height = image_size230 log_ratio = math.log(original_width / original_height)231 ratio = original_width * original_height / (self.scale_resolution * self.scale_resolution)232 multiple = min(math.ceil(ratio), max_slice_nums)233 if multiple <= 1 or nerver_split:234 return None235 candidate_split_grids_nums = []236 for i in [multiple - 1, multiple, multiple + 1]:237 if i == 1 or i > max_slice_nums:238 continue239 candidate_split_grids_nums.append(i)240 241 candidate_grids = []242 for split_grids_nums in candidate_split_grids_nums:243 m = 1244 while m <= split_grids_nums:245 if split_grids_nums % m == 0:246 candidate_grids.append([m, split_grids_nums // m])247 m += 1248 249 best_grid = [1, 1]250 min_error = float("inf")251 for grid in candidate_grids:252 error = abs(log_ratio - math.log(grid[0] / grid[1]))253 if error < min_error:254 best_grid = grid255 min_error = error256 257 return best_grid258 259 def get_slice_image_placeholder(self, image_size, image_idx=0, max_slice_nums=None, use_image_id=None):260 max_slice_nums = self.max_slice_nums if max_slice_nums is None else int(max_slice_nums)261 assert max_slice_nums > 0262 grid = self.get_sliced_grid(image_size=image_size, max_slice_nums=max_slice_nums)263 264 image_placeholder = self.im_start_token + self.unk_token * self.image_feature_size + self.im_end_token265 use_image_id = self.use_image_id if use_image_id is None else bool(use_image_id)266 if use_image_id:267 final_placeholder = self.get_image_id_placeholder(image_idx) + image_placeholder268 else:269 final_placeholder = image_placeholder270 271 if self.slice_mode:272 final_placeholder = final_placeholder + self.get_grid_placeholder(grid=grid)273 return final_placeholder274 275 def to_pil_image(self, image, rescale=None) -> PIL.Image.Image:276 """277 Converts `image` to a PIL Image. Optionally rescales it and puts the channel dimension back as the last axis if278 needed.279 280 Args:281 image (`PIL.Image.Image` or `numpy.ndarray` or `torch.Tensor`):282 The image to convert to the PIL Image format.283 rescale (`bool`, *optional*):284 Whether or not to apply the scaling factor (to make pixel values integers between 0 and 255). Will285 default to `True` if the image type is a floating type, `False` otherwise.286 """287 if isinstance(image, PIL.Image.Image):288 return image289 if is_torch_tensor(image):290 image = image.numpy()291 292 if isinstance(image, np.ndarray):293 if rescale is None:294 # rescale default to the array being of floating type.295 rescale = isinstance(image.flat[0], np.floating)296 # If the channel as been moved to first dim, we put it back at the end.297 if image.ndim == 3 and image.shape[0] in [1, 3]:298 image = image.transpose(1, 2, 0)299 if rescale:300 image = image * 255301 image = image.astype(np.uint8)302 return PIL.Image.fromarray(image)303 return image304 305 def reshape_by_patch(self, image):306 """307 :param image: shape [3, H, W]308 :param patch_size:309 :return: [3, patch_size, HW/patch_size]310 """311 image = torch.from_numpy(image)312 patch_size = self.patch_size313 patches = torch.nn.functional.unfold(image, (patch_size, patch_size), stride=(patch_size, patch_size))314 315 patches = patches.reshape(image.size(0), patch_size, patch_size, -1)316 patches = patches.permute(0, 1, 3, 2).reshape(image.size(0), patch_size, -1)317 return patches.numpy()318 319 def preprocess(320 self,321 images: Union[Image.Image, List[Image.Image], List[List[Image.Image]]],322 do_pad: Optional[bool] = True, # TODO: add pad for MiniCPM-Llama3-V-2_5323 max_slice_nums: int = None,324 return_tensors: Optional[Union[str, TensorType]] = None,325 **kwargs,326 ) -> MiniCPMVBatchFeature:327 if isinstance(images, Image.Image):328 images_list = [[images]]329 elif isinstance(images[0], Image.Image):330 images_list = [images]331 else:332 images_list = images333 334 new_images_list = []335 image_sizes_list = []336 tgt_sizes_list = []337 338 for _images in images_list:339 if _images is None or len(_images) == 0:340 new_images_list.append([])341 image_sizes_list.append([])342 tgt_sizes_list.append([])343 continue344 if not valid_images(_images):345 raise ValueError(346 "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "347 "torch.Tensor, tf.Tensor or jax.ndarray."348 )349 350 _images = [self.to_pil_image(image).convert("RGB") for image in _images]351 input_data_format = infer_channel_dimension_format(np.array(_images[0]))352 353 new_images = []354 image_sizes = [image.size for image in _images]355 tgt_sizes = []356 for image in _images:357 image_patches = self.get_sliced_images(image, max_slice_nums)358 image_patches = [to_numpy_array(image).astype(np.float32) / 255 for image in image_patches]359 image_patches = [360 self.normalize(image=image, mean=self.mean, std=self.std, input_data_format=input_data_format)361 for image in image_patches362 ]363 image_patches = [364 to_channel_dimension_format(image, ChannelDimension.FIRST, input_channel_dim=input_data_format)365 for image in image_patches366 ]367 for slice_image in image_patches:368 new_images.append(self.reshape_by_patch(slice_image))369 tgt_sizes.append(370 np.array((slice_image.shape[1] // self.patch_size, slice_image.shape[2] // self.patch_size))371 )372 373 if tgt_sizes:374 tgt_sizes = np.vstack(tgt_sizes)375 376 new_images_list.append(new_images)377 image_sizes_list.append(image_sizes)378 tgt_sizes_list.append(tgt_sizes)379 return MiniCPMVBatchFeature(380 data={"pixel_values": new_images_list, "image_sizes": image_sizes_list, "tgt_sizes": tgt_sizes_list},381 tensor_type=return_tensors,382 )383 384 385AutoImageProcessor.register("MiniCPMVImageProcessor", MiniCPMVImageProcessor)386