Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2024 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16Processor class for LLaVa-Onevision.17"""18 19import math20from collections.abc import Iterable21from typing import Optional, Union22 23import numpy as np24 25from ...feature_extraction_utils import BatchFeature26from ...image_processing_utils import select_best_resolution27from ...image_utils import ImageInput, get_image_size, to_numpy_array28from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack29from ...tokenization_utils_base import PreTokenizedInput, TextInput30from ...utils import logging31from ...video_utils import VideoInput32 33 34logger = logging.get_logger(__name__)35 36 37class LlavaOnevisionProcessorKwargs(ProcessingKwargs, total=False):38 # see processing_utils.ProcessingKwargs documentation for usage.39 _defaults = {40 "text_kwargs": {41 "padding": False,42 "return_mm_token_type_ids": False,43 },44 "image_kwargs": {},45 "videos_kwargs": {},46 }47 48 49class LlavaOnevisionProcessor(ProcessorMixin):50 r"""51 Constructs a LLaVa-Onevision processor which wraps a LLaVa-Onevision video processor, LLaVa-NeXT image processor and a LLaMa tokenizer into a single processor.52 53 [`LlavaNextProcessor`] offers all the functionalities of [`LlavaOnevisionVideoProcessor`], [`LlavaOnevisionImageProcessor`] and [`LlamaTokenizerFast`]. See the54 [`~LlavaOnevisionVideoProcessor.__call__`], [`~LlavaNextProcessor.__call__`] and [`~LlavaNextProcessor.decode`] for more information.55 56 Args:57 image_processor ([`LlavaOnevisionImageProcessor`], *optional*):58 The image processor is a required input.59 tokenizer ([`LlamaTokenizerFast`], *optional*):60 The tokenizer is a required input.61 video_processor ([`LlavaOnevisionVideoProcessor`], *optional*):62 The video processor is a required input.63 num_image_tokens (`int`, *optional*):64 Number of image tokens for one imagethat will be returned by vision tower.65 vision_feature_select_strategy (`str`, *optional*):66 The feature selection strategy used to select the vision feature from the vision backbone.67 Should be same as in model's config68 chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages69 in a chat into a tokenizable string.70 image_token (`str`, *optional*, defaults to `"<image>"`):71 Special token used to denote image location.72 video_token (`str`, *optional*, defaults to `"<video>"`):73 Special token used to denote video location.74 vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`):75 Aspect ratio used when processong image features. The default value is "anyres_max_9".76 """77 78 attributes = ["image_processor", "tokenizer", "video_processor"]79 image_processor_class = "AutoImageProcessor"80 tokenizer_class = "AutoTokenizer"81 video_processor_class = "AutoVideoProcessor"82 83 def __init__(84 self,85 image_processor=None,86 tokenizer=None,87 video_processor=None,88 num_image_tokens=None,89 vision_feature_select_strategy=None,90 chat_template=None,91 image_token="<image>",92 video_token="<video>",93 vision_aspect_ratio="anyres_max_9",94 **kwargs,95 ):96 self.num_image_tokens = num_image_tokens97 self.vision_feature_select_strategy = vision_feature_select_strategy98 self.image_token = tokenizer.image_token if hasattr(tokenizer, "image_token") else image_token99 self.video_token = tokenizer.video_token if hasattr(tokenizer, "video_token") else video_token100 self.image_token_id = (101 tokenizer.image_token_id102 if getattr(tokenizer, "image_token_id", None)103 else tokenizer.convert_tokens_to_ids(self.image_token)104 )105 self.video_token_id = (106 tokenizer.video_token_id107 if getattr(tokenizer, "video_token_id", None)108 else tokenizer.convert_tokens_to_ids(self.video_token)109 )110 self.vision_aspect_ratio = vision_aspect_ratio111 super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template)112 113 def __call__(114 self,115 images: Optional[ImageInput] = None,116 text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,117 audio=None,118 videos: Optional[VideoInput] = None,119 **kwargs: Unpack[LlavaOnevisionProcessorKwargs],120 ) -> BatchFeature:121 """122 Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`123 and `kwargs` arguments to LlamaTokenizerFast's [`~LlamaTokenizerFast.__call__`] if `text` is not `None` to encode124 the text. To prepare the image(s), this method forwards the `images` and `kwargs` arguments to125 LlavaNextImageProcessor's [`~LlavaNextImageProcessor.__call__`] if `images` is not `None`. Please refer to the docstring126 of the above two methods for more information.127 128 Args:129 images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):130 The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch131 tensor. Both channels-first and channels-last formats are supported.132 text (`str`, `list[str]`, `list[list[str]]`):133 The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings134 (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set135 `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).136 videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`):137 The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch138 139 Returns:140 [`BatchFeature`]: A [`BatchFeature`] with the following fields:141 142 - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.143 - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when144 `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not145 `None`).146 - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.147 - **pixel_values_videos** -- Pixel values of a video input to be fed to a model. Returned when `videos` is not `None`.148 - **image_sizes** -- Size of each image that will be used to unpad an image. Returned when `images` is not `None`.149 """150 151 output_kwargs = self._merge_kwargs(152 LlavaOnevisionProcessorKwargs,153 tokenizer_init_kwargs=self.tokenizer.init_kwargs,154 **kwargs,155 )156 157 if isinstance(text, str):158 text = [text]159 elif not isinstance(text, list) and not isinstance(text[0], str):160 raise TypeError("Invalid input text. Please provide a string, or a list of strings")161 162 image_inputs = video_inputs = {}163 164 if images is not None:165 image_inputs = self.image_processor(images, **output_kwargs["images_kwargs"])166 167 batch_num_images = iter(image_inputs["batch_num_images"])168 image_sizes = iter(image_inputs["image_sizes"])169 height, width = get_image_size(170 to_numpy_array(image_inputs["pixel_values"][0][0]),171 channel_dim=output_kwargs["images_kwargs"].get("data_format"),172 )173 text, num_image_tokens = self._expand_image_tokens(174 text, image_sizes, height, width, self.image_token, batch_num_images175 )176 177 if videos is not None:178 video_inputs = self.video_processor(videos, **output_kwargs["videos_kwargs"])179 180 one_video = video_inputs.get("pixel_values_videos")[0]181 if isinstance(video_inputs.get("pixel_values_videos")[0], (list, tuple)):182 one_video = np.array(one_video)183 else:184 one_video = to_numpy_array(one_video)185 height, width = get_image_size(one_video[0], channel_dim=output_kwargs["images_kwargs"].get("data_format"))186 num_frames = one_video.shape[0] # frame dim is always after batch dim187 patches_height_width = int(math.sqrt(self.num_image_tokens))188 pooled_height_width = math.ceil(patches_height_width / 2)189 num_video_tokens = (num_frames * pooled_height_width * pooled_height_width) + 1 # +1 for newline token190 text = [sample.replace(self.video_token, self.video_token * num_video_tokens) for sample in text]191 192 return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)193 return_mm_token_type_ids = output_kwargs["text_kwargs"].pop("return_mm_token_type_ids", None)194 text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])195 self._check_special_mm_tokens(text, text_inputs, modalities=["image"])196 197 if return_mm_token_type_ids:198 array_ids = np.array(text_inputs["input_ids"])199 mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])200 mm_token_type_ids[array_ids == self.image_token_id] = 1201 text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()202 203 return BatchFeature(data={**text_inputs, **image_inputs, **video_inputs}, tensor_type=return_tensors)204 205 def _expand_image_tokens(206 self,207 text: list[TextInput],208 image_sizes: Iterable[Union[list[int], int]],209 height: int,210 width: int,211 special_token: str,212 batch_num_images: Iterable[int],213 ):214 prompt_strings = []215 max_num_vision_tokens = 0216 for sample in text:217 if special_token in sample:218 num_images = next(batch_num_images) # should consume iterable219 is_multi_image = num_images != 1220 else:221 is_multi_image = False222 while special_token in sample:223 original_size = next(image_sizes) # should consume iterable224 if is_multi_image:225 num_image_tokens = self.num_image_tokens + 1 # one for image_newline226 else:227 if not isinstance(original_size, (list, tuple)):228 # cast to list to avoid numerical precision errors when calculating unpadding229 original_size = original_size.tolist()230 orig_height, orig_width = original_size231 num_image_tokens = self._get_number_of_features(orig_height, orig_width, height, width)232 max_num_vision_tokens = max(max_num_vision_tokens, num_image_tokens)233 if self.vision_feature_select_strategy == "default":234 num_image_tokens -= 1235 sample = sample.replace(special_token, "<placeholder>" * num_image_tokens, 1)236 prompt_strings.append(sample)237 text = [sample.replace("<placeholder>", special_token) for sample in prompt_strings]238 return text, max_num_vision_tokens239 240 def _get_number_of_features(self, orig_height: int, orig_width: int, height: int, width: int) -> int:241 image_grid_pinpoints = self.image_processor.image_grid_pinpoints242 243 height_best_resolution, width_best_resolution = select_best_resolution(244 [orig_height, orig_width], image_grid_pinpoints245 )246 scale_height, scale_width = height_best_resolution // height, width_best_resolution // width247 248 patches_height = patches_width = int(math.sqrt(self.num_image_tokens))249 unpadded_features, newline_features = self._get_unpadded_features(250 orig_height, orig_width, patches_height, patches_width, scale_height, scale_width251 )252 253 # The base patch covers the entire image (no CLS for SigLIP)254 base_features = self.num_image_tokens255 num_image_tokens = unpadded_features + newline_features + base_features256 return num_image_tokens257 258 # Adapted from transformers.models.llava_next.processing_llava_next.LlavaNextProcessor._get_unpadded_features259 def _get_unpadded_features(self, height, width, patches_height, patches_width, scale_height, scale_width):260 """261 Get number of features for a given image with height/width. LLaVA-NeXT is different from LLaVA262 because it divided each image into patches depending on its resolution. Therefore we need to calculate how many263 patches an image is divided into and get the number of features from that.264 """265 current_height = patches_height * scale_height266 current_width = patches_width * scale_width267 268 original_aspect_ratio = width / height269 current_aspect_ratio = current_width / current_height270 if original_aspect_ratio > current_aspect_ratio:271 new_height = int(round(height * (current_width / width), 7))272 padding = (current_height - new_height) // 2273 current_height -= padding * 2274 else:275 new_width = int(round(width * (current_height / height), 7))276 padding = (current_width - new_width) // 2277 current_width -= padding * 2278 279 unpadded_features = current_height * current_width280 newline_features = current_height281 282 max_num_patches = int(self.vision_aspect_ratio.strip("anyres_max_"))283 ratio = math.sqrt(current_height * current_width / (max_num_patches * patches_height**2))284 if ratio > 1.1:285 unpadded_features = int(current_height // ratio) * int(current_width // ratio)286 newline_features = int(current_height // ratio)287 288 return (unpadded_features, newline_features)289 290 def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):291 """292 Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.293 Args:294 image_sizes (list[list[str]], *optional*):295 The input sizes formatted as (height, width) per each image.296 video_sizes (list[list[str]], *optional*):297 The input sizes formatted as (num_frames, height, width) per each video.298 audio_lengths (list[int], *optional*):299 The input length formatted as per each audio.300 Returns:301 dict[str, list[int]]: A dictionary mapping each modality ("image", "video", "audio")302 to a list containing the number of placeholder tokens required. If the model doesn't accept303 a certain modality or no input sizes are provided, the dict value is set to an empty list.304 """305 vision_data = {}306 if image_sizes is not None:307 images_kwargs = LlavaOnevisionProcessorKwargs._defaults.get("images_kwargs", {})308 images_kwargs.update(kwargs)309 310 size = images_kwargs.get("size", None) or self.image_processor.size311 size = (312 (size["shortest_edge"], size["shortest_edge"])313 if "shortest_edge" in size314 else (min(size["height"], size["width"]), min(size["height"], size["width"]))315 )316 processed_height, processed_width = size317 318 batch_num_image_tokens = []319 num_image_patches = [1] * len(image_sizes) # llava-ov doesn't batch pixels as Idefics, thus `1` patch`320 for image_size in image_sizes:321 orig_height, orig_width = image_size322 num_image_tokens = self._get_number_of_features(323 orig_height, orig_width, processed_height, processed_width324 )325 if self.vision_feature_select_strategy == "default":326 num_image_tokens -= 1327 batch_num_image_tokens.append(num_image_tokens)328 vision_data.update({"num_image_tokens": batch_num_image_tokens, "num_image_patches": num_image_patches})329 330 return MultiModalData(**vision_data)331 332 333__all__ = ["LlavaOnevisionProcessor"]334