Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2024 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16Processor class for Idefics3.17"""18 19import re20from itertools import accumulate21from typing import TYPE_CHECKING, Optional, Union22 23import numpy as np24 25from ...feature_extraction_utils import BatchFeature26from ...image_utils import ImageInput, is_valid_image, load_image27from ...processing_utils import ImagesKwargs, MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack28from ...tokenization_utils_base import AddedToken, BatchEncoding, TextInput29from ...utils import logging30 31 32if TYPE_CHECKING:33 from ...tokenization_utils_base import PreTokenizedInput34 35logger = logging.get_logger(__name__)36 37 38def is_url(val) -> bool:39 return isinstance(val, str) and val.startswith("http")40 41 42def is_image_or_image_url(elem):43 return is_url(elem) or is_valid_image(elem)44 45 46def _prompt_split_image(image_seq_len, image_rows, image_cols, fake_token_around_image, image_token, global_img_token):47 """Prompt with expanded image tokens for when the image is split into patches."""48 text_split_images = ""49 for n_h in range(image_rows):50 for n_w in range(image_cols):51 text_split_images += (52 f"{fake_token_around_image}" + f"<row_{n_h + 1}_col_{n_w + 1}>" + f"{image_token}" * image_seq_len53 )54 text_split_images += "\n"55 56 text_split_images += (57 f"\n{fake_token_around_image}"58 + f"{global_img_token}"59 + f"{image_token}" * image_seq_len60 + f"{fake_token_around_image}"61 )62 return text_split_images63 64 65def _prompt_single_image(image_seq_len, fake_token_around_image, image_token, global_img_token):66 """Prompt with expanded image tokens for a single image."""67 return (68 f"{fake_token_around_image}"69 + f"{global_img_token}"70 + f"{image_token}" * image_seq_len71 + f"{fake_token_around_image}"72 )73 74 75def get_image_prompt_string(76 image_rows, image_cols, image_seq_len, fake_token_around_image, image_token, global_img_token77):78 if image_rows == 0 and image_cols == 0:79 return _prompt_single_image(80 image_seq_len,81 fake_token_around_image=fake_token_around_image,82 image_token=image_token,83 global_img_token=global_img_token,84 )85 return _prompt_split_image(86 image_seq_len, image_rows, image_cols, fake_token_around_image, image_token, global_img_token87 )88 89 90class Idefics3ImagesKwargs(ImagesKwargs, total=False):91 return_row_col_info: Optional[bool]92 max_image_size: Optional[dict[str, int]]93 94 95class Idefics3ProcessorKwargs(ProcessingKwargs, total=False):96 images_kwargs: Idefics3ImagesKwargs97 98 _defaults = {99 "text_kwargs": {100 "add_special_tokens": True,101 "padding": False,102 "is_split_into_words": False,103 "return_mm_token_type_ids": False,104 },105 "images_kwargs": {106 "return_row_col_info": True,107 },108 }109 110 111class Idefics3Processor(ProcessorMixin):112 r"""113 Constructs a Idefics3 processor which wraps a LLama tokenizer and Idefics3 image processor into a single processor.114 115 [`Idefics3Processor`] offers all the functionalities of [`Idefics3ImageProcessor`] and [`Idefics3TokenizerFast`]. See116 the docstring of [`~IdeficsProcessor.__call__`] and [`~IdeficsProcessor.decode`] for more information.117 118 Args:119 image_processor (`Idefics3ImageProcessor`):120 An instance of [`Idefics3ImageProcessor`]. The image processor is a required input.121 tokenizer (`PreTrainedTokenizerBase`, *optional*):122 An instance of [`PreTrainedTokenizerBase`]. This should correspond with the model's text model. The tokenizer is a required input.123 image_seq_len (`int`, *optional*, defaults to 169):124 The length of the image sequence i.e. the number of <image> tokens per image in the input.125 This parameter is used to build the string from the input prompt and image tokens and should match the126 value the model used. It is computed as: image_seq_len = int(((image_size // patch_size) ** 2) / (scale_factor**2))127 chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages128 in a chat into a tokenizable string.129 """130 131 attributes = ["image_processor", "tokenizer"]132 image_processor_class = "Idefics3ImageProcessor"133 tokenizer_class = "AutoTokenizer"134 135 def __init__(136 self, image_processor, tokenizer=None, image_seq_len: int = 169, chat_template: Optional[str] = None, **kwargs137 ):138 self.fake_image_token = AddedToken("<fake_token_around_image>", normalized=False, special=True).content139 self.image_token = AddedToken("<image>", normalized=False, special=True).content140 self.end_of_utterance_token = AddedToken("<end_of_utterance>", normalized=False, special=True).content141 self.global_image_tag = "<global-img>" # https://github.com/huggingface/transformers/pull/32473/files/8063e5e17362571b693f1db95167f5443a3be1b2#r1734825341142 self.image_seq_len = image_seq_len143 self.image_token_id = tokenizer.convert_tokens_to_ids(self.image_token)144 self.fake_image_token_id = tokenizer.convert_tokens_to_ids(self.fake_image_token)145 self.global_image_token_id = tokenizer.convert_tokens_to_ids(self.global_image_tag)146 self.row_col_ids = [147 tokenizer.convert_tokens_to_ids(f"<row_{i + 1}_col_{j + 1}>") for i in range(6) for j in range(6)148 ]149 150 # This regex matches one or more occurrences of <global-img> tags (optionally surrounded by newline characters)151 # or <row_x_col_y> tags (where x and y are digits, also optionally surrounded by newline characters).152 self._regex_to_remove_extra_special_tokens = re.compile(r"(\n?<global-img>\n?|<row_\d+_col_\d+>\n?)+")153 154 tokens_to_add = {155 "additional_special_tokens": [156 self.fake_image_token,157 self.image_token,158 self.end_of_utterance_token,159 ]160 }161 tokenizer.add_special_tokens(tokens_to_add)162 self.image_token_id = tokenizer.convert_tokens_to_ids(self.image_token)163 164 super().__init__(image_processor, tokenizer, chat_template=chat_template, **kwargs)165 166 def _extract_images_from_prompts(self, prompts):167 prompt_images = []168 for prompt in prompts:169 images = []170 for elem in prompt:171 if is_valid_image(elem):172 images.append(elem)173 elif is_url(elem):174 images.append(load_image(elem))175 prompt_images.append(images)176 return prompt_images177 178 def __call__(179 self,180 images: Union[ImageInput, list[ImageInput], list[list[ImageInput]]] = None,181 text: Union[TextInput, "PreTokenizedInput", list[TextInput], list["PreTokenizedInput"]] = None,182 audio=None,183 videos=None,184 image_seq_len: Optional[int] = None,185 **kwargs: Unpack[Idefics3ProcessorKwargs],186 ) -> BatchEncoding:187 """188 Processes the input prompts and returns a BatchEncoding.189 190 Example:191 192 ```python193 >>> import requests194 >>> from transformers import Idefics3Processor195 >>> from transformers.image_utils import load_image196 197 >>> processor = Idefics3Processor.from_pretrained("HuggingFaceM4/Idefics3-8B-Llama3")198 >>> processor.image_processor.do_image_splitting = False # Force as False to simplify the example199 200 >>> url1 = "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"201 >>> url2 = "https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg"202 203 >>> image1, image2 = load_image(url1), load_image(url2)204 >>> images = [[image1], [image2]]205 206 >>> text = [207 ... "<image>In this image, we see",208 ... "bla bla bla<image>",209 ... ]210 >>> outputs = processor(images=images, text=text, return_tensors="pt", padding=True)211 >>> input_ids = outputs.input_ids212 >>> input_tokens = processor.tokenizer.batch_decode(input_ids)213 >>> print(input_tokens)214 ['<|begin_of_text|><fake_token_around_image><global-img>((<image>)*169)<fake_token_around_image> In this image, we see', '<|reserved_special_token_0|><|reserved_special_token_0|><|reserved_special_token_0|><|begin_of_text|>bla bla bla<fake_token_around_image><global-img>((<image>)*169)<fake_token_around_image>']215 ```216 217 Args:218 images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`, *optional*):219 The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch220 tensor. If is of type `list[ImageInput]`, it's assumed that this is for a single prompt i.e. of batch size 1.221 text (`Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]]`, *optional*):222 The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings223 (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set224 `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).225 Wherever an image token, `<image>` is encountered it is expanded to226 `<fake_token_around_image>` + `<row_x_col_y>` + `<image>` * `image_seq_len` * <fake_token_around_image>`.227 image_seq_len (`int`, *optional*):228 The length of the image sequence. If not provided, the default value of self.image_seq_len is used.229 image_seq_len should be equal to int(((image_size // patch_size) ** 2) / (scale_factor**2))230 return_tensors (`Union[str, TensorType]`, *optional*):231 If set, will return tensors of a particular framework. See [`PreTrainedTokenizerFast.__call__`] for more232 information.233 """234 if text is None and images is None:235 raise ValueError("You must provide either `text` or `images`.")236 237 output_kwargs = self._merge_kwargs(238 Idefics3ProcessorKwargs,239 tokenizer_init_kwargs=self.tokenizer.init_kwargs,240 **kwargs,241 )242 243 image_seq_len = image_seq_len if image_seq_len is not None else self.image_seq_len244 return_mm_token_type_ids = output_kwargs["text_kwargs"].pop("return_mm_token_type_ids", False)245 return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)246 247 n_images_in_text = []248 n_images_in_images = []249 inputs = {}250 251 if text is not None:252 if isinstance(text, str):253 text = [text]254 elif not isinstance(text, list) and not isinstance(text[0], str):255 raise ValueError("Invalid input text. Please provide a string, or a list of strings")256 n_images_in_text = [sample.count(self.image_token) for sample in text]257 258 if images is not None:259 if is_image_or_image_url(images):260 images = [[images]]261 elif isinstance(images, (list, tuple)) and is_image_or_image_url(images[0]):262 if text is not None:263 if sum(n_images_in_text) != len(images):264 raise ValueError(265 f"The total number of {self.image_token} tokens in the prompts should be the same as the number of images passed."266 f" Found {sum(n_images_in_text)} {self.image_token} tokens and {len(images)} images."267 )268 # Reorganize the images to match the prompts269 cumsum_images_in_text = [0] + list(accumulate(n_images_in_text))270 images = [271 images[cumsum_images_in_text[i] : cumsum_images_in_text[i + 1]]272 for i in range(len(n_images_in_text))273 ]274 else:275 images = [images]276 elif (277 not isinstance(images, (list, tuple))278 and not isinstance(images[0], (list, tuple))279 and not is_image_or_image_url(images[0][0])280 ):281 raise ValueError(282 "Invalid input images. Please provide a single image or a list of images or a list of list of images."283 )284 n_images_in_images = [len(sample) for sample in images]285 286 # Load images if they are URLs287 images = [[load_image(im) if is_url(im) else im for im in sample] for sample in images]288 289 image_inputs = self.image_processor(images, **output_kwargs["images_kwargs"])290 inputs.update(image_inputs)291 292 if text is not None:293 if n_images_in_images != n_images_in_text:294 raise ValueError(295 f"The number of images in the text {n_images_in_text} and images {n_images_in_images} should be the same."296 )297 298 image_rows = inputs.pop("rows", [[0] * len(text)])299 image_cols = inputs.pop("cols", [[0] * len(text)])300 301 fake_image_token = self.fake_image_token302 image_token = self.image_token303 global_img_token = self.global_image_tag304 305 prompt_strings = []306 batch_image_seq_lengths = []307 for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols):308 # Replace the image token with fake tokens around the expanded image token sequence of length `image_seq_len`309 image_prompt_strings = []310 image_seq_lengths = []311 for n_rows, n_cols in zip(sample_rows, sample_cols):312 image_prompt_string = get_image_prompt_string(313 n_rows,314 n_cols,315 image_seq_len,316 image_token=image_token,317 fake_token_around_image=fake_image_token,318 global_img_token=global_img_token,319 )320 # Add +2 and +3 for special BOI/EOI/fake_image_wrapper tokens321 row_length = (self.image_seq_len + 2) * n_cols + 1322 image_seq_lengths.append((self.image_seq_len + 3) + row_length * n_rows)323 image_prompt_strings.append(image_prompt_string)324 325 batch_image_seq_lengths.append(image_seq_lengths)326 split_sample = sample.split(image_token)327 if len(split_sample) == 0:328 raise ValueError("The image token should be present in the text.")329 330 # Place in the image prompt strings where the image tokens are331 sample = split_sample[0]332 for i, image_prompt_string in enumerate(image_prompt_strings):333 sample += image_prompt_string + split_sample[i + 1]334 prompt_strings.append(sample)335 336 text_inputs = self.tokenizer(prompt_strings, **output_kwargs["text_kwargs"])337 self._check_special_mm_tokens(prompt_strings, text_inputs, modalities=["image"])338 inputs.update(text_inputs)339 340 elif text is not None:341 if any(n_images_in_text):342 raise ValueError(343 f"Found {sum(n_images_in_text)} {self.image_token} tokens in the text but no images were passed."344 )345 text_inputs = self.tokenizer(text=text, **output_kwargs["text_kwargs"])346 inputs.update(text_inputs)347 348 if return_mm_token_type_ids:349 array_ids = np.array(inputs["input_ids"])350 mm_token_type_ids = np.zeros_like(array_ids)351 for i, seq_lengths in enumerate(batch_image_seq_lengths):352 image_start_positions = np.where(array_ids[i] == self.fake_image_token_id)[0]353 j = 0354 for seq_len in seq_lengths:355 if j >= len(image_start_positions):356 break357 start = image_start_positions[j]358 end = start + seq_len359 mm_token_type_ids[i, start:end] = 1360 j = np.searchsorted(image_start_positions, end)361 362 inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()363 364 return BatchFeature(data=inputs, tensor_type=return_tensors)365 366 def _get_num_multimodal_tokens(self, image_sizes=None, **kwargs):367 """368 Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.369 370 Args:371 image_sizes (`list[list[int]]`, *optional*):372 The input sizes formatted as (height, width) per each image.373 374 Returns:375 `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided376 input modalities, along with other useful data.377 """378 379 vision_data = {}380 if image_sizes is not None:381 images_kwargs = Idefics3ProcessorKwargs._defaults.get("images_kwargs", {})382 images_kwargs.update(kwargs)383 384 num_image_row_cols = [385 self.image_processor.get_number_of_image_patches(*image_size, images_kwargs)386 for image_size in image_sizes387 ]388 389 base_image_length = self.image_seq_len + 3390 col_length = self.image_seq_len + 2391 num_image_tokens = []392 num_image_patches = []393 394 for num_patches, num_rows, num_cols in num_image_row_cols:395 row_length = col_length * num_cols + 1396 num_image_tokens.append(base_image_length + (row_length * num_rows))397 num_image_patches.append(num_patches)398 399 vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches})400 401 return MultiModalData(**vision_data)402 403 404__all__ = ["Idefics3Processor"]405