Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2024 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16Processor class for PaliGemma.17"""18 19from typing import Optional, Union20 21import numpy as np22 23from ...feature_extraction_utils import BatchFeature24from ...image_utils import ImageInput, is_valid_image25from ...processing_utils import (26    ImagesKwargs,27    MultiModalData,28    ProcessingKwargs,29    ProcessorMixin,30    TextKwargs,31    Unpack,32)33from ...tokenization_utils_base import AddedToken, PreTokenizedInput, TextInput34from ...utils import logging35 36 37logger = logging.get_logger(__name__)38 39IMAGE_TOKEN = "<image>"40EXTRA_TOKENS = [f"<loc{i:0>4}>" for i in range(1024)] + [f"<seg{i:0>3}>" for i in range(128)]41 42 43class PaliGemmaTextKwargs(TextKwargs):44    suffix: Optional[Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]]]45 46 47class PaliGemmaImagesKwargs(ImagesKwargs):48    do_convert_rgb: Optional[bool]49 50 51class PaliGemmaProcessorKwargs(ProcessingKwargs, total=False):52    text_kwargs: PaliGemmaTextKwargs53    images_kwargs: PaliGemmaImagesKwargs54    _defaults = {55        "text_kwargs": {56            "padding": False,57            "return_mm_token_type_ids": False,58        },59        "images_kwargs": {60            "data_format": "channels_first",61        },62    }63 64 65# Copied from transformers.models.idefics2.processing_idefics2.is_url66def is_url(val) -> bool:67    return isinstance(val, str) and val.startswith("http")68 69 70# Copied from transformers.models.idefics2.processing_idefics2.is_image_or_image_url71def is_image_or_image_url(elem):72    return is_url(elem) or is_valid_image(elem)73 74 75def _is_str_or_image(elem):76    return isinstance(elem, (str)) or is_image_or_image_url(elem)77 78 79def build_string_from_input(prompt, bos_token, image_seq_len, image_token, num_images):80    """81    Builds a string from the input prompt and image tokens.82    For example, for the call:83    build_string_from_input(84        prompt="Prefix str"85        bos_token="<s>",86        image_seq_len=3,87        image_token="<im>",88    )89    The output will be:90    "<im><im><im><s>Initial str"91    Args:92        prompt (`list[Union[str, ImageInput]]`): The input prompt.93        bos_token (`str`): The beginning of sentence token.94        image_seq_len (`int`): The length of the image sequence.95        image_token (`str`): The image token.96        num_images (`int`): Number of images in the prompt.97    """98    return f"{image_token * image_seq_len * num_images}{bos_token}{prompt}\n"99 100 101class PaliGemmaProcessor(ProcessorMixin):102    r"""103    Constructs a PaliGemma processor which wraps a PaliGemma image processor and a PaliGemma tokenizer into a single processor.104 105    [`PaliGemmaProcessor`] offers all the functionalities of [`SiglipImageProcessor`] and [`GemmaTokenizerFast`]. See the106    [`~PaliGemmaProcessor.__call__`] and [`~PaliGemmaProcessor.decode`] for more information.107 108    Args:109        image_processor ([`SiglipImageProcessor`], *optional*):110            The image processor is a required input.111        tokenizer ([`GemmaTokenizerFast`], *optional*):112            The tokenizer is a required input.113        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages114            in a chat into a tokenizable string.115    """116 117    attributes = ["image_processor", "tokenizer"]118    image_processor_class = ("SiglipImageProcessor", "SiglipImageProcessorFast")119    tokenizer_class = ("GemmaTokenizer", "GemmaTokenizerFast")120 121    def __init__(122        self,123        image_processor=None,124        tokenizer=None,125        chat_template=None,126        **kwargs,127    ):128        if not hasattr(image_processor, "image_seq_length"):129            raise ValueError("Image processor is missing an `image_seq_length` attribute.")130 131        self.image_seq_length = image_processor.image_seq_length132 133        if not hasattr(tokenizer, "image_token"):134            image_token = AddedToken(IMAGE_TOKEN, normalized=False, special=True)135            tokens_to_add = {"additional_special_tokens": [image_token]}136            tokenizer.add_special_tokens(tokens_to_add)137            self.image_token_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)138            self.image_token = IMAGE_TOKEN139        else:140            self.image_token_id = tokenizer.image_token_id141            self.image_token = tokenizer.image_token142 143        tokenizer.add_tokens(EXTRA_TOKENS)144        tokenizer.add_bos_token = False145        tokenizer.add_eos_token = False146 147        super().__init__(image_processor, tokenizer, chat_template=chat_template)148 149    def __call__(150        self,151        images: Optional[ImageInput] = None,152        text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,153        audio=None,154        videos=None,155        **kwargs: Unpack[PaliGemmaProcessorKwargs],156    ) -> BatchFeature:157        """158        Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`159        and `kwargs` arguments to GemmaTokenizerFast's [`~GemmaTokenizerFast.__call__`] if `text` is not `None` to encode160        the text. To prepare the image(s), this method forwards the `images` and `kwargs` arguments to161        SiglipImageProcessor's [`~SiglipImageProcessor.__call__`] if `images` is not `None`. Please refer to the docstring162        of the above two methods for more information.163 164        The usage for PaliGemma fine-tuning preparation is slightly different than usual. suffix passed are suffixes to165        the prompt in `text`, and will be placed after the prompt. This is because attention is handled differently for166        the prefix and the suffix. For instance,167        ```python168        image = PIL_cow_image169        prompt = "answer en Where is the cow standing?"170        suffix = "on the beach"171        inputs = processor(text=prompt, images=image, suffix=suffix)172        ```173        Here `inputs` will contain the `input_ids` and `token_type_ids` that follow174        ```python175        inputs["input_ids"][:, 256:]176        # tensor([[     2,   6006,    603,    573,  13910,   9980, 235336,    108,    477,   573,   8318]])177        inputs["token_type_ids"][:, 256:]178        tensor([[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1]])179        ```180        Meaning the last three tokens are of "label" ("suffix") type while the other ones are of "prefix" type.181 182 183        Args:184            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):185                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch186                tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a187                number of channels, H and W are image height and width.188            text (`str`, `list[str]`, `list[list[str]]`):189                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings190                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set191                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).192            return_tensors (`str` or [`~utils.TensorType`], *optional*):193                If set, will return tensors of a particular framework. Acceptable values are:194 195                - `'tf'`: Return TensorFlow `tf.constant` objects.196                - `'pt'`: Return PyTorch `torch.Tensor` objects.197                - `'np'`: Return NumPy `np.ndarray` objects.198                - `'jax'`: Return JAX `jnp.ndarray` objects.199            suffix (`str`, `list[str]`, `list[list[str]]`):200                The suffixes or batch of suffixes to be encoded. Only necessary for finetuning. See https://github.com/google-research/big_vision/blob/main/big_vision/configs/proj/paligemma/README.md201                for more information. If your prompt is "<image> What is on the image", the suffix corresponds to the expected prediction "a cow sitting on a bench".202 203        Returns:204            [`BatchFeature`]: A [`BatchFeature`] with the following fields:205 206            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. If `suffix`207              is provided, the `input_ids` will also contain the suffix input ids.208            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when209              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not210              `None`).211            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.212            - **labels** -- Labels compatible with training if `suffix` is not None213        """214 215        output_kwargs = self._merge_kwargs(216            PaliGemmaProcessorKwargs,217            tokenizer_init_kwargs=self.tokenizer.init_kwargs,218            **kwargs,219        )220        suffix = output_kwargs["text_kwargs"].pop("suffix", None)221 222        return_token_type_ids = suffix is not None223 224        if images is None:225            raise ValueError("`images` are expected as arguments to a `PaliGemmaProcessor` instance.")226        if text is None:227            logger.warning_once(228                "You are using PaliGemma without a text prefix. It will perform as a picture-captioning model."229            )230            text = ""231 232        if _is_str_or_image(text):233            text = [text]234        elif isinstance(text, list) and _is_str_or_image(text[0]):235            pass236 237        if text is not None and images is not None:238            if not any(IMAGE_TOKEN in sample for sample in text):239                logger.warning(240                    "You are passing both `text` and `images` to `PaliGemmaProcessor`. The processor expects special "241                    "image tokens in the text, as many tokens as there are images per each text. It is recommended to "242                    "add `<image>` tokens in the very beginning of your text. For this call, we will infer how many images "243                    "each text has and add special tokens."244                )245 246                if isinstance(text, list) and isinstance(images, list):247                    if len(images) != len(text):248                        raise ValueError(249                            f"Received {len(images)} images for {len(text)} prompts. Each prompt should be associated with an image or list of images."250                        )251 252                # make a nested list of lists to be able to iterate over the images and text below253                if is_valid_image(images):254                    images = [[images]]255                elif isinstance(images, (list, tuple)) and is_valid_image(images[0]):256                    images = [[image] for image in images]257                elif not (258                    isinstance(images, (list, tuple))259                    and isinstance(images[0], (list, tuple))260                    and is_valid_image(images[0][0])261                ):262                    raise ValueError("images must be an image, list of images or list of list of images")263 264                input_strings = [265                    build_string_from_input(266                        prompt=prompt,267                        bos_token=self.tokenizer.bos_token,268                        image_seq_len=self.image_seq_length,269                        image_token=IMAGE_TOKEN,270                        num_images=len(image_list) if isinstance(image_list, list) else 1,271                    )272                    for prompt, image_list in zip(text, images)273                ]274            else:275                expanded_samples = []276                for sample in text:277                    expanded_sample = sample.replace(IMAGE_TOKEN, IMAGE_TOKEN * self.image_seq_length)278                    bos_rfind_index = expanded_sample.rfind(IMAGE_TOKEN)279                    bos_index = bos_rfind_index + len(IMAGE_TOKEN) if bos_rfind_index != -1 else 0280                    expanded_sample = (281                        expanded_sample[:bos_index] + self.tokenizer.bos_token + expanded_sample[bos_index:]282                    )283                    expanded_samples.append(expanded_sample)284                input_strings = [f"{sample}\n" for sample in expanded_samples]285 286        if suffix is not None and _is_str_or_image(suffix):287            suffix = [suffix]288        if suffix is not None:289            suffix = [sfx + self.tokenizer.eos_token for sfx in suffix]290        pixel_values = self.image_processor(images, **output_kwargs["images_kwargs"])["pixel_values"]291 292        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)293        return_mm_token_type_ids = output_kwargs["text_kwargs"].pop("return_mm_token_type_ids", None)294        inputs = self.tokenizer(295            input_strings,296            text_pair=suffix,297            return_token_type_ids=return_token_type_ids,298            **output_kwargs["text_kwargs"],299        )300        self._check_special_mm_tokens(input_strings, inputs, modalities=["image"])301 302        return_data = {**inputs, "pixel_values": pixel_values}303 304        if return_token_type_ids:305            labels = np.array(inputs["input_ids"])306            labels[np.array(inputs["token_type_ids"]) == 0] = -100307            return_data.update({"labels": labels})308 309        if return_mm_token_type_ids:310            array_ids = np.array(return_data["input_ids"])311            mm_token_type_ids = np.zeros_like(return_data["input_ids"])312            mm_token_type_ids[array_ids == self.image_token_id] = 1313            return_data["mm_token_type_ids"] = mm_token_type_ids.tolist()314 315        return BatchFeature(data=return_data, tensor_type=return_tensors)316 317    def _get_num_multimodal_tokens(self, image_sizes=None, **kwargs):318        """319        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.320 321        Args:322            image_sizes (list[list[str]], *optional*):323                The input sizes formatted as (height, width) per each image.324        Returns:325            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided326            input modalities, along with other useful data.327        """328        vision_data = {}329        if image_sizes is not None:330            num_image_tokens = [self.image_seq_length] * len(image_sizes)331            num_image_patches = [1] * len(image_sizes)332            vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches})333        return MultiModalData(**vision_data)334 335 336__all__ = ["PaliGemmaProcessor"]337