Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2025 Meta Platforms, Inc. and the HuggingFace Inc. team. All rights reserved.3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""15Processor class for PerceptionLM.16"""17 18from collections.abc import Iterable19from typing import Optional, Union20 21import numpy as np22 23from ...feature_extraction_utils import BatchFeature24from ...image_utils import ImageInput, get_image_size, to_numpy_array25from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack26from ...tokenization_utils_base import PreTokenizedInput, TextInput27from ...utils import logging28from ...video_utils import VideoInput29 30 31logger = logging.get_logger(__name__)32 33 34class PerceptionLMProcessorKwargs(ProcessingKwargs, total=False):35    _defaults = {36        "text_kwargs": {37            "padding": False,38            "return_mm_token_type_ids": False,39        },40    }41 42 43class PerceptionLMProcessor(ProcessorMixin):44    r"""45    Constructs a PerceptionLM processor which wraps a PerceptionLM image processor, a PerceptionLM video processor, and a tokenizer into a single processor.46 47    [`PerceptionLMProcessor`] offers all the functionalities of [`PerceptionLMImageProcessorFast`], [`PerceptionLMVideoProcessor`], and the tokenizer (e.g. [`LlamaTokenizerFast`]). See the48    [`~PerceptionLMProcessor.__call__`] and [`~PerceptionLMProcessor.decode`] for more information.49 50    Args:51        video_processor ([`PerceptionLMVideoProcessor`], *optional*):52            The video processor to process video inputs.53        image_processor ([`PerceptionLMImageProcessorFast`], *optional*):54            The image processor to process image inputs.55        tokenizer ([`LlamaTokenizerFast`] or similar, *optional*):56            The tokenizer to process text inputs.57        patch_size (`int`, *optional*):58            Patch size from the vision tower.59        chat_template (`str`, *optional*):60            A Jinja template which will be used to convert lists of messages in a chat into a tokenizable string.61        pooling_ratio (`int`, *optional*, defaults to 2):62            Pooling ratio for vision tokens. If not 1, 2D adaptive pooling is applied over projected vision tokens.63    """64 65    attributes = ["video_processor", "image_processor", "tokenizer"]66    image_processor_class = "AutoImageProcessor"67    video_processor_class = "AutoVideoProcessor"68    tokenizer_class = "AutoTokenizer"69 70    def __init__(71        self,72        video_processor=None,73        image_processor=None,74        tokenizer=None,75        patch_size=None,76        chat_template=None,77        pooling_ratio=2,78        **kwargs,79    ):80        self.patch_size = patch_size81        self.pooling_ratio = pooling_ratio82        self.image_token = tokenizer.image_token83        self.video_token = tokenizer.video_token84        self.image_token_id = tokenizer.image_token_id85        self.video_token_id = tokenizer.video_token_id86        super().__init__(video_processor, image_processor, tokenizer, chat_template=chat_template)87 88    def __call__(89        self,90        images: Optional[ImageInput] = None,91        text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,92        audio=None,93        videos: Optional[VideoInput] = None,94        **kwargs: Unpack[PerceptionLMProcessorKwargs],95    ) -> BatchFeature:96        """97        Prepares a batch containing one or more sequences of text and/or images and/or videos.98 99        If `text` is provided, it is tokenized using the tokenizer.100        If `images` is provided, they are processed using the image processor.101        If `videos` is provided, they are processed using the video processor.102 103        Args:104            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):105                The image or batch of images to be processed. Each image can be a PIL image, NumPy array, or PyTorch tensor.106                Both channels-first and channels-last formats are supported.107            text (`str`, `List[str]`, *optional*):108                The sequence or batch of sequences to be tokenized. Each sequence can be a string.109            videos (`Any`, *optional*):110                The video or batch of videos to be processed.111            return_tensors (`str` or [`~utils.TensorType`], *optional*):112                If set, will return tensors of a particular framework. Acceptable values are:113                - `'tf'`: Return TensorFlow `tf.constant` objects.114                - `'pt'`: Return PyTorch `torch.Tensor` objects.115                - `'np'`: Return NumPy `np.ndarray` objects.116                - `'jax'`: Return JAX `jnp.ndarray` objects.117 118        Returns:119            [`BatchFeature`]: A [`BatchFeature`] with the following fields:120 121            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is provided.122            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when123              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is provided).124            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is provided.125            - **pixel_values_videos** -- Video pixel values to be fed to a model. Returned when `videos` is provided.126        """127        if text is None:128            raise ValueError(129                "You have to specify at least `text` input. Optionally, you can also specify `images` or `videos`."130            )131 132        output_kwargs = self._merge_kwargs(133            PerceptionLMProcessorKwargs,134            tokenizer_init_kwargs=self.tokenizer.init_kwargs,135            **kwargs,136        )137        if images is not None:138            image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])139        else:140            image_inputs = {}141 142        if videos is not None:143            videos_inputs = self.video_processor(videos, **output_kwargs["videos_kwargs"])144        else:145            videos_inputs = {}146 147        if isinstance(text, str):148            text = [text]149        elif not isinstance(text, list) and not isinstance(text[0], str):150            raise ValueError("Invalid input text. Please provide a string, or a list of strings")151 152        # try to expand inputs in processing if we have the necessary parts153        prompt_strings = []154 155        pixel_values = iter(image_inputs.get("pixel_values", []))156        pixel_values_videos = iter(videos_inputs.get("pixel_values_videos", []))157        for sample in text:158            # Replace the media token with the expanded media token sequence159            sample = self._expand_media_tokens(sample, self.tokenizer.image_token, pixel_values)160            sample = self._expand_media_tokens(sample, self.tokenizer.video_token, pixel_values_videos)161            prompt_strings.append(sample)162 163        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)164        return_mm_token_type_ids = output_kwargs["text_kwargs"].pop("return_mm_token_type_ids", False)165        text_inputs = self.tokenizer(prompt_strings, **output_kwargs["text_kwargs"], return_tensors=None)166        self._check_special_mm_tokens(prompt_strings, text_inputs, modalities=["image", "video"])167 168        if return_mm_token_type_ids:169            array_ids = np.array(text_inputs["input_ids"])170            mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])171            mm_token_type_ids[array_ids == self.image_token_id] = 1172            text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()173 174        return BatchFeature(data={**text_inputs, **image_inputs, **videos_inputs}, tensor_type=return_tensors)175 176    def _expand_media_tokens(self, sample, media_token: str, media_iter: Iterable):177        media_count = sample.count(media_token)178        if media_count > 0:179            media_list = [next(media_iter) for _ in range(media_count)]180            sample_splits = sample.split(media_token)181            media_token_list = []182            for media in media_list:183                height, width = get_image_size(to_numpy_array(media))184                num_tiles = media.shape[0]185                num_media_tokens = (186                    (height // self.patch_size // self.pooling_ratio)187                    * (width // self.patch_size // self.pooling_ratio)188                    * num_tiles189                )190                media_token_list.append(num_media_tokens)191            sample = ""192            for i, num_media_tokens in enumerate(media_token_list):193                sample += sample_splits[i]194                sample += media_token * num_media_tokens195            sample += sample_splits[-1]196        return sample197 198    def _get_num_multimodal_tokens(self, image_sizes=None, **kwargs):199        """200        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.201 202        Args:203            image_sizes (`list[list[int]]`, *optional*):204                The input sizes formatted as (height, width) per each image.205 206        Returns:207            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided208            input modalities, along with other useful data.209        """210 211        vision_data = {}212        if image_sizes is not None:213            images_kwargs = PerceptionLMProcessorKwargs._defaults.get("images_kwargs", {})214            images_kwargs.update(kwargs)215            tile_size = images_kwargs.get("tile_size", None) or self.image_processor.tile_size216            vision_input_type = images_kwargs.get("vision_input_type", None) or self.image_processor.vision_input_type217 218            num_image_tokens = []219            num_image_patches = []220            for height, width in image_sizes:221                if vision_input_type == "thumb+tile":222                    aspect_ratio = self.image_processor._fit_image_to_canvas(223                        img_width=width, img_height=height, tile_size=tile_size224                    )225                    if aspect_ratio is None:226                        aspect_ratio = self.image_processor._find_closest_aspect_ratio(227                            img_width=width, img_height=height, tile_size=tile_size228                        )229                    num_tiles = aspect_ratio[0] * aspect_ratio[1] + 1  # base image and tiles230                else:231                    num_tiles = 1232 233                num_image_tokens.append(234                    (tile_size // self.patch_size // self.pooling_ratio)235                    * (tile_size // self.patch_size // self.pooling_ratio)236                    * num_tiles237                )238                num_image_patches.append(num_tiles)239 240            vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches})241        return MultiModalData(**vision_data)242 243 244__all__ = ["PerceptionLMProcessor"]245