Aluode/PerceptionLabPortable

1# Copyright 2025 Microsoft and the HuggingFace Inc. team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15"""16Processor class for Phi4Multimodal17"""18 19import re20from typing import Optional, Union21 22from ...audio_utils import AudioInput23from ...image_processing_utils import BatchFeature24from ...image_utils import ImageInput25from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack26from ...tokenization_utils_base import TextInput27from ...utils import logging28 29 30logger = logging.get_logger(__name__)31 32 33class Phi4MultimodalProcessorKwargs(ProcessingKwargs, total=False):34    _defaults = {35        "audio_kwargs": {36            "device": "cpu",37        },38    }39 40 41class Phi4MultimodalProcessor(ProcessorMixin):42    r"""43    Constructs a Phi4Multimodal processor which raps an image processor, a audio processor, and a GPT tokenizer into a single processor.44 45    [`Phi4MultimodalProcessor`] offers all the functionalities of [`Phi4MultimodalImageProcessorFast`] and [`GPT2Tokenizer`]. See the46    [`~Phi4MultimodalProcessor.__call__`] and [`~Phi4MultimodalProcessor.decode`] for more information.47 48    Args:49        image_processor (`Phi4MultimodalImageProcessorFast`):50            The image processor to use for images.51        audio_processor (`Phi4MultimodalFeatureExtractor`):52            The audio processor to use for audio inputs.53        tokenizer (`GPT2TokenizerFast`):54            The tokenizer to use for text.55        fake_image_token_pattern (`str`, *optional*, defaults to `r"<\|image_\d+\|>"`):56            The fake image token pattern.57        fake_audio_token_pattern (`str`, *optional*, defaults to `r"<\|audio_\d+\|>"`):58            The fake audio token pattern.59    """60 61    attributes = ["image_processor", "audio_processor", "tokenizer"]62    tokenizer_class = "GPT2TokenizerFast"63    image_processor_class = "Phi4MultimodalImageProcessorFast"64    audio_processor_class = "Phi4MultimodalFeatureExtractor"65 66    def __init__(67        self,68        image_processor,69        audio_processor,70        tokenizer,71        **kwargs,72    ):73        self.image_token = tokenizer.image_token74        self.image_token_id = tokenizer.image_token_id75        self.audio_token = tokenizer.audio_token76        self.audio_token_id = tokenizer.audio_token_id77        super().__init__(image_processor, audio_processor, tokenizer, **kwargs)78 79    def __call__(80        self,81        text: Union[TextInput, list[TextInput]],82        images: Optional[ImageInput] = None,83        audio: Optional[AudioInput] = None,84        **kwargs: Unpack[ProcessingKwargs],85    ) -> BatchFeature:86        """87        Main method to prepare for the model one or several sequences(s) and image(s). This method forards the `text`88        and `kwargs` arguments to GPT2Tokenizer's [`~GPT2Tokenizer.__call__`] if `text` is not `None` to encode89        the text. To prepare the image(s), this method forwards the `images` and `kwargs` arguments to90        Phi4MultimodalImageProcessorFast's [`~Phi4MultimodalImageProcessorFast.__call__`] if `images` is not `None`. Please refer to the doctsring91        of the above two methods for more information.92 93        Args:94            text (`str`, `list[str]`, `list[list[str]]`):95                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings96                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set97                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).98            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):99                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch100                tensor. Both channels-first and channels-last formats are supported.101            audio (`list[Union[np.ndarray, torch.Tensor]]`):102                List of the audios to be prepared.103 104        Returns:105            [`BatchFeature`]: A [`BatchFeature`] with the following fields:106 107            - **input_ids** -- List of token ids to be fed to a model.108            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model.109            - **input_image_embeds** -- Pixel values to be fed to a model.110            - **image_sizes** -- List of tuples specifying the size of each image in `input_image_embeds`.111            - **image_attention_mask** -- List of attention masks for each image in `input_image_embeds`.112            - **input_audio_embeds** -- Audio embeddings to be fed to a model.113            - **audio_embed_sizes** -- List of integers specifying the size of each audio in `input_audio_embeds`.114        """115 116        output_kwargs = self._merge_kwargs(Phi4MultimodalProcessorKwargs, self.tokenizer.init_kwargs, **kwargs)117        image_kwargs = output_kwargs["images_kwargs"]118        audio_kwargs = output_kwargs["audio_kwargs"]119 120        image_inputs = self.image_processor(images, **image_kwargs) if images is not None else {}121        audio_inputs = self.audio_processor(audio, **audio_kwargs) if audio is not None else {}122 123        # We pop here for images as we don't need it later124        num_img_tokens = image_inputs.pop("num_img_tokens", [])125        audio_embed_sizes = audio_inputs.get("audio_embed_sizes", [])126 127        # Replace certain special tokens for compatibility128        if isinstance(text, str):129            text = [text]130        elif not isinstance(text, list) and not isinstance(text[0], str):131            raise TypeError("Invalid input text. Please provide a string, or a list of strings")132 133        image_token = self.tokenizer.image_token134        audio_token = self.tokenizer.audio_token135 136        # Check that the number of special tokens is sound137        concatenated_prompt = "".join(text)138        if concatenated_prompt.count(image_token) != len(num_img_tokens):139            raise ValueError(140                "You should add as much image tokens `<|image|>` in your prompt as you pass `images` to the processor. ",141                f"Input contains {concatenated_prompt.count(image_token)} tokens != {len(num_img_tokens)} images",142            )143        if concatenated_prompt.count(audio_token) != len(audio_embed_sizes):144            raise ValueError(145                "You should add as much audio tokens `<|audio|>` in your prompt as you pass `audios` to the processor. "146                f"Input contains {concatenated_prompt.count(audio_token)} tokens != {len(audio_embed_sizes)} audios"147            )148 149        # Add appropriate number of image/audio tokens (note that the count of replacement is dynamic)150        image_count_iter = iter(num_img_tokens)151        audio_count_iter = iter(audio_embed_sizes)152        processed_text = [153            re.sub(re.escape(image_token), lambda _: image_token * next(image_count_iter), t) for t in text154        ]155        processed_text = [156            re.sub(re.escape(audio_token), lambda _: audio_token * next(audio_count_iter), t) for t in processed_text157        ]158 159        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)160        text_inputs = self.tokenizer(processed_text, **output_kwargs["text_kwargs"])161        self._check_special_mm_tokens(processed_text, text_inputs, modalities=["image"])162 163        # prepare batch feature164        data = {165            **text_inputs,166            **image_inputs,167            **audio_inputs,168        }169 170        return BatchFeature(data=data, tensor_type=return_tensors)171 172 173__all__ = ["Phi4MultimodalProcessor"]174