Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2023 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16Processor class for InstructBLIP. Largely copy of Blip2Processor with addition of a tokenizer for the Q-Former.17"""18 19import os20from typing import Optional, Union21 22from ...image_processing_utils import BatchFeature23from ...image_utils import ImageInput24from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack25from ...tokenization_utils_base import AddedToken, PreTokenizedInput, TextInput26from ...utils import logging27from ..auto import AutoTokenizer28 29 30logger = logging.get_logger(__name__)31 32 33class InstructBlipProcessorKwargs(ProcessingKwargs, total=False):34    _defaults = {35        "text_kwargs": {36            "add_special_tokens": True,37            "padding": False,38            "stride": 0,39            "return_overflowing_tokens": False,40            "return_special_tokens_mask": False,41            "return_offsets_mapping": False,42            "return_token_type_ids": False,43            "return_length": False,44            "verbose": True,45        },46        "images_kwargs": {},47    }48 49 50class InstructBlipProcessor(ProcessorMixin):51    r"""52    Constructs an InstructBLIP processor which wraps a BLIP image processor and a LLaMa/T5 tokenizer into a single53    processor.54 55    [`InstructBlipProcessor`] offers all the functionalities of [`BlipImageProcessor`] and [`AutoTokenizer`]. See the56    docstring of [`~BlipProcessor.__call__`] and [`~BlipProcessor.decode`] for more information.57 58    Args:59        image_processor (`BlipImageProcessor`):60            An instance of [`BlipImageProcessor`]. The image processor is a required input.61        tokenizer (`AutoTokenizer`):62            An instance of ['PreTrainedTokenizer`]. The tokenizer is a required input.63        qformer_tokenizer (`AutoTokenizer`):64            An instance of ['PreTrainedTokenizer`]. The Q-Former tokenizer is a required input.65        num_query_tokens (`int`, *optional*):"66            Number of tokens used by the Qformer as queries, should be same as in model's config.67    """68 69    attributes = ["image_processor", "tokenizer", "qformer_tokenizer"]70    image_processor_class = ("BlipImageProcessor", "BlipImageProcessorFast")71    tokenizer_class = "AutoTokenizer"72    qformer_tokenizer_class = "AutoTokenizer"73 74    def __init__(self, image_processor, tokenizer, qformer_tokenizer, num_query_tokens=None, **kwargs):75        if not hasattr(tokenizer, "image_token"):76            self.image_token = AddedToken("<image>", normalized=False, special=True)77            tokenizer.add_tokens([self.image_token], special_tokens=True)78        else:79            self.image_token = tokenizer.image_token80        self.num_query_tokens = num_query_tokens81 82        super().__init__(image_processor, tokenizer, qformer_tokenizer)83 84    def __call__(85        self,86        images: Optional[ImageInput] = None,87        text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,88        audio=None,89        videos=None,90        **kwargs: Unpack[InstructBlipProcessorKwargs],91    ) -> BatchFeature:92        """93        This method uses [`BlipImageProcessor.__call__`] method to prepare image(s) for the model, and94        [`BertTokenizerFast.__call__`] to prepare text for the model.95 96        Please refer to the docstring of the above two methods for more information.97        Args:98            images (`ImageInput`):99                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch100                tensor. Both channels-first and channels-last formats are supported.101            text (`TextInput`, `PreTokenizedInput`, `list[TextInput]`, `list[PreTokenizedInput]`):102                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings103                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set104                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).105        """106        if images is None and text is None:107            raise ValueError("You have to specify at least images or text.")108 109        output_kwargs = self._merge_kwargs(110            InstructBlipProcessorKwargs,111            tokenizer_init_kwargs=self.tokenizer.init_kwargs,112            **kwargs,113        )114 115        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)116        encoding = {}117        if text is not None:118            if isinstance(text, str):119                text = [text]120            elif not isinstance(text, list) and not isinstance(text[0], str):121                raise ValueError("Invalid input text. Please provide a string, or a list of strings")122 123            qformer_text_encoding = self.qformer_tokenizer(text, **output_kwargs["text_kwargs"])124            encoding["qformer_input_ids"] = qformer_text_encoding.pop("input_ids")125            encoding["qformer_attention_mask"] = qformer_text_encoding.pop("attention_mask")126 127            # We need this hacky manipulation because BLIP expects image tokens to be at the beginning even before BOS token128            if output_kwargs["text_kwargs"].get("max_length") is not None:129                output_kwargs["text_kwargs"]["max_length"] -= self.num_query_tokens130            text_encoding = self.tokenizer(text, **output_kwargs["text_kwargs"])131 132            if images is not None:133                # Image tokens should not be padded/truncated or prepended with special BOS token134                image_tokens = self.image_token.content * self.num_query_tokens135                output_kwargs["text_kwargs"]["add_special_tokens"] = False136                output_kwargs["text_kwargs"]["padding"] = False137                output_kwargs["text_kwargs"]["truncation"] = False138                image_text_encoding = self.tokenizer(image_tokens, **output_kwargs["text_kwargs"])139                for k in text_encoding:140                    text_encoding[k] = [image_text_encoding[k] + sample for sample in text_encoding[k]]141            encoding.update(text_encoding)142 143        if images is not None:144            image_encoding = self.image_processor(images, **output_kwargs["images_kwargs"])145            encoding.update(image_encoding)146 147        # Cast to desired return tensors type148        encoding = BatchFeature(encoding, tensor_type=return_tensors)149        return encoding150 151    @property152    def model_input_names(self):153        tokenizer_input_names = self.tokenizer.model_input_names154        image_processor_input_names = self.image_processor.model_input_names155        qformer_input_names = ["qformer_input_ids", "qformer_attention_mask"]156        return tokenizer_input_names + image_processor_input_names + qformer_input_names157 158    # overwrite to save the Q-Former tokenizer in a separate folder159    def save_pretrained(self, save_directory, **kwargs):160        if os.path.isfile(save_directory):161            raise ValueError(f"Provided path ({save_directory}) should be a directory, not a file")162        os.makedirs(save_directory, exist_ok=True)163        qformer_tokenizer_path = os.path.join(save_directory, "qformer_tokenizer")164        self.qformer_tokenizer.save_pretrained(qformer_tokenizer_path)165 166        # We modify the attributes so that only the tokenizer and image processor are saved in the main folder167        qformer_present = "qformer_tokenizer" in self.attributes168        if qformer_present:169            self.attributes.remove("qformer_tokenizer")170 171        outputs = super().save_pretrained(save_directory, **kwargs)172 173        if qformer_present:174            self.attributes += ["qformer_tokenizer"]175        return outputs176 177    # overwrite to load the Q-Former tokenizer from a separate folder178    @classmethod179    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):180        processor = super().from_pretrained(pretrained_model_name_or_path, **kwargs)181 182        # if return_unused_kwargs a tuple is returned where the second element is 'unused_kwargs'183        if isinstance(processor, tuple):184            processor = processor[0]185        qformer_tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, subfolder="qformer_tokenizer")186        processor.qformer_tokenizer = qformer_tokenizer187        return processor188 189 190__all__ = ["InstructBlipProcessor"]191