optimum-intel-internal-testing/tiny-random-minicpmv-2_6

1# coding=utf-82# Copyright 2024 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16Processor class for MiniCPMV.17"""18 19import re20from typing import List, Optional, Union21 22import torch23from transformers.image_utils import ImageInput24from transformers.processing_utils import ProcessorMixin25from transformers.tokenization_utils_base import PreTokenizedInput, TextInput26from transformers.utils import TensorType27 28from .image_processing_minicpmv import MiniCPMVBatchFeature29 30 31class MiniCPMVProcessor(ProcessorMixin):32    r"""33    Constructs a MiniCPMV processor which wraps a MiniCPMV image processor and a MiniCPMV tokenizer into a single processor.34 35    [`MiniCPMVProcessor`] offers all the functionalities of [`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the36    [`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`] for more information.37 38    Args:39        image_processor ([`MiniCPMVImageProcessor`], *optional*):40            The image processor is a required input.41        tokenizer ([`LlamaTokenizerWrapper`], *optional*):42            The tokenizer is a required input.43    """44    attributes = ["image_processor", "tokenizer"]45    image_processor_class = "AutoImageProcessor"46    tokenizer_class = "AutoTokenizer"47 48    def __init__(self, image_processor=None, tokenizer=None, **kwargs):49        super().__init__(image_processor, tokenizer)50        self.version = image_processor.version51 52    def __call__(53        self,54        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],55        images: ImageInput = None,56        max_length: Optional[int] = None,57        do_pad: Optional[bool] = True,58        max_slice_nums: int = None,59        use_image_id: bool = None,60        return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,61        **kwargs,62    ) -> MiniCPMVBatchFeature:63        image_inputs = None64        if images is not None:65            image_inputs = self.image_processor(66                images, do_pad=do_pad, max_slice_nums=max_slice_nums, return_tensors=return_tensors67            )68        return self._convert_images_texts_to_inputs(69            image_inputs,70            text,71            max_slice_nums=max_slice_nums,72            use_image_id=use_image_id,73            max_length=max_length,74            **kwargs,75            return_tensors=return_tensors,76        )77 78    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama79    def batch_decode(self, *args, **kwargs):80        """81        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please82        refer to the docstring of this method for more information.83        """84        output_ids = args[0]85        result_text = []86        for result in output_ids:87            result = result[result != 0]88            if result[0] == self.tokenizer.bos_id:89                result = result[1:]90            if result[-1] == self.tokenizer.eos_id:91                result = result[:-1]92            result_text.append(self.tokenizer.decode(result, *args[1:], **kwargs).strip())93        return result_text94        # return self.tokenizer.batch_decode(*args, **kwargs)95 96    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama97    def decode(self, *args, **kwargs):98        """99        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to100        the docstring of this method for more information.101        """102        result = args[0]103        result = result[result != 0]104        if result[0] == self.tokenizer.bos_id:105            result = result[1:]106        if result[-1] == self.tokenizer.eos_id or (107            hasattr(self.tokenizer, "eot_id") and result[-1] == self.tokenizer.eot_id108        ):109            result = result[:-1]110        return self.tokenizer.decode(result, *args[1:], **kwargs).strip()111 112    def _convert(self, input_str, max_inp_length: Optional[int] = None):113        if self.version > 2.5 or not getattr(self.tokenizer, "add_bos_token", False):114            input_ids = self.tokenizer.encode(input_str)115        else:116            input_ids = [self.tokenizer.bos_id] + self.tokenizer.encode(input_str)117        if max_inp_length is not None:118            input_ids = input_ids[:max_inp_length]119        input_ids = torch.tensor(input_ids, dtype=torch.int32)120 121        start_cond = (input_ids == self.tokenizer.im_start_id) | (input_ids == self.tokenizer.slice_start_id)122        end_cond = (input_ids == self.tokenizer.im_end_id) | (input_ids == self.tokenizer.slice_end_id)123 124        image_start_tokens = torch.where(start_cond)[0]125        image_start_tokens += 1126        image_end_tokens = torch.where(end_cond)[0]127 128        valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))129 130        image_bounds = torch.hstack(131            [132                image_start_tokens[:valid_image_nums].unsqueeze(-1),133                image_end_tokens[:valid_image_nums].unsqueeze(-1),134            ]135        )136        return input_ids, image_bounds137 138    def _convert_images_texts_to_inputs(139        self,140        images,141        texts: Union[str, List[str]],142        truncation=None,143        max_length=None,144        max_slice_nums=None,145        use_image_id=None,146        return_tensors=None,147        **kwargs,148    ):149        if images is None or not len(images):150            model_inputs = self.tokenizer(151                texts, return_tensors=return_tensors, truncation=truncation, max_length=max_length, **kwargs152            )153            return MiniCPMVBatchFeature(data={**model_inputs})154 155        pattern = "(<image>./</image>)"156        images, image_sizes, tgt_sizes = images["pixel_values"], images["image_sizes"], images["tgt_sizes"]157 158        if isinstance(texts, str):159            texts = [texts]160        input_ids_list = []161        image_bounds_list = []162        for index, text in enumerate(texts):163            image_tags = re.findall(pattern, text)164            assert len(image_tags) == len(image_sizes[index])165            text_chunks = text.split(pattern)166            final_text = ""167            for i in range(len(image_tags)):168                final_text = (169                    final_text170                    + text_chunks[i]171                    + self.image_processor.get_slice_image_placeholder(172                        image_sizes[index][i], i, max_slice_nums, use_image_id173                    )174                )175            final_text += text_chunks[-1]176            input_ids, image_bounds = self._convert(final_text, max_length)177            input_ids_list.append(input_ids)178            image_bounds_list.append(image_bounds)179        padded_input_ids, padding_lengths = self.pad(input_ids_list, padding_side="left")180        for i, length in enumerate(padding_lengths):181            image_bounds_list[i] = image_bounds_list[i] + length182        attention_mask = padded_input_ids.ne(0)183 184        return MiniCPMVBatchFeature(185            data={186                "input_ids": padded_input_ids,187                "attention_mask": attention_mask,188                "pixel_values": images,189                "image_sizes": image_sizes,190                "image_bound": image_bounds_list,191                "tgt_sizes": tgt_sizes,192            }193        )194 195    @property196    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names197    def model_input_names(self):198        tokenizer_input_names = self.tokenizer.model_input_names199        image_processor_input_names = self.image_processor.model_input_names200        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))201 202    def pad(self, inputs, max_length=None, padding_value=0, padding_side="left"):203        items = []204        if isinstance(inputs[0], list):205            assert isinstance(inputs[0][0], torch.Tensor)206            for it in inputs:207                for tr in it:208                    items.append(tr)209        else:210            assert isinstance(inputs[0], torch.Tensor)211            items = inputs212 213        batch_size = len(items)214        shape = items[0].shape215        dim = len(shape)216        assert dim <= 2217        if max_length is None:218            max_length = 0219        max_length = max(max_length, max(item.shape[-1] for item in items))220        min_length = min(item.shape[-1] for item in items)221        dtype = items[0].dtype222 223        if dim == 0:224            return torch.stack([item for item in items], dim=0), [0]225        elif dim == 1:226            if max_length == min_length:227                return torch.stack([item for item in items], dim=0), [0] * batch_size228            tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value229        else:230            tensor = torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype) + padding_value231 232        padding_length = []233        for i, item in enumerate(items):234            if dim == 1:235                if padding_side == "left":236                    tensor[i, -len(item) :] = item.clone()237                else:238                    tensor[i, : len(item)] = item.clone()239            elif dim == 2:240                if padding_side == "left":241                    tensor[i, -len(item) :, :] = item.clone()242                else:243                    tensor[i, : len(item), :] = item.clone()244            padding_length.append(tensor.shape[-1] - len(item))245 246        return tensor, padding_length247