MiniMaxAI/MiniMax-VL-01

1"""2Processor class for MiniMaxVL01.3"""4 5from typing import List, Union6 7from transformers.feature_extraction_utils import BatchFeature8from transformers.image_utils import ImageInput, get_image_size, to_numpy_array9from transformers.processing_utils import ProcessingKwargs, ProcessorMixin#, _validate_images_text_input_order10from transformers.tokenization_utils_base import PreTokenizedInput, TextInput11from transformers.utils import logging12 13from .image_processor import CustomBatchFeature14logger = logging.get_logger(__name__)15 16import os17 18LEGACY_PROCESSING = int(os.getenv('LEGACY_PROCESSING', 1))19 20class MiniMaxVL01ProcessorKwargs(ProcessingKwargs, total=False):21    _defaults = {22        "text_kwargs": {23            "padding": False,24        },25        "images_kwargs": {},26    }27 28def get_hw_multiple_of(image_size, multiple, max_size=None):29    w, h = image_size30    new_w = w if w % multiple == 0 else w + (multiple - w % multiple)31    new_h = h if h % multiple == 0 else h + (multiple - h % multiple)32    if max_size is not None:33        assert isinstance(max_size, (list, tuple)) and len(max_size) == 234        max_w, max_h = max_size35        assert max_w % multiple == 0 and max_h % multiple == 036        if new_w > max_w or new_h > max_h:37            # ratio = min(max_w / new_w, max_h / new_h)38            # new_w = int(new_w * ratio)39            # new_h = int(new_h * ratio)40            new_w = min((new_w * max_w) // new_w, (new_w * max_h) // new_h)41            new_h = min((new_h * max_w) // new_w, (new_h * max_h) // new_h)42 43            new_w = new_w if new_w % multiple == 0 else new_w + (multiple - new_w % multiple)44            new_h = new_h if new_h % multiple == 0 else new_h + (multiple - new_h % multiple)45        assert new_w % multiple == 0 and new_h % multiple == 046        assert new_w <= max_w and new_h <= max_h47    return new_w, new_h48 49def split_special_tokens(text, special_tokens):50    # 使用正则表达式匹配所有特殊标记及其前后内容51    import re52    pattern = '|'.join(map(re.escape, special_tokens))53    return re.split(f'({pattern})', text)54 55 56def select_best_resolution(original_size, possible_resolutions):57    """58    Selects the best resolution from a list of possible resolutions based on the original size.59 60    Args:61        original_size (tuple): The original size of the image in the format (width, height).62        possible_resolutions (list): A list of possible resolutions in the format [(width1, height1), (width2, height2), ...].63 64    Returns:65        tuple: The best fit resolution in the format (width, height).66    """67    original_width, original_height = original_size68    best_fit = None69    max_effective_resolution = 070    min_wasted_resolution = float("inf")71 72    for width, height in possible_resolutions:73        # Calculate the downscaled size to keep the aspect ratio74        scale = min(width / original_width, height / original_height)75        downscaled_width, downscaled_height = int(original_width * scale), int(original_height * scale)76 77        # Calculate effective and wasted resolutions78        effective_resolution = min(downscaled_width * downscaled_height, original_width * original_height)79        wasted_resolution = (width * height) - effective_resolution80 81        if effective_resolution > max_effective_resolution or (effective_resolution == max_effective_resolution and wasted_resolution < min_wasted_resolution):82            max_effective_resolution = effective_resolution83            min_wasted_resolution = wasted_resolution84            best_fit = (width, height)85 86    return best_fit 87 88 89def get_w_h_num(resolution, best_resolution):90    original_width, original_height = resolution91    current_width, current_height = best_resolution92 93    current_height = int(current_height)94    current_width = int(current_width)95    original_height = int(original_height)96    original_width = int(original_width)97 98    original_aspect_ratio = original_width / original_height99    current_aspect_ratio = current_width / current_height100 101    if original_aspect_ratio > current_aspect_ratio:102        scale_factor = current_width / original_width103        new_height = int(original_height * current_width) // original_width104        padding = (current_height - new_height) // 2105        w_num = current_width106        h_num = current_height - 2*padding107    else:108        scale_factor = current_height / original_height109        new_width = int(original_width * current_height) // original_height110        111        padding = (current_width - new_width) // 2112        w_num = current_width - 2*padding113        h_num = current_height114 115    return (w_num, h_num)116    117def get_num_token(img_h, img_w, grid_pinpoints, patch_size):118    #patch_size = 14119    #grid_pinpoints = eval("[(336, 336), (336, 672), (336, 1008), (336, 1344), (336, 1680), (336, 2016), (672, 336), (672, 672), (672, 1008), (672, 1344), (672, 1680), (672, 2016), (1008, 336), (1008, 672), (1008, 1008), (1008, 1344), (1008, 1680), (1008, 2016), (1344, 336), (1344, 672), (1344, 1008), (1344, 1344), (1344, 1680), (1344, 2016), (1680, 336), (1680, 672), (1680, 1008), (1680, 1344), (1680, 1680), (1680, 2016), (2016, 336), (2016, 672), (2016, 1008), (2016, 1344), (2016, 1680), (2016, 2016)]")120    best_resolution = select_best_resolution((img_w,img_h), grid_pinpoints)121    resized_w, resized_h = best_resolution122    w_num, h_num = get_w_h_num((img_w, img_h), (resized_w// patch_size, resized_h// patch_size))123    total_token = int((w_num+1) * h_num) + (336//patch_size)**2124    return total_token125 126 127class MiniMaxVL01Processor(ProcessorMixin):128    r"""129    Constructs a MiniMaxVL01 processor which wraps a MiniMaxVL01 image processor and a MiniMaxVL01 tokenizer into a single processor.130 131    [`MiniMaxVL01Processor`] offers all the functionalities of [`CLIPImageProcessor`] and [`LlamaTokenizerFast`]. See the132    [`~MiniMaxVL01Processor.__call__`] and [`~MiniMaxVL01Processor.decode`] for more information.133 134    Args:135        image_processor ([`CLIPImageProcessor`], *optional*):136            The image processor is a required input.137        tokenizer ([`LlamaTokenizerFast`], *optional*):138            The tokenizer is a required input.139        patch_size (`int`, *optional*):140            Patch size from the vision tower.141        vision_feature_select_strategy (`str`, *optional*):142            The feature selection strategy used to select the vision feature from the vision backbone.143            Shoudl be same as in model's config144        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages145            in a chat into a tokenizable string.146        image_token (`str`, *optional*, defaults to `"<image>"`):147            Special token used to denote image location.148    """149 150    attributes = ["image_processor", "tokenizer"]151    valid_kwargs = ["chat_template", "patch_size", "vision_feature_select_strategy", "image_token"]152    image_processor_class = "AutoImageProcessor"153    tokenizer_class = "AutoTokenizer"154 155    def __init__(156        self,157        image_processor=None,158        tokenizer=None,159        patch_size=None,160        vision_feature_select_strategy=None,161        chat_template=None,162        image_token="<image>",  # set the default and let users change if they have peculiar special tokens in rare cases163        **kwargs,164    ):165        self.patch_size = patch_size166        self.vision_feature_select_strategy = vision_feature_select_strategy167        self.image_token = image_token168        super().__init__(image_processor, tokenizer, chat_template=chat_template)169        self.patch_size = image_processor.patch_size170        self.grid_pinpoints = image_processor.image_grid_pinpoints171        self.max_size = image_processor.size172        self.process_image_mode = image_processor.process_image_mode173 174    def __call__(175        self,176        images: ImageInput = None,177        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,178        audio=None,179        videos=None,180        **kwargs,181    ) -> BatchFeature:182        """183        Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`184        and `kwargs` arguments to LlamaTokenizerFast's [`~LlamaTokenizerFast.__call__`] if `text` is not `None` to encode185        the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to186        CLIPImageProcessor's [`~CLIPImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring187        of the above two methods for more information.188 189        Args:190            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):191                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch192                tensor. Both channels-first and channels-last formats are supported.193            text (`str`, `List[str]`, `List[List[str]]`):194                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings195                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set196                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).197            return_tensors (`str` or [`~utils.TensorType`], *optional*):198                If set, will return tensors of a particular framework. Acceptable values are:199                - `'tf'`: Return TensorFlow `tf.constant` objects.200                - `'pt'`: Return PyTorch `torch.Tensor` objects.201                - `'np'`: Return NumPy `np.ndarray` objects.202                - `'jax'`: Return JAX `jnp.ndarray` objects.203 204        Returns:205            [`BatchFeature`]: A [`BatchFeature`] with the following fields:206 207            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.208            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when209              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not210              `None`).211            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.212        """213        if images is None and text is None:214            raise ValueError("You have to specify at least one of `images` or `text`.")215 216        # check if images and text inputs are reversed for BC217        #images, text = _validate_images_text_input_order(images, text)218        output_kwargs = self._merge_kwargs(219            MiniMaxVL01ProcessorKwargs,220            tokenizer_init_kwargs=self.tokenizer.init_kwargs,221            **kwargs,222        )223        if images is not None:224            image_inputs = self.image_processor(images, **output_kwargs["images_kwargs"])225        else:226            image_inputs = {}227 228        if isinstance(text, str):229            text = [text]230        elif not isinstance(text, list) and not isinstance(text[0], str):231            raise ValueError("Invalid input text. Please provide a string, or a list of strings")232 233        # try to expand inputs in processing if we have the necessary parts234        prompt_strings = text235        if image_inputs.get("pixel_values") is not None:236            if self.process_image_mode == 'anyres':237                if LEGACY_PROCESSING:# 推理时不提前替换image token238                    pixel_values = image_inputs["pixel_values"]239                    image_sizes = image_inputs["image_sizes"]240                    # height, width = get_image_size(to_numpy_array(pixel_values[0]))241                    # num_image_tokens = (height // self.patch_size) * (width // self.patch_size) + 1242                    # if self.vision_feature_select_strategy == "default":243                    #     num_image_tokens -= 1244                    all_image_tokens = []245                    for pixel_value, image_size in zip(pixel_values, image_sizes):246                        height, width = image_size247                        num_image_tokens = get_num_token(height, width, self.grid_pinpoints, self.patch_size)248                        # if self.vision_feature_select_strategy == "default":249                        #     num_image_tokens -= 1250                        all_image_tokens.append(num_image_tokens)251                    prompt_strings = []252                    image_index = 0253                    for sample in text:254                        split_text = split_special_tokens(sample, [self.image_token])255                        final_text = ''256                        for i, _sample in enumerate(split_text):257                            if _sample == self.image_token:258                                final_text += _sample * all_image_tokens[image_index]259                                image_index += 1260                            else:261                                final_text += _sample262                        #sample = sample.replace(self.image_token, self.image_token * all_image_tokens)263                        prompt_strings.append(final_text)264            elif self.process_image_mode == 'resize':265                pixel_values = image_inputs["pixel_values"]266                # height, width = get_image_size(to_numpy_array(pixel_values[0]))267                # num_image_tokens = (height // self.patch_size) * (width // self.patch_size) + 1268                # if self.vision_feature_select_strategy == "default":269                #     num_image_tokens -= 1270                all_image_tokens = []271                for pixel_value in pixel_values:272                    height, width = get_image_size(to_numpy_array(pixel_value))273                    all_image_tokens.append(int(height*width/self.patch_size**2))274                275                prompt_strings = []276                image_index = 0277                for sample in text:278                    split_text = split_special_tokens(sample, [self.image_token])279                    final_text = ''280                    for i, _sample in enumerate(split_text):281                        if _sample == self.image_token:282                            final_text += _sample * all_image_tokens[image_index]283                            image_index += 1284                        else:285                            final_text += _sample286                    #sample = sample.replace(self.image_token, self.image_token * all_image_tokens)287                    prompt_strings.append(final_text)288            else:289                290                if self.patch_size is not None:291                    # Replace the image token with the expanded image token sequence292                    pixel_values = image_inputs["pixel_values"]293                    # height, width = get_image_size(to_numpy_array(pixel_values[0]))294                    # num_image_tokens = (height // self.patch_size) * (width // self.patch_size) + 1295                    # if self.vision_feature_select_strategy == "default":296                    #     num_image_tokens -= 1297                    all_image_tokens = []298                    for pixel_value in pixel_values:299                        height, width = get_image_size(to_numpy_array(pixel_value))300                        new_width, new_height = get_hw_multiple_of((width, height), self.patch_size, self.max_size)301                        num_image_tokens = (new_height // self.patch_size) * (new_width // self.patch_size)# + 1302                        # if self.vision_feature_select_strategy == "default":303                        #     num_image_tokens -= 1304                        all_image_tokens.append(num_image_tokens)305                    306                    prompt_strings = []307                    image_index = 0308                    for sample in text:309                        split_text = split_special_tokens(sample, [self.image_token])310                        final_text = ''311                        for i, _sample in enumerate(split_text):312                            if _sample == self.image_token:313                                final_text += _sample * all_image_tokens[image_index]314                                image_index += 1315                            else:316                                final_text += _sample317                        #sample = sample.replace(self.image_token, self.image_token * all_image_tokens)318                        prompt_strings.append(final_text)319                else:320                    logger.warning_once(321                        "Expanding inputs for image tokens in MiniMaxVL01 should be done in processing. "322                        "Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly "323                        "with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. "324                        "Using processors without these attributes in the config is deprecated and will throw an error in v4.47."325                    )326                    raise ValueError(327                        "You need to provide `patch_size` and `vision_feature_select_strategy` in the model's processing config to expand inputs for image tokens."328                    )329 330        text_inputs = self.tokenizer(prompt_strings, **output_kwargs["text_kwargs"])331        #return {**text_inputs, **image_inputs}332        return CustomBatchFeature(data={**text_inputs, **image_inputs})333 334    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama335    def batch_decode(self, *args, **kwargs):336        """337        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please338        refer to the docstring of this method for more information.339        """340        return self.tokenizer.batch_decode(*args, **kwargs)341 342    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama343    def decode(self, *args, **kwargs):344        """345        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to346        the docstring of this method for more information.347        """348        return self.tokenizer.decode(*args, **kwargs)349 350    @property351    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names352    def model_input_names(self):353        tokenizer_input_names = self.tokenizer.model_input_names354        image_processor_input_names = self.image_processor.model_input_names355        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))356