yanziang/InternVideo3-8B-Instruct

1# coding=utf-82# Copyright 2025 The InternVideo Team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Processor class for InternVideo3."""16 17from typing import Optional, Union18 19import numpy as np20 21from transformers.feature_extraction_utils import BatchFeature22from transformers.image_utils import ImageInput23from transformers.processing_utils import ImagesKwargs, ProcessingKwargs, ProcessorMixin, Unpack, VideosKwargs24from transformers.tokenization_utils_base import PreTokenizedInput, TextInput25from transformers.utils import logging26from transformers.video_utils import VideoInput27 28 29logger = logging.get_logger(__name__)30 31 32class InternVideo3VideosProcessorKwargs(VideosKwargs, total=False):33    pass34 35 36class InternVideo3ImagesKwargs(ImagesKwargs):37    min_pixels: Optional[int]38    max_pixels: Optional[int]39    patch_size: Optional[int]40    temporal_patch_size: Optional[int]41    merge_size: Optional[int]42 43 44class InternVideo3ProcessorKwargs(ProcessingKwargs, total=False):45    images_kwargs: InternVideo3ImagesKwargs46    videos_kwargs: InternVideo3VideosProcessorKwargs47    _defaults = {48        "text_kwargs": {49            "padding": False,50            "return_token_type_ids": False,51        },52        "videos_kwargs": {"return_metadata": True},53    }54 55 56class InternVideo3Processor(ProcessorMixin):57    r"""58    Constructs an InternVideo3 processor which wraps an image processor, a video processor,59    and a tokenizer into a single processor.60 61    Args:62        image_processor: The image processor.63        tokenizer: The tokenizer.64        video_processor: The video processor.65        chat_template (`str`, *optional*): A Jinja template for chat formatting.66    """67 68    attributes = ["image_processor", "tokenizer", "video_processor"]69    image_processor_class = "AutoImageProcessor"70    video_processor_class = "AutoVideoProcessor"71    tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")72 73    def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs):74        super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template)75        self.image_token = "<|image_pad|>" if not hasattr(tokenizer, "image_token") else tokenizer.image_token76        self.video_token = "<|video_pad|>" if not hasattr(tokenizer, "video_token") else tokenizer.video_token77        self.image_token_id = (78            tokenizer.image_token_id79            if getattr(tokenizer, "image_token_id", None)80            else tokenizer.convert_tokens_to_ids(self.image_token)81        )82        self.video_token_id = (83            tokenizer.video_token_id84            if getattr(tokenizer, "video_token_id", None)85            else tokenizer.convert_tokens_to_ids(self.video_token)86        )87        self.vision_start_token = (88            "<|vision_start|>" if not hasattr(tokenizer, "vision_start_token") else tokenizer.vision_start_token89        )90        self.vision_end_token = (91            "<|vision_end|>" if not hasattr(tokenizer, "vision_end_token") else tokenizer.vision_end_token92        )93        self.vision_start_token_id = (94            tokenizer.vision_start_token_id95            if getattr(tokenizer, "vision_start_token_id", None)96            else tokenizer.convert_tokens_to_ids(self.vision_start_token)97        )98        self.vision_end_token_id = (99            tokenizer.vision_end_token_id100            if getattr(tokenizer, "vision_end_token_id", None)101            else tokenizer.convert_tokens_to_ids(self.vision_end_token)102        )103 104    def __call__(105        self,106        images: ImageInput = None,107        text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,108        videos: VideoInput = None,109        **kwargs: Unpack[InternVideo3ProcessorKwargs],110    ) -> BatchFeature:111        """112        Main method to prepare inputs for the model.113 114        Args:115            images: The image or batch of images to be prepared.116            text: The sequence or batch of sequences to be encoded.117            videos: The video or batch of videos to be prepared.118            return_tensors: If set, will return tensors of a particular framework.119 120        Returns:121            [`BatchFeature`]: A [`BatchFeature`] with the following fields:122            - **input_ids** -- Token ids to be fed to a model.123            - **attention_mask** -- Attention mask.124            - **pixel_values** -- Pixel values for images.125            - **pixel_values_videos** -- Pixel values for videos.126            - **image_grid_thw** -- Image 3D grid dimensions.127            - **video_grid_thw** -- Video 3D grid dimensions.128        """129        output_kwargs = self._merge_kwargs(130            InternVideo3ProcessorKwargs,131            tokenizer_init_kwargs=self.tokenizer.init_kwargs,132            **kwargs,133        )134 135        if images is not None:136            image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])137            image_grid_thw = image_inputs["image_grid_thw"]138        else:139            image_inputs = {}140            image_grid_thw = None141 142        if videos is not None:143            videos_inputs = self.video_processor(videos=videos, **output_kwargs["videos_kwargs"])144            video_grid_thw = videos_inputs["video_grid_thw"]145            # If user has not requested video metadata, pop it146            if "return_metadata" not in kwargs:147                video_metadata = videos_inputs.pop("video_metadata", None)148            else:149                video_metadata = videos_inputs.get("video_metadata", None)150            video_grid_thw = videos_inputs["video_grid_thw"]151        else:152            videos_inputs = {}153            video_grid_thw = None154            video_metadata = None155 156        if not isinstance(text, list):157            text = [text]158 159        text = text.copy()160        if image_grid_thw is not None:161            merge_length = self.image_processor.merge_size**2162            index = 0163            for i in range(len(text)):164                while self.image_token in text[i]:165                    num_image_tokens = image_grid_thw[index].prod() // merge_length166                    text[i] = text[i].replace(self.image_token, "<|placeholder|>" * num_image_tokens, 1)167                    index += 1168                text[i] = text[i].replace("<|placeholder|>", self.image_token)169 170        if video_grid_thw is not None:171            merge_length = self.video_processor.merge_size**2172            index = 0173            for i in range(len(text)):174                while self.video_token in text[i]:175                    metadata = video_metadata[index] if video_metadata else None176                    if metadata is not None:177                        if metadata.fps is None:178                            logger.warning_once(179                                "InternVideo3 requires frame timestamps to construct prompts, but the `fps` of the input video could not be inferred. "180                                "Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results."181                            )182                            metadata.fps = 24183 184                        curr_timestamp = self._calculate_timestamps(185                            metadata.frames_indices,186                            metadata.fps,187                            self.video_processor.merge_size,188                        )189 190                        video_placeholder = ""191                        frame_seqlen = video_grid_thw[index][1:].prod() // merge_length192                        for frame_idx in range(video_grid_thw[index][0]):193                            curr_time = curr_timestamp[frame_idx]194                            video_placeholder += f"<{curr_time:.1f} seconds>"195                            video_placeholder += (196                                self.vision_start_token + "<|placeholder|>" * frame_seqlen + self.vision_end_token197                            )198                        if f"{self.vision_start_token}{self.video_token}{self.vision_end_token}" in text[i]:199                            text[i] = text[i].replace(200                                f"{self.vision_start_token}{self.video_token}{self.vision_end_token}", video_placeholder, 1201                            )202                        else:203                            text[i] = text[i].replace(self.video_token, video_placeholder, 1)204                    else:205                        num_video_tokens = video_grid_thw[index].prod() // merge_length206                        text[i] = text[i].replace(self.video_token, "<|placeholder|>" * num_video_tokens, 1)207                    index += 1208                text[i] = text[i].replace("<|placeholder|>", self.video_token)209 210        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)211        text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])212 213        return BatchFeature(data={**text_inputs, **image_inputs, **videos_inputs}, tensor_type=return_tensors)214 215    def _calculate_timestamps(self, indices: Union[list[int], np.ndarray], video_fps: float, merge_size: int = 2):216        if not isinstance(indices, list):217            indices = indices.tolist()218        if len(indices) % merge_size != 0:219            indices.extend(indices[-1] for _ in range(merge_size - len(indices) % merge_size))220        timestamps = [idx / video_fps for idx in indices]221        timestamps = [222            (timestamps[i] + timestamps[i + merge_size - 1]) / 2 for i in range(0, len(timestamps), merge_size)223        ]224        return timestamps225 226    def post_process_image_text_to_text(227        self, generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False, **kwargs228    ):229        return self.tokenizer.batch_decode(230            generated_outputs,231            skip_special_tokens=skip_special_tokens,232            clean_up_tokenization_spaces=clean_up_tokenization_spaces,233            **kwargs,234        )235 236 237__all__ = ["InternVideo3Processor"]238