ecoxial2007/CheX-Phi4MM-SFT

1# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15"""16Processor class for Phi4MM17"""18import re19from typing import List, Optional, Tuple, Union20import math21from enum import Enum22 23import numpy as np24import scipy25import torch26import torchvision27 28from transformers import AutoFeatureExtractor, AutoImageProcessor29from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor30from transformers.image_processing_utils import BaseImageProcessor, BatchFeature31from transformers.image_utils import (32    ImageInput,33    make_list_of_images,34    valid_images,35)36from transformers.processing_utils import ProcessorMixin37from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy38from transformers.utils import TensorType, logging39from torch.nn.utils.rnn import pad_sequence40 41 42logger = logging.get_logger(__name__)43 44# Special tokens45_COMPATIBLE_IMAGE_SPECIAL_TOKEN_PATTERN = r'<\|image_\d+\|>'  # For backward compatibility46_COMPATIBLE_AUDIO_SPECIAL_TOKEN_PATTERN = r'<\|audio_\d+\|>'  # For backward compatibility47_IMAGE_SPECIAL_TOKEN = '<|endoftext10|>'48_AUDIO_SPECIAL_TOKEN = '<|endoftext11|>'49_IMAGE_SPECIAL_TOKEN_ID = 200010  # '<|endoftext10|>', or we can better name it (in `tokenizer_config.json`)50_AUDIO_SPECIAL_TOKEN_ID = 200011  # '<|endoftext11|>'51 52 53class InputMode(Enum):54    LANGUAGE = 055    VISION = 156    SPEECH = 257    VISION_SPEECH = 358 59 60class Phi4MMImageProcessor(BaseImageProcessor):61    r"""62    Constructs a Phi4MM image processor.63    """64    model_input_names = ["input_image_embeds", "image_sizes", "image_attention_mask"]65 66    def __init__(67        self,68        dynamic_hd,69        **kwargs,70    ) -> None:71        super().__init__(**kwargs)72        self.dynamic_hd = dynamic_hd73 74    def find_closest_aspect_ratio(self, aspect_ratio, target_ratios, width, height, image_size):75        best_ratio_diff = float('inf')76        best_ratio = (1, 1)77        area = width * height78        for ratio in target_ratios:79            target_aspect_ratio = ratio[0] / ratio[1]80            ratio_diff = abs(aspect_ratio - target_aspect_ratio)81            if ratio_diff < best_ratio_diff:82                best_ratio_diff = ratio_diff83                best_ratio = ratio84            elif ratio_diff == best_ratio_diff:85                if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:86                    best_ratio = ratio87        return best_ratio88 89    def dynamic_preprocess(self, image, min_num=1, max_num=12, image_size=384, mask_size=27, use_thumbnail=True):90        orig_width, orig_height = image.size91 92        w_crop_num = math.ceil(orig_width/float(image_size))93        h_crop_num = math.ceil(orig_height/float(image_size))94        if w_crop_num * h_crop_num > max_num:95 96            aspect_ratio = orig_width / orig_height97 98            # calculate the existing image aspect ratio99            target_ratios = set(100                (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if101                i * j <= max_num and i * j >= min_num)102            target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])103 104            # find the closest aspect ratio to the target105            target_aspect_ratio = self.find_closest_aspect_ratio(106                aspect_ratio, target_ratios, orig_width, orig_height, image_size)107 108            # calculate the target width and height109            target_width = image_size * target_aspect_ratio[0]110            target_height = image_size * target_aspect_ratio[1]111        else:112            target_width = image_size * w_crop_num113            target_height = image_size * h_crop_num114            target_aspect_ratio = (w_crop_num, h_crop_num)115 116        # Calculate the ratio117        ratio_width = target_width / orig_width118        ratio_height = target_height / orig_height119        if ratio_width < ratio_height:120            new_size = (target_width, int(orig_height * ratio_width))121            padding_width = 0122            padding_height = target_height - int(orig_height * ratio_width)123        else:124            new_size = (int(orig_width * ratio_height), target_height)125            padding_width = target_width - int(orig_width * ratio_height)126            padding_height = 0127 128        attention_mask = torch.ones((int(mask_size*target_aspect_ratio[1]), int(mask_size*target_aspect_ratio[0])))129        if padding_width >= 14:130            attention_mask[:, -math.floor(padding_width/14):] = 0131        if padding_height >= 14:132            attention_mask[-math.floor(padding_height/14):,:] = 0133        assert attention_mask.sum() > 0134 135        if min(new_size[1], target_height) < 10 or min(new_size[0], target_width) < 10:136            raise ValueError(f'the aspect ratio is very extreme {new_size}')137 138        image = torchvision.transforms.functional.resize(image, [new_size[1], new_size[0]],)139 140        resized_img = torchvision.transforms.functional.pad(image, [0, 0, padding_width, padding_height], fill=[255,255,255])141 142        return resized_img, attention_mask143 144    def pad_to_max_num_crops(self, images, max_crops=5):145        """146        images: B x 3 x H x W, B<=max_crops147        """148        B, _, H, W = images.shape149        if B < max_crops:150            pad = torch.zeros(max_crops - B, 3, H, W, dtype=images.dtype, device=images.device)151            images = torch.cat([images, pad], dim=0)152        return images153 154    def pad_mask_to_max_num_crops(self, masks, max_crops=5):155        B, H, W = masks.shape156        if B < max_crops:157            pad = torch.ones(max_crops - B, H, W, dtype=masks.dtype, device=masks.device)158            masks = torch.cat([masks, pad], dim=0)159        return masks160 161    def preprocess(162        self,163        images: ImageInput,164        return_tensors: Optional[Union[str, TensorType]] = None,165    ):166        """167        Args:168            images (`ImageInput`):169                Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If170                passing in images with pixel values between 0 and 1, set `do_rescale=False`.171            return_tensors (`str` or `TensorType`, *optional*):172                The type of tensors to return. Can be one of:173                - Unset: Return a list of `np.ndarray`.174                - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.175                - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.176                - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.177                - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.178        """179        images = make_list_of_images(images)180 181        if not valid_images(images):182            raise ValueError(183                "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "184                "torch.Tensor, tf.Tensor or jax.ndarray."185            )186 187        # Basic settings.188        img_processor = torchvision.transforms.Compose([189            torchvision.transforms.ToTensor(),190            torchvision.transforms.Normalize(191                (0.5, 0.5, 0.5),192                (0.5, 0.5, 0.5)193            ),194        ])195        dyhd_base_resolution = 448196 197        # Dynamic HD198        base_resolution = dyhd_base_resolution199        images = [image.convert('RGB') for image in images]200        # cover 384 and 448 resolution201        mask_resolution = base_resolution // 14202        elems, image_attention_masks = [], []203        for im in images:204            elem, attention_mask = self.dynamic_preprocess(im, max_num=self.dynamic_hd, image_size=base_resolution, mask_size=mask_resolution)205            elems.append(elem)206            image_attention_masks.append(attention_mask)207        hd_images = [img_processor(im) for im in elems]208        global_image = [torch.nn.functional.interpolate(im.unsqueeze(0).float(), size=(base_resolution, base_resolution), mode='bicubic',).to(im.dtype) for im in hd_images]209        shapes = [[im.size(1), im.size(2)] for im in hd_images]210        mask_shapes = [[mask.size(0), mask.size(1)] for mask in image_attention_masks]211        global_attention_mask = [torch.ones((1, mask_resolution, mask_resolution)) for _ in hd_images]212        hd_images_reshape = [im.reshape(1, 3,213                                            h//base_resolution,214                                            base_resolution,215                                            w//base_resolution,216                                            base_resolution217                                            ).permute(0,2,4,1,3,5).reshape(-1, 3, base_resolution, base_resolution).contiguous() for im, (h, w) in zip(hd_images, shapes)]218        attention_masks_reshape = [mask.reshape(1,219                                            h//mask_resolution,220                                            mask_resolution,221                                            w//mask_resolution,222                                            mask_resolution223                                            ).permute(0,1,3,2,4).reshape(-1, mask_resolution, mask_resolution).contiguous() for mask, (h, w) in zip(image_attention_masks, mask_shapes)]224        downsample_attention_masks = [mask[:,0::2,0::2].reshape(1,225                                            h//mask_resolution,226                                            w//mask_resolution,227                                            mask_resolution//2+mask_resolution%2,228                                            mask_resolution//2+mask_resolution%2229                                            ).permute(0,1,3,2,4) for mask, (h,w) in zip(attention_masks_reshape, mask_shapes)]230        downsample_attention_masks = [mask.reshape(mask.size(1)*mask.size(2), mask.size(3)*mask.size(4))for mask in downsample_attention_masks]231        # 计算 num_img_tokens232        num_img_tokens = [256 + 1 + int(mask.sum().item()) + int(mask[:,0].sum().item()) + 16 for mask in downsample_attention_masks]233 234        hd_images_reshape = [torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape)]235        hd_masks_reshape = [torch.cat([_global_mask] + [_mask], dim=0) for _global_mask, _mask in zip(global_attention_mask, attention_masks_reshape)]236        max_crops = max([img.size(0) for img in hd_images_reshape])237        image_transformed = [self.pad_to_max_num_crops(im, max_crops) for im in hd_images_reshape]238        image_transformed = torch.stack(image_transformed, dim=0)239        mask_transformed = [self.pad_mask_to_max_num_crops(mask, max_crops) for mask in hd_masks_reshape]240        mask_transformed = torch.stack(mask_transformed, dim=0)241 242        returned_input_image_embeds = image_transformed243        returned_image_sizes = torch.tensor(shapes, dtype=torch.long)244        returned_image_attention_mask = mask_transformed245        returned_num_img_tokens = num_img_tokens246 247        data = {248            "input_image_embeds": returned_input_image_embeds,249            "image_sizes": returned_image_sizes,250            "image_attention_mask": returned_image_attention_mask,251            "num_img_tokens": returned_num_img_tokens,252        }253 254        return BatchFeature(data=data, tensor_type=return_tensors)255 256 257AudioInput = Tuple[Union[np.ndarray, torch.Tensor], int]258AudioInputs = List[AudioInput]259 260 261def speechlib_mel(sample_rate, n_fft, n_mels, fmin=None, fmax=None):262    """Create a Mel filter-bank the same as SpeechLib FbankFC.263 264    Args:265        sample_rate (int): Sample rate in Hz. number > 0 [scalar]266        n_fft (int): FFT size. int > 0 [scalar]267        n_mel (int): Mel filter size. int > 0 [scalar]268        fmin (float): lowest frequency (in Hz). If None use 0.0.269            float >= 0 [scalar]270        fmax: highest frequency (in Hz). If None use sample_rate / 2.271            float >= 0 [scalar]272 273    Returns274        out (numpy.ndarray): Mel transform matrix275            [shape=(n_mels, 1 + n_fft/2)]276    """277 278    bank_width = int(n_fft // 2 + 1)279    if fmax is None:280        fmax = sample_rate / 2281    if fmin is None:282        fmin = 0283    assert fmin >= 0, "fmin cannot be negtive"284    assert fmin < fmax <= sample_rate / 2, "fmax must be between (fmin, samplerate / 2]"285 286    def mel(f):287        return 1127.0 * np.log(1.0 + f / 700.0)288 289    def bin2mel(fft_bin):290        return 1127.0 * np.log(1.0 + fft_bin * sample_rate / (n_fft * 700.0))291 292    def f2bin(f):293        return int((f * n_fft / sample_rate) + 0.5)294 295    # Spec 1: FFT bin range [f2bin(fmin) + 1, f2bin(fmax) - 1]296    klo = f2bin(fmin) + 1297    khi = f2bin(fmax)298 299    khi = max(khi, klo)300 301    # Spec 2: SpeechLib uses trianges in Mel space302    mlo = mel(fmin)303    mhi = mel(fmax)304    m_centers = np.linspace(mlo, mhi, n_mels + 2)305    ms = (mhi - mlo) / (n_mels + 1)306 307    matrix = np.zeros((n_mels, bank_width), dtype=np.float32)308    for m in range(0, n_mels):309        left = m_centers[m]310        center = m_centers[m + 1]311        right = m_centers[m + 2]312        for fft_bin in range(klo, khi):313            mbin = bin2mel(fft_bin)314            if left < mbin < right:315                matrix[m, fft_bin] = 1.0 - abs(center - mbin) / ms316 317    return matrix318 319 320class Phi4MMAudioFeatureExtractor(SequenceFeatureExtractor):321    model_input_names = ["input_audio_embeds", "audio_embed_sizes", "audio_attention_mask"]322 323    def __init__(self, audio_compression_rate, audio_downsample_rate, audio_feat_stride, **kwargs):324        feature_size = 80325        sampling_rate = 16000326        padding_value = 0.0327        super().__init__(feature_size, sampling_rate, padding_value, **kwargs)328 329        self.compression_rate = audio_compression_rate330        self.qformer_compression_rate = audio_downsample_rate331        self.feat_stride = audio_feat_stride332 333        self._eightk_method = "fillzero"334        self._mel = speechlib_mel(16000, 512, 80, fmin=None, fmax=7690).T335 336        self._hamming400 = np.hamming(400)  # for 16k audio337        self._hamming200 = np.hamming(200)  # for 8k audio338 339    def duration_to_frames(self, duration):340        """duration in s, estimated frames"""341        frame_rate = 10342 343        num_frames = duration * 1000 // frame_rate344        return num_frames345 346    def __call__(347        self,348        audios: List[AudioInput],349        return_tensors: Optional[Union[str, TensorType]] = None,350    ):351        # Ref: https://github.com/huggingface/transformers/blob/v4.47.0/src/transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py#L161352        returned_input_audio_embeds = []353        returned_audio_embed_sizes = []354        audio_frames_list = []355 356        for audio_data, sample_rate in audios:357            audio_embeds = self._extract_features(audio_data, sample_rate)358            audio_frames = len(audio_embeds) * self.feat_stride359            audio_embed_size = self._compute_audio_embed_size(audio_frames)360 361            returned_input_audio_embeds.append(torch.tensor(audio_embeds))362            returned_audio_embed_sizes.append(torch.tensor(audio_embed_size).long())363            audio_frames_list.append(audio_frames)364 365        returned_input_audio_embeds = pad_sequence(366            returned_input_audio_embeds, batch_first=True367        )368        returned_audio_embed_sizes = torch.stack(returned_audio_embed_sizes, dim=0)369        audio_frames = torch.tensor(audio_frames_list)370        returned_audio_attention_mask = torch.arange(0, audio_frames.max()).unsqueeze(0) < audio_frames.unsqueeze(1) if len(audios) > 1 else None371 372        data = {373            "input_audio_embeds": returned_input_audio_embeds,374            "audio_embed_sizes": returned_audio_embed_sizes,375        }376        if returned_audio_attention_mask is not None:377            data["audio_attention_mask"] = returned_audio_attention_mask378 379        return BatchFeature(data=data, tensor_type=return_tensors)380 381    def _extract_spectrogram(self, wav, fs):382        """Extract spectrogram features from waveform.383        Args:384            wav (1D array): waveform of the input385            fs (int): sampling rate of the waveform, 16000 or 8000.386                If fs=8000, the waveform will be resampled to 16000Hz.387        Output:388            log_fbank (2D array): a TxD matrix of log Mel filterbank features.389                D=80, and T is the number of frames.390        """391        if wav.ndim > 1:392            wav = np.squeeze(wav)393 394        # by default, we extract the mean if stereo395        if len(wav.shape) == 2:396            wav = wav.mean(1)397 398        # Resample to 16000 or 8000 if needed399        if fs > 16000:400            wav = scipy.signal.resample_poly(wav, 1, fs // 16000)401            fs = 16000402        elif 8000 < fs < 16000:403            wav = scipy.signal.resample_poly(wav, 1, fs // 8000)404            fs = 8000405        elif fs < 8000:406            raise RuntimeError(f"Unsupported sample rate {fs}")407 408        if fs == 8000:409            if self._eightk_method == "resample":410                # Input audio is 8 kHz. Convert to 16 kHz before feature411                # extraction412                wav = scipy.signal.resample_poly(wav, 2, 1)413                fs = 16000414            # Do nothing here for fillzero method415        elif fs != 16000:416            # Input audio is not a supported sample rate.417            raise RuntimeError(f"Input data using an unsupported sample rate: {fs}")418 419        preemphasis = 0.97420 421        if fs == 8000:422            n_fft = 256423            win_length = 200424            hop_length = 80425            fft_window = self._hamming200426        elif fs == 16000:427            n_fft = 512428            win_length = 400429            hop_length = 160430            fft_window = self._hamming400431 432        # Spec 1: SpeechLib cut remaining sample insufficient for a hop433        n_batch = (wav.shape[0] - win_length) // hop_length + 1434        # Here we don't use stride_tricks since the input array may not satisfy435        # memory layout requirement and we need writeable output436        # Here we only use list of views before copy to desination437        # so it is more efficient than broadcasting438        y_frames = np.array(439            [wav[_stride : _stride + win_length] for _stride in range(0, hop_length * n_batch, hop_length)],440            dtype=np.float32,441        )442 443        # Spec 2: SpeechLib applies preemphasis within each batch444        y_frames_prev = np.roll(y_frames, 1, axis=1)445        y_frames_prev[:, 0] = y_frames_prev[:, 1]446        y_frames = (y_frames - preemphasis * y_frames_prev) * 32768447 448        S = np.fft.rfft(fft_window * y_frames, n=n_fft, axis=1).astype(np.complex64)449 450        if fs == 8000:451            # Need to pad the output to look like 16 kHz data but with zeros in452            # the 4 to 8 kHz bins.453            frames, bins = S.shape454            padarray = np.zeros((frames, bins))455            S = np.concatenate((S[:, 0:-1], padarray), axis=1)  # Nyquist bin gets set to zero456 457        spec = np.abs(S).astype(np.float32)458        return spec459 460    def _extract_features(self, wav, fs):461        """Extract log filterbank features from waveform.462        Args:463            wav (1D array): waveform of the input464            fs (int): sampling rate of the waveform, 16000 or 8000.465                If fs=8000, the waveform will be resampled to 16000Hz.466        Output:467            log_fbank (2D array): a TxD matrix of log Mel filterbank features.468                D=80, and T is the number of frames.469        """470        spec = self._extract_spectrogram(wav, fs)471        spec_power = spec**2472 473        fbank_power = np.clip(spec_power.dot(self._mel), 1.0, None)474        log_fbank = np.log(fbank_power).astype(np.float32)475 476        return log_fbank477 478    def _compute_audio_embed_size(self, audio_frames):479        integer = audio_frames // self.compression_rate480        remainder = audio_frames % self.compression_rate481 482        result = integer if remainder == 0 else integer + 1483 484        integer = result // self.qformer_compression_rate485        remainder = result % self.qformer_compression_rate486        result = integer if remainder == 0 else integer + 1  # qformer compression487 488        return result489 490 491class Phi4MMProcessor(ProcessorMixin):492    r"""493    Constructs a Phi4MM processor which raps an image processor, a audio processor, and a GPT tokenizer into a single processor.494 495    [`Phi4MMProcessor`] offers all the functionalities of [`Phi4MMImageProcessor`] and [`GPT2Tokenizer`]. See the496    [`~Phi4MMProcessor.__call__`] and [`~Phi4MMProcessor.decode`] for more information.497 498    Args:499        image_processor ([`Phi4MMImageProcessor`], *optional*):500            The image processor is a required input.501        tokenizer ([`GPT2Tokenizer`], *optional*):502            The tokenizer is a required input.503    """504 505    attributes = ["image_processor", "audio_processor", "tokenizer"]506    tokenizer_class = "GPT2TokenizerFast"507    image_processor_class = "AutoImageProcessor"  # Phi4MMImageProcessor will be registered later508    audio_processor_class = "AutoFeatureExtractor"  # Phi4MMAudioFeatureExtractor will be registered later509 510    def __init__(self, image_processor, audio_processor, tokenizer):511        self.image_processor = image_processor512        self.audio_processor = audio_processor513        self.tokenizer = tokenizer514 515    def __call__(516        self,517        text: Union[TextInput, List[TextInput]],518        images: Optional[ImageInput] = None,519        audios: Optional[AudioInputs] = None,520        padding: Union[bool, str, PaddingStrategy] = False,521        truncation: Optional[Union[bool, str, TruncationStrategy]] = None,522        max_length=None,523        return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,524    ) -> BatchFeature:525        """526        Main method to prepare for the model one or several sequences(s) and image(s). This method forards the `text`527        and `kwargs` arguments to GPT2Tokenizer's [`~GPT2Tokenizer.__call__`] if `text` is not `None` to encode528        the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to529        Phi4MMImageProcessor's [`~Phi4MMImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring530        of the above two methods for more information.531 532        Args:533            text (`str`, `List[str]`, `List[List[str]]`):534                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings535                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set536                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).537            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):538                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch539                tensor. Both channels-first and channels-last formats are supported.540            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):541                Select a strategy to pad the returned sequences (according to the model's padding side and padding542                index) among:543                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single544                  sequence if provided).545                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum546                  acceptable input length for the model if that argument is not provided.547                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different548                  lengths).549            max_length (`int`, *optional*):550                Maximum length of the returned list and optionally padding length (see above).551            truncation (`bool`, *optional*):552                Activates truncation to cut input sequences longer than `max_length` to `max_length`.553            return_tensors (`str` or [`~utils.TensorType`], *optional*):554                If set, will return tensors of a particular framework. Acceptable values are:555 556                - `'tf'`: Return TensorFlow `tf.constant` objects.557                - `'pt'`: Return PyTorch `torch.Tensor` objects.558                - `'np'`: Return NumPy `np.ndarray` objects.559                - `'jax'`: Return JAX `jnp.ndarray` objects.560 561        Returns:562            [`BatchFeature`]: A [`BatchFeature`] with the following fields:563 564            - **input_ids** -- List of token ids to be fed to a model.565            - **input_image_embeds** -- Pixel values to be fed to a model.566            - **image_sizes** -- List of tuples specifying the size of each image in `input_image_embeds`.567            - **image_attention_mask** -- List of attention masks for each image in `input_image_embeds`.568            - **input_audio_embeds** -- Audio embeddings to be fed to a model.569            - **audio_embed_sizes** -- List of integers specifying the size of each audio in `input_audio_embeds`.570            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model.571        """572        image_inputs = self.image_processor(images, return_tensors=return_tensors) if images is not None else {}573        audio_inputs = self.audio_processor(audios, return_tensors=return_tensors) if audios is not None else {}574        inputs = self._convert_images_audios_text_to_inputs(575            image_inputs,576            audio_inputs,577            text,578            padding=padding,579            truncation=truncation,580            max_length=max_length,581            return_tensors=return_tensors,582        )583 584        # idenfity the input mode585        if len(image_inputs) > 0 and len(audio_inputs) > 0:586            input_mode = InputMode.VISION_SPEECH587        elif len(image_inputs) > 0:588            input_mode = InputMode.VISION589        elif len(audio_inputs) > 0:590            input_mode = InputMode.SPEECH591        else:592            input_mode = InputMode.LANGUAGE593        inputs["input_mode"] = torch.tensor([input_mode.value], dtype=torch.long)594 595        return inputs596 597    @property598    def special_image_token_id(self):599        return self.tokenizer.convert_tokens_to_ids(self.special_image_token)600 601    def get_special_image_token_id(self):602        return self.tokenizer.convert_tokens_to_ids(self.special_image_token)603 604    @property605    def chat_template(self):606        return self.tokenizer.chat_template607 608    def _convert_images_audios_text_to_inputs(609        self, images, audios, text, padding=False, truncation=None, max_length=None, return_tensors=None610    ):611        # prepare image id to image input ids612        if len(images) > 0:613            input_image_embeds = images["input_image_embeds"]614            image_sizes = images["image_sizes"]615            image_attention_mask = images["image_attention_mask"]616            num_img_tokens = images['num_img_tokens']617        else:618            input_image_embeds = torch.tensor([])619            image_sizes = torch.tensor([])620            image_attention_mask = torch.tensor([])621            num_img_tokens = []622 623        # prepare audio id to audio input ids624        if len(audios) > 0:625            input_audio_embeds = audios["input_audio_embeds"]626            audio_embed_sizes = audios["audio_embed_sizes"]627            audio_attention_mask = audios.get("audio_attention_mask", None)628        else:629            input_audio_embeds = torch.tensor([])630            audio_embed_sizes = torch.tensor([])631            audio_attention_mask = None632 633        # Replace certain special tokens for compatibility634        # Ref: https://stackoverflow.com/questions/11475885/python-replace-regex635        if isinstance(text, str):636            text = [text]637        assert isinstance(text, list)638        processed_text = [re.sub(_COMPATIBLE_IMAGE_SPECIAL_TOKEN_PATTERN, _IMAGE_SPECIAL_TOKEN, t) for t in text]639        processed_text = [re.sub(_COMPATIBLE_AUDIO_SPECIAL_TOKEN_PATTERN, _AUDIO_SPECIAL_TOKEN, t) for t in processed_text]640 641        input_ids_list = [self.tokenizer(t).input_ids for t in processed_text]642 643        img_cnt, audio_cnt = 0, 0  # only needed for later assertion644        image_token_count_iter = iter(num_img_tokens)645        audio_embed_size_iter = iter(audio_embed_sizes.tolist())646        new_input_ids_list = []647        for input_ids in input_ids_list:648            i = 0649            while i < len(input_ids):650                token_id = input_ids[i]651                if token_id == _AUDIO_SPECIAL_TOKEN_ID:652                    token_count = next(audio_embed_size_iter)653                    audio_cnt += 1654                elif token_id == _IMAGE_SPECIAL_TOKEN_ID:655                    token_count = next(image_token_count_iter)656                    img_cnt += 1657                else:658                    i += 1659                    continue660                tokens = [token_id] * token_count661                input_ids = input_ids[:i] + tokens + input_ids[i + 1:]662                i += token_count663            input_ids = torch.tensor(input_ids, dtype=torch.long)664            new_input_ids_list.append(input_ids)665        lengths = torch.tensor([len(input_ids) for input_ids in new_input_ids_list])666        max_len = lengths.max()667        input_ids = input_ids.new_full((len(new_input_ids_list), max_len), self.tokenizer.pad_token_id)668        # batched inference requires left padding669        for i in range(len(new_input_ids_list)):670            input_ids[i, max_len - len(new_input_ids_list[i]):] = new_input_ids_list[i]671 672        # If the below assertion fails, it might be that input pure-text673        # messages contain image/audio special tokens literally674        # (<|endoftext10|>, <|endoftext11|>).675        assert (676            img_cnt == len(num_img_tokens)677        ), (678            f"Number of image tokens in prompt_token_ids ({img_cnt}) "679            f"does not match number of images ({len(num_img_tokens)})"680        )681        assert (682            audio_cnt == len(audio_embed_sizes)683        ), (684            f"Number of audio tokens in prompt_token_ids ({audio_cnt}) "685            f"does not match number of audios ({len(audio_embed_sizes)})"686        )687 688        # prepare attention mask689        seq_range = torch.arange(max_len - 1, -1, -1)690        attention_mask = seq_range.unsqueeze(0) < lengths.unsqueeze(1)691 692        # prepare batch feature693        data = {694            "input_ids": input_ids,695            "input_image_embeds": input_image_embeds,696            "image_sizes": image_sizes,697            "image_attention_mask": image_attention_mask,698            "input_audio_embeds": input_audio_embeds,699            "audio_embed_sizes": audio_embed_sizes,700            "audio_attention_mask": audio_attention_mask,701            "attention_mask": attention_mask,702        }703 704        return BatchFeature(705            data=data706        )707 708    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama709    def batch_decode(self, *args, **kwargs):710        """711        This method forwards all its arguments to GPT2Tokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please712        refer to the docstring of this method for more information.713        """714        return self.tokenizer.batch_decode(*args, **kwargs)715 716    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama717    def decode(self, *args, **kwargs):718        """719        This method forwards all its arguments to GPT2Tokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to720        the docstring of this method for more information.721        """722        return self.tokenizer.decode(*args, **kwargs)723 724    @property725    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names726    def model_input_names(self):727        tokenizer_input_names = self.tokenizer.model_input_names728        image_processor_input_names = self.image_processor.model_input_names729        audio_processor_input_names = self.audio_processor.model_input_names730        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names + audio_processor_input_names))731 732 733AutoImageProcessor.register("Phi4MMImageProcessor", Phi4MMImageProcessor)734AutoFeatureExtractor.register("Phi4MMAudioFeatureExtractor", Phi4MMAudioFeatureExtractor)735