ManishThota/CustomModel

1 2# ------------------------------- Phi-2 ---------------------------------------------3# Copyright (c) Microsoft Corporation.4# Licensed under the MIT license.5# https://huggingface.co/google/siglip-so400m-patch14-3846# 7# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.8# Licensed under the BSD 3-Clause License.9# ------------------------------- SigLIP --------------------------------------------10# Copyright 2024 Google AI and The HuggingFace Team. All rights reserved.11#12# Licensed under the Apache License, Version 2.0 (the "License");13# you may not use this file except in compliance with the License.14# You may obtain a copy of the License at15#16#     http://www.apache.org/licenses/LICENSE-2.017#18# Unless required by applicable law or agreed to in writing, software19# distributed under the License is distributed on an "AS IS" BASIS,20# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.21# See the License for the specific language governing permissions and22# limitations under the License.23# ------------------------------- Llava ---------------------------------------------24# Copyright 2023 Haotian Liu25#26# Licensed under the Apache License, Version 2.0 (the "License");27# you may not use this file except in compliance with the License.28# You may obtain a copy of the License at29#30# http://www.apache.org/licenses/LICENSE-2.031#32# Unless required by applicable law or agreed to in writing, software33# distributed under the License is distributed on an "AS IS" BASIS,34# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.35# See the License for the specific language governing permissions and36# limitations under the License.37# -----------------------------------------------------------------------------------38 39 40import os41import math42from typing import Optional, Union43 44from transformers import PretrainedConfig45from transformers.utils import logging46 47logger = logging.get_logger(__name__)48 49 50class PhiConfig(PretrainedConfig):51    """Phi configuration."""52 53    model_type = "phi-msft"54    attribute_map = {55        "max_position_embeddings": "n_positions",56        "hidden_size": "n_embd",57        "num_attention_heads": "n_head",58        "num_hidden_layers": "n_layer",59    }60 61    def __init__(62        self,63        vocab_size: int = 50304,64        n_positions: int = 2048,65        n_embd: int = 1024,66        n_layer: int = 20,67        n_inner: Optional[int] = None,68        n_head: int = 16,69        n_head_kv: Optional[int] = None,70        rotary_dim: Optional[int] = 32,71        activation_function: Optional[str] = "gelu_new",72        flash_attn: bool = False,73        flash_rotary: bool = False,74        fused_dense: bool = False,75        attn_pdrop: float = 0.0,76        embd_pdrop: float = 0.0,77        resid_pdrop: float = 0.0,78        layer_norm_epsilon: float = 1e-5,79        initializer_range: float = 0.02,80        tie_word_embeddings: bool = False,81        pad_vocab_size_multiple: int = 64,82        **kwargs83    ) -> None:84        self.vocab_size = int(math.ceil(vocab_size / pad_vocab_size_multiple) * pad_vocab_size_multiple)85        self.n_positions = n_positions86        self.n_embd = n_embd87        self.n_layer = n_layer88        self.n_inner = n_inner89        self.n_head = n_head90        self.n_head_kv = n_head_kv91        self.rotary_dim = min(rotary_dim, n_embd // n_head)92        self.activation_function = activation_function93        self.flash_attn = flash_attn94        self.flash_rotary = flash_rotary95        self.fused_dense = fused_dense96        self.attn_pdrop = attn_pdrop97        self.embd_pdrop = embd_pdrop98        self.resid_pdrop = resid_pdrop99        self.layer_norm_epsilon = layer_norm_epsilon100        self.initializer_range = initializer_range101 102        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)103 104 105 106class SiglipVisionConfig(PretrainedConfig):107 108    model_type = "siglip_vision_model"109 110    def __init__(111        self,112        hidden_size=768,113        intermediate_size=3072,114        num_hidden_layers=12,115        num_attention_heads=12,116        num_channels=3,117        image_size=224,118        patch_size=16,119        hidden_act="gelu_pytorch_tanh",120        layer_norm_eps=1e-6,121        attention_dropout=0.0,122        **kwargs,123    ):124        super().__init__(**kwargs)125 126        self.hidden_size = hidden_size127        self.intermediate_size = intermediate_size128        self.num_hidden_layers = num_hidden_layers129        self.num_attention_heads = num_attention_heads130        self.num_channels = num_channels131        self.patch_size = patch_size132        self.image_size = image_size133        self.attention_dropout = attention_dropout134        self.layer_norm_eps = layer_norm_eps135        self.hidden_act = hidden_act136 137    @classmethod138    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":139        cls._set_token_in_kwargs(kwargs)140 141        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)142 143        # get the vision config dict if we are loading from SiglipConfig144        if config_dict.get("model_type") == "siglip":145            config_dict = config_dict["vision_config"]146 147        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:148            logger.warning(149                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "150                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."151            )152 153        return cls.from_dict(config_dict, **kwargs)154 155 156class ImpConfig(PhiConfig):157    model_type = "imp"158 159    def __init__(self, **kwargs):160        super().__init__(**kwargs)161        self.image_token_index = getattr(self, "image_token_index", 50296)162        self.image_token = getattr(self, "image_token", "<image>")163 164        if not hasattr(self, "vision_tower_config") and hasattr(self, "mm_vision_tower"):165            vision_tower_config = SiglipVisionConfig.from_pretrained(self.mm_vision_tower)166            self.vision_tower_config = vision_tower_config.to_diff_dict()167    168    @property169    def vision_tower_cfg(self):170        cfg = SiglipVisionConfig.from_dict(self.vision_tower_config)171        # imp-v1 only supports `patch` feature for now w/o cls token172        # cfg.mm_vision_select_feature = self.mm_vision_select_feature173        cfg.mm_vision_select_layer = self.mm_vision_select_layer174        cfg.mm_vision_tower = self.mm_vision_tower175        return cfg176