XaviXva/Video-LLaVA
0
1import copy2import os3from typing import Union4 5from transformers import PretrainedConfig6from transformers.utils import logging7 8logger = logging.get_logger(__name__)9 10 11 12 13 14 15 16class CLIPTextConfig(PretrainedConfig):17 r"""18 This is the configuration class to store the configuration of a [`CLIPTextModel`]. It is used to instantiate a CLIP19 text encoder according to the specified arguments, defining the model architecture. Instantiating a configuration20 with the defaults will yield a similar configuration to that of the text encoder of the CLIP21 [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.22 23 Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the24 documentation from [`PretrainedConfig`] for more information.25 26 Args:27 vocab_size (`int`, *optional*, defaults to 49408):28 Vocabulary size of the CLIP text model. Defines the number of different tokens that can be represented by29 the `inputs_ids` passed when calling [`CLIPModel`].30 hidden_size (`int`, *optional*, defaults to 512):31 Dimensionality of the encoder layers and the pooler layer.32 intermediate_size (`int`, *optional*, defaults to 2048):33 Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.34 num_hidden_layers (`int`, *optional*, defaults to 12):35 Number of hidden layers in the Transformer encoder.36 num_attention_heads (`int`, *optional*, defaults to 8):37 Number of attention heads for each attention layer in the Transformer encoder.38 max_position_embeddings (`int`, *optional*, defaults to 77):39 The maximum sequence length that this model might ever be used with. Typically set this to something large40 just in case (e.g., 512 or 1024 or 2048).41 hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):42 The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,43 `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.44 layer_norm_eps (`float`, *optional*, defaults to 1e-5):45 The epsilon used by the layer normalization layers.46 attention_dropout (`float`, *optional*, defaults to 0.0):47 The dropout ratio for the attention probabilities.48 initializer_range (`float`, *optional*, defaults to 0.02):49 The standard deviation of the truncated_normal_initializer for initializing all weight matrices.50 initializer_factor (`float`, *optional*, defaults to 1):51 A factor for initializing all weight matrices (should be kept to 1, used internally for initialization52 testing).53 54 Example:55 56 ```python57 >>> from transformers import CLIPTextConfig, CLIPTextModel58 59 >>> # Initializing a CLIPTextConfig with openai/clip-vit-base-patch32 style configuration60 >>> configuration = CLIPTextConfig()61 62 >>> # Initializing a CLIPTextModel (with random weights) from the openai/clip-vit-base-patch32 style configuration63 >>> model = CLIPTextModel(configuration)64 65 >>> # Accessing the model configuration66 >>> configuration = model.config67 ```"""68 model_type = "clip_text_model"69 70 def __init__(71 self,72 vocab_size=49408,73 hidden_size=512,74 intermediate_size=2048,75 projection_dim=512,76 num_hidden_layers=12,77 num_attention_heads=8,78 max_position_embeddings=77,79 hidden_act="quick_gelu",80 layer_norm_eps=1e-5,81 attention_dropout=0.0,82 initializer_range=0.02,83 initializer_factor=1.0,84 # This differs from `CLIPTokenizer`'s default and from openai/clip85 # See https://github.com/huggingface/transformers/pull/24773#issuecomment-163228753886 pad_token_id=1,87 bos_token_id=49406,88 eos_token_id=49407,89 **kwargs,90 ):91 super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)92 93 self.vocab_size = vocab_size94 self.hidden_size = hidden_size95 self.intermediate_size = intermediate_size96 self.projection_dim = projection_dim97 self.num_hidden_layers = num_hidden_layers98 self.num_attention_heads = num_attention_heads99 self.max_position_embeddings = max_position_embeddings100 self.layer_norm_eps = layer_norm_eps101 self.hidden_act = hidden_act102 self.initializer_range = initializer_range103 self.initializer_factor = initializer_factor104 self.attention_dropout = attention_dropout105 self.add_time_attn = False ######################################106 107 @classmethod108 def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":109 cls._set_token_in_kwargs(kwargs)110 111 config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)112 113 # get the text config dict if we are loading from CLIPConfig114 if config_dict.get("model_type") == "clip":115 config_dict = config_dict["text_config"]116 117 if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:118 logger.warning(119 f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "120 f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."121 )122 123 return cls.from_dict(config_dict, **kwargs)124 125 126 127 128class CLIPVisionConfig(PretrainedConfig):129 r"""130 This is the configuration class to store the configuration of a [`CLIPVisionModel`]. It is used to instantiate a131 CLIP vision encoder according to the specified arguments, defining the model architecture. Instantiating a132 configuration with the defaults will yield a similar configuration to that of the vision encoder of the CLIP133 [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.134 135 Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the136 documentation from [`PretrainedConfig`] for more information.137 138 Args:139 hidden_size (`int`, *optional*, defaults to 768):140 Dimensionality of the encoder layers and the pooler layer.141 intermediate_size (`int`, *optional*, defaults to 3072):142 Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.143 num_hidden_layers (`int`, *optional*, defaults to 12):144 Number of hidden layers in the Transformer encoder.145 num_attention_heads (`int`, *optional*, defaults to 12):146 Number of attention heads for each attention layer in the Transformer encoder.147 image_size (`int`, *optional*, defaults to 224):148 The size (resolution) of each image.149 patch_size (`int`, *optional*, defaults to 32):150 The size (resolution) of each patch.151 hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):152 The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,153 `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.154 layer_norm_eps (`float`, *optional*, defaults to 1e-5):155 The epsilon used by the layer normalization layers.156 attention_dropout (`float`, *optional*, defaults to 0.0):157 The dropout ratio for the attention probabilities.158 initializer_range (`float`, *optional*, defaults to 0.02):159 The standard deviation of the truncated_normal_initializer for initializing all weight matrices.160 initializer_factor (`float`, *optional*, defaults to 1):161 A factor for initializing all weight matrices (should be kept to 1, used internally for initialization162 testing).163 164 Example:165 166 ```python167 >>> from transformers import CLIPVisionConfig, CLIPVisionModel168 169 >>> # Initializing a CLIPVisionConfig with openai/clip-vit-base-patch32 style configuration170 >>> configuration = CLIPVisionConfig()171 172 >>> # Initializing a CLIPVisionModel (with random weights) from the openai/clip-vit-base-patch32 style configuration173 >>> model = CLIPVisionModel(configuration)174 175 >>> # Accessing the model configuration176 >>> configuration = model.config177 ```"""178 179 model_type = "clip_vision_model"180 181 def __init__(182 self,183 hidden_size=768,184 intermediate_size=3072,185 projection_dim=512,186 num_hidden_layers=12,187 num_attention_heads=12,188 num_channels=3,189 image_size=224,190 patch_size=32,191 hidden_act="quick_gelu",192 layer_norm_eps=1e-5,193 attention_dropout=0.0,194 initializer_range=0.02,195 initializer_factor=1.0,196 197 add_time_attn=False, ################################198 num_frames=1, ################################199 force_patch_dropout=0.0, ################################200 lora_r=2, ################################201 lora_alpha=16, ################################202 lora_dropout=0.0, ################################203 num_mel_bins=0.0, ################################204 target_length=0.0, ################################205 max_depth=10,206 video_decode_backend='decord', #########################207 **kwargs,208 ):209 super().__init__(**kwargs)210 211 self.hidden_size = hidden_size212 self.intermediate_size = intermediate_size213 self.projection_dim = projection_dim214 self.num_hidden_layers = num_hidden_layers215 self.num_attention_heads = num_attention_heads216 self.num_channels = num_channels217 self.patch_size = patch_size218 self.image_size = image_size219 self.initializer_range = initializer_range220 self.initializer_factor = initializer_factor221 self.attention_dropout = attention_dropout222 self.layer_norm_eps = layer_norm_eps223 self.hidden_act = hidden_act224 225 self.add_time_attn = add_time_attn ################226 self.num_frames = num_frames ################227 self.force_patch_dropout = force_patch_dropout ################228 self.lora_r = lora_r ################229 self.lora_alpha = lora_alpha ################230 self.lora_dropout = lora_dropout ################231 self.num_mel_bins = num_mel_bins ################232 self.target_length = target_length ################233 self.max_depth = max_depth ################234 self.video_decode_backend = video_decode_backend ################235 236 @classmethod237 def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":238 cls._set_token_in_kwargs(kwargs)239 240 config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)241 242 # get the vision config dict if we are loading from CLIPConfig243 if config_dict.get("model_type") == "clip":244 config_dict = config_dict["vision_config"]245 246 if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:247 logger.warning(248 f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "249 f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."250 )251 252 return cls.from_dict(config_dict, **kwargs)253 254 255class LanguageBindDepthConfig(PretrainedConfig):256 r"""257 [`CLIPConfig`] is the configuration class to store the configuration of a [`CLIPModel`]. It is used to instantiate258 a CLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating259 a configuration with the defaults will yield a similar configuration to that of the CLIP260 [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.261 262 Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the263 documentation from [`PretrainedConfig`] for more information.264 265 Args:266 text_config (`dict`, *optional*):267 Dictionary of configuration options used to initialize [`CLIPTextConfig`].268 vision_config (`dict`, *optional*):269 Dictionary of configuration options used to initialize [`CLIPVisionConfig`].270 projection_dim (`int`, *optional*, defaults to 512):271 Dimentionality of text and vision projection layers.272 logit_scale_init_value (`float`, *optional*, defaults to 2.6592):273 The inital value of the *logit_scale* paramter. Default is used as per the original CLIP implementation.274 kwargs (*optional*):275 Dictionary of keyword arguments.276 277 Example:278 279 ```python280 >>> from transformers import CLIPConfig, CLIPModel281 282 >>> # Initializing a CLIPConfig with openai/clip-vit-base-patch32 style configuration283 >>> configuration = CLIPConfig()284 285 >>> # Initializing a CLIPModel (with random weights) from the openai/clip-vit-base-patch32 style configuration286 >>> model = CLIPModel(configuration)287 288 >>> # Accessing the model configuration289 >>> configuration = model.config290 291 >>> # We can also initialize a CLIPConfig from a CLIPTextConfig and a CLIPVisionConfig292 >>> from transformers import CLIPTextConfig, CLIPVisionConfig293 294 >>> # Initializing a CLIPText and CLIPVision configuration295 >>> config_text = CLIPTextConfig()296 >>> config_vision = CLIPVisionConfig()297 298 >>> config = CLIPConfig.from_text_vision_configs(config_text, config_vision)299 ```"""300 301 model_type = "LanguageBindDepth"302 is_composition = True303 304 def __init__(305 self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs306 ):307 # If `_config_dict` exist, we use them for the backward compatibility.308 # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot309 # of confusion!).310 text_config_dict = kwargs.pop("text_config_dict", None)311 vision_config_dict = kwargs.pop("vision_config_dict", None)312 313 super().__init__(**kwargs)314 315 # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in316 # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most317 # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.318 if text_config_dict is not None:319 if text_config is None:320 text_config = {}321 322 # This is the complete result when using `text_config_dict`.323 _text_config_dict = CLIPTextConfig(**text_config_dict).to_dict()324 325 # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.326 for key, value in _text_config_dict.items():327 if key in text_config and value != text_config[key] and key not in ["transformers_version"]:328 # If specified in `text_config_dict`329 if key in text_config_dict:330 message = (331 f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "332 f'The value `text_config_dict["{key}"]` will be used instead.'333 )334 # If inferred from default argument values (just to be super careful)335 else:336 message = (337 f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The "338 f'value `text_config["{key}"]` will be overriden.'339 )340 logger.warning(message)341 342 # Update all values in `text_config` with the ones in `_text_config_dict`.343 text_config.update(_text_config_dict)344 345 if vision_config_dict is not None:346 if vision_config is None:347 vision_config = {}348 349 # This is the complete result when using `vision_config_dict`.350 _vision_config_dict = CLIPVisionConfig(**vision_config_dict).to_dict()351 # convert keys to string instead of integer352 if "id2label" in _vision_config_dict:353 _vision_config_dict["id2label"] = {354 str(key): value for key, value in _vision_config_dict["id2label"].items()355 }356 357 # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.358 for key, value in _vision_config_dict.items():359 if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:360 # If specified in `vision_config_dict`361 if key in vision_config_dict:362 message = (363 f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "364 f'values. The value `vision_config_dict["{key}"]` will be used instead.'365 )366 # If inferred from default argument values (just to be super careful)367 else:368 message = (369 f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. "370 f'The value `vision_config["{key}"]` will be overriden.'371 )372 logger.warning(message)373 374 # Update all values in `vision_config` with the ones in `_vision_config_dict`.375 vision_config.update(_vision_config_dict)376 377 if text_config is None:378 text_config = {}379 logger.info("`text_config` is `None`. Initializing the `CLIPTextConfig` with default values.")380 381 if vision_config is None:382 vision_config = {}383 logger.info("`vision_config` is `None`. initializing the `CLIPVisionConfig` with default values.")384 385 self.text_config = CLIPTextConfig(**text_config)386 self.vision_config = CLIPVisionConfig(**vision_config)387 388 self.projection_dim = projection_dim389 self.logit_scale_init_value = logit_scale_init_value390 self.initializer_factor = 1.0391 392 @classmethod393 def from_text_vision_configs(cls, text_config: CLIPTextConfig, vision_config: CLIPVisionConfig, **kwargs):394 r"""395 Instantiate a [`CLIPConfig`] (or a derived class) from clip text model configuration and clip vision model396 configuration.397 398 Returns:399 [`CLIPConfig`]: An instance of a configuration object400 """401 402 return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)403 404 def to_dict(self):405 """406 Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].407 408 Returns:409 `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,410 """411 output = copy.deepcopy(self.__dict__)412 output["text_config"] = self.text_config.to_dict()413 output["vision_config"] = self.vision_config.to_dict()414 output["model_type"] = self.__class__.model_type415 return output416 417 418 419 420 421 422 423 424 425 426