optimum-intel-internal-testing/tiny-random-internvl2

1# --------------------------------------------------------2# InternVL3# Copyright (c) 2024 OpenGVLab4# Licensed under The MIT License [see LICENSE for details]5# --------------------------------------------------------6 7import copy8 9from transformers import LlamaConfig, Qwen2Config10from transformers.configuration_utils import PretrainedConfig11from transformers.utils import logging12 13from .configuration_intern_vit import InternVisionConfig14 15 16logger = logging.get_logger(__name__)17 18 19class InternVLChatConfig(PretrainedConfig):20    model_type = "internvl_chat"21    is_composition = True22 23    def __init__(24        self,25        vision_config=None,26        llm_config=None,27        use_backbone_lora=0,28        use_llm_lora=0,29        select_layer=-1,30        force_image_size=None,31        downsample_ratio=0.5,32        template=None,33        dynamic_image_size=False,34        use_thumbnail=False,35        ps_version="v1",36        min_dynamic_patch=1,37        max_dynamic_patch=6,38        **kwargs,39    ):40        super().__init__(**kwargs)41 42        if vision_config is None:43            vision_config = {}44            logger.info("vision_config is None. Initializing the InternVisionConfig with default values.")45 46        if llm_config is None:47            llm_config = {"architectures": ["Qwen2ForCausalLM"]}48            logger.info("llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).")49 50        self.vision_config = InternVisionConfig(**vision_config)51        if llm_config["architectures"][0] == "LlamaForCausalLM":52            self.llm_config = LlamaConfig(**llm_config)53        elif llm_config["architectures"][0] == "Qwen2ForCausalLM":54            self.llm_config = Qwen2Config(**llm_config)55        else:56            raise ValueError("Unsupported architecture: {}".format(llm_config["architectures"][0]))57        self.use_backbone_lora = use_backbone_lora58        self.use_llm_lora = use_llm_lora59        self.select_layer = select_layer60        self.force_image_size = force_image_size61        self.downsample_ratio = downsample_ratio62        self.template = template63        self.dynamic_image_size = dynamic_image_size64        self.use_thumbnail = use_thumbnail65        self.ps_version = ps_version  # pixel shuffle version66        self.min_dynamic_patch = min_dynamic_patch67        self.max_dynamic_patch = max_dynamic_patch68 69        logger.info(f"vision_select_layer: {self.select_layer}")70        logger.info(f"ps_version: {self.ps_version}")71        logger.info(f"min_dynamic_patch: {self.min_dynamic_patch}")72        logger.info(f"max_dynamic_patch: {self.max_dynamic_patch}")73 74    def to_dict(self):75        """76        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].77 78        Returns:79            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,80        """81        output = copy.deepcopy(self.__dict__)82        output["vision_config"] = self.vision_config.to_dict()83        output["llm_config"] = self.llm_config.to_dict()84        output["model_type"] = self.__class__.model_type85        output["use_backbone_lora"] = self.use_backbone_lora86        output["use_llm_lora"] = self.use_llm_lora87        output["select_layer"] = self.select_layer88        output["force_image_size"] = self.force_image_size89        output["downsample_ratio"] = self.downsample_ratio90        output["template"] = self.template91        output["dynamic_image_size"] = self.dynamic_image_size92        output["use_thumbnail"] = self.use_thumbnail93        output["ps_version"] = self.ps_version94        output["min_dynamic_patch"] = self.min_dynamic_patch95        output["max_dynamic_patch"] = self.max_dynamic_patch96 97        return output98