OpenGVLab/InternVL3-1B

1# --------------------------------------------------------2# InternVL3# Copyright (c) 2024 OpenGVLab4# Licensed under The MIT License [see LICENSE for details]5# --------------------------------------------------------6 7import copy8 9from transformers import AutoConfig, LlamaConfig, Qwen2Config10from transformers.configuration_utils import PretrainedConfig11from transformers.utils import logging12 13from .configuration_intern_vit import InternVisionConfig14 15logger = logging.get_logger(__name__)16 17 18class InternVLChatConfig(PretrainedConfig):19    model_type = 'internvl_chat'20    is_composition = True21 22    def __init__(23            self,24            vision_config=None,25            llm_config=None,26            use_backbone_lora=0,27            use_llm_lora=0,28            select_layer=-1,29            force_image_size=None,30            downsample_ratio=0.5,31            template=None,32            dynamic_image_size=False,33            use_thumbnail=False,34            ps_version='v1',35            min_dynamic_patch=1,36            max_dynamic_patch=6,37            **kwargs):38        super().__init__(**kwargs)39 40        if vision_config is None:41            vision_config = {'architectures': ['InternVisionModel']}42            logger.info('vision_config is None. Initializing the InternVisionConfig with default values.')43 44        if llm_config is None:45            llm_config = {'architectures': ['Qwen2ForCausalLM']}46            logger.info('llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).')47 48        self.vision_config = InternVisionConfig(**vision_config)49        if llm_config.get('architectures')[0] == 'LlamaForCausalLM':50            self.llm_config = LlamaConfig(**llm_config)51        elif llm_config.get('architectures')[0] == 'Qwen2ForCausalLM':52            self.llm_config = Qwen2Config(**llm_config)53        else:54            raise ValueError('Unsupported architecture: {}'.format(llm_config.get('architectures')[0]))55        self.use_backbone_lora = use_backbone_lora56        self.use_llm_lora = use_llm_lora57        self.select_layer = select_layer58        self.force_image_size = force_image_size59        self.downsample_ratio = downsample_ratio60        self.template = template61        self.dynamic_image_size = dynamic_image_size62        self.use_thumbnail = use_thumbnail63        self.ps_version = ps_version  # pixel shuffle version64        self.min_dynamic_patch = min_dynamic_patch65        self.max_dynamic_patch = max_dynamic_patch66        # By default, we use tie_word_embeddings=False for models of all sizes.67        self.tie_word_embeddings = self.llm_config.tie_word_embeddings68 69        logger.info(f'vision_select_layer: {self.select_layer}')70        logger.info(f'ps_version: {self.ps_version}')71        logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')72        logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')73 74    def to_dict(self):75        """76        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].77 78        Returns:79            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,80        """81        output = copy.deepcopy(self.__dict__)82        output['vision_config'] = self.vision_config.to_dict()83        output['llm_config'] = self.llm_config.to_dict()84        output['model_type'] = self.__class__.model_type85        output['use_backbone_lora'] = self.use_backbone_lora86        output['use_llm_lora'] = self.use_llm_lora87        output['select_layer'] = self.select_layer88        output['force_image_size'] = self.force_image_size89        output['downsample_ratio'] = self.downsample_ratio90        output['template'] = self.template91        output['dynamic_image_size'] = self.dynamic_image_size92        output['use_thumbnail'] = self.use_thumbnail93        output['ps_version'] = self.ps_version94        output['min_dynamic_patch'] = self.min_dynamic_patch95        output['max_dynamic_patch'] = self.max_dynamic_patch96 97        return output98