OpenGVLab/InternVL2_5-26B

1# --------------------------------------------------------2# InternVL3# Copyright (c) 2024 OpenGVLab4# Licensed under The MIT License [see LICENSE for details]5# --------------------------------------------------------6 7import copy8 9from transformers import AutoConfig, LlamaConfig10from transformers.configuration_utils import PretrainedConfig11from transformers.utils import logging12 13from .configuration_intern_vit import InternVisionConfig14from .configuration_internlm2 import InternLM2Config15 16logger = logging.get_logger(__name__)17 18 19class InternVLChatConfig(PretrainedConfig):20    model_type = 'internvl_chat'21    is_composition = True22 23    def __init__(24            self,25            vision_config=None,26            llm_config=None,27            use_backbone_lora=0,28            use_llm_lora=0,29            select_layer=-1,30            force_image_size=None,31            downsample_ratio=0.5,32            template=None,33            dynamic_image_size=False,34            use_thumbnail=False,35            ps_version='v1',36            min_dynamic_patch=1,37            max_dynamic_patch=6,38            **kwargs):39        super().__init__(**kwargs)40 41        if vision_config is None:42            vision_config = {'architectures': ['InternVisionModel']}43            logger.info('vision_config is None. Initializing the InternVisionConfig with default values.')44 45        if llm_config is None:46            llm_config = {'architectures': ['InternLM2ForCausalLM']}47            logger.info('llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).')48 49        self.vision_config = InternVisionConfig(**vision_config)50        if llm_config.get('architectures')[0] == 'LlamaForCausalLM':51            self.llm_config = LlamaConfig(**llm_config)52        elif llm_config.get('architectures')[0] == 'InternLM2ForCausalLM':53            self.llm_config = InternLM2Config(**llm_config)54        else:55            raise ValueError('Unsupported architecture: {}'.format(llm_config.get('architectures')[0]))56        self.use_backbone_lora = use_backbone_lora57        self.use_llm_lora = use_llm_lora58        self.select_layer = select_layer59        self.force_image_size = force_image_size60        self.downsample_ratio = downsample_ratio61        self.template = template62        self.dynamic_image_size = dynamic_image_size63        self.use_thumbnail = use_thumbnail64        self.ps_version = ps_version  # pixel shuffle version65        self.min_dynamic_patch = min_dynamic_patch66        self.max_dynamic_patch = max_dynamic_patch67        # By default, we use tie_word_embeddings=False for models of all sizes.68        self.tie_word_embeddings = self.llm_config.tie_word_embeddings69 70        logger.info(f'vision_select_layer: {self.select_layer}')71        logger.info(f'ps_version: {self.ps_version}')72        logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')73        logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')74 75    def to_dict(self):76        """77        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].78 79        Returns:80            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,81        """82        output = copy.deepcopy(self.__dict__)83        output['vision_config'] = self.vision_config.to_dict()84        output['llm_config'] = self.llm_config.to_dict()85        output['model_type'] = self.__class__.model_type86        output['use_backbone_lora'] = self.use_backbone_lora87        output['use_llm_lora'] = self.use_llm_lora88        output['select_layer'] = self.select_layer89        output['force_image_size'] = self.force_image_size90        output['downsample_ratio'] = self.downsample_ratio91        output['template'] = self.template92        output['dynamic_image_size'] = self.dynamic_image_size93        output['use_thumbnail'] = self.use_thumbnail94        output['ps_version'] = self.ps_version95        output['min_dynamic_patch'] = self.min_dynamic_patch96        output['max_dynamic_patch'] = self.max_dynamic_patch97 98        return output99