OpenGVLab/InternVideo2_5_Chat_8B

1# --------------------------------------------------------2# InternVL3# Copyright (c) 2024 OpenGVLab4# Licensed under The MIT License [see LICENSE for details]5# --------------------------------------------------------6 7import copy8 9from transformers import AutoConfig, LlamaConfig10from transformers.configuration_utils import PretrainedConfig11from transformers.utils import logging12 13from .configuration_intern_vit import InternVisionConfig14from .configuration_internlm2 import InternLM2Config15 16logger = logging.get_logger(__name__)17 18 19class InternVLChatConfig(PretrainedConfig):20    model_type = 'internvl_chat'21    is_composition = True22 23    def __init__(24            self,25            vision_config=None,26            llm_config=None,27            use_backbone_lora=0,28            use_llm_lora=0,29            select_layer=-1,30            force_image_size=None,31            downsample_ratio=0.5,32            template=None,33            dynamic_image_size=False,34            use_thumbnail=False,35            ps_version='v1',36            min_dynamic_patch=1,37            max_dynamic_patch=6,38            **kwargs):39        super().__init__(**kwargs)40 41        if vision_config is None:42            vision_config = {}43            logger.info('vision_config is None. Initializing the InternVisionConfig with default values.')44 45        if llm_config is None:46            llm_config = {}47            logger.info('llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).')48 49        self.vision_config = InternVisionConfig(**vision_config)50        if llm_config.get('architectures', None) is not None:51            if llm_config.get('architectures')[0] == 'LlamaForCausalLM':52                self.llm_config = LlamaConfig(**llm_config)53            elif llm_config.get('architectures')[0] == 'InternLM2ForCausalLM':54                self.llm_config = InternLM2Config(**llm_config)55            else:56                pass57            # self.llm_config = InternLM2Config(**llm_config)58            # raise ValueError('Unsupported architecture: {}'.format(llm_config.get('architectures')[0]))59        self.use_backbone_lora = use_backbone_lora60        self.use_llm_lora = use_llm_lora61        self.select_layer = select_layer62        self.force_image_size = force_image_size63        self.downsample_ratio = downsample_ratio64        self.template = template65        self.dynamic_image_size = dynamic_image_size66        self.use_thumbnail = use_thumbnail67        self.ps_version = ps_version  # pixel shuffle version68        self.min_dynamic_patch = min_dynamic_patch69        self.max_dynamic_patch = max_dynamic_patch70 71        logger.info(f'vision_select_layer: {self.select_layer}')72        logger.info(f'ps_version: {self.ps_version}')73        logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')74        logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')75 76    def to_dict(self):77        """78        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].79 80        Returns:81            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,82        """83        output = copy.deepcopy(self.__dict__)84        output['vision_config'] = self.vision_config.to_dict()85        output['llm_config'] = self.llm_config.to_dict()86        output['model_type'] = self.__class__.model_type87        output['use_backbone_lora'] = self.use_backbone_lora88        output['use_llm_lora'] = self.use_llm_lora89        output['select_layer'] = self.select_layer90        output['force_image_size'] = self.force_image_size91        output['downsample_ratio'] = self.downsample_ratio92        output['template'] = self.template93        output['dynamic_image_size'] = self.dynamic_image_size94        output['use_thumbnail'] = self.use_thumbnail95        output['ps_version'] = self.ps_version96        output['min_dynamic_patch'] = self.min_dynamic_patch97        output['max_dynamic_patch'] = self.max_dynamic_patch98 99        return output100