OpenGVLab/InternVL-Chat-V1-1

1# --------------------------------------------------------2# InternVL3# Copyright (c) 2024 OpenGVLab4# Licensed under The MIT License [see LICENSE for details]5# --------------------------------------------------------6 7import copy8 9from transformers import AutoConfig, LlamaConfig10from transformers.configuration_utils import PretrainedConfig11from transformers.utils import logging12 13from .configuration_intern_vit import InternVisionConfig14 15logger = logging.get_logger(__name__)16 17 18class InternVLChatConfig(PretrainedConfig):19    model_type = 'internvl_chat'20    is_composition = True21 22    def __init__(23            self,24            vision_config=None,25            llm_config=None,26            use_backbone_lora=0,27            use_llm_lora=0,28            select_layer=-1,29            force_image_size=None,30            downsample_ratio=0.5,31            template=None,32            dynamic_image_size=False,33            use_thumbnail=False,34            ps_version='v1',35            min_dynamic_patch=1,36            max_dynamic_patch=6,37            **kwargs):38        super().__init__(**kwargs)39 40        if vision_config is None:41            vision_config = {'architectures': ['InternVisionModel']}42            logger.info('vision_config is None. Initializing the InternVisionConfig with default values.')43 44        if llm_config is None:45            llm_config = {'architectures': ['LlamaForCausalLM']}46            logger.info('llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).')47 48        self.vision_config = InternVisionConfig(**vision_config)49        if llm_config['architectures'][0] == 'LlamaForCausalLM':50            self.llm_config = LlamaConfig(**llm_config)51        else:52            raise ValueError('Unsupported architecture: {}'.format(llm_config['architectures'][0]))53        self.use_backbone_lora = use_backbone_lora54        self.use_llm_lora = use_llm_lora55        self.select_layer = select_layer56        self.force_image_size = force_image_size57        self.downsample_ratio = downsample_ratio58        self.template = template59        self.dynamic_image_size = dynamic_image_size60        self.use_thumbnail = use_thumbnail61        self.ps_version = ps_version  # pixel shuffle version62        self.min_dynamic_patch = min_dynamic_patch63        self.max_dynamic_patch = max_dynamic_patch64        # By default, we use tie_word_embeddings=False for models of all sizes.65        self.tie_word_embeddings = self.llm_config.tie_word_embeddings66 67        logger.info(f'vision_select_layer: {self.select_layer}')68        logger.info(f'ps_version: {self.ps_version}')69        logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')70        logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')71 72    def to_dict(self):73        """74        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].75 76        Returns:77            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,78        """79        output = copy.deepcopy(self.__dict__)80        output['vision_config'] = self.vision_config.to_dict()81        output['llm_config'] = self.llm_config.to_dict()82        output['model_type'] = self.__class__.model_type83        output['use_backbone_lora'] = self.use_backbone_lora84        output['use_llm_lora'] = self.use_llm_lora85        output['select_layer'] = self.select_layer86        output['force_image_size'] = self.force_image_size87        output['downsample_ratio'] = self.downsample_ratio88        output['template'] = self.template89        output['dynamic_image_size'] = self.dynamic_image_size90        output['use_thumbnail'] = self.use_thumbnail91        output['ps_version'] = self.ps_version92        output['min_dynamic_patch'] = self.min_dynamic_patch93        output['max_dynamic_patch'] = self.max_dynamic_patch94 95        return output96