OpenGVLab/InternVL3-2B-Pretrained
150
1{2 "_commit_hash": null,3 "_name_or_path": "/mnt/petrelfs/share_data/wangweiyun/share_internvl_preview/InternVL3-2B-Pretrain",4 "architectures": [5 "InternVLChatModel"6 ],7 "auto_map": {8 "AutoConfig": "configuration_internvl_chat.InternVLChatConfig",9 "AutoModel": "modeling_internvl_chat.InternVLChatModel",10 "AutoModelForCausalLM": "modeling_internvl_chat.InternVLChatModel"11 },12 "downsample_ratio": 0.5,13 "dynamic_image_size": true,14 "force_image_size": 448,15 "image_fold": null,16 "llm_config": {17 "_name_or_path": "./pretrained/Qwen2.5-32B-Instruct",18 "add_cross_attention": false,19 "architectures": [20 "Qwen2ForCausalLM"21 ],22 "attention_dropout": 0.0,23 "bad_words_ids": null,24 "begin_suppress_tokens": null,25 "bos_token_id": 151643,26 "chunk_size_feed_forward": 0,27 "cross_attention_hidden_size": null,28 "decoder_start_token_id": null,29 "diversity_penalty": 0.0,30 "do_sample": false,31 "early_stopping": false,32 "encoder_no_repeat_ngram_size": 0,33 "eos_token_id": 151643,34 "exponential_decay_length_penalty": null,35 "finetuning_task": null,36 "forced_bos_token_id": null,37 "forced_eos_token_id": null,38 "hidden_act": "silu",39 "hidden_size": 1536,40 "id2label": {41 "0": "LABEL_0",42 "1": "LABEL_1"43 },44 "initializer_range": 0.02,45 "intermediate_size": 8960,46 "is_decoder": false,47 "is_encoder_decoder": false,48 "label2id": {49 "LABEL_0": 0,50 "LABEL_1": 151 },52 "length_penalty": 1.0,53 "max_length": 20,54 "max_position_embeddings": 32768,55 "max_window_layers": 70,56 "min_length": 0,57 "model_type": "qwen2",58 "moe_config": null,59 "no_repeat_ngram_size": 0,60 "num_attention_heads": 12,61 "num_beam_groups": 1,62 "num_beams": 1,63 "num_hidden_layers": 28,64 "num_key_value_heads": 2,65 "num_return_sequences": 1,66 "output_attentions": false,67 "output_hidden_states": false,68 "output_scores": false,69 "pad_token_id": null,70 "prefix": null,71 "problem_type": null,72 "pruned_heads": {},73 "remove_invalid_values": false,74 "repetition_penalty": 1.0,75 "return_dict": true,76 "return_dict_in_generate": false,77 "rms_norm_eps": 1e-06,78 "rope_scaling": {79 "factor": 2.0,80 "rope_type": "dynamic",81 "type": "dynamic"82 },83 "rope_theta": 1000000.0,84 "sep_token_id": null,85 "sliding_window": null,86 "suppress_tokens": null,87 "task_specific_params": null,88 "temperature": 1.0,89 "tf_legacy_loss": false,90 "tie_encoder_decoder": false,91 "tie_word_embeddings": false,92 "tokenizer_class": null,93 "top_k": 50,94 "top_p": 1.0,95 "torch_dtype": "bfloat16",96 "torchscript": false,97 "transformers_version": "4.45.1",98 "typical_p": 1.0,99 "use_bfloat16": true,100 "use_cache": true,101 "use_sliding_window": false,102 "vocab_size": 151674103 },104 "max_dynamic_patch": 12,105 "min_dynamic_patch": 1,106 "model_type": "internvl_chat",107 "pad2square": false,108 "ps_version": "v2",109 "select_layer": -1,110 "template": "internvl2_5",111 "tie_word_embeddings": false,112 "torch_dtype": "bfloat16",113 "transformers_version": null,114 "use_backbone_lora": 0,115 "use_llm_lora": 0,116 "use_thumbnail": true,117 "vision_config": {118 "_name_or_path": "OpenGVLab/InternViT-6B-448px-V1-5",119 "add_cross_attention": false,120 "architectures": [121 "InternVisionModel"122 ],123 "attention_dropout": 0.0,124 "auto_map": {125 "AutoConfig": "configuration_intern_vit.InternVisionConfig",126 "AutoModel": "modeling_intern_vit.InternVisionModel"127 },128 "bad_words_ids": null,129 "begin_suppress_tokens": null,130 "bos_token_id": null,131 "capacity_factor": 1.2,132 "chunk_size_feed_forward": 0,133 "cross_attention_hidden_size": null,134 "decoder_start_token_id": null,135 "diversity_penalty": 0.0,136 "do_sample": false,137 "drop_path_rate": 0.05,138 "dropout": 0.0,139 "early_stopping": false,140 "encoder_no_repeat_ngram_size": 0,141 "eos_token_id": null,142 "eval_capacity_factor": 1.4,143 "exponential_decay_length_penalty": null,144 "finetuning_task": null,145 "forced_bos_token_id": null,146 "forced_eos_token_id": null,147 "hidden_act": "gelu",148 "hidden_size": 1024,149 "id2label": {150 "0": "LABEL_0",151 "1": "LABEL_1"152 },153 "image_size": 448,154 "initializer_factor": 0.1,155 "initializer_range": 1e-10,156 "intermediate_size": 4096,157 "is_decoder": false,158 "is_encoder_decoder": false,159 "label2id": {160 "LABEL_0": 0,161 "LABEL_1": 1162 },163 "laux_allreduce": "all_nodes",164 "layer_norm_eps": 1e-06,165 "length_penalty": 1.0,166 "max_length": 20,167 "min_length": 0,168 "model_type": "intern_vit_6b",169 "moe_coeff_ratio": 0.5,170 "moe_intermediate_size": 768,171 "moe_output_scale": 4.0,172 "no_repeat_ngram_size": 0,173 "noisy_gate_policy": "RSample_before",174 "norm_type": "layer_norm",175 "num_attention_heads": 16,176 "num_beam_groups": 1,177 "num_beams": 1,178 "num_channels": 3,179 "num_experts": 8,180 "num_hidden_layers": 24,181 "num_return_sequences": 1,182 "num_routed_experts": 4,183 "num_shared_experts": 4,184 "output_attentions": false,185 "output_hidden_states": false,186 "output_scores": false,187 "pad_token_id": null,188 "patch_size": 14,189 "prefix": null,190 "problem_type": null,191 "pruned_heads": {},192 "qk_normalization": false,193 "qkv_bias": true,194 "remove_invalid_values": false,195 "repetition_penalty": 1.0,196 "return_dict": true,197 "return_dict_in_generate": false,198 "sep_token_id": null,199 "shared_expert_intermediate_size": 3072,200 "suppress_tokens": null,201 "task_specific_params": null,202 "temperature": 1.0,203 "tf_legacy_loss": false,204 "tie_encoder_decoder": false,205 "tie_word_embeddings": true,206 "tokenizer_class": null,207 "top_k": 50,208 "top_p": 1.0,209 "torch_dtype": "bfloat16",210 "torchscript": false,211 "transformers_version": "4.45.1",212 "typical_p": 1.0,213 "use_bfloat16": true,214 "use_flash_attn": true,215 "use_moe": false,216 "use_residual": true,217 "use_rts": false,218 "use_weighted_residual": false219 }220}221