trl-internal-testing/tiny-InternVLForConditionalGeneration

1{2  "architectures": [3    "InternVLForConditionalGeneration"4  ],5  "downsample_ratio": 0.5,6  "dtype": "bfloat16",7  "image_seq_length": 256,8  "image_token_id": 151667,9  "model_type": "internvl",10  "projector_hidden_act": "gelu",11  "text_config": {12    "architectures": [13      "Qwen2ForCausalLM"14    ],15    "attention_dropout": 0.0,16    "bos_token_id": 151643,17    "dtype": "bfloat16",18    "eos_token_id": 151645,19    "hidden_act": "silu",20    "hidden_size": 16,21    "initializer_range": 0.02,22    "intermediate_size": 18944,23    "layer_types": [24      "full_attention",25      "full_attention"26    ],27    "max_position_embeddings": 32768,28    "max_window_layers": 70,29    "model_type": "qwen2",30    "num_attention_heads": 4,31    "num_hidden_layers": 2,32    "num_key_value_heads": 2,33    "rms_norm_eps": 1e-06,34    "rope_scaling": {35      "factor": 2.0,36      "rope_type": "dynamic",37      "type": "dynamic"38    },39    "rope_theta": 1000000.0,40    "sliding_window": null,41    "use_cache": true,42    "use_sliding_window": false,43    "vocab_size": 15167444  },45  "transformers_version": "4.57.3",46  "vision_config": {47    "architectures": [48      "InternVisionModel"49    ],50    "attention_bias": true,51    "attention_dropout": 0.0,52    "dropout": 0.0,53    "dtype": "bfloat16",54    "embed_dim": 64,55    "hidden_act": "gelu",56    "hidden_dropout_prob": 0.0,57    "hidden_size": 16,58    "image_size": [59      448,60      44861    ],62    "initializer_factor": 0.1,63    "initializer_range": 1e-10,64    "intermediate_size": 4096,65    "layer_norm_eps": 1e-06,66    "layer_scale_init_value": 0.1,67    "model_type": "internvl_vision",68    "norm_type": "layer_norm",69    "num_attention_heads": 4,70    "num_channels": 3,71    "num_hidden_layers": 2,72    "num_key_value_heads": 2,73    "patch_size": [74      14,75      1476    ],77    "projection_dropout": 0.0,78    "use_absolute_position_embeddings": true,79    "use_mask_token": false,80    "use_mean_pooling": true,81    "use_qk_norm": false82  },83  "vision_feature_layer": -1,84  "vision_feature_select_strategy": "default"85}86