fudan-generative-ai/WAM-Diff2

1{2  "architectures": [3    "WAMDiff2ForConditionalGeneration"4  ],5  "image_token_id": 151655,6  "model_type": "wam_diff2",7  "text_config": {8    "attention_bias": false,9    "attention_dropout": 0.0,10    "bos_token_id": 151643,11    "dtype": "bfloat16",12    "eos_token_id": 151645,13    "head_dim": 128,14    "hidden_act": "silu",15    "hidden_size": 2048,16    "initializer_range": 0.02,17    "intermediate_size": 6144,18    "max_position_embeddings": 262144,19    "model_type": "wam_diff2_text",20    "num_attention_heads": 16,21    "num_hidden_layers": 28,22    "num_key_value_heads": 8,23    "rms_norm_eps": 1e-06,24    "rope_scaling": {25      "mrope_interleaved": true,26      "mrope_section": [27        24,28        20,29        2030      ],31      "rope_type": "default"32    },33    "rope_theta": 5000000,34    "tie_word_embeddings": true,35    "use_cache": true,36    "vocab_size": 15193637  },38  "tie_word_embeddings": true,39  "transformers_version": "4.57.0.dev0",40  "video_token_id": 151656,41  "vision_config": {42    "deepstack_visual_indexes": [43      5,44      11,45      1746    ],47    "depth": 24,48    "hidden_act": "gelu_pytorch_tanh",49    "hidden_size": 1024,50    "in_channels": 3,51    "initializer_range": 0.02,52    "intermediate_size": 4096,53    "model_type": "wam_diff2",54    "num_heads": 16,55    "num_position_embeddings": 2304,56    "out_hidden_size": 2048,57    "patch_size": 16,58    "spatial_merge_size": 2,59    "temporal_patch_size": 260  },61  "vision_end_token_id": 151653,62  "vision_start_token_id": 15165263}64