MiniMaxAI/MiniMax-M3

1{2  "architectures": [3    "MiniMaxM3SparseForConditionalGeneration"4  ],5  "auto_map": {6    "AutoConfig": "configuration_minimax_m3_vl.MiniMaxM3VLConfig"7  },8  "model_type": "minimax_m3_vl",9  "text_config": {10    "hidden_size": 6144,11    "intermediate_size": 3072,12    "num_hidden_layers": 60,13    "num_attention_heads": 64,14    "num_key_value_heads": 4,15    "head_dim": 128,16    "vocab_size": 200064,17    "max_position_embeddings": 1048576,18    "rms_norm_eps": 1e-06,19    "use_gemma_norm": true,20    "attention_output_gate": false,21    "rope_theta": 5000000,22    "rotary_dim": 64,23    "partial_rotary_factor": 0.5,24    "hidden_act": "swigluoai",25    "use_qk_norm": true,26    "tie_word_embeddings": false,27    "dense_intermediate_size": 12288,28    "shared_intermediate_size": 3072,29    "num_local_experts": 128,30    "num_experts_per_tok": 4,31    "n_shared_experts": 1,32    "scoring_func": "sigmoid",33    "use_routing_bias": true,34    "moe_layer_freq": [35      0,36      0,37      0,38      1,39      1,40      1,41      1,42      1,43      1,44      1,45      1,46      1,47      1,48      1,49      1,50      1,51      1,52      1,53      1,54      1,55      1,56      1,57      1,58      1,59      1,60      1,61      1,62      1,63      1,64      1,65      1,66      1,67      1,68      1,69      1,70      1,71      1,72      1,73      1,74      1,75      1,76      1,77      1,78      1,79      1,80      1,81      1,82      1,83      1,84      1,85      1,86      1,87      1,88      1,89      1,90      1,91      1,92      1,93      1,94      195    ],96    "qk_norm_type": "per_head",97    "num_mtp_modules": 7,98    "num_nextn_predict_layers": 1,99    "swiglu_alpha": 1.702,100    "swiglu_limit": 7.0,101    "routed_scaling_factor": 2.0,102    "sparse_attention_config": {103      "use_sparse_attention": true,104      "sparse_index_dim": 128,105      "sparse_num_index_heads": 4,106      "sparse_topk_blocks": 16,107      "sparse_block_size": 128,108      "sparse_disable_index_value": [109        0,110        0,111        0,112        1,113        1,114        1,115        1,116        1,117        1,118        1,119        1,120        1,121        1,122        1,123        1,124        1,125        1,126        1,127        1,128        1,129        1,130        1,131        1,132        1,133        1,134        1,135        1,136        1,137        1,138        1,139        1,140        1,141        1,142        1,143        1,144        1,145        1,146        1,147        1,148        1,149        1,150        1,151        1,152        1,153        1,154        1,155        1,156        1,157        1,158        1,159        1,160        1,161        1,162        1,163        1,164        1,165        1,166        1,167        1,168        1169      ],170      "sparse_score_type": "max",171      "sparse_init_block": 0,172      "sparse_local_block": 1,173      "sparse_attention_freq": [174        0,175        0,176        0,177        1,178        1,179        1,180        1,181        1,182        1,183        1,184        1,185        1,186        1,187        1,188        1,189        1,190        1,191        1,192        1,193        1,194        1,195        1,196        1,197        1,198        1,199        1,200        1,201        1,202        1,203        1,204        1,205        1,206        1,207        1,208        1,209        1,210        1,211        1,212        1,213        1,214        1,215        1,216        1,217        1,218        1,219        1,220        1,221        1,222        1,223        1,224        1,225        1,226        1,227        1,228        1,229        1,230        1,231        1,232        1,233        1234      ]235    },236    "architectures": [237      "MiniMaxM3SparseForCausalLM"238    ]239  },240  "vision_config": {241    "hidden_size": 1280,242    "num_attention_heads": 16,243    "num_hidden_layers": 32,244    "intermediate_size": 5120,245    "patch_size": 14,246    "image_size": 2016,247    "projection_dim": 6144,248    "position_embedding_type": "rope",249    "rope_mode": "3d",250    "rope_theta": 10000.0,251    "attention_dropout": 0.0,252    "hidden_act": "gelu",253    "initializer_factor": 1.0,254    "initializer_range": 0.02,255    "layer_norm_eps": 1e-05,256    "model_type": "clip_vision_model",257    "num_channels": 3,258    "vocab_size": 32000,259    "img_token_compression_config": {260      "image_token_compression_method": "patch_merge",261      "spatial_merge_size": 2,262      "temporal_patch_size": 2263    },264    "vision_segment_max_frames": 4265  },266  "img_token_compression_config": {267    "image_token_compression_method": "patch_merge",268    "spatial_merge_size": 2,269    "temporal_patch_size": 2270  },271  "image_grid_pinpoints": "[(336, 336), (336, 672), (336, 1008), (336, 1344), (336, 1680), (336, 2016), (672, 336), (672, 672), (672, 1008), (672, 1344), (672, 1680), (672, 2016), (1008, 336), (1008, 672), (1008, 1008), (1008, 1344), (1008, 1680), (1008, 2016), (1344, 336), (1344, 672), (1344, 1008), (1344, 1344), (1344, 1680), (1344, 2016), (1680, 336), (1680, 672), (1680, 1008), (1680, 1344), (1680, 1680), (1680, 2016), (2016, 336), (2016, 672), (2016, 1008), (2016, 1344), (2016, 1680), (2016, 2016)]",272  "image_seq_length": 576,273  "image_token_index": 200025,274  "video_token_index": 200026,275  "multimodal_projector_bias": true,276  "num_reward_heads": 0,277  "process_image_mode": "dynamic_res",278  "projector_hidden_act": "gelu",279  "vision_feature_layer": -1,280  "vision_feature_select_strategy": "full",281  "torch_dtype": "bfloat16",282  "transformers_version": "4.52.4",283  "projector_hidden_size": 6144284}285