amd/MiniMax-M2.7-MXFP4

1{2  "architectures": [3    "MiniMaxM2ForCausalLM"4  ],5  "attention_dropout": 0.0,6  "attn_type_list": [7    1,8    1,9    1,10    1,11    1,12    1,13    1,14    1,15    1,16    1,17    1,18    1,19    1,20    1,21    1,22    1,23    1,24    1,25    1,26    1,27    1,28    1,29    1,30    1,31    1,32    1,33    1,34    1,35    1,36    1,37    1,38    1,39    1,40    1,41    1,42    1,43    1,44    1,45    1,46    1,47    1,48    1,49    1,50    1,51    1,52    1,53    1,54    1,55    1,56    1,57    1,58    1,59    1,60    1,61    1,62    1,63    1,64    1,65    1,66    1,67    1,68    169  ],70  "auto_map": {71    "AutoConfig": "configuration_minimax_m2.MiniMaxM2Config",72    "AutoModelForCausalLM": "modeling_minimax_m2.MiniMaxM2ForCausalLM"73  },74  "bos_token_id": 1,75  "dtype": "bfloat16",76  "eos_token_id": 2,77  "head_dim": 128,78  "hidden_act": "silu",79  "hidden_size": 3072,80  "initializer_range": 0.02,81  "intermediate_size": 1536,82  "max_position_embeddings": 204800,83  "model_type": "minimax_m2",84  "mtp_transformer_layers": 1,85  "num_attention_heads": 48,86  "num_experts_per_tok": 8,87  "num_hidden_layers": 62,88  "num_key_value_heads": 8,89  "num_local_experts": 256,90  "num_mtp_modules": 3,91  "output_router_logits": false,92  "partial_rotary_factor": 0.5,93  "qk_norm_type": "per_layer",94  "quantization_config": {95    "algo_config": null,96    "exclude": [97      "model.layers.0.self_attn.q_proj",98      "model.layers.0.self_attn.k_proj",99      "model.layers.0.self_attn.v_proj",100      "model.layers.0.self_attn.o_proj",101      "model.layers.0.block_sparse_moe.gate",102      "model.layers.1.self_attn.q_proj",103      "model.layers.1.self_attn.k_proj",104      "model.layers.1.self_attn.v_proj",105      "model.layers.1.self_attn.o_proj",106      "model.layers.1.block_sparse_moe.gate",107      "model.layers.2.self_attn.q_proj",108      "model.layers.2.self_attn.k_proj",109      "model.layers.2.self_attn.v_proj",110      "model.layers.2.self_attn.o_proj",111      "model.layers.2.block_sparse_moe.gate",112      "model.layers.3.self_attn.q_proj",113      "model.layers.3.self_attn.k_proj",114      "model.layers.3.self_attn.v_proj",115      "model.layers.3.self_attn.o_proj",116      "model.layers.3.block_sparse_moe.gate",117      "model.layers.4.self_attn.q_proj",118      "model.layers.4.self_attn.k_proj",119      "model.layers.4.self_attn.v_proj",120      "model.layers.4.self_attn.o_proj",121      "model.layers.4.block_sparse_moe.gate",122      "model.layers.5.self_attn.q_proj",123      "model.layers.5.self_attn.k_proj",124      "model.layers.5.self_attn.v_proj",125      "model.layers.5.self_attn.o_proj",126      "model.layers.5.block_sparse_moe.gate",127      "model.layers.6.self_attn.q_proj",128      "model.layers.6.self_attn.k_proj",129      "model.layers.6.self_attn.v_proj",130      "model.layers.6.self_attn.o_proj",131      "model.layers.6.block_sparse_moe.gate",132      "model.layers.7.self_attn.q_proj",133      "model.layers.7.self_attn.k_proj",134      "model.layers.7.self_attn.v_proj",135      "model.layers.7.self_attn.o_proj",136      "model.layers.7.block_sparse_moe.gate",137      "model.layers.8.self_attn.q_proj",138      "model.layers.8.self_attn.k_proj",139      "model.layers.8.self_attn.v_proj",140      "model.layers.8.self_attn.o_proj",141      "model.layers.8.block_sparse_moe.gate",142      "model.layers.9.self_attn.q_proj",143      "model.layers.9.self_attn.k_proj",144      "model.layers.9.self_attn.v_proj",145      "model.layers.9.self_attn.o_proj",146      "model.layers.9.block_sparse_moe.gate",147      "model.layers.10.self_attn.q_proj",148      "model.layers.10.self_attn.k_proj",149      "model.layers.10.self_attn.v_proj",150      "model.layers.10.self_attn.o_proj",151      "model.layers.10.block_sparse_moe.gate",152      "model.layers.11.self_attn.q_proj",153      "model.layers.11.self_attn.k_proj",154      "model.layers.11.self_attn.v_proj",155      "model.layers.11.self_attn.o_proj",156      "model.layers.11.block_sparse_moe.gate",157      "model.layers.12.self_attn.q_proj",158      "model.layers.12.self_attn.k_proj",159      "model.layers.12.self_attn.v_proj",160      "model.layers.12.self_attn.o_proj",161      "model.layers.12.block_sparse_moe.gate",162      "model.layers.13.self_attn.q_proj",163      "model.layers.13.self_attn.k_proj",164      "model.layers.13.self_attn.v_proj",165      "model.layers.13.self_attn.o_proj",166      "model.layers.13.block_sparse_moe.gate",167      "model.layers.14.self_attn.q_proj",168      "model.layers.14.self_attn.k_proj",169      "model.layers.14.self_attn.v_proj",170      "model.layers.14.self_attn.o_proj",171      "model.layers.14.block_sparse_moe.gate",172      "model.layers.15.self_attn.q_proj",173      "model.layers.15.self_attn.k_proj",174      "model.layers.15.self_attn.v_proj",175      "model.layers.15.self_attn.o_proj",176      "model.layers.15.block_sparse_moe.gate",177      "model.layers.16.self_attn.q_proj",178      "model.layers.16.self_attn.k_proj",179      "model.layers.16.self_attn.v_proj",180      "model.layers.16.self_attn.o_proj",181      "model.layers.16.block_sparse_moe.gate",182      "model.layers.17.self_attn.q_proj",183      "model.layers.17.self_attn.k_proj",184      "model.layers.17.self_attn.v_proj",185      "model.layers.17.self_attn.o_proj",186      "model.layers.17.block_sparse_moe.gate",187      "model.layers.18.self_attn.q_proj",188      "model.layers.18.self_attn.k_proj",189      "model.layers.18.self_attn.v_proj",190      "model.layers.18.self_attn.o_proj",191      "model.layers.18.block_sparse_moe.gate",192      "model.layers.19.self_attn.q_proj",193      "model.layers.19.self_attn.k_proj",194      "model.layers.19.self_attn.v_proj",195      "model.layers.19.self_attn.o_proj",196      "model.layers.19.block_sparse_moe.gate",197      "model.layers.20.self_attn.q_proj",198      "model.layers.20.self_attn.k_proj",199      "model.layers.20.self_attn.v_proj",200      "model.layers.20.self_attn.o_proj",201      "model.layers.20.block_sparse_moe.gate",202      "model.layers.21.self_attn.q_proj",203      "model.layers.21.self_attn.k_proj",204      "model.layers.21.self_attn.v_proj",205      "model.layers.21.self_attn.o_proj",206      "model.layers.21.block_sparse_moe.gate",207      "model.layers.22.self_attn.q_proj",208      "model.layers.22.self_attn.k_proj",209      "model.layers.22.self_attn.v_proj",210      "model.layers.22.self_attn.o_proj",211      "model.layers.22.block_sparse_moe.gate",212      "model.layers.23.self_attn.q_proj",213      "model.layers.23.self_attn.k_proj",214      "model.layers.23.self_attn.v_proj",215      "model.layers.23.self_attn.o_proj",216      "model.layers.23.block_sparse_moe.gate",217      "model.layers.24.self_attn.q_proj",218      "model.layers.24.self_attn.k_proj",219      "model.layers.24.self_attn.v_proj",220      "model.layers.24.self_attn.o_proj",221      "model.layers.24.block_sparse_moe.gate",222      "model.layers.25.self_attn.q_proj",223      "model.layers.25.self_attn.k_proj",224      "model.layers.25.self_attn.v_proj",225      "model.layers.25.self_attn.o_proj",226      "model.layers.25.block_sparse_moe.gate",227      "model.layers.26.self_attn.q_proj",228      "model.layers.26.self_attn.k_proj",229      "model.layers.26.self_attn.v_proj",230      "model.layers.26.self_attn.o_proj",231      "model.layers.26.block_sparse_moe.gate",232      "model.layers.27.self_attn.q_proj",233      "model.layers.27.self_attn.k_proj",234      "model.layers.27.self_attn.v_proj",235      "model.layers.27.self_attn.o_proj",236      "model.layers.27.block_sparse_moe.gate",237      "model.layers.28.self_attn.q_proj",238      "model.layers.28.self_attn.k_proj",239      "model.layers.28.self_attn.v_proj",240      "model.layers.28.self_attn.o_proj",241      "model.layers.28.block_sparse_moe.gate",242      "model.layers.29.self_attn.q_proj",243      "model.layers.29.self_attn.k_proj",244      "model.layers.29.self_attn.v_proj",245      "model.layers.29.self_attn.o_proj",246      "model.layers.29.block_sparse_moe.gate",247      "model.layers.30.self_attn.q_proj",248      "model.layers.30.self_attn.k_proj",249      "model.layers.30.self_attn.v_proj",250      "model.layers.30.self_attn.o_proj",251      "model.layers.30.block_sparse_moe.gate",252      "model.layers.31.self_attn.q_proj",253      "model.layers.31.self_attn.k_proj",254      "model.layers.31.self_attn.v_proj",255      "model.layers.31.self_attn.o_proj",256      "model.layers.31.block_sparse_moe.gate",257      "model.layers.32.self_attn.q_proj",258      "model.layers.32.self_attn.k_proj",259      "model.layers.32.self_attn.v_proj",260      "model.layers.32.self_attn.o_proj",261      "model.layers.32.block_sparse_moe.gate",262      "model.layers.33.self_attn.q_proj",263      "model.layers.33.self_attn.k_proj",264      "model.layers.33.self_attn.v_proj",265      "model.layers.33.self_attn.o_proj",266      "model.layers.33.block_sparse_moe.gate",267      "model.layers.34.self_attn.q_proj",268      "model.layers.34.self_attn.k_proj",269      "model.layers.34.self_attn.v_proj",270      "model.layers.34.self_attn.o_proj",271      "model.layers.34.block_sparse_moe.gate",272      "model.layers.35.self_attn.q_proj",273      "model.layers.35.self_attn.k_proj",274      "model.layers.35.self_attn.v_proj",275      "model.layers.35.self_attn.o_proj",276      "model.layers.35.block_sparse_moe.gate",277      "model.layers.36.self_attn.q_proj",278      "model.layers.36.self_attn.k_proj",279      "model.layers.36.self_attn.v_proj",280      "model.layers.36.self_attn.o_proj",281      "model.layers.36.block_sparse_moe.gate",282      "model.layers.37.self_attn.q_proj",283      "model.layers.37.self_attn.k_proj",284      "model.layers.37.self_attn.v_proj",285      "model.layers.37.self_attn.o_proj",286      "model.layers.37.block_sparse_moe.gate",287      "model.layers.38.self_attn.q_proj",288      "model.layers.38.self_attn.k_proj",289      "model.layers.38.self_attn.v_proj",290      "model.layers.38.self_attn.o_proj",291      "model.layers.38.block_sparse_moe.gate",292      "model.layers.39.self_attn.q_proj",293      "model.layers.39.self_attn.k_proj",294      "model.layers.39.self_attn.v_proj",295      "model.layers.39.self_attn.o_proj",296      "model.layers.39.block_sparse_moe.gate",297      "model.layers.40.self_attn.q_proj",298      "model.layers.40.self_attn.k_proj",299      "model.layers.40.self_attn.v_proj",300      "model.layers.40.self_attn.o_proj",301      "model.layers.40.block_sparse_moe.gate",302      "model.layers.41.self_attn.q_proj",303      "model.layers.41.self_attn.k_proj",304      "model.layers.41.self_attn.v_proj",305      "model.layers.41.self_attn.o_proj",306      "model.layers.41.block_sparse_moe.gate",307      "model.layers.42.self_attn.q_proj",308      "model.layers.42.self_attn.k_proj",309      "model.layers.42.self_attn.v_proj",310      "model.layers.42.self_attn.o_proj",311      "model.layers.42.block_sparse_moe.gate",312      "model.layers.43.self_attn.q_proj",313      "model.layers.43.self_attn.k_proj",314      "model.layers.43.self_attn.v_proj",315      "model.layers.43.self_attn.o_proj",316      "model.layers.43.block_sparse_moe.gate",317      "model.layers.44.self_attn.q_proj",318      "model.layers.44.self_attn.k_proj",319      "model.layers.44.self_attn.v_proj",320      "model.layers.44.self_attn.o_proj",321      "model.layers.44.block_sparse_moe.gate",322      "model.layers.45.self_attn.q_proj",323      "model.layers.45.self_attn.k_proj",324      "model.layers.45.self_attn.v_proj",325      "model.layers.45.self_attn.o_proj",326      "model.layers.45.block_sparse_moe.gate",327      "model.layers.46.self_attn.q_proj",328      "model.layers.46.self_attn.k_proj",329      "model.layers.46.self_attn.v_proj",330      "model.layers.46.self_attn.o_proj",331      "model.layers.46.block_sparse_moe.gate",332      "model.layers.47.self_attn.q_proj",333      "model.layers.47.self_attn.k_proj",334      "model.layers.47.self_attn.v_proj",335      "model.layers.47.self_attn.o_proj",336      "model.layers.47.block_sparse_moe.gate",337      "model.layers.48.self_attn.q_proj",338      "model.layers.48.self_attn.k_proj",339      "model.layers.48.self_attn.v_proj",340      "model.layers.48.self_attn.o_proj",341      "model.layers.48.block_sparse_moe.gate",342      "model.layers.49.self_attn.q_proj",343      "model.layers.49.self_attn.k_proj",344      "model.layers.49.self_attn.v_proj",345      "model.layers.49.self_attn.o_proj",346      "model.layers.49.block_sparse_moe.gate",347      "model.layers.50.self_attn.q_proj",348      "model.layers.50.self_attn.k_proj",349      "model.layers.50.self_attn.v_proj",350      "model.layers.50.self_attn.o_proj",351      "model.layers.50.block_sparse_moe.gate",352      "model.layers.51.self_attn.q_proj",353      "model.layers.51.self_attn.k_proj",354      "model.layers.51.self_attn.v_proj",355      "model.layers.51.self_attn.o_proj",356      "model.layers.51.block_sparse_moe.gate",357      "model.layers.52.self_attn.q_proj",358      "model.layers.52.self_attn.k_proj",359      "model.layers.52.self_attn.v_proj",360      "model.layers.52.self_attn.o_proj",361      "model.layers.52.block_sparse_moe.gate",362      "model.layers.53.self_attn.q_proj",363      "model.layers.53.self_attn.k_proj",364      "model.layers.53.self_attn.v_proj",365      "model.layers.53.self_attn.o_proj",366      "model.layers.53.block_sparse_moe.gate",367      "model.layers.54.self_attn.q_proj",368      "model.layers.54.self_attn.k_proj",369      "model.layers.54.self_attn.v_proj",370      "model.layers.54.self_attn.o_proj",371      "model.layers.54.block_sparse_moe.gate",372      "model.layers.55.self_attn.q_proj",373      "model.layers.55.self_attn.k_proj",374      "model.layers.55.self_attn.v_proj",375      "model.layers.55.self_attn.o_proj",376      "model.layers.55.block_sparse_moe.gate",377      "model.layers.56.self_attn.q_proj",378      "model.layers.56.self_attn.k_proj",379      "model.layers.56.self_attn.v_proj",380      "model.layers.56.self_attn.o_proj",381      "model.layers.56.block_sparse_moe.gate",382      "model.layers.57.self_attn.q_proj",383      "model.layers.57.self_attn.k_proj",384      "model.layers.57.self_attn.v_proj",385      "model.layers.57.self_attn.o_proj",386      "model.layers.57.block_sparse_moe.gate",387      "model.layers.58.self_attn.q_proj",388      "model.layers.58.self_attn.k_proj",389      "model.layers.58.self_attn.v_proj",390      "model.layers.58.self_attn.o_proj",391      "model.layers.58.block_sparse_moe.gate",392      "model.layers.59.self_attn.q_proj",393      "model.layers.59.self_attn.k_proj",394      "model.layers.59.self_attn.v_proj",395      "model.layers.59.self_attn.o_proj",396      "model.layers.59.block_sparse_moe.gate",397      "model.layers.60.self_attn.q_proj",398      "model.layers.60.self_attn.k_proj",399      "model.layers.60.self_attn.v_proj",400      "model.layers.60.self_attn.o_proj",401      "model.layers.60.block_sparse_moe.gate",402      "model.layers.61.self_attn.q_proj",403      "model.layers.61.self_attn.k_proj",404      "model.layers.61.self_attn.v_proj",405      "model.layers.61.self_attn.o_proj",406      "model.layers.61.block_sparse_moe.gate",407      "lm_head"408    ],409    "export": {410      "kv_cache_group": [],411      "min_kv_scale": 0.0,412      "pack_method": "reorder",413      "weight_format": "real_quantized",414      "weight_merge_groups": null415    },416    "global_quant_config": {417      "bias": null,418      "input_tensors": {419        "block_size": null,420        "ch_axis": -1,421        "dtype": "fp4",422        "enable_buffer_reuse": false,423        "group_size": 32,424        "is_dynamic": true,425        "is_scale_quant": false,426        "max_input_numel": 4194304,427        "mx_element_dtype": null,428        "observer_cls": "PerBlockMXObserver",429        "qscheme": "per_group",430        "round_method": "half_even",431        "scale_calculation_mode": "even",432        "scale_format": "e8m0",433        "scale_type": "float",434        "symmetric": null435      },436      "output_tensors": null,437      "target_device": null,438      "weight": {439        "block_size": null,440        "ch_axis": -1,441        "dtype": "fp4",442        "enable_buffer_reuse": false,443        "group_size": 32,444        "is_dynamic": false,445        "is_scale_quant": false,446        "max_input_numel": 4194304,447        "mx_element_dtype": null,448        "observer_cls": "PerBlockMXObserver",449        "qscheme": "per_group",450        "round_method": "half_even",451        "scale_calculation_mode": "even",452        "scale_format": "e8m0",453        "scale_type": "float",454        "symmetric": null455      }456    },457    "kv_cache_post_rope": false,458    "kv_cache_quant_config": {},459    "layer_quant_config": {},460    "layer_type_quant_config": {},461    "quant_method": "quark",462    "quant_mode": "eager_mode",463    "softmax_quant_spec": null,464    "version": "0.12+810893dea1"465  },466  "rms_norm_eps": 1e-06,467  "rope_theta": 5000000,468  "rotary_dim": 64,469  "router_aux_loss_coef": 0.001,470  "router_jitter_noise": 0.0,471  "scoring_func": "sigmoid",472  "shared_intermediate_size": 0,473  "sliding_window": null,474  "tie_word_embeddings": false,475  "transformers_version": "4.57.1",476  "use_cache": true,477  "use_mtp": true,478  "use_qk_norm": true,479  "use_routing_bias": true,480  "vocab_size": 200064481}482