nvidia/MiniMax-M2.5-NVFP4

1{2    "architectures": [3        "MiniMaxM2ForCausalLM"4    ],5    "attention_dropout": 0.0,6    "attn_type_list": [7        1,8        1,9        1,10        1,11        1,12        1,13        1,14        1,15        1,16        1,17        1,18        1,19        1,20        1,21        1,22        1,23        1,24        1,25        1,26        1,27        1,28        1,29        1,30        1,31        1,32        1,33        1,34        1,35        1,36        1,37        1,38        1,39        1,40        1,41        1,42        1,43        1,44        1,45        1,46        1,47        1,48        1,49        1,50        1,51        1,52        1,53        1,54        1,55        1,56        1,57        1,58        1,59        1,60        1,61        1,62        1,63        1,64        1,65        1,66        1,67        1,68        169    ],70    "auto_map": {71        "AutoConfig": "configuration_minimax_m2.MiniMaxM2Config",72        "AutoModelForCausalLM": "modeling_minimax_m2.MiniMaxM2ForCausalLM"73    },74    "bos_token_id": 1,75    "dtype": "bfloat16",76    "eos_token_id": 2,77    "head_dim": 128,78    "hidden_act": "silu",79    "hidden_size": 3072,80    "initializer_range": 0.02,81    "intermediate_size": 1536,82    "max_position_embeddings": 196608,83    "model_type": "minimax_m2",84    "mtp_transformer_layers": 1,85    "num_attention_heads": 48,86    "num_experts_per_tok": 8,87    "num_hidden_layers": 62,88    "num_key_value_heads": 8,89    "num_local_experts": 256,90    "num_mtp_modules": 3,91    "output_router_logits": false,92    "partial_rotary_factor": 0.5,93    "qk_norm_type": "per_layer",94    "quantization_config": {95        "config_groups": {96            "group_0": {97                "input_activations": {98                    "dynamic": false,99                    "num_bits": 4,100                    "type": "float",101                    "group_size": 16102                },103                "weights": {104                    "dynamic": false,105                    "num_bits": 4,106                    "type": "float",107                    "group_size": 16108                },109                "targets": [110                    "Linear"111                ]112            }113        },114        "ignore": [115            "lm_head",116            "model.layers.0.block_sparse_moe.gate",117            "model.layers.0.self_attn*",118            "model.layers.1.block_sparse_moe.gate",119            "model.layers.1.self_attn*",120            "model.layers.10.block_sparse_moe.gate",121            "model.layers.10.self_attn*",122            "model.layers.11.block_sparse_moe.gate",123            "model.layers.11.self_attn*",124            "model.layers.12.block_sparse_moe.gate",125            "model.layers.12.self_attn*",126            "model.layers.13.block_sparse_moe.gate",127            "model.layers.13.self_attn*",128            "model.layers.14.block_sparse_moe.gate",129            "model.layers.14.self_attn*",130            "model.layers.15.block_sparse_moe.gate",131            "model.layers.15.self_attn*",132            "model.layers.16.block_sparse_moe.gate",133            "model.layers.16.self_attn*",134            "model.layers.17.block_sparse_moe.gate",135            "model.layers.17.self_attn*",136            "model.layers.18.block_sparse_moe.gate",137            "model.layers.18.self_attn*",138            "model.layers.19.block_sparse_moe.gate",139            "model.layers.19.self_attn*",140            "model.layers.2.block_sparse_moe.gate",141            "model.layers.2.self_attn*",142            "model.layers.20.block_sparse_moe.gate",143            "model.layers.20.self_attn*",144            "model.layers.21.block_sparse_moe.gate",145            "model.layers.21.self_attn*",146            "model.layers.22.block_sparse_moe.gate",147            "model.layers.22.self_attn*",148            "model.layers.23.block_sparse_moe.gate",149            "model.layers.23.self_attn*",150            "model.layers.24.block_sparse_moe.gate",151            "model.layers.24.self_attn*",152            "model.layers.25.block_sparse_moe.gate",153            "model.layers.25.self_attn*",154            "model.layers.26.block_sparse_moe.gate",155            "model.layers.26.self_attn*",156            "model.layers.27.block_sparse_moe.gate",157            "model.layers.27.self_attn*",158            "model.layers.28.block_sparse_moe.gate",159            "model.layers.28.self_attn*",160            "model.layers.29.block_sparse_moe.gate",161            "model.layers.29.self_attn*",162            "model.layers.3.block_sparse_moe.gate",163            "model.layers.3.self_attn*",164            "model.layers.30.block_sparse_moe.gate",165            "model.layers.30.self_attn*",166            "model.layers.31.block_sparse_moe.gate",167            "model.layers.31.self_attn*",168            "model.layers.32.block_sparse_moe.gate",169            "model.layers.32.self_attn*",170            "model.layers.33.block_sparse_moe.gate",171            "model.layers.33.self_attn*",172            "model.layers.34.block_sparse_moe.gate",173            "model.layers.34.self_attn*",174            "model.layers.35.block_sparse_moe.gate",175            "model.layers.35.self_attn*",176            "model.layers.36.block_sparse_moe.gate",177            "model.layers.36.self_attn*",178            "model.layers.37.block_sparse_moe.gate",179            "model.layers.37.self_attn*",180            "model.layers.38.block_sparse_moe.gate",181            "model.layers.38.self_attn*",182            "model.layers.39.block_sparse_moe.gate",183            "model.layers.39.self_attn*",184            "model.layers.4.block_sparse_moe.gate",185            "model.layers.4.self_attn*",186            "model.layers.40.block_sparse_moe.gate",187            "model.layers.40.self_attn*",188            "model.layers.41.block_sparse_moe.gate",189            "model.layers.41.self_attn*",190            "model.layers.42.block_sparse_moe.gate",191            "model.layers.42.self_attn*",192            "model.layers.43.block_sparse_moe.gate",193            "model.layers.43.self_attn*",194            "model.layers.44.block_sparse_moe.gate",195            "model.layers.44.self_attn*",196            "model.layers.45.block_sparse_moe.gate",197            "model.layers.45.self_attn*",198            "model.layers.46.block_sparse_moe.gate",199            "model.layers.46.self_attn*",200            "model.layers.47.block_sparse_moe.gate",201            "model.layers.47.self_attn*",202            "model.layers.48.block_sparse_moe.gate",203            "model.layers.48.self_attn*",204            "model.layers.49.block_sparse_moe.gate",205            "model.layers.49.self_attn*",206            "model.layers.5.block_sparse_moe.gate",207            "model.layers.5.self_attn*",208            "model.layers.50.block_sparse_moe.gate",209            "model.layers.50.self_attn*",210            "model.layers.51.block_sparse_moe.gate",211            "model.layers.51.self_attn*",212            "model.layers.52.block_sparse_moe.gate",213            "model.layers.52.self_attn*",214            "model.layers.53.block_sparse_moe.gate",215            "model.layers.53.self_attn*",216            "model.layers.54.block_sparse_moe.gate",217            "model.layers.54.self_attn*",218            "model.layers.55.block_sparse_moe.gate",219            "model.layers.55.self_attn*",220            "model.layers.56.block_sparse_moe.gate",221            "model.layers.56.self_attn*",222            "model.layers.57.block_sparse_moe.gate",223            "model.layers.57.self_attn*",224            "model.layers.58.block_sparse_moe.gate",225            "model.layers.58.self_attn*",226            "model.layers.59.block_sparse_moe.gate",227            "model.layers.59.self_attn*",228            "model.layers.6.block_sparse_moe.gate",229            "model.layers.6.self_attn*",230            "model.layers.60.block_sparse_moe.gate",231            "model.layers.60.self_attn*",232            "model.layers.61.block_sparse_moe.gate",233            "model.layers.61.self_attn*",234            "model.layers.7.block_sparse_moe.gate",235            "model.layers.7.self_attn*",236            "model.layers.8.block_sparse_moe.gate",237            "model.layers.8.self_attn*",238            "model.layers.9.block_sparse_moe.gate",239            "model.layers.9.self_attn*"240        ],241        "quant_algo": "NVFP4",242        "kv_cache_scheme": {243            "dynamic": false,244            "num_bits": 8,245            "type": "float"246        },247        "producer": {248            "name": "modelopt",249            "version": "0.0.1.dev507+gbb6639d33"250        },251        "quant_method": "modelopt"252    },253    "rms_norm_eps": 1e-06,254    "rope_theta": 5000000,255    "rotary_dim": 64,256    "router_aux_loss_coef": 0.001,257    "router_jitter_noise": 0.0,258    "scoring_func": "sigmoid",259    "shared_intermediate_size": 0,260    "sliding_window": null,261    "tie_word_embeddings": false,262    "transformers_version": "4.57.1",263    "use_cache": true,264    "use_mtp": true,265    "use_qk_norm": true,266    "use_routing_bias": true,267    "vocab_size": 200064,268    "sparse_attention_config": {269        "config_groups": {270            "group_0": {271                "sparse_algo": "softmax_skip",272                "targets": [273                    "MiniMaxM2Attention"274                ]275            }276        },277        "threshold_scale_factor": {278            "formula": "a * exp(b * target_sparsity)",279            "prefill": {280                "a": 87.239981,281                "b": 5.3221282            },283            "decode": {284                "a": 0.032140,285                "b": 11.7877286            }287        },288        "producer": {289            "name": "modelopt",290            "version": "0.43.0.dev27+g26aa1d0b1"291        }292    }293}294