nvidia/MiniMax-M3-NVFP4
8489k
1{2 "architectures": [3 "MiniMaxM3SparseForConditionalGeneration"4 ],5 "auto_map": {6 "AutoConfig": "configuration_minimax_m3_vl.MiniMaxM3VLConfig"7 },8 "model_type": "minimax_m3_vl",9 "text_config": {10 "dtype": "bfloat16",11 "hidden_size": 6144,12 "intermediate_size": 3072,13 "num_hidden_layers": 60,14 "num_attention_heads": 64,15 "num_key_value_heads": 4,16 "head_dim": 128,17 "vocab_size": 200064,18 "max_position_embeddings": 1048576,19 "rms_norm_eps": 1e-06,20 "use_gemma_norm": true,21 "attention_output_gate": false,22 "rope_theta": 5000000,23 "rotary_dim": 64,24 "partial_rotary_factor": 0.5,25 "hidden_act": "swigluoai",26 "use_qk_norm": true,27 "tie_word_embeddings": false,28 "dense_intermediate_size": 12288,29 "shared_intermediate_size": 3072,30 "num_local_experts": 128,31 "num_experts_per_tok": 4,32 "n_shared_experts": 1,33 "scoring_func": "sigmoid",34 "use_routing_bias": true,35 "moe_layer_freq": [36 0,37 0,38 0,39 1,40 1,41 1,42 1,43 1,44 1,45 1,46 1,47 1,48 1,49 1,50 1,51 1,52 1,53 1,54 1,55 1,56 1,57 1,58 1,59 1,60 1,61 1,62 1,63 1,64 1,65 1,66 1,67 1,68 1,69 1,70 1,71 1,72 1,73 1,74 1,75 1,76 1,77 1,78 1,79 1,80 1,81 1,82 1,83 1,84 1,85 1,86 1,87 1,88 1,89 1,90 1,91 1,92 1,93 1,94 1,95 196 ],97 "qk_norm_type": "per_head",98 "num_mtp_modules": 1,99 "swiglu_alpha": 1.702,100 "swiglu_limit": 7.0,101 "routed_scaling_factor": 2.0,102 "sparse_attention_config": {103 "use_sparse_attention": true,104 "sparse_index_dim": 128,105 "sparse_num_index_heads": 4,106 "sparse_topk_blocks": 16,107 "sparse_block_size": 128,108 "sparse_disable_index_value": [109 0,110 0,111 0,112 1,113 1,114 1,115 1,116 1,117 1,118 1,119 1,120 1,121 1,122 1,123 1,124 1,125 1,126 1,127 1,128 1,129 1,130 1,131 1,132 1,133 1,134 1,135 1,136 1,137 1,138 1,139 1,140 1,141 1,142 1,143 1,144 1,145 1,146 1,147 1,148 1,149 1,150 1,151 1,152 1,153 1,154 1,155 1,156 1,157 1,158 1,159 1,160 1,161 1,162 1,163 1,164 1,165 1,166 1,167 1,168 1169 ],170 "sparse_score_type": "max",171 "sparse_init_block": 0,172 "sparse_local_block": 1,173 "sparse_attention_freq": [174 0,175 0,176 0,177 1,178 1,179 1,180 1,181 1,182 1,183 1,184 1,185 1,186 1,187 1,188 1,189 1,190 1,191 1,192 1,193 1,194 1,195 1,196 1,197 1,198 1,199 1,200 1,201 1,202 1,203 1,204 1,205 1,206 1,207 1,208 1,209 1,210 1,211 1,212 1,213 1,214 1,215 1,216 1,217 1,218 1,219 1,220 1,221 1,222 1,223 1,224 1,225 1,226 1,227 1,228 1,229 1,230 1,231 1,232 1,233 1234 ]235 },236 "architectures": [237 "MiniMaxM3SparseForCausalLM"238 ]239 },240 "vision_config": {241 "hidden_size": 1280,242 "num_attention_heads": 16,243 "num_hidden_layers": 32,244 "intermediate_size": 5120,245 "patch_size": 14,246 "image_size": 2016,247 "projection_dim": 6144,248 "position_embedding_type": "rope",249 "rope_mode": "3d",250 "rope_theta": 10000.0,251 "attention_dropout": 0.0,252 "hidden_act": "gelu",253 "initializer_factor": 1.0,254 "initializer_range": 0.02,255 "layer_norm_eps": 1e-05,256 "model_type": "clip_vision_model",257 "num_channels": 3,258 "vocab_size": 32000,259 "img_token_compression_config": {260 "image_token_compression_method": "patch_merge",261 "spatial_merge_size": 2,262 "temporal_patch_size": 2263 },264 "vision_segment_max_frames": 4265 },266 "img_token_compression_config": {267 "image_token_compression_method": "patch_merge",268 "spatial_merge_size": 2,269 "temporal_patch_size": 2270 },271 "image_grid_pinpoints": "[(336, 336), (336, 672), (336, 1008), (336, 1344), (336, 1680), (336, 2016), (672, 336), (672, 672), (672, 1008), (672, 1344), (672, 1680), (672, 2016), (1008, 336), (1008, 672), (1008, 1008), (1008, 1344), (1008, 1680), (1008, 2016), (1344, 336), (1344, 672), (1344, 1008), (1344, 1344), (1344, 1680), (1344, 2016), (1680, 336), (1680, 672), (1680, 1008), (1680, 1344), (1680, 1680), (1680, 2016), (2016, 336), (2016, 672), (2016, 1008), (2016, 1344), (2016, 1680), (2016, 2016)]",272 "image_seq_length": 576,273 "image_token_index": 200025,274 "video_token_index": 200026,275 "multimodal_projector_bias": true,276 "num_reward_heads": 0,277 "process_image_mode": "dynamic_res",278 "projector_hidden_act": "gelu",279 "vision_feature_layer": -1,280 "vision_feature_select_strategy": "full",281 "torch_dtype": "bfloat16",282 "transformers_version": "4.52.4",283 "projector_hidden_size": 6144,284 "quantization_config": {285 "quant_algo": "MIXED_PRECISION",286 "kv_cache_quant_algo": null,287 "exclude_modules": [288 "lm_head",289 "model.embed_tokens",290 "vision_tower",291 "multi_modal_projector",292 "patch_merge_mlp",293 "language_model.model.layers.10.block_sparse_moe.gate",294 "language_model.model.layers.11.block_sparse_moe.gate",295 "language_model.model.layers.12.block_sparse_moe.gate",296 "language_model.model.layers.13.block_sparse_moe.gate",297 "language_model.model.layers.14.block_sparse_moe.gate",298 "language_model.model.layers.15.block_sparse_moe.gate",299 "language_model.model.layers.16.block_sparse_moe.gate",300 "language_model.model.layers.17.block_sparse_moe.gate",301 "language_model.model.layers.18.block_sparse_moe.gate",302 "language_model.model.layers.19.block_sparse_moe.gate",303 "language_model.model.layers.20.block_sparse_moe.gate",304 "language_model.model.layers.21.block_sparse_moe.gate",305 "language_model.model.layers.22.block_sparse_moe.gate",306 "language_model.model.layers.23.block_sparse_moe.gate",307 "language_model.model.layers.24.block_sparse_moe.gate",308 "language_model.model.layers.25.block_sparse_moe.gate",309 "language_model.model.layers.26.block_sparse_moe.gate",310 "language_model.model.layers.27.block_sparse_moe.gate",311 "language_model.model.layers.28.block_sparse_moe.gate",312 "language_model.model.layers.29.block_sparse_moe.gate",313 "language_model.model.layers.3.block_sparse_moe.gate",314 "language_model.model.layers.30.block_sparse_moe.gate",315 "language_model.model.layers.31.block_sparse_moe.gate",316 "language_model.model.layers.32.block_sparse_moe.gate",317 "language_model.model.layers.33.block_sparse_moe.gate",318 "language_model.model.layers.34.block_sparse_moe.gate",319 "language_model.model.layers.35.block_sparse_moe.gate",320 "language_model.model.layers.36.block_sparse_moe.gate",321 "language_model.model.layers.37.block_sparse_moe.gate",322 "language_model.model.layers.38.block_sparse_moe.gate",323 "language_model.model.layers.39.block_sparse_moe.gate",324 "language_model.model.layers.4.block_sparse_moe.gate",325 "language_model.model.layers.40.block_sparse_moe.gate",326 "language_model.model.layers.41.block_sparse_moe.gate",327 "language_model.model.layers.42.block_sparse_moe.gate",328 "language_model.model.layers.43.block_sparse_moe.gate",329 "language_model.model.layers.44.block_sparse_moe.gate",330 "language_model.model.layers.45.block_sparse_moe.gate",331 "language_model.model.layers.46.block_sparse_moe.gate",332 "language_model.model.layers.47.block_sparse_moe.gate",333 "language_model.model.layers.48.block_sparse_moe.gate",334 "language_model.model.layers.49.block_sparse_moe.gate",335 "language_model.model.layers.5.block_sparse_moe.gate",336 "language_model.model.layers.50.block_sparse_moe.gate",337 "language_model.model.layers.51.block_sparse_moe.gate",338 "language_model.model.layers.52.block_sparse_moe.gate",339 "language_model.model.layers.53.block_sparse_moe.gate",340 "language_model.model.layers.54.block_sparse_moe.gate",341 "language_model.model.layers.55.block_sparse_moe.gate",342 "language_model.model.layers.56.block_sparse_moe.gate",343 "language_model.model.layers.57.block_sparse_moe.gate",344 "language_model.model.layers.58.block_sparse_moe.gate",345 "language_model.model.layers.59.block_sparse_moe.gate",346 "language_model.model.layers.6.block_sparse_moe.gate",347 "language_model.model.layers.7.block_sparse_moe.gate",348 "language_model.model.layers.8.block_sparse_moe.gate",349 "language_model.model.layers.9.block_sparse_moe.gate"350 ],351 "quantized_layers": {352 "language_model.model.layers.0.mlp.down_proj": {353 "quant_algo": "MXFP8"354 },355 "language_model.model.layers.0.mlp.gate_proj": {356 "quant_algo": "MXFP8"357 },358 "language_model.model.layers.0.mlp.up_proj": {359 "quant_algo": "MXFP8"360 },361 "language_model.model.layers.0.self_attn.k_proj": {362 "quant_algo": "MXFP8"363 },364 "language_model.model.layers.0.self_attn.o_proj": {365 "quant_algo": "MXFP8"366 },367 "language_model.model.layers.0.self_attn.q_proj": {368 "quant_algo": "MXFP8"369 },370 "language_model.model.layers.0.self_attn.v_proj": {371 "quant_algo": "MXFP8"372 },373 "language_model.model.layers.1.mlp.down_proj": {374 "quant_algo": "MXFP8"375 },376 "language_model.model.layers.1.mlp.gate_proj": {377 "quant_algo": "MXFP8"378 },379 "language_model.model.layers.1.mlp.up_proj": {380 "quant_algo": "MXFP8"381 },382 "language_model.model.layers.1.self_attn.k_proj": {383 "quant_algo": "MXFP8"384 },385 "language_model.model.layers.1.self_attn.o_proj": {386 "quant_algo": "MXFP8"387 },388 "language_model.model.layers.1.self_attn.q_proj": {389 "quant_algo": "MXFP8"390 },391 "language_model.model.layers.1.self_attn.v_proj": {392 "quant_algo": "MXFP8"393 },394 "language_model.model.layers.2.mlp.down_proj": {395 "quant_algo": "MXFP8"396 },397 "language_model.model.layers.2.mlp.gate_proj": {398 "quant_algo": "MXFP8"399 },400 "language_model.model.layers.2.mlp.up_proj": {401 "quant_algo": "MXFP8"402 },403 "language_model.model.layers.2.self_attn.k_proj": {404 "quant_algo": "MXFP8"405 },406 "language_model.model.layers.2.self_attn.o_proj": {407 "quant_algo": "MXFP8"408 },409 "language_model.model.layers.2.self_attn.q_proj": {410 "quant_algo": "MXFP8"411 },412 "language_model.model.layers.2.self_attn.v_proj": {413 "quant_algo": "MXFP8"414 },415 "language_model.model.layers.3.block_sparse_moe.shared_experts.down_proj": {416 "quant_algo": "MXFP8"417 },418 "language_model.model.layers.3.block_sparse_moe.shared_experts.gate_proj": {419 "quant_algo": "MXFP8"420 },421 "language_model.model.layers.3.block_sparse_moe.shared_experts.up_proj": {422 "quant_algo": "MXFP8"423 },424 "language_model.model.layers.3.self_attn.index_k_proj": {425 "quant_algo": "MXFP8"426 },427 "language_model.model.layers.3.self_attn.index_q_proj": {428 "quant_algo": "MXFP8"429 },430 "language_model.model.layers.3.self_attn.k_proj": {431 "quant_algo": "MXFP8"432 },433 "language_model.model.layers.3.self_attn.o_proj": {434 "quant_algo": "MXFP8"435 },436 "language_model.model.layers.3.self_attn.q_proj": {437 "quant_algo": "MXFP8"438 },439 "language_model.model.layers.3.self_attn.v_proj": {440 "quant_algo": "MXFP8"441 },442 "language_model.model.layers.4.block_sparse_moe.shared_experts.down_proj": {443 "quant_algo": "MXFP8"444 },445 "language_model.model.layers.4.block_sparse_moe.shared_experts.gate_proj": {446 "quant_algo": "MXFP8"447 },448 "language_model.model.layers.4.block_sparse_moe.shared_experts.up_proj": {449 "quant_algo": "MXFP8"450 },451 "language_model.model.layers.4.self_attn.index_k_proj": {452 "quant_algo": "MXFP8"453 },454 "language_model.model.layers.4.self_attn.index_q_proj": {455 "quant_algo": "MXFP8"456 },457 "language_model.model.layers.4.self_attn.k_proj": {458 "quant_algo": "MXFP8"459 },460 "language_model.model.layers.4.self_attn.o_proj": {461 "quant_algo": "MXFP8"462 },463 "language_model.model.layers.4.self_attn.q_proj": {464 "quant_algo": "MXFP8"465 },466 "language_model.model.layers.4.self_attn.v_proj": {467 "quant_algo": "MXFP8"468 },469 "language_model.model.layers.5.block_sparse_moe.shared_experts.down_proj": {470 "quant_algo": "MXFP8"471 },472 "language_model.model.layers.5.block_sparse_moe.shared_experts.gate_proj": {473 "quant_algo": "MXFP8"474 },475 "language_model.model.layers.5.block_sparse_moe.shared_experts.up_proj": {476 "quant_algo": "MXFP8"477 },478 "language_model.model.layers.5.self_attn.index_k_proj": {479 "quant_algo": "MXFP8"480 },481 "language_model.model.layers.5.self_attn.index_q_proj": {482 "quant_algo": "MXFP8"483 },484 "language_model.model.layers.5.self_attn.k_proj": {485 "quant_algo": "MXFP8"486 },487 "language_model.model.layers.5.self_attn.o_proj": {488 "quant_algo": "MXFP8"489 },490 "language_model.model.layers.5.self_attn.q_proj": {491 "quant_algo": "MXFP8"492 },493 "language_model.model.layers.5.self_attn.v_proj": {494 "quant_algo": "MXFP8"495 },496 "language_model.model.layers.6.block_sparse_moe.shared_experts.down_proj": {497 "quant_algo": "MXFP8"498 },499 "language_model.model.layers.6.block_sparse_moe.shared_experts.gate_proj": {500 "quant_algo": "MXFP8"501 },502 "language_model.model.layers.6.block_sparse_moe.shared_experts.up_proj": {503 "quant_algo": "MXFP8"504 },505 "language_model.model.layers.6.self_attn.index_k_proj": {506 "quant_algo": "MXFP8"507 },508 "language_model.model.layers.6.self_attn.index_q_proj": {509 "quant_algo": "MXFP8"510 },511 "language_model.model.layers.6.self_attn.k_proj": {512 "quant_algo": "MXFP8"513 },514 "language_model.model.layers.6.self_attn.o_proj": {515 "quant_algo": "MXFP8"516 },517 "language_model.model.layers.6.self_attn.q_proj": {518 "quant_algo": "MXFP8"519 },520 "language_model.model.layers.6.self_attn.v_proj": {521 "quant_algo": "MXFP8"522 },523 "language_model.model.layers.7.block_sparse_moe.shared_experts.down_proj": {524 "quant_algo": "MXFP8"525 },526 "language_model.model.layers.7.block_sparse_moe.shared_experts.gate_proj": {527 "quant_algo": "MXFP8"528 },529 "language_model.model.layers.7.block_sparse_moe.shared_experts.up_proj": {530 "quant_algo": "MXFP8"531 },532 "language_model.model.layers.7.self_attn.index_k_proj": {533 "quant_algo": "MXFP8"534 },535 "language_model.model.layers.7.self_attn.index_q_proj": {536 "quant_algo": "MXFP8"537 },538 "language_model.model.layers.7.self_attn.k_proj": {539 "quant_algo": "MXFP8"540 },541 "language_model.model.layers.7.self_attn.o_proj": {542 "quant_algo": "MXFP8"543 },544 "language_model.model.layers.7.self_attn.q_proj": {545 "quant_algo": "MXFP8"546 },547 "language_model.model.layers.7.self_attn.v_proj": {548 "quant_algo": "MXFP8"549 },550 "language_model.model.layers.8.block_sparse_moe.shared_experts.down_proj": {551 "quant_algo": "MXFP8"552 },553 "language_model.model.layers.8.block_sparse_moe.shared_experts.gate_proj": {554 "quant_algo": "MXFP8"555 },556 "language_model.model.layers.8.block_sparse_moe.shared_experts.up_proj": {557 "quant_algo": "MXFP8"558 },559 "language_model.model.layers.8.self_attn.index_k_proj": {560 "quant_algo": "MXFP8"561 },562 "language_model.model.layers.8.self_attn.index_q_proj": {563 "quant_algo": "MXFP8"564 },565 "language_model.model.layers.8.self_attn.k_proj": {566 "quant_algo": "MXFP8"567 },568 "language_model.model.layers.8.self_attn.o_proj": {569 "quant_algo": "MXFP8"570 },571 "language_model.model.layers.8.self_attn.q_proj": {572 "quant_algo": "MXFP8"573 },574 "language_model.model.layers.8.self_attn.v_proj": {575 "quant_algo": "MXFP8"576 },577 "language_model.model.layers.9.block_sparse_moe.shared_experts.down_proj": {578 "quant_algo": "MXFP8"579 },580 "language_model.model.layers.9.block_sparse_moe.shared_experts.gate_proj": {581 "quant_algo": "MXFP8"582 },583 "language_model.model.layers.9.block_sparse_moe.shared_experts.up_proj": {584 "quant_algo": "MXFP8"585 },586 "language_model.model.layers.9.self_attn.index_k_proj": {587 "quant_algo": "MXFP8"588 },589 "language_model.model.layers.9.self_attn.index_q_proj": {590 "quant_algo": "MXFP8"591 },592 "language_model.model.layers.9.self_attn.k_proj": {593 "quant_algo": "MXFP8"594 },595 "language_model.model.layers.9.self_attn.o_proj": {596 "quant_algo": "MXFP8"597 },598 "language_model.model.layers.9.self_attn.q_proj": {599 "quant_algo": "MXFP8"600 },601 "language_model.model.layers.9.self_attn.v_proj": {602 "quant_algo": "MXFP8"603 },604 "language_model.model.layers.10.block_sparse_moe.shared_experts.down_proj": {605 "quant_algo": "MXFP8"606 },607 "language_model.model.layers.10.block_sparse_moe.shared_experts.gate_proj": {608 "quant_algo": "MXFP8"609 },610 "language_model.model.layers.10.block_sparse_moe.shared_experts.up_proj": {611 "quant_algo": "MXFP8"612 },613 "language_model.model.layers.10.self_attn.index_k_proj": {614 "quant_algo": "MXFP8"615 },616 "language_model.model.layers.10.self_attn.index_q_proj": {617 "quant_algo": "MXFP8"618 },619 "language_model.model.layers.10.self_attn.k_proj": {620 "quant_algo": "MXFP8"621 },622 "language_model.model.layers.10.self_attn.o_proj": {623 "quant_algo": "MXFP8"624 },625 "language_model.model.layers.10.self_attn.q_proj": {626 "quant_algo": "MXFP8"627 },628 "language_model.model.layers.10.self_attn.v_proj": {629 "quant_algo": "MXFP8"630 },631 "language_model.model.layers.11.block_sparse_moe.shared_experts.down_proj": {632 "quant_algo": "MXFP8"633 },634 "language_model.model.layers.11.block_sparse_moe.shared_experts.gate_proj": {635 "quant_algo": "MXFP8"636 },637 "language_model.model.layers.11.block_sparse_moe.shared_experts.up_proj": {638 "quant_algo": "MXFP8"639 },640 "language_model.model.layers.11.self_attn.index_k_proj": {641 "quant_algo": "MXFP8"642 },643 "language_model.model.layers.11.self_attn.index_q_proj": {644 "quant_algo": "MXFP8"645 },646 "language_model.model.layers.11.self_attn.k_proj": {647 "quant_algo": "MXFP8"648 },649 "language_model.model.layers.11.self_attn.o_proj": {650 "quant_algo": "MXFP8"651 },652 "language_model.model.layers.11.self_attn.q_proj": {653 "quant_algo": "MXFP8"654 },655 "language_model.model.layers.11.self_attn.v_proj": {656 "quant_algo": "MXFP8"657 },658 "language_model.model.layers.12.block_sparse_moe.shared_experts.down_proj": {659 "quant_algo": "MXFP8"660 },661 "language_model.model.layers.12.block_sparse_moe.shared_experts.gate_proj": {662 "quant_algo": "MXFP8"663 },664 "language_model.model.layers.12.block_sparse_moe.shared_experts.up_proj": {665 "quant_algo": "MXFP8"666 },667 "language_model.model.layers.12.self_attn.index_k_proj": {668 "quant_algo": "MXFP8"669 },670 "language_model.model.layers.12.self_attn.index_q_proj": {671 "quant_algo": "MXFP8"672 },673 "language_model.model.layers.12.self_attn.k_proj": {674 "quant_algo": "MXFP8"675 },676 "language_model.model.layers.12.self_attn.o_proj": {677 "quant_algo": "MXFP8"678 },679 "language_model.model.layers.12.self_attn.q_proj": {680 "quant_algo": "MXFP8"681 },682 "language_model.model.layers.12.self_attn.v_proj": {683 "quant_algo": "MXFP8"684 },685 "language_model.model.layers.13.block_sparse_moe.shared_experts.down_proj": {686 "quant_algo": "MXFP8"687 },688 "language_model.model.layers.13.block_sparse_moe.shared_experts.gate_proj": {689 "quant_algo": "MXFP8"690 },691 "language_model.model.layers.13.block_sparse_moe.shared_experts.up_proj": {692 "quant_algo": "MXFP8"693 },694 "language_model.model.layers.13.self_attn.index_k_proj": {695 "quant_algo": "MXFP8"696 },697 "language_model.model.layers.13.self_attn.index_q_proj": {698 "quant_algo": "MXFP8"699 },700 "language_model.model.layers.13.self_attn.k_proj": {701 "quant_algo": "MXFP8"702 },703 "language_model.model.layers.13.self_attn.o_proj": {704 "quant_algo": "MXFP8"705 },706 "language_model.model.layers.13.self_attn.q_proj": {707 "quant_algo": "MXFP8"708 },709 "language_model.model.layers.13.self_attn.v_proj": {710 "quant_algo": "MXFP8"711 },712 "language_model.model.layers.14.block_sparse_moe.shared_experts.down_proj": {713 "quant_algo": "MXFP8"714 },715 "language_model.model.layers.14.block_sparse_moe.shared_experts.gate_proj": {716 "quant_algo": "MXFP8"717 },718 "language_model.model.layers.14.block_sparse_moe.shared_experts.up_proj": {719 "quant_algo": "MXFP8"720 },721 "language_model.model.layers.14.self_attn.index_k_proj": {722 "quant_algo": "MXFP8"723 },724 "language_model.model.layers.14.self_attn.index_q_proj": {725 "quant_algo": "MXFP8"726 },727 "language_model.model.layers.14.self_attn.k_proj": {728 "quant_algo": "MXFP8"729 },730 "language_model.model.layers.14.self_attn.o_proj": {731 "quant_algo": "MXFP8"732 },733 "language_model.model.layers.14.self_attn.q_proj": {734 "quant_algo": "MXFP8"735 },736 "language_model.model.layers.14.self_attn.v_proj": {737 "quant_algo": "MXFP8"738 },739 "language_model.model.layers.15.block_sparse_moe.shared_experts.down_proj": {740 "quant_algo": "MXFP8"741 },742 "language_model.model.layers.15.block_sparse_moe.shared_experts.gate_proj": {743 "quant_algo": "MXFP8"744 },745 "language_model.model.layers.15.block_sparse_moe.shared_experts.up_proj": {746 "quant_algo": "MXFP8"747 },748 "language_model.model.layers.15.self_attn.index_k_proj": {749 "quant_algo": "MXFP8"750 },751 "language_model.model.layers.15.self_attn.index_q_proj": {752 "quant_algo": "MXFP8"753 },754 "language_model.model.layers.15.self_attn.k_proj": {755 "quant_algo": "MXFP8"756 },757 "language_model.model.layers.15.self_attn.o_proj": {758 "quant_algo": "MXFP8"759 },760 "language_model.model.layers.15.self_attn.q_proj": {761 "quant_algo": "MXFP8"762 },763 "language_model.model.layers.15.self_attn.v_proj": {764 "quant_algo": "MXFP8"765 },766 "language_model.model.layers.16.block_sparse_moe.shared_experts.down_proj": {767 "quant_algo": "MXFP8"768 },769 "language_model.model.layers.16.block_sparse_moe.shared_experts.gate_proj": {770 "quant_algo": "MXFP8"771 },772 "language_model.model.layers.16.block_sparse_moe.shared_experts.up_proj": {773 "quant_algo": "MXFP8"774 },775 "language_model.model.layers.16.self_attn.index_k_proj": {776 "quant_algo": "MXFP8"777 },778 "language_model.model.layers.16.self_attn.index_q_proj": {779 "quant_algo": "MXFP8"780 },781 "language_model.model.layers.16.self_attn.k_proj": {782 "quant_algo": "MXFP8"783 },784 "language_model.model.layers.16.self_attn.o_proj": {785 "quant_algo": "MXFP8"786 },787 "language_model.model.layers.16.self_attn.q_proj": {788 "quant_algo": "MXFP8"789 },790 "language_model.model.layers.16.self_attn.v_proj": {791 "quant_algo": "MXFP8"792 },793 "language_model.model.layers.17.block_sparse_moe.shared_experts.down_proj": {794 "quant_algo": "MXFP8"795 },796 "language_model.model.layers.17.block_sparse_moe.shared_experts.gate_proj": {797 "quant_algo": "MXFP8"798 },799 "language_model.model.layers.17.block_sparse_moe.shared_experts.up_proj": {800 "quant_algo": "MXFP8"801 },802 "language_model.model.layers.17.self_attn.index_k_proj": {803 "quant_algo": "MXFP8"804 },805 "language_model.model.layers.17.self_attn.index_q_proj": {806 "quant_algo": "MXFP8"807 },808 "language_model.model.layers.17.self_attn.k_proj": {809 "quant_algo": "MXFP8"810 },811 "language_model.model.layers.17.self_attn.o_proj": {812 "quant_algo": "MXFP8"813 },814 "language_model.model.layers.17.self_attn.q_proj": {815 "quant_algo": "MXFP8"816 },817 "language_model.model.layers.17.self_attn.v_proj": {818 "quant_algo": "MXFP8"819 },820 "language_model.model.layers.18.block_sparse_moe.shared_experts.down_proj": {821 "quant_algo": "MXFP8"822 },823 "language_model.model.layers.18.block_sparse_moe.shared_experts.gate_proj": {824 "quant_algo": "MXFP8"825 },826 "language_model.model.layers.18.block_sparse_moe.shared_experts.up_proj": {827 "quant_algo": "MXFP8"828 },829 "language_model.model.layers.18.self_attn.index_k_proj": {830 "quant_algo": "MXFP8"831 },832 "language_model.model.layers.18.self_attn.index_q_proj": {833 "quant_algo": "MXFP8"834 },835 "language_model.model.layers.18.self_attn.k_proj": {836 "quant_algo": "MXFP8"837 },838 "language_model.model.layers.18.self_attn.o_proj": {839 "quant_algo": "MXFP8"840 },841 "language_model.model.layers.18.self_attn.q_proj": {842 "quant_algo": "MXFP8"843 },844 "language_model.model.layers.18.self_attn.v_proj": {845 "quant_algo": "MXFP8"846 },847 "language_model.model.layers.19.block_sparse_moe.shared_experts.down_proj": {848 "quant_algo": "MXFP8"849 },850 "language_model.model.layers.19.block_sparse_moe.shared_experts.gate_proj": {851 "quant_algo": "MXFP8"852 },853 "language_model.model.layers.19.block_sparse_moe.shared_experts.up_proj": {854 "quant_algo": "MXFP8"855 },856 "language_model.model.layers.19.self_attn.index_k_proj": {857 "quant_algo": "MXFP8"858 },859 "language_model.model.layers.19.self_attn.index_q_proj": {860 "quant_algo": "MXFP8"861 },862 "language_model.model.layers.19.self_attn.k_proj": {863 "quant_algo": "MXFP8"864 },865 "language_model.model.layers.19.self_attn.o_proj": {866 "quant_algo": "MXFP8"867 },868 "language_model.model.layers.19.self_attn.q_proj": {869 "quant_algo": "MXFP8"870 },871 "language_model.model.layers.19.self_attn.v_proj": {872 "quant_algo": "MXFP8"873 },874 "language_model.model.layers.20.block_sparse_moe.shared_experts.down_proj": {875 "quant_algo": "MXFP8"876 },877 "language_model.model.layers.20.block_sparse_moe.shared_experts.gate_proj": {878 "quant_algo": "MXFP8"879 },880 "language_model.model.layers.20.block_sparse_moe.shared_experts.up_proj": {881 "quant_algo": "MXFP8"882 },883 "language_model.model.layers.20.self_attn.index_k_proj": {884 "quant_algo": "MXFP8"885 },886 "language_model.model.layers.20.self_attn.index_q_proj": {887 "quant_algo": "MXFP8"888 },889 "language_model.model.layers.20.self_attn.k_proj": {890 "quant_algo": "MXFP8"891 },892 "language_model.model.layers.20.self_attn.o_proj": {893 "quant_algo": "MXFP8"894 },895 "language_model.model.layers.20.self_attn.q_proj": {896 "quant_algo": "MXFP8"897 },898 "language_model.model.layers.20.self_attn.v_proj": {899 "quant_algo": "MXFP8"900 },901 "language_model.model.layers.21.block_sparse_moe.shared_experts.down_proj": {902 "quant_algo": "MXFP8"903 },904 "language_model.model.layers.21.block_sparse_moe.shared_experts.gate_proj": {905 "quant_algo": "MXFP8"906 },907 "language_model.model.layers.21.block_sparse_moe.shared_experts.up_proj": {908 "quant_algo": "MXFP8"909 },910 "language_model.model.layers.21.self_attn.index_k_proj": {911 "quant_algo": "MXFP8"912 },913 "language_model.model.layers.21.self_attn.index_q_proj": {914 "quant_algo": "MXFP8"915 },916 "language_model.model.layers.21.self_attn.k_proj": {917 "quant_algo": "MXFP8"918 },919 "language_model.model.layers.21.self_attn.o_proj": {920 "quant_algo": "MXFP8"921 },922 "language_model.model.layers.21.self_attn.q_proj": {923 "quant_algo": "MXFP8"924 },925 "language_model.model.layers.21.self_attn.v_proj": {926 "quant_algo": "MXFP8"927 },928 "language_model.model.layers.22.block_sparse_moe.shared_experts.down_proj": {929 "quant_algo": "MXFP8"930 },931 "language_model.model.layers.22.block_sparse_moe.shared_experts.gate_proj": {932 "quant_algo": "MXFP8"933 },934 "language_model.model.layers.22.block_sparse_moe.shared_experts.up_proj": {935 "quant_algo": "MXFP8"936 },937 "language_model.model.layers.22.self_attn.index_k_proj": {938 "quant_algo": "MXFP8"939 },940 "language_model.model.layers.22.self_attn.index_q_proj": {941 "quant_algo": "MXFP8"942 },943 "language_model.model.layers.22.self_attn.k_proj": {944 "quant_algo": "MXFP8"945 },946 "language_model.model.layers.22.self_attn.o_proj": {947 "quant_algo": "MXFP8"948 },949 "language_model.model.layers.22.self_attn.q_proj": {950 "quant_algo": "MXFP8"951 },952 "language_model.model.layers.22.self_attn.v_proj": {953 "quant_algo": "MXFP8"954 },955 "language_model.model.layers.23.block_sparse_moe.shared_experts.down_proj": {956 "quant_algo": "MXFP8"957 },958 "language_model.model.layers.23.block_sparse_moe.shared_experts.gate_proj": {959 "quant_algo": "MXFP8"960 },961 "language_model.model.layers.23.block_sparse_moe.shared_experts.up_proj": {962 "quant_algo": "MXFP8"963 },964 "language_model.model.layers.23.self_attn.index_k_proj": {965 "quant_algo": "MXFP8"966 },967 "language_model.model.layers.23.self_attn.index_q_proj": {968 "quant_algo": "MXFP8"969 },970 "language_model.model.layers.23.self_attn.k_proj": {971 "quant_algo": "MXFP8"972 },973 "language_model.model.layers.23.self_attn.o_proj": {974 "quant_algo": "MXFP8"975 },976 "language_model.model.layers.23.self_attn.q_proj": {977 "quant_algo": "MXFP8"978 },979 "language_model.model.layers.23.self_attn.v_proj": {980 "quant_algo": "MXFP8"981 },982 "language_model.model.layers.24.block_sparse_moe.shared_experts.down_proj": {983 "quant_algo": "MXFP8"984 },985 "language_model.model.layers.24.block_sparse_moe.shared_experts.gate_proj": {986 "quant_algo": "MXFP8"987 },988 "language_model.model.layers.24.block_sparse_moe.shared_experts.up_proj": {989 "quant_algo": "MXFP8"990 },991 "language_model.model.layers.24.self_attn.index_k_proj": {992 "quant_algo": "MXFP8"993 },994 "language_model.model.layers.24.self_attn.index_q_proj": {995 "quant_algo": "MXFP8"996 },997 "language_model.model.layers.24.self_attn.k_proj": {998 "quant_algo": "MXFP8"999 },1000 "language_model.model.layers.24.self_attn.o_proj": {1001 "quant_algo": "MXFP8"1002 },1003 "language_model.model.layers.24.self_attn.q_proj": {1004 "quant_algo": "MXFP8"1005 },1006 "language_model.model.layers.24.self_attn.v_proj": {1007 "quant_algo": "MXFP8"1008 },1009 "language_model.model.layers.25.block_sparse_moe.shared_experts.down_proj": {1010 "quant_algo": "MXFP8"1011 },1012 "language_model.model.layers.25.block_sparse_moe.shared_experts.gate_proj": {1013 "quant_algo": "MXFP8"1014 },1015 "language_model.model.layers.25.block_sparse_moe.shared_experts.up_proj": {1016 "quant_algo": "MXFP8"1017 },1018 "language_model.model.layers.25.self_attn.index_k_proj": {1019 "quant_algo": "MXFP8"1020 },1021 "language_model.model.layers.25.self_attn.index_q_proj": {1022 "quant_algo": "MXFP8"1023 },1024 "language_model.model.layers.25.self_attn.k_proj": {1025 "quant_algo": "MXFP8"1026 },1027 "language_model.model.layers.25.self_attn.o_proj": {1028 "quant_algo": "MXFP8"1029 },1030 "language_model.model.layers.25.self_attn.q_proj": {1031 "quant_algo": "MXFP8"1032 },1033 "language_model.model.layers.25.self_attn.v_proj": {1034 "quant_algo": "MXFP8"1035 },1036 "language_model.model.layers.26.block_sparse_moe.shared_experts.down_proj": {1037 "quant_algo": "MXFP8"1038 },1039 "language_model.model.layers.26.block_sparse_moe.shared_experts.gate_proj": {1040 "quant_algo": "MXFP8"1041 },1042 "language_model.model.layers.26.block_sparse_moe.shared_experts.up_proj": {1043 "quant_algo": "MXFP8"1044 },1045 "language_model.model.layers.26.self_attn.index_k_proj": {1046 "quant_algo": "MXFP8"1047 },1048 "language_model.model.layers.26.self_attn.index_q_proj": {1049 "quant_algo": "MXFP8"1050 },1051 "language_model.model.layers.26.self_attn.k_proj": {1052 "quant_algo": "MXFP8"1053 },1054 "language_model.model.layers.26.self_attn.o_proj": {1055 "quant_algo": "MXFP8"1056 },1057 "language_model.model.layers.26.self_attn.q_proj": {1058 "quant_algo": "MXFP8"1059 },1060 "language_model.model.layers.26.self_attn.v_proj": {1061 "quant_algo": "MXFP8"1062 },1063 "language_model.model.layers.27.block_sparse_moe.shared_experts.down_proj": {1064 "quant_algo": "MXFP8"1065 },1066 "language_model.model.layers.27.block_sparse_moe.shared_experts.gate_proj": {1067 "quant_algo": "MXFP8"1068 },1069 "language_model.model.layers.27.block_sparse_moe.shared_experts.up_proj": {1070 "quant_algo": "MXFP8"1071 },1072 "language_model.model.layers.27.self_attn.index_k_proj": {1073 "quant_algo": "MXFP8"1074 },1075 "language_model.model.layers.27.self_attn.index_q_proj": {1076 "quant_algo": "MXFP8"1077 },1078 "language_model.model.layers.27.self_attn.k_proj": {1079 "quant_algo": "MXFP8"1080 },1081 "language_model.model.layers.27.self_attn.o_proj": {1082 "quant_algo": "MXFP8"1083 },1084 "language_model.model.layers.27.self_attn.q_proj": {1085 "quant_algo": "MXFP8"1086 },1087 "language_model.model.layers.27.self_attn.v_proj": {1088 "quant_algo": "MXFP8"1089 },1090 "language_model.model.layers.28.block_sparse_moe.shared_experts.down_proj": {1091 "quant_algo": "MXFP8"1092 },1093 "language_model.model.layers.28.block_sparse_moe.shared_experts.gate_proj": {1094 "quant_algo": "MXFP8"1095 },1096 "language_model.model.layers.28.block_sparse_moe.shared_experts.up_proj": {1097 "quant_algo": "MXFP8"1098 },1099 "language_model.model.layers.28.self_attn.index_k_proj": {1100 "quant_algo": "MXFP8"1101 },1102 "language_model.model.layers.28.self_attn.index_q_proj": {1103 "quant_algo": "MXFP8"1104 },1105 "language_model.model.layers.28.self_attn.k_proj": {1106 "quant_algo": "MXFP8"1107 },1108 "language_model.model.layers.28.self_attn.o_proj": {1109 "quant_algo": "MXFP8"1110 },1111 "language_model.model.layers.28.self_attn.q_proj": {1112 "quant_algo": "MXFP8"1113 },1114 "language_model.model.layers.28.self_attn.v_proj": {1115 "quant_algo": "MXFP8"1116 },1117 "language_model.model.layers.29.block_sparse_moe.shared_experts.down_proj": {1118 "quant_algo": "MXFP8"1119 },1120 "language_model.model.layers.29.block_sparse_moe.shared_experts.gate_proj": {1121 "quant_algo": "MXFP8"1122 },1123 "language_model.model.layers.29.block_sparse_moe.shared_experts.up_proj": {1124 "quant_algo": "MXFP8"1125 },1126 "language_model.model.layers.29.self_attn.index_k_proj": {1127 "quant_algo": "MXFP8"1128 },1129 "language_model.model.layers.29.self_attn.index_q_proj": {1130 "quant_algo": "MXFP8"1131 },1132 "language_model.model.layers.29.self_attn.k_proj": {1133 "quant_algo": "MXFP8"1134 },1135 "language_model.model.layers.29.self_attn.o_proj": {1136 "quant_algo": "MXFP8"1137 },1138 "language_model.model.layers.29.self_attn.q_proj": {1139 "quant_algo": "MXFP8"1140 },1141 "language_model.model.layers.29.self_attn.v_proj": {1142 "quant_algo": "MXFP8"1143 },1144 "language_model.model.layers.30.block_sparse_moe.shared_experts.down_proj": {1145 "quant_algo": "MXFP8"1146 },1147 "language_model.model.layers.30.block_sparse_moe.shared_experts.gate_proj": {1148 "quant_algo": "MXFP8"1149 },1150 "language_model.model.layers.30.block_sparse_moe.shared_experts.up_proj": {1151 "quant_algo": "MXFP8"1152 },1153 "language_model.model.layers.30.self_attn.index_k_proj": {1154 "quant_algo": "MXFP8"1155 },1156 "language_model.model.layers.30.self_attn.index_q_proj": {1157 "quant_algo": "MXFP8"1158 },1159 "language_model.model.layers.30.self_attn.k_proj": {1160 "quant_algo": "MXFP8"1161 },1162 "language_model.model.layers.30.self_attn.o_proj": {1163 "quant_algo": "MXFP8"1164 },1165 "language_model.model.layers.30.self_attn.q_proj": {1166 "quant_algo": "MXFP8"1167 },1168 "language_model.model.layers.30.self_attn.v_proj": {1169 "quant_algo": "MXFP8"1170 },1171 "language_model.model.layers.31.block_sparse_moe.shared_experts.down_proj": {1172 "quant_algo": "MXFP8"1173 },1174 "language_model.model.layers.31.block_sparse_moe.shared_experts.gate_proj": {1175 "quant_algo": "MXFP8"1176 },1177 "language_model.model.layers.31.block_sparse_moe.shared_experts.up_proj": {1178 "quant_algo": "MXFP8"1179 },1180 "language_model.model.layers.31.self_attn.index_k_proj": {1181 "quant_algo": "MXFP8"1182 },1183 "language_model.model.layers.31.self_attn.index_q_proj": {1184 "quant_algo": "MXFP8"1185 },1186 "language_model.model.layers.31.self_attn.k_proj": {1187 "quant_algo": "MXFP8"1188 },1189 "language_model.model.layers.31.self_attn.o_proj": {1190 "quant_algo": "MXFP8"1191 },1192 "language_model.model.layers.31.self_attn.q_proj": {1193 "quant_algo": "MXFP8"1194 },1195 "language_model.model.layers.31.self_attn.v_proj": {1196 "quant_algo": "MXFP8"1197 },1198 "language_model.model.layers.32.block_sparse_moe.shared_experts.down_proj": {1199 "quant_algo": "MXFP8"1200 },