zai-org/GLM-5.3-Flash
2.6k4m
1{2 "architectures": [3 "Glm5NextForConditionalGeneration"4 ],5 "text_config": {6 "attention_bias": false,7 "attention_dropout": 0.0,8 "dtype": "bfloat16",9 "eos_token_id": [10 154820,11 154827,12 15482913 ],14 "first_k_dense_replace": 3,15 "hc_eps": 1e-06,16 "hc_mult": 4,17 "hc_sinkhorn_iters": 20,18 "head_dim": 0,19 "hidden_act": "silu",20 "hidden_size": 4096,21 "index_head_dim": 128,22 "index_kpool": 4,23 "index_kpool_always_select_tail": true,24 "index_kpool_compress": true,25 "index_n_heads": 32,26 "index_topk": 2048,27 "index_share_for_mtp_iteration": true,28 "indexer_rope_interleave": true,29 "indexer_types": [30 "full",31 "full",32 "full",33 "full",34 "full",35 "full",36 "full",37 "full",38 "full",39 "full",40 "full",41 "full",42 "full",43 "full",44 "full",45 "full",46 "full",47 "full",48 "full",49 "full",50 "full",51 "full",52 "full",53 "full",54 "full",55 "full",56 "full",57 "full",58 "full",59 "full",60 "full",61 "full",62 "full",63 "full",64 "full",65 "full",66 "full",67 "full",68 "full",69 "full",70 "full",71 "full",72 "full",73 "full",74 "full"75 ],76 "initializer_range": 0.02,77 "intermediate_size": 12288,78 "kv_lora_rank": 512,79 "layer_types": [80 "linear_attention",81 "linear_attention",82 "linear_attention",83 "deepseek_sparse_attention",84 "linear_attention",85 "linear_attention",86 "linear_attention",87 "deepseek_sparse_attention",88 "linear_attention",89 "linear_attention",90 "linear_attention",91 "deepseek_sparse_attention",92 "linear_attention",93 "linear_attention",94 "linear_attention",95 "deepseek_sparse_attention",96 "linear_attention",97 "linear_attention",98 "linear_attention",99 "deepseek_sparse_attention",100 "linear_attention",101 "linear_attention",102 "linear_attention",103 "deepseek_sparse_attention",104 "linear_attention",105 "linear_attention",106 "linear_attention",107 "deepseek_sparse_attention",108 "linear_attention",109 "linear_attention",110 "linear_attention",111 "deepseek_sparse_attention",112 "linear_attention",113 "linear_attention",114 "linear_attention",115 "deepseek_sparse_attention",116 "linear_attention",117 "linear_attention",118 "linear_attention",119 "deepseek_sparse_attention",120 "linear_attention",121 "linear_attention",122 "linear_attention",123 "deepseek_sparse_attention",124 "linear_attention"125 ],126 "linear_attn_config": {127 "num_heads": 64,128 "gate_lower_bound": -5.0,129 "head_dim": 128,130 "short_conv_kernel_size": 4,131 "kda_layers": [132 0,133 1,134 2,135 4,136 5,137 6,138 8,139 9,140 10,141 12,142 13,143 14,144 16,145 17,146 18,147 20,148 21,149 22,150 24,151 25,152 26,153 28,154 29,155 30,156 32,157 33,158 34,159 36,160 37,161 38,162 40,163 41,164 42,165 44166 ],167 "full_attn_layers": [168 3,169 7,170 11,171 15,172 19,173 23,174 27,175 31,176 35,177 39,178 43179 ]180 },181 "max_position_embeddings": 1048576,182 "mhc": true,183 "mla_use_nope": true,184 "mlp_layer_types": [185 "dense",186 "dense",187 "dense",188 "sparse",189 "sparse",190 "sparse",191 "sparse",192 "sparse",193 "sparse",194 "sparse",195 "sparse",196 "sparse",197 "sparse",198 "sparse",199 "sparse",200 "sparse",201 "sparse",202 "sparse",203 "sparse",204 "sparse",205 "sparse",206 "sparse",207 "sparse",208 "sparse",209 "sparse",210 "sparse",211 "sparse",212 "sparse",213 "sparse",214 "sparse",215 "sparse",216 "sparse",217 "sparse",218 "sparse",219 "sparse",220 "sparse",221 "sparse",222 "sparse",223 "sparse",224 "sparse",225 "sparse",226 "sparse",227 "sparse",228 "sparse",229 "sparse"230 ],231 "model_type": "glm5_next_text",232 "moe_intermediate_size": 2048,233 "moe_router_dtype": "float32",234 "n_group": 1,235 "n_routed_experts": 288,236 "n_shared_experts": 1,237 "norm_topk_prob": true,238 "num_attention_heads": 64,239 "num_experts_per_tok": 8,240 "num_hidden_layers": 45,241 "num_key_value_heads": 64,242 "num_nextn_predict_layers": 1,243 "output_router_logits": false,244 "pad_token_id": 154820,245 "q_lora_rank": 1536,246 "qk_head_dim": 256,247 "qk_nope_head_dim": 256,248 "qk_rope_head_dim": 0,249 "rms_norm_eps": 1e-05,250 "routed_scaling_factor": 2.5,251 "router_aux_loss_coef": 0.001,252 "scoring_func": "sigmoid",253 "swiglu_limit": 10.0,254 "tie_word_embeddings": false,255 "topk_group": 1,256 "topk_method": "noaux_tc",257 "use_cache": true,258 "v_head_dim": 256,259 "vocab_size": 154880260 },261 "vision_config": {262 "attention_bias": true,263 "attention_dropout": 0.0,264 "depth": 24,265 "hidden_act": "silu",266 "hidden_size": 1024,267 "image_size": 448,268 "in_channels": 3,269 "initializer_range": 0.02,270 "intermediate_size": 4096,271 "model_type": "glm5_next_vision",272 "num_heads": 16,273 "out_hidden_size": 4096,274 "patch_size": 14,275 "projection_intermediate_size": 10240,276 "rms_norm_eps": 1e-05,277 "spatial_merge_size": 2,278 "swiglu_limit": 10.0,279 "temporal_patch_size": 2280 },281 "image_token_id": 154854,282 "video_token_id": 154855,283 "image_start_token_id": 154830,284 "image_end_token_id": 154831,285 "video_start_token_id": 154832,286 "video_end_token_id": 154833,287 "tie_word_embeddings": false,288 "model_type": "glm5_next",289 "transformers_version": "5.16.0",290 "quantization_config": {291 "activation_scheme": "dynamic",292 "fmt": "e4m3",293 "modules_to_not_convert": [294 "attn_mha",295 "attn_mqa",296 "dt_bias",297 "hyper_connection",298 "lm_head",299 "mapping_proj",300 "model.embed_tokens",301 "model.layers.0.hc_attn_base",302 "model.layers.0.hc_attn_fn",303 "model.layers.0.hc_attn_scale",304 "model.layers.0.hc_ffn_base",305 "model.layers.0.hc_ffn_fn",306 "model.layers.0.hc_ffn_scale",307 "model.layers.0.input_layernorm",308 "model.layers.0.post_attention_layernorm",309 "model.layers.0.self_attn.A_log",310 "model.layers.0.self_attn.b_proj",311 "model.layers.0.self_attn.dt_bias",312 "model.layers.0.self_attn.f_a_proj",313 "model.layers.0.self_attn.f_b_proj",314 "model.layers.0.self_attn.fused_qkvbfg_a_proj",315 "model.layers.0.self_attn.g_a_proj",316 "model.layers.0.self_attn.g_b_proj",317 "model.layers.0.self_attn.k_conv1d",318 "model.layers.0.self_attn.k_proj",319 "model.layers.0.self_attn.o_norm",320 "model.layers.0.self_attn.o_proj",321 "model.layers.0.self_attn.q_conv1d",322 "model.layers.0.self_attn.q_proj",323 "model.layers.0.self_attn.qkv_proj",324 "model.layers.0.self_attn.v_conv1d",325 "model.layers.0.self_attn.v_proj",326 "model.layers.1.hc_attn_base",327 "model.layers.1.hc_attn_fn",328 "model.layers.1.hc_attn_scale",329 "model.layers.1.hc_ffn_base",330 "model.layers.1.hc_ffn_fn",331 "model.layers.1.hc_ffn_scale",332 "model.layers.1.input_layernorm",333 "model.layers.1.post_attention_layernorm",334 "model.layers.1.self_attn.A_log",335 "model.layers.1.self_attn.b_proj",336 "model.layers.1.self_attn.dt_bias",337 "model.layers.1.self_attn.f_a_proj",338 "model.layers.1.self_attn.f_b_proj",339 "model.layers.1.self_attn.fused_qkvbfg_a_proj",340 "model.layers.1.self_attn.g_a_proj",341 "model.layers.1.self_attn.g_b_proj",342 "model.layers.1.self_attn.k_conv1d",343 "model.layers.1.self_attn.k_proj",344 "model.layers.1.self_attn.o_norm",345 "model.layers.1.self_attn.o_proj",346 "model.layers.1.self_attn.q_conv1d",347 "model.layers.1.self_attn.q_proj",348 "model.layers.1.self_attn.qkv_proj",349 "model.layers.1.self_attn.v_conv1d",350 "model.layers.1.self_attn.v_proj",351 "model.layers.10.hc_attn_base",352 "model.layers.10.hc_attn_fn",353 "model.layers.10.hc_attn_scale",354 "model.layers.10.hc_ffn_base",355 "model.layers.10.hc_ffn_fn",356 "model.layers.10.hc_ffn_scale",357 "model.layers.10.input_layernorm",358 "model.layers.10.mlp.gate",359 "model.layers.10.mlp.gate.e_score_correction_bias",360 "model.layers.10.post_attention_layernorm",361 "model.layers.10.self_attn.A_log",362 "model.layers.10.self_attn.b_proj",363 "model.layers.10.self_attn.dt_bias",364 "model.layers.10.self_attn.f_a_proj",365 "model.layers.10.self_attn.f_b_proj",366 "model.layers.10.self_attn.fused_qkvbfg_a_proj",367 "model.layers.10.self_attn.g_a_proj",368 "model.layers.10.self_attn.g_b_proj",369 "model.layers.10.self_attn.k_conv1d",370 "model.layers.10.self_attn.k_proj",371 "model.layers.10.self_attn.o_norm",372 "model.layers.10.self_attn.o_proj",373 "model.layers.10.self_attn.q_conv1d",374 "model.layers.10.self_attn.q_proj",375 "model.layers.10.self_attn.qkv_proj",376 "model.layers.10.self_attn.v_conv1d",377 "model.layers.10.self_attn.v_proj",378 "model.layers.11.hc_attn_base",379 "model.layers.11.hc_attn_fn",380 "model.layers.11.hc_attn_scale",381 "model.layers.11.hc_ffn_base",382 "model.layers.11.hc_ffn_fn",383 "model.layers.11.hc_ffn_scale",384 "model.layers.11.input_layernorm",385 "model.layers.11.mlp.gate",386 "model.layers.11.mlp.gate.e_score_correction_bias",387 "model.layers.11.post_attention_layernorm",388 "model.layers.11.self_attn.indexer.index_kpool_compress_ape",389 "model.layers.11.self_attn.indexer.index_kpool_compress_gate",390 "model.layers.11.self_attn.indexer.k_norm",391 "model.layers.11.self_attn.indexer.k_norm.bias",392 "model.layers.11.self_attn.indexer.weights_proj",393 "model.layers.11.self_attn.indexer.wk",394 "model.layers.11.self_attn.indexer.wq_b",395 "model.layers.11.self_attn.kv_a_layernorm",396 "model.layers.11.self_attn.kv_b_proj",397 "model.layers.11.self_attn.q_a_layernorm",398 "model.layers.12.hc_attn_base",399 "model.layers.12.hc_attn_fn",400 "model.layers.12.hc_attn_scale",401 "model.layers.12.hc_ffn_base",402 "model.layers.12.hc_ffn_fn",403 "model.layers.12.hc_ffn_scale",404 "model.layers.12.input_layernorm",405 "model.layers.12.mlp.gate",406 "model.layers.12.mlp.gate.e_score_correction_bias",407 "model.layers.12.post_attention_layernorm",408 "model.layers.12.self_attn.A_log",409 "model.layers.12.self_attn.b_proj",410 "model.layers.12.self_attn.dt_bias",411 "model.layers.12.self_attn.f_a_proj",412 "model.layers.12.self_attn.f_b_proj",413 "model.layers.12.self_attn.fused_qkvbfg_a_proj",414 "model.layers.12.self_attn.g_a_proj",415 "model.layers.12.self_attn.g_b_proj",416 "model.layers.12.self_attn.k_conv1d",417 "model.layers.12.self_attn.k_proj",418 "model.layers.12.self_attn.o_norm",419 "model.layers.12.self_attn.o_proj",420 "model.layers.12.self_attn.q_conv1d",421 "model.layers.12.self_attn.q_proj",422 "model.layers.12.self_attn.qkv_proj",423 "model.layers.12.self_attn.v_conv1d",424 "model.layers.12.self_attn.v_proj",425 "model.layers.13.hc_attn_base",426 "model.layers.13.hc_attn_fn",427 "model.layers.13.hc_attn_scale",428 "model.layers.13.hc_ffn_base",429 "model.layers.13.hc_ffn_fn",430 "model.layers.13.hc_ffn_scale",431 "model.layers.13.input_layernorm",432 "model.layers.13.mlp.gate",433 "model.layers.13.mlp.gate.e_score_correction_bias",434 "model.layers.13.post_attention_layernorm",435 "model.layers.13.self_attn.A_log",436 "model.layers.13.self_attn.b_proj",437 "model.layers.13.self_attn.dt_bias",438 "model.layers.13.self_attn.f_a_proj",439 "model.layers.13.self_attn.f_b_proj",440 "model.layers.13.self_attn.fused_qkvbfg_a_proj",441 "model.layers.13.self_attn.g_a_proj",442 "model.layers.13.self_attn.g_b_proj",443 "model.layers.13.self_attn.k_conv1d",444 "model.layers.13.self_attn.k_proj",445 "model.layers.13.self_attn.o_norm",446 "model.layers.13.self_attn.o_proj",447 "model.layers.13.self_attn.q_conv1d",448 "model.layers.13.self_attn.q_proj",449 "model.layers.13.self_attn.qkv_proj",450 "model.layers.13.self_attn.v_conv1d",451 "model.layers.13.self_attn.v_proj",452 "model.layers.14.hc_attn_base",453 "model.layers.14.hc_attn_fn",454 "model.layers.14.hc_attn_scale",455 "model.layers.14.hc_ffn_base",456 "model.layers.14.hc_ffn_fn",457 "model.layers.14.hc_ffn_scale",458 "model.layers.14.input_layernorm",459 "model.layers.14.mlp.gate",460 "model.layers.14.mlp.gate.e_score_correction_bias",461 "model.layers.14.post_attention_layernorm",462 "model.layers.14.self_attn.A_log",463 "model.layers.14.self_attn.b_proj",464 "model.layers.14.self_attn.dt_bias",465 "model.layers.14.self_attn.f_a_proj",466 "model.layers.14.self_attn.f_b_proj",467 "model.layers.14.self_attn.fused_qkvbfg_a_proj",468 "model.layers.14.self_attn.g_a_proj",469 "model.layers.14.self_attn.g_b_proj",470 "model.layers.14.self_attn.k_conv1d",471 "model.layers.14.self_attn.k_proj",472 "model.layers.14.self_attn.o_norm",473 "model.layers.14.self_attn.o_proj",474 "model.layers.14.self_attn.q_conv1d",475 "model.layers.14.self_attn.q_proj",476 "model.layers.14.self_attn.qkv_proj",477 "model.layers.14.self_attn.v_conv1d",478 "model.layers.14.self_attn.v_proj",479 "model.layers.15.hc_attn_base",480 "model.layers.15.hc_attn_fn",481 "model.layers.15.hc_attn_scale",482 "model.layers.15.hc_ffn_base",483 "model.layers.15.hc_ffn_fn",484 "model.layers.15.hc_ffn_scale",485 "model.layers.15.input_layernorm",486 "model.layers.15.mlp.gate",487 "model.layers.15.mlp.gate.e_score_correction_bias",488 "model.layers.15.post_attention_layernorm",489 "model.layers.15.self_attn.indexer.index_kpool_compress_ape",490 "model.layers.15.self_attn.indexer.index_kpool_compress_gate",491 "model.layers.15.self_attn.indexer.k_norm",492 "model.layers.15.self_attn.indexer.k_norm.bias",493 "model.layers.15.self_attn.indexer.weights_proj",494 "model.layers.15.self_attn.indexer.wk",495 "model.layers.15.self_attn.indexer.wq_b",496 "model.layers.15.self_attn.kv_a_layernorm",497 "model.layers.15.self_attn.kv_b_proj",498 "model.layers.15.self_attn.q_a_layernorm",499 "model.layers.16.hc_attn_base",500 "model.layers.16.hc_attn_fn",501 "model.layers.16.hc_attn_scale",502 "model.layers.16.hc_ffn_base",503 "model.layers.16.hc_ffn_fn",504 "model.layers.16.hc_ffn_scale",505 "model.layers.16.input_layernorm",506 "model.layers.16.mlp.gate",507 "model.layers.16.mlp.gate.e_score_correction_bias",508 "model.layers.16.post_attention_layernorm",509 "model.layers.16.self_attn.A_log",510 "model.layers.16.self_attn.b_proj",511 "model.layers.16.self_attn.dt_bias",512 "model.layers.16.self_attn.f_a_proj",513 "model.layers.16.self_attn.f_b_proj",514 "model.layers.16.self_attn.fused_qkvbfg_a_proj",515 "model.layers.16.self_attn.g_a_proj",516 "model.layers.16.self_attn.g_b_proj",517 "model.layers.16.self_attn.k_conv1d",518 "model.layers.16.self_attn.k_proj",519 "model.layers.16.self_attn.o_norm",520 "model.layers.16.self_attn.o_proj",521 "model.layers.16.self_attn.q_conv1d",522 "model.layers.16.self_attn.q_proj",523 "model.layers.16.self_attn.qkv_proj",524 "model.layers.16.self_attn.v_conv1d",525 "model.layers.16.self_attn.v_proj",526 "model.layers.17.hc_attn_base",527 "model.layers.17.hc_attn_fn",528 "model.layers.17.hc_attn_scale",529 "model.layers.17.hc_ffn_base",530 "model.layers.17.hc_ffn_fn",531 "model.layers.17.hc_ffn_scale",532 "model.layers.17.input_layernorm",533 "model.layers.17.mlp.gate",534 "model.layers.17.mlp.gate.e_score_correction_bias",535 "model.layers.17.post_attention_layernorm",536 "model.layers.17.self_attn.A_log",537 "model.layers.17.self_attn.b_proj",538 "model.layers.17.self_attn.dt_bias",539 "model.layers.17.self_attn.f_a_proj",540 "model.layers.17.self_attn.f_b_proj",541 "model.layers.17.self_attn.fused_qkvbfg_a_proj",542 "model.layers.17.self_attn.g_a_proj",543 "model.layers.17.self_attn.g_b_proj",544 "model.layers.17.self_attn.k_conv1d",545 "model.layers.17.self_attn.k_proj",546 "model.layers.17.self_attn.o_norm",547 "model.layers.17.self_attn.o_proj",548 "model.layers.17.self_attn.q_conv1d",549 "model.layers.17.self_attn.q_proj",550 "model.layers.17.self_attn.qkv_proj",551 "model.layers.17.self_attn.v_conv1d",552 "model.layers.17.self_attn.v_proj",553 "model.layers.18.hc_attn_base",554 "model.layers.18.hc_attn_fn",555 "model.layers.18.hc_attn_scale",556 "model.layers.18.hc_ffn_base",557 "model.layers.18.hc_ffn_fn",558 "model.layers.18.hc_ffn_scale",559 "model.layers.18.input_layernorm",560 "model.layers.18.mlp.gate",561 "model.layers.18.mlp.gate.e_score_correction_bias",562 "model.layers.18.post_attention_layernorm",563 "model.layers.18.self_attn.A_log",564 "model.layers.18.self_attn.b_proj",565 "model.layers.18.self_attn.dt_bias",566 "model.layers.18.self_attn.f_a_proj",567 "model.layers.18.self_attn.f_b_proj",568 "model.layers.18.self_attn.fused_qkvbfg_a_proj",569 "model.layers.18.self_attn.g_a_proj",570 "model.layers.18.self_attn.g_b_proj",571 "model.layers.18.self_attn.k_conv1d",572 "model.layers.18.self_attn.k_proj",573 "model.layers.18.self_attn.o_norm",574 "model.layers.18.self_attn.o_proj",575 "model.layers.18.self_attn.q_conv1d",576 "model.layers.18.self_attn.q_proj",577 "model.layers.18.self_attn.qkv_proj",578 "model.layers.18.self_attn.v_conv1d",579 "model.layers.18.self_attn.v_proj",580 "model.layers.19.hc_attn_base",581 "model.layers.19.hc_attn_fn",582 "model.layers.19.hc_attn_scale",583 "model.layers.19.hc_ffn_base",584 "model.layers.19.hc_ffn_fn",585 "model.layers.19.hc_ffn_scale",586 "model.layers.19.input_layernorm",587 "model.layers.19.mlp.gate",588 "model.layers.19.mlp.gate.e_score_correction_bias",589 "model.layers.19.post_attention_layernorm",590 "model.layers.19.self_attn.indexer.index_kpool_compress_ape",591 "model.layers.19.self_attn.indexer.index_kpool_compress_gate",592 "model.layers.19.self_attn.indexer.k_norm",593 "model.layers.19.self_attn.indexer.k_norm.bias",594 "model.layers.19.self_attn.indexer.weights_proj",595 "model.layers.19.self_attn.indexer.wk",596 "model.layers.19.self_attn.indexer.wq_b",597 "model.layers.19.self_attn.kv_a_layernorm",598 "model.layers.19.self_attn.kv_b_proj",599 "model.layers.19.self_attn.q_a_layernorm",600 "model.layers.2.hc_attn_base",601 "model.layers.2.hc_attn_fn",602 "model.layers.2.hc_attn_scale",603 "model.layers.2.hc_ffn_base",604 "model.layers.2.hc_ffn_fn",605 "model.layers.2.hc_ffn_scale",606 "model.layers.2.input_layernorm",607 "model.layers.2.post_attention_layernorm",608 "model.layers.2.self_attn.A_log",609 "model.layers.2.self_attn.b_proj",610 "model.layers.2.self_attn.dt_bias",611 "model.layers.2.self_attn.f_a_proj",612 "model.layers.2.self_attn.f_b_proj",613 "model.layers.2.self_attn.fused_qkvbfg_a_proj",614 "model.layers.2.self_attn.g_a_proj",615 "model.layers.2.self_attn.g_b_proj",616 "model.layers.2.self_attn.k_conv1d",617 "model.layers.2.self_attn.k_proj",618 "model.layers.2.self_attn.o_norm",619 "model.layers.2.self_attn.o_proj",620 "model.layers.2.self_attn.q_conv1d",621 "model.layers.2.self_attn.q_proj",622 "model.layers.2.self_attn.qkv_proj",623 "model.layers.2.self_attn.v_conv1d",624 "model.layers.2.self_attn.v_proj",625 "model.layers.20.hc_attn_base",626 "model.layers.20.hc_attn_fn",627 "model.layers.20.hc_attn_scale",628 "model.layers.20.hc_ffn_base",629 "model.layers.20.hc_ffn_fn",630 "model.layers.20.hc_ffn_scale",631 "model.layers.20.input_layernorm",632 "model.layers.20.mlp.gate",633 "model.layers.20.mlp.gate.e_score_correction_bias",634 "model.layers.20.post_attention_layernorm",635 "model.layers.20.self_attn.A_log",636 "model.layers.20.self_attn.b_proj",637 "model.layers.20.self_attn.dt_bias",638 "model.layers.20.self_attn.f_a_proj",639 "model.layers.20.self_attn.f_b_proj",640 "model.layers.20.self_attn.fused_qkvbfg_a_proj",641 "model.layers.20.self_attn.g_a_proj",642 "model.layers.20.self_attn.g_b_proj",643 "model.layers.20.self_attn.k_conv1d",644 "model.layers.20.self_attn.k_proj",645 "model.layers.20.self_attn.o_norm",646 "model.layers.20.self_attn.o_proj",647 "model.layers.20.self_attn.q_conv1d",648 "model.layers.20.self_attn.q_proj",649 "model.layers.20.self_attn.qkv_proj",650 "model.layers.20.self_attn.v_conv1d",651 "model.layers.20.self_attn.v_proj",652 "model.layers.21.hc_attn_base",653 "model.layers.21.hc_attn_fn",654 "model.layers.21.hc_attn_scale",655 "model.layers.21.hc_ffn_base",656 "model.layers.21.hc_ffn_fn",657 "model.layers.21.hc_ffn_scale",658 "model.layers.21.input_layernorm",659 "model.layers.21.mlp.gate",660 "model.layers.21.mlp.gate.e_score_correction_bias",661 "model.layers.21.post_attention_layernorm",662 "model.layers.21.self_attn.A_log",663 "model.layers.21.self_attn.b_proj",664 "model.layers.21.self_attn.dt_bias",665 "model.layers.21.self_attn.f_a_proj",666 "model.layers.21.self_attn.f_b_proj",667 "model.layers.21.self_attn.fused_qkvbfg_a_proj",668 "model.layers.21.self_attn.g_a_proj",669 "model.layers.21.self_attn.g_b_proj",670 "model.layers.21.self_attn.k_conv1d",671 "model.layers.21.self_attn.k_proj",672 "model.layers.21.self_attn.o_norm",673 "model.layers.21.self_attn.o_proj",674 "model.layers.21.self_attn.q_conv1d",675 "model.layers.21.self_attn.q_proj",676 "model.layers.21.self_attn.qkv_proj",677 "model.layers.21.self_attn.v_conv1d",678 "model.layers.21.self_attn.v_proj",679 "model.layers.22.hc_attn_base",680 "model.layers.22.hc_attn_fn",681 "model.layers.22.hc_attn_scale",682 "model.layers.22.hc_ffn_base",683 "model.layers.22.hc_ffn_fn",684 "model.layers.22.hc_ffn_scale",685 "model.layers.22.input_layernorm",686 "model.layers.22.mlp.gate",687 "model.layers.22.mlp.gate.e_score_correction_bias",688 "model.layers.22.post_attention_layernorm",689 "model.layers.22.self_attn.A_log",690 "model.layers.22.self_attn.b_proj",691 "model.layers.22.self_attn.dt_bias",692 "model.layers.22.self_attn.f_a_proj",693 "model.layers.22.self_attn.f_b_proj",694 "model.layers.22.self_attn.fused_qkvbfg_a_proj",695 "model.layers.22.self_attn.g_a_proj",696 "model.layers.22.self_attn.g_b_proj",697 "model.layers.22.self_attn.k_conv1d",698 "model.layers.22.self_attn.k_proj",699 "model.layers.22.self_attn.o_norm",700 "model.layers.22.self_attn.o_proj",701 "model.layers.22.self_attn.q_conv1d",702 "model.layers.22.self_attn.q_proj",703 "model.layers.22.self_attn.qkv_proj",704 "model.layers.22.self_attn.v_conv1d",705 "model.layers.22.self_attn.v_proj",706 "model.layers.23.hc_attn_base",707 "model.layers.23.hc_attn_fn",708 "model.layers.23.hc_attn_scale",709 "model.layers.23.hc_ffn_base",710 "model.layers.23.hc_ffn_fn",711 "model.layers.23.hc_ffn_scale",712 "model.layers.23.input_layernorm",713 "model.layers.23.mlp.gate",714 "model.layers.23.mlp.gate.e_score_correction_bias",715 "model.layers.23.post_attention_layernorm",716 "model.layers.23.self_attn.indexer.index_kpool_compress_ape",717 "model.layers.23.self_attn.indexer.index_kpool_compress_gate",718 "model.layers.23.self_attn.indexer.k_norm",719 "model.layers.23.self_attn.indexer.k_norm.bias",720 "model.layers.23.self_attn.indexer.weights_proj",721 "model.layers.23.self_attn.indexer.wk",722 "model.layers.23.self_attn.indexer.wq_b",723 "model.layers.23.self_attn.kv_a_layernorm",724 "model.layers.23.self_attn.kv_b_proj",725 "model.layers.23.self_attn.q_a_layernorm",726 "model.layers.24.hc_attn_base",727 "model.layers.24.hc_attn_fn",728 "model.layers.24.hc_attn_scale",729 "model.layers.24.hc_ffn_base",730 "model.layers.24.hc_ffn_fn",731 "model.layers.24.hc_ffn_scale",732 "model.layers.24.input_layernorm",733 "model.layers.24.mlp.gate",734 "model.layers.24.mlp.gate.e_score_correction_bias",735 "model.layers.24.post_attention_layernorm",736 "model.layers.24.self_attn.A_log",737 "model.layers.24.self_attn.b_proj",738 "model.layers.24.self_attn.dt_bias",739 "model.layers.24.self_attn.f_a_proj",740 "model.layers.24.self_attn.f_b_proj",741 "model.layers.24.self_attn.fused_qkvbfg_a_proj",742 "model.layers.24.self_attn.g_a_proj",743 "model.layers.24.self_attn.g_b_proj",744 "model.layers.24.self_attn.k_conv1d",745 "model.layers.24.self_attn.k_proj",746 "model.layers.24.self_attn.o_norm",747 "model.layers.24.self_attn.o_proj",748 "model.layers.24.self_attn.q_conv1d",749 "model.layers.24.self_attn.q_proj",750 "model.layers.24.self_attn.qkv_proj",751 "model.layers.24.self_attn.v_conv1d",752 "model.layers.24.self_attn.v_proj",753 "model.layers.25.hc_attn_base",754 "model.layers.25.hc_attn_fn",755 "model.layers.25.hc_attn_scale",756 "model.layers.25.hc_ffn_base",757 "model.layers.25.hc_ffn_fn",758 "model.layers.25.hc_ffn_scale",759 "model.layers.25.input_layernorm",760 "model.layers.25.mlp.gate",761 "model.layers.25.mlp.gate.e_score_correction_bias",762 "model.layers.25.post_attention_layernorm",763 "model.layers.25.self_attn.A_log",764 "model.layers.25.self_attn.b_proj",765 "model.layers.25.self_attn.dt_bias",766 "model.layers.25.self_attn.f_a_proj",767 "model.layers.25.self_attn.f_b_proj",768 "model.layers.25.self_attn.fused_qkvbfg_a_proj",769 "model.layers.25.self_attn.g_a_proj",770 "model.layers.25.self_attn.g_b_proj",771 "model.layers.25.self_attn.k_conv1d",772 "model.layers.25.self_attn.k_proj",773 "model.layers.25.self_attn.o_norm",774 "model.layers.25.self_attn.o_proj",775 "model.layers.25.self_attn.q_conv1d",776 "model.layers.25.self_attn.q_proj",777 "model.layers.25.self_attn.qkv_proj",778 "model.layers.25.self_attn.v_conv1d",779 "model.layers.25.self_attn.v_proj",780 "model.layers.26.hc_attn_base",781 "model.layers.26.hc_attn_fn",782 "model.layers.26.hc_attn_scale",783 "model.layers.26.hc_ffn_base",784 "model.layers.26.hc_ffn_fn",785 "model.layers.26.hc_ffn_scale",786 "model.layers.26.input_layernorm",787 "model.layers.26.mlp.gate",788 "model.layers.26.mlp.gate.e_score_correction_bias",789 "model.layers.26.post_attention_layernorm",790 "model.layers.26.self_attn.A_log",791 "model.layers.26.self_attn.b_proj",792 "model.layers.26.self_attn.dt_bias",793 "model.layers.26.self_attn.f_a_proj",794 "model.layers.26.self_attn.f_b_proj",795 "model.layers.26.self_attn.fused_qkvbfg_a_proj",796 "model.layers.26.self_attn.g_a_proj",797 "model.layers.26.self_attn.g_b_proj",798 "model.layers.26.self_attn.k_conv1d",799 "model.layers.26.self_attn.k_proj",800 "model.layers.26.self_attn.o_norm",801 "model.layers.26.self_attn.o_proj",802 "model.layers.26.self_attn.q_conv1d",803 "model.layers.26.self_attn.q_proj",804 "model.layers.26.self_attn.qkv_proj",805 "model.layers.26.self_attn.v_conv1d",806 "model.layers.26.self_attn.v_proj",807 "model.layers.27.hc_attn_base",808 "model.layers.27.hc_attn_fn",809 "model.layers.27.hc_attn_scale",810 "model.layers.27.hc_ffn_base",811 "model.layers.27.hc_ffn_fn",812 "model.layers.27.hc_ffn_scale",813 "model.layers.27.input_layernorm",814 "model.layers.27.mlp.gate",815 "model.layers.27.mlp.gate.e_score_correction_bias",816 "model.layers.27.post_attention_layernorm",817 "model.layers.27.self_attn.indexer.index_kpool_compress_ape",818 "model.layers.27.self_attn.indexer.index_kpool_compress_gate",819 "model.layers.27.self_attn.indexer.k_norm",820 "model.layers.27.self_attn.indexer.k_norm.bias",821 "model.layers.27.self_attn.indexer.weights_proj",822 "model.layers.27.self_attn.indexer.wk",823 "model.layers.27.self_attn.indexer.wq_b",824 "model.layers.27.self_attn.kv_a_layernorm",825 "model.layers.27.self_attn.kv_b_proj",826 "model.layers.27.self_attn.q_a_layernorm",827 "model.layers.28.hc_attn_base",828 "model.layers.28.hc_attn_fn",829 "model.layers.28.hc_attn_scale",830 "model.layers.28.hc_ffn_base",831 "model.layers.28.hc_ffn_fn",832 "model.layers.28.hc_ffn_scale",833 "model.layers.28.input_layernorm",834 "model.layers.28.mlp.gate",835 "model.layers.28.mlp.gate.e_score_correction_bias",836 "model.layers.28.post_attention_layernorm",837 "model.layers.28.self_attn.A_log",838 "model.layers.28.self_attn.b_proj",839 "model.layers.28.self_attn.dt_bias",840 "model.layers.28.self_attn.f_a_proj",841 "model.layers.28.self_attn.f_b_proj",842 "model.layers.28.self_attn.fused_qkvbfg_a_proj",843 "model.layers.28.self_attn.g_a_proj",844 "model.layers.28.self_attn.g_b_proj",845 "model.layers.28.self_attn.k_conv1d",846 "model.layers.28.self_attn.k_proj",847 "model.layers.28.self_attn.o_norm",848 "model.layers.28.self_attn.o_proj",849 "model.layers.28.self_attn.q_conv1d",850 "model.layers.28.self_attn.q_proj",851 "model.layers.28.self_attn.qkv_proj",852 "model.layers.28.self_attn.v_conv1d",853 "model.layers.28.self_attn.v_proj",854 "model.layers.29.hc_attn_base",855 "model.layers.29.hc_attn_fn",856 "model.layers.29.hc_attn_scale",857 "model.layers.29.hc_ffn_base",858 "model.layers.29.hc_ffn_fn",859 "model.layers.29.hc_ffn_scale",860 "model.layers.29.input_layernorm",861 "model.layers.29.mlp.gate",862 "model.layers.29.mlp.gate.e_score_correction_bias",863 "model.layers.29.post_attention_layernorm",864 "model.layers.29.self_attn.A_log",865 "model.layers.29.self_attn.b_proj",866 "model.layers.29.self_attn.dt_bias",867 "model.layers.29.self_attn.f_a_proj",868 "model.layers.29.self_attn.f_b_proj",869 "model.layers.29.self_attn.fused_qkvbfg_a_proj",870 "model.layers.29.self_attn.g_a_proj",871 "model.layers.29.self_attn.g_b_proj",872 "model.layers.29.self_attn.k_conv1d",873 "model.layers.29.self_attn.k_proj",874 "model.layers.29.self_attn.o_norm",875 "model.layers.29.self_attn.o_proj",876 "model.layers.29.self_attn.q_conv1d",877 "model.layers.29.self_attn.q_proj",878 "model.layers.29.self_attn.qkv_proj",879 "model.layers.29.self_attn.v_conv1d",880 "model.layers.29.self_attn.v_proj",881 "model.layers.3.hc_attn_base",882 "model.layers.3.hc_attn_fn",883 "model.layers.3.hc_attn_scale",884 "model.layers.3.hc_ffn_base",885 "model.layers.3.hc_ffn_fn",886 "model.layers.3.hc_ffn_scale",887 "model.layers.3.input_layernorm",888 "model.layers.3.mlp.gate",889 "model.layers.3.mlp.gate.e_score_correction_bias",890 "model.layers.3.post_attention_layernorm",891 "model.layers.3.self_attn.indexer.index_kpool_compress_ape",892 "model.layers.3.self_attn.indexer.index_kpool_compress_gate",893 "model.layers.3.self_attn.indexer.k_norm",894 "model.layers.3.self_attn.indexer.k_norm.bias",895 "model.layers.3.self_attn.indexer.weights_proj",896 "model.layers.3.self_attn.indexer.wk",897 "model.layers.3.self_attn.indexer.wq_b",898 "model.layers.3.self_attn.kv_a_layernorm",899 "model.layers.3.self_attn.kv_b_proj",900 "model.layers.3.self_attn.q_a_layernorm",901 "model.layers.30.hc_attn_base",902 "model.layers.30.hc_attn_fn",903 "model.layers.30.hc_attn_scale",904 "model.layers.30.hc_ffn_base",905 "model.layers.30.hc_ffn_fn",906 "model.layers.30.hc_ffn_scale",907 "model.layers.30.input_layernorm",908 "model.layers.30.mlp.gate",909 "model.layers.30.mlp.gate.e_score_correction_bias",910 "model.layers.30.post_attention_layernorm",911 "model.layers.30.self_attn.A_log",912 "model.layers.30.self_attn.b_proj",913 "model.layers.30.self_attn.dt_bias",914 "model.layers.30.self_attn.f_a_proj",915 "model.layers.30.self_attn.f_b_proj",916 "model.layers.30.self_attn.fused_qkvbfg_a_proj",917 "model.layers.30.self_attn.g_a_proj",918 "model.layers.30.self_attn.g_b_proj",919 "model.layers.30.self_attn.k_conv1d",920 "model.layers.30.self_attn.k_proj",921 "model.layers.30.self_attn.o_norm",922 "model.layers.30.self_attn.o_proj",923 "model.layers.30.self_attn.q_conv1d",924 "model.layers.30.self_attn.q_proj",925 "model.layers.30.self_attn.qkv_proj",926 "model.layers.30.self_attn.v_conv1d",927 "model.layers.30.self_attn.v_proj",928 "model.layers.31.hc_attn_base",929 "model.layers.31.hc_attn_fn",930 "model.layers.31.hc_attn_scale",931 "model.layers.31.hc_ffn_base",932 "model.layers.31.hc_ffn_fn",933 "model.layers.31.hc_ffn_scale",934 "model.layers.31.input_layernorm",935 "model.layers.31.mlp.gate",936 "model.layers.31.mlp.gate.e_score_correction_bias",937 "model.layers.31.post_attention_layernorm",938 "model.layers.31.self_attn.indexer.index_kpool_compress_ape",939 "model.layers.31.self_attn.indexer.index_kpool_compress_gate",940 "model.layers.31.self_attn.indexer.k_norm",941 "model.layers.31.self_attn.indexer.k_norm.bias",942 "model.layers.31.self_attn.indexer.weights_proj",943 "model.layers.31.self_attn.indexer.wk",944 "model.layers.31.self_attn.indexer.wq_b",945 "model.layers.31.self_attn.kv_a_layernorm",946 "model.layers.31.self_attn.kv_b_proj",947 "model.layers.31.self_attn.q_a_layernorm",948 "model.layers.32.hc_attn_base",949 "model.layers.32.hc_attn_fn",950 "model.layers.32.hc_attn_scale",951 "model.layers.32.hc_ffn_base",952 "model.layers.32.hc_ffn_fn",953 "model.layers.32.hc_ffn_scale",954 "model.layers.32.input_layernorm",955 "model.layers.32.mlp.gate",956 "model.layers.32.mlp.gate.e_score_correction_bias",957 "model.layers.32.post_attention_layernorm",958 "model.layers.32.self_attn.A_log",959 "model.layers.32.self_attn.b_proj",960 "model.layers.32.self_attn.dt_bias",961 "model.layers.32.self_attn.f_a_proj",962 "model.layers.32.self_attn.f_b_proj",963 "model.layers.32.self_attn.fused_qkvbfg_a_proj",964 "model.layers.32.self_attn.g_a_proj",965 "model.layers.32.self_attn.g_b_proj",966 "model.layers.32.self_attn.k_conv1d",967 "model.layers.32.self_attn.k_proj",968 "model.layers.32.self_attn.o_norm",969 "model.layers.32.self_attn.o_proj",970 "model.layers.32.self_attn.q_conv1d",971 "model.layers.32.self_attn.q_proj",972 "model.layers.32.self_attn.qkv_proj",973 "model.layers.32.self_attn.v_conv1d",974 "model.layers.32.self_attn.v_proj",975 "model.layers.33.hc_attn_base",976 "model.layers.33.hc_attn_fn",977 "model.layers.33.hc_attn_scale",978 "model.layers.33.hc_ffn_base",979 "model.layers.33.hc_ffn_fn",980 "model.layers.33.hc_ffn_scale",981 "model.layers.33.input_layernorm",982 "model.layers.33.mlp.gate",983 "model.layers.33.mlp.gate.e_score_correction_bias",984 "model.layers.33.post_attention_layernorm",985 "model.layers.33.self_attn.A_log",986 "model.layers.33.self_attn.b_proj",987 "model.layers.33.self_attn.dt_bias",988 "model.layers.33.self_attn.f_a_proj",989 "model.layers.33.self_attn.f_b_proj",990 "model.layers.33.self_attn.fused_qkvbfg_a_proj",991 "model.layers.33.self_attn.g_a_proj",992 "model.layers.33.self_attn.g_b_proj",993 "model.layers.33.self_attn.k_conv1d",994 "model.layers.33.self_attn.k_proj",995 "model.layers.33.self_attn.o_norm",996 "model.layers.33.self_attn.o_proj",997 "model.layers.33.self_attn.q_conv1d",998 "model.layers.33.self_attn.q_proj",999 "model.layers.33.self_attn.qkv_proj",1000 "model.layers.33.self_attn.v_conv1d",1001 "model.layers.33.self_attn.v_proj",1002 "model.layers.34.hc_attn_base",1003 "model.layers.34.hc_attn_fn",1004 "model.layers.34.hc_attn_scale",1005 "model.layers.34.hc_ffn_base",1006 "model.layers.34.hc_ffn_fn",1007 "model.layers.34.hc_ffn_scale",1008 "model.layers.34.input_layernorm",1009 "model.layers.34.mlp.gate",1010 "model.layers.34.mlp.gate.e_score_correction_bias",1011 "model.layers.34.post_attention_layernorm",1012 "model.layers.34.self_attn.A_log",1013 "model.layers.34.self_attn.b_proj",1014 "model.layers.34.self_attn.dt_bias",1015 "model.layers.34.self_attn.f_a_proj",1016 "model.layers.34.self_attn.f_b_proj",1017 "model.layers.34.self_attn.fused_qkvbfg_a_proj",1018 "model.layers.34.self_attn.g_a_proj",1019 "model.layers.34.self_attn.g_b_proj",1020 "model.layers.34.self_attn.k_conv1d",1021 "model.layers.34.self_attn.k_proj",1022 "model.layers.34.self_attn.o_norm",1023 "model.layers.34.self_attn.o_proj",1024 "model.layers.34.self_attn.q_conv1d",1025 "model.layers.34.self_attn.q_proj",1026 "model.layers.34.self_attn.qkv_proj",1027 "model.layers.34.self_attn.v_conv1d",1028 "model.layers.34.self_attn.v_proj",1029 "model.layers.35.hc_attn_base",1030 "model.layers.35.hc_attn_fn",1031 "model.layers.35.hc_attn_scale",1032 "model.layers.35.hc_ffn_base",1033 "model.layers.35.hc_ffn_fn",1034 "model.layers.35.hc_ffn_scale",1035 "model.layers.35.input_layernorm",1036 "model.layers.35.mlp.gate",1037 "model.layers.35.mlp.gate.e_score_correction_bias",1038 "model.layers.35.post_attention_layernorm",1039 "model.layers.35.self_attn.indexer.index_kpool_compress_ape",1040 "model.layers.35.self_attn.indexer.index_kpool_compress_gate",1041 "model.layers.35.self_attn.indexer.k_norm",1042 "model.layers.35.self_attn.indexer.k_norm.bias",1043 "model.layers.35.self_attn.indexer.weights_proj",1044 "model.layers.35.self_attn.indexer.wk",1045 "model.layers.35.self_attn.indexer.wq_b",1046 "model.layers.35.self_attn.kv_a_layernorm",1047 "model.layers.35.self_attn.kv_b_proj",1048 "model.layers.35.self_attn.q_a_layernorm",1049 "model.layers.36.hc_attn_base",1050 "model.layers.36.hc_attn_fn",1051 "model.layers.36.hc_attn_scale",1052 "model.layers.36.hc_ffn_base",1053 "model.layers.36.hc_ffn_fn",1054 "model.layers.36.hc_ffn_scale",1055 "model.layers.36.input_layernorm",1056 "model.layers.36.mlp.gate",1057 "model.layers.36.mlp.gate.e_score_correction_bias",1058 "model.layers.36.post_attention_layernorm",1059 "model.layers.36.self_attn.A_log",1060 "model.layers.36.self_attn.b_proj",1061 "model.layers.36.self_attn.dt_bias",1062 "model.layers.36.self_attn.f_a_proj",1063 "model.layers.36.self_attn.f_b_proj",1064 "model.layers.36.self_attn.fused_qkvbfg_a_proj",1065 "model.layers.36.self_attn.g_a_proj",1066 "model.layers.36.self_attn.g_b_proj",1067 "model.layers.36.self_attn.k_conv1d",1068 "model.layers.36.self_attn.k_proj",1069 "model.layers.36.self_attn.o_norm",1070 "model.layers.36.self_attn.o_proj",1071 "model.layers.36.self_attn.q_conv1d",1072 "model.layers.36.self_attn.q_proj",1073 "model.layers.36.self_attn.qkv_proj",1074 "model.layers.36.self_attn.v_conv1d",1075 "model.layers.36.self_attn.v_proj",1076 "model.layers.37.hc_attn_base",1077 "model.layers.37.hc_attn_fn",1078 "model.layers.37.hc_attn_scale",1079 "model.layers.37.hc_ffn_base",1080 "model.layers.37.hc_ffn_fn",1081 "model.layers.37.hc_ffn_scale",1082 "model.layers.37.input_layernorm",1083 "model.layers.37.mlp.gate",1084 "model.layers.37.mlp.gate.e_score_correction_bias",1085 "model.layers.37.post_attention_layernorm",1086 "model.layers.37.self_attn.A_log",1087 "model.layers.37.self_attn.b_proj",1088 "model.layers.37.self_attn.dt_bias",1089 "model.layers.37.self_attn.f_a_proj",1090 "model.layers.37.self_attn.f_b_proj",1091 "model.layers.37.self_attn.fused_qkvbfg_a_proj",1092 "model.layers.37.self_attn.g_a_proj",1093 "model.layers.37.self_attn.g_b_proj",1094 "model.layers.37.self_attn.k_conv1d",1095 "model.layers.37.self_attn.k_proj",1096 "model.layers.37.self_attn.o_norm",1097 "model.layers.37.self_attn.o_proj",1098 "model.layers.37.self_attn.q_conv1d",1099 "model.layers.37.self_attn.q_proj",1100 "model.layers.37.self_attn.qkv_proj",1101 "model.layers.37.self_attn.v_conv1d",1102 "model.layers.37.self_attn.v_proj",1103 "model.layers.38.hc_attn_base",1104 "model.layers.38.hc_attn_fn",1105 "model.layers.38.hc_attn_scale",1106 "model.layers.38.hc_ffn_base",1107 "model.layers.38.hc_ffn_fn",1108 "model.layers.38.hc_ffn_scale",1109 "model.layers.38.input_layernorm",1110 "model.layers.38.mlp.gate",1111 "model.layers.38.mlp.gate.e_score_correction_bias",1112 "model.layers.38.post_attention_layernorm",1113 "model.layers.38.self_attn.A_log",1114 "model.layers.38.self_attn.b_proj",1115 "model.layers.38.self_attn.dt_bias",1116 "model.layers.38.self_attn.f_a_proj",1117 "model.layers.38.self_attn.f_b_proj",1118 "model.layers.38.self_attn.fused_qkvbfg_a_proj",1119 "model.layers.38.self_attn.g_a_proj",1120 "model.layers.38.self_attn.g_b_proj",1121 "model.layers.38.self_attn.k_conv1d",1122 "model.layers.38.self_attn.k_proj",1123 "model.layers.38.self_attn.o_norm",1124 "model.layers.38.self_attn.o_proj",1125 "model.layers.38.self_attn.q_conv1d",1126 "model.layers.38.self_attn.q_proj",1127 "model.layers.38.self_attn.qkv_proj",1128 "model.layers.38.self_attn.v_conv1d",1129 "model.layers.38.self_attn.v_proj",1130 "model.layers.39.hc_attn_base",1131 "model.layers.39.hc_attn_fn",1132 "model.layers.39.hc_attn_scale",1133 "model.layers.39.hc_ffn_base",1134 "model.layers.39.hc_ffn_fn",1135 "model.layers.39.hc_ffn_scale",1136 "model.layers.39.input_layernorm",1137 "model.layers.39.mlp.gate",1138 "model.layers.39.mlp.gate.e_score_correction_bias",1139 "model.layers.39.post_attention_layernorm",1140 "model.layers.39.self_attn.indexer.index_kpool_compress_ape",1141 "model.layers.39.self_attn.indexer.index_kpool_compress_gate",1142 "model.layers.39.self_attn.indexer.k_norm",1143 "model.layers.39.self_attn.indexer.k_norm.bias",1144 "model.layers.39.self_attn.indexer.weights_proj",1145 "model.layers.39.self_attn.indexer.wk",1146 "model.layers.39.self_attn.indexer.wq_b",1147 "model.layers.39.self_attn.kv_a_layernorm",1148 "model.layers.39.self_attn.kv_b_proj",1149 "model.layers.39.self_attn.q_a_layernorm",1150 "model.layers.4.hc_attn_base",1151 "model.layers.4.hc_attn_fn",1152 "model.layers.4.hc_attn_scale",1153 "model.layers.4.hc_ffn_base",1154 "model.layers.4.hc_ffn_fn",1155 "model.layers.4.hc_ffn_scale",1156 "model.layers.4.input_layernorm",1157 "model.layers.4.mlp.gate",1158 "model.layers.4.mlp.gate.e_score_correction_bias",1159 "model.layers.4.post_attention_layernorm",1160 "model.layers.4.self_attn.A_log",1161 "model.layers.4.self_attn.b_proj",1162 "model.layers.4.self_attn.dt_bias",1163 "model.layers.4.self_attn.f_a_proj",1164 "model.layers.4.self_attn.f_b_proj",1165 "model.layers.4.self_attn.fused_qkvbfg_a_proj",1166 "model.layers.4.self_attn.g_a_proj",1167 "model.layers.4.self_attn.g_b_proj",1168 "model.layers.4.self_attn.k_conv1d",1169 "model.layers.4.self_attn.k_proj",1170 "model.layers.4.self_attn.o_norm",1171 "model.layers.4.self_attn.o_proj",1172 "model.layers.4.self_attn.q_conv1d",1173 "model.layers.4.self_attn.q_proj",1174 "model.layers.4.self_attn.qkv_proj",1175 "model.layers.4.self_attn.v_conv1d",1176 "model.layers.4.self_attn.v_proj",1177 "model.layers.40.hc_attn_base",1178 "model.layers.40.hc_attn_fn",1179 "model.layers.40.hc_attn_scale",1180 "model.layers.40.hc_ffn_base",1181 "model.layers.40.hc_ffn_fn",1182 "model.layers.40.hc_ffn_scale",1183 "model.layers.40.input_layernorm",1184 "model.layers.40.mlp.gate",1185 "model.layers.40.mlp.gate.e_score_correction_bias",1186 "model.layers.40.post_attention_layernorm",1187 "model.layers.40.self_attn.A_log",1188 "model.layers.40.self_attn.b_proj",1189 "model.layers.40.self_attn.dt_bias",1190 "model.layers.40.self_attn.f_a_proj",1191 "model.layers.40.self_attn.f_b_proj",1192 "model.layers.40.self_attn.fused_qkvbfg_a_proj",1193 "model.layers.40.self_attn.g_a_proj",1194 "model.layers.40.self_attn.g_b_proj",1195 "model.layers.40.self_attn.k_conv1d",1196 "model.layers.40.self_attn.k_proj",1197 "model.layers.40.self_attn.o_norm",1198 "model.layers.40.self_attn.o_proj",1199 "model.layers.40.self_attn.q_conv1d",1200 "model.layers.40.self_attn.q_proj",