echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0755
1from __future__ import annotations2 3from typing import Sequence4 5from .constants import MODEL_ARCH, MODEL_TENSOR, MODEL_TENSORS, TENSOR_NAMES6 7 8class TensorNameMap:9 mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {10 # Token embeddings11 MODEL_TENSOR.TOKEN_EMBD: (12 "gpt_neox.embed_in", # gptneox13 "transformer.wte", # gpt2 gpt-j mpt refact qwen dbrx jais exaone14 "transformer.word_embeddings", # falcon15 "word_embeddings", # bloom16 "model.embed_tokens", # llama-hf nemotron olmoe olmo2 rwkv6qwen2 glm4-0414 plamo2 granite-hybrid17 "embed_tokens", # embeddinggemma18 "tok_embeddings", # llama-pth19 "embeddings.word_embeddings", # bert nomic-bert20 "embeddings.tok_embeddings", # modern-bert21 "language_model.embedding.word_embeddings", # persimmon22 "wte", # gpt223 "transformer.embd.wte", # phi224 "model.tok_embeddings", # internlm225 "model.embedding", # mamba-qbert26 "backbone.embedding", # mamba27 "backbone.embeddings", # mamba-hf28 "transformer.in_out_embed", # Grok29 "embedding.word_embeddings", # chatglm30 "transformer.token_embeddings", # openelm31 "shared", # t532 "rwkv.embeddings", # rwkv633 "model.embeddings", # rwkv734 "model.word_embeddings", # bailingmoe35 "language_model.model.embed_tokens", # llama436 "encoder", # neobert37 "model.transformer.wte", # llada38 "embed_tokens", # qwen3-embedding39 ),40 41 # Token type embeddings42 MODEL_TENSOR.TOKEN_TYPES: (43 "embeddings.token_type_embeddings", # bert nomic-bert44 ),45 46 # Normalization of token embeddings47 MODEL_TENSOR.TOKEN_EMBD_NORM: (48 "word_embeddings_layernorm", # bloom49 "embeddings.LayerNorm", # bert50 "embeddings.norm", # modern-bert51 "emb_ln", # nomic-bert52 "transformer.norm", # openelm53 "rwkv.blocks.0.pre_ln", # rwkv54 "rwkv.blocks.0.pre_ln", # rwkv655 "model.pre_ln", # rwkv756 "model.layers.0.pre_norm", # rwkv757 "backbone.norm", # wavtokenizer58 "model.embedding_norm", # lfm259 ),60 61 # Position embeddings62 MODEL_TENSOR.POS_EMBD: (63 "transformer.wpe", # gpt264 "embeddings.position_embeddings", # bert65 "wpe", # gpt266 "model.embed_positions", # rugpt3xl67 ),68 69 # Output70 MODEL_TENSOR.OUTPUT: (71 "embed_out", # gptneox72 "lm_head", # gpt2 mpt falcon llama-hf baichuan qwen mamba dbrx jais nemotron exaone olmoe olmo2 phimoe plamo273 "output", # llama-pth bloom internlm274 "word_embeddings_for_head", # persimmon75 "lm_head.linear", # phi276 "output_layer", # chatglm77 "head", # rwkv78 "head.out", # wavtokenizer79 "lm_head", # llama480 "model.transformer.ff_out", # llada81 "head.decoder", # modern-bert82 ),83 MODEL_TENSOR.DENSE_2_OUT: (84 "dense_2_out", # embeddinggemma85 ),86 MODEL_TENSOR.DENSE_3_OUT: (87 "dense_3_out", # embeddinggemma88 ),89 # Output norm90 MODEL_TENSOR.OUTPUT_NORM: (91 "gpt_neox.final_layer_norm", # gptneox92 "transformer.ln_f", # gpt2 gpt-j falcon jais exaone93 "model.norm", # llama-hf baichuan internlm2 olmoe olmo2 phimoe plamo294 "norm", # llama-pth95 "transformer.norm_f", # mpt dbrx96 "ln_f", # refact bloom qwen gpt297 "language_model.encoder.final_layernorm", # persimmon98 "model.final_layernorm", # persimmon99 "lm_head.ln", # phi2100 "model.norm_f", # mamba-qbert101 "backbone.norm_f", # mamba102 "transformer.rms_norm", # Grok103 "encoder.final_layernorm", # chatglm104 "transformer.norm", # openelm105 "model.norm", # nemotron106 "rwkv.ln_out", # rwkv6107 "model.ln_out", # rwkv7108 "backbone.final_layer_norm", # wavtokenizer109 "model.norm", # llama4110 "model.transformer.ln_f", # llada111 "final_norm", # modern-bert112 "model.norm", # cogvlm113 ),114 115 # Rope frequencies116 MODEL_TENSOR.ROPE_FREQS: (117 "rope.freqs", # llama-pth118 "rotary_pos_emb.inv_freq", # chatglm119 ),120 121 MODEL_TENSOR.ROPE_FACTORS_LONG: (),122 MODEL_TENSOR.ROPE_FACTORS_SHORT: (),123 124 MODEL_TENSOR.CONV1D: (125 "backbone.embed", # roberta126 ),127 128 MODEL_TENSOR.V_MM_EMBEDDING: (129 "model.embed_vision.embedding", # gemma3n130 ),131 MODEL_TENSOR.V_MM_HARD_EMB_NORM: (132 "model.embed_vision.hard_embedding_norm", # gemma3n133 ),134 MODEL_TENSOR.V_MM_INP_PROJ: (135 "model.embed_vision.embedding_projection", # gemma3n136 ),137 MODEL_TENSOR.V_MM_SOFT_EMB_NORM: (138 "model.embed_vision.soft_embedding_norm", # gemma3n139 ),140 MODEL_TENSOR.V_ENC_CONV_STEM: (141 "model.vision_tower.timm_model.conv_stem.conv", # gemma3n142 ),143 MODEL_TENSOR.V_ENC_CONV_STEM_NORM: (144 "model.vision_tower.timm_model.conv_stem.bn", # gemma3n145 ),146 MODEL_TENSOR.V_ENC_MSFA_EXP: (147 "model.vision_tower.timm_model.msfa.ffn.pw_exp.conv", # gemma3n148 ),149 MODEL_TENSOR.V_ENC_MSFA_EXP_NORM: (150 "model.vision_tower.timm_model.msfa.ffn.pw_exp.bn", # gemma3n151 ),152 MODEL_TENSOR.V_ENC_MSFA_PROJ: (153 "model.vision_tower.timm_model.msfa.ffn.pw_proj.conv", # gemma3n154 ),155 MODEL_TENSOR.V_ENC_MSFA_PROJ_NORM: (156 "model.vision_tower.timm_model.msfa.ffn.pw_proj.bn", # gemma3n157 ),158 MODEL_TENSOR.V_ENC_MSFA_NORM: (159 "model.vision_tower.timm_model.msfa.norm", # gemma3n160 ),161 }162 163 block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {164 # Attention norm165 MODEL_TENSOR.ATTN_NORM: (166 "gpt_neox.layers.{bid}.input_layernorm", # gptneox167 "transformer.h.{bid}.ln_1", # gpt2 gpt-j refact qwen jais exaone168 "transformer.blocks.{bid}.norm_1", # mpt169 "transformer.h.{bid}.input_layernorm", # falcon7b170 "h.{bid}.input_layernorm", # bloom171 "transformer.h.{bid}.ln_mlp", # falcon40b172 "model.layers.{bid}.input_layernorm", # llama-hf nemotron olmoe phimoe granite-hybrid173 "layers.{bid}.attention_norm", # llama-pth174 "language_model.encoder.layers.{bid}.input_layernorm", # persimmon175 "model.layers.{bid}.ln1", # yi176 "h.{bid}.ln_1", # gpt2177 "transformer.h.{bid}.ln", # phi2178 "model.layers.layers.{bid}.norm", # plamo179 "model.layers.layers.{bid}.pre_mixer_norm", # plamo2180 "model.layers.{bid}.attention_norm", # internlm2181 "model.layers.{bid}.norm", # mamba-qbert182 "backbone.layers.{bid}.norm", # mamba183 "transformer.decoder_layer.{bid}.rms_norm", # Grok184 "model.layers.{bid}.pre_attn_norm", # grok-2185 "transformer.blocks.{bid}.norm_attn_norm.norm_1", # dbrx186 "encoder.layers.{bid}.input_layernorm", # chatglm187 "transformer.layers.{bid}.attn_norm", # openelm188 "rwkv.blocks.{bid}.ln1", # rwkv6189 "model.layers.{bid}.ln1", # rwkv7190 "model.layers.{bid}.input_layernorm", # llama4191 "layers.{bid}.input_layernorm", # embeddinggemma192 "transformer_encoder.{bid}.attention_norm", # neobert193 "layers.{bid}.attn_norm", # modern-bert194 "model.layers.{bid}.operator_norm", # lfm2195 "model.transformer.blocks.{bid}.attn_norm", # llada196 "layers.{bid}.input_layernorm", # qwen3-embedding197 "model.layers.{bid}.attention_layernorm", # apertus198 "model.layers.{bid}.pre_attention_layernorm", # kormo199 ),200 201 # Attention norm 2202 MODEL_TENSOR.ATTN_NORM_2: (203 "transformer.h.{bid}.ln_attn", # falcon40b204 "encoder.layer.{bid}.layer_norm_1", # jina-v2-code205 "rwkv.blocks.{bid}.ln2", # rwkv6206 "model.layers.{bid}.ln2", # rwkv7207 "model.layers.{bid}.post_attention_layernorm", # cogvlm208 ),209 210 # Attention query-key-value211 MODEL_TENSOR.ATTN_QKV: (212 "gpt_neox.layers.{bid}.attention.query_key_value", # gptneox213 "transformer.h.{bid}.attn.c_attn", # gpt2 qwen jais214 "transformer.blocks.{bid}.attn.Wqkv", # mpt215 "transformer.blocks.{bid}.norm_attn_norm.attn.Wqkv", # dbrx216 "transformer.h.{bid}.self_attention.query_key_value", # falcon217 "h.{bid}.self_attention.query_key_value", # bloom218 "language_model.encoder.layers.{bid}.self_attention.query_key_value", # persimmon219 "model.layers.{bid}.self_attn.query_key_value", # persimmon220 "model.layers.{bid}.attention.query_key_value", # bailingmoe2221 "h.{bid}.attn.c_attn", # gpt2222 "transformer.h.{bid}.mixer.Wqkv", # phi2223 "encoder.layers.{bid}.attn.Wqkv", # nomic-bert224 "encoder.layers.{bid}.mixer.Wqkv", # jina225 "model.layers.{bid}.self_attn.qkv_proj", # phi3226 "model.layers.layers.{bid}.mixer.qkv_proj", # plamo2227 "encoder.layers.{bid}.self_attention.query_key_value", # chatglm228 "transformer.layers.{bid}.attn.qkv_proj", # openelm229 "transformer_encoder.{bid}.qkv", # neobert230 "layers.{bid}.attn.Wqkv", # modern-bert231 "model.layers.{bid}.self_attn.language_expert_query_key_value", # cogvlm232 "model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5233 ),234 235 # Attention query236 MODEL_TENSOR.ATTN_Q: (237 "model.layers.{bid}.self_attn.q_proj", # llama-hf nemotron olmoe olmo2 phimoe238 "layers.{bid}.self_attn.q_proj", # embeddinggemma239 "model.layers.{bid}.self_attn.q_proj_no_perm", # llama-custom240 "layers.{bid}.attention.wq", # llama-pth241 "encoder.layer.{bid}.attention.self.query", # bert242 "transformer.layer.{bid}.attention.q_lin", # distillbert243 "transformer.h.{bid}.attn.q_proj", # gpt-j244 "model.layers.layers.{bid}.self_attn.q_proj", # plamo245 "model.layers.{bid}.attention.wq", # internlm2246 "transformer.decoder_layer.{bid}.multi_head_attention.query",# Grok247 "transformer.h.{bid}.attn.attention.q_proj", # exaone248 "model.layers.{bid}.self_attn.q_proj", # llama4249 "model.transformer.blocks.{bid}.q_proj", # llada250 "layers.{bid}.self_attn.q_proj", # qwen3-embedding251 "backbone.layers.{bid}.mixer.q_proj", # nemotron-h252 ),253 254 # Attention key255 MODEL_TENSOR.ATTN_K: (256 "model.layers.{bid}.self_attn.k_proj", # llama-hf nemotron olmoe olmo2 phimoe257 "layers.{bid}.self_attn.k_proj", # embeddinggemma258 "model.layers.{bid}.self_attn.k_proj_no_perm", # llama-custom259 "layers.{bid}.attention.wk", # llama-pth260 "encoder.layer.{bid}.attention.self.key", # bert261 "transformer.layer.{bid}.attention.k_lin", # distillbert262 "transformer.h.{bid}.attn.k_proj", # gpt-j263 "transformer.h.{bid}.attn.k", # refact264 "model.layers.layers.{bid}.self_attn.k_proj", # plamo265 "model.layers.{bid}.attention.wk", # internlm2266 "transformer.decoder_layer.{bid}.multi_head_attention.key",# Grok267 "transformer.h.{bid}.attn.attention.k_proj", # exaone268 "model.layers.{bid}.self_attn.k_proj", # llama4269 "model.transformer.blocks.{bid}.k_proj", # llada270 "layers.{bid}.self_attn.k_proj", # qwen3-embedding271 "backbone.layers.{bid}.mixer.k_proj", # nemotron-h272 ),273 274 # Attention value275 MODEL_TENSOR.ATTN_V: (276 "model.layers.{bid}.self_attn.v_proj", # llama-hf nemotron olmoe olmo2 phimoe277 "layers.{bid}.self_attn.v_proj", # embeddinggemma278 "layers.{bid}.attention.wv", # llama-pth279 "encoder.layer.{bid}.attention.self.value", # bert280 "transformer.layer.{bid}.attention.v_lin", # distillbert281 "transformer.h.{bid}.attn.v_proj", # gpt-j282 "transformer.h.{bid}.attn.v", # refact283 "model.layers.layers.{bid}.self_attn.v_proj", # plamo284 "model.layers.{bid}.attention.wv", # internlm2285 "transformer.decoder_layer.{bid}.multi_head_attention.value",# Grok286 "transformer.h.{bid}.attn.attention.v_proj", # exaone287 "model.layers.{bid}.self_attn.v_proj", # llama4288 "model.transformer.blocks.{bid}.v_proj", # llada289 "layers.{bid}.self_attn.v_proj", # qwen3-embedding290 "backbone.layers.{bid}.mixer.v_proj", # nemotron-h291 ),292 293 # Attention output294 MODEL_TENSOR.ATTN_OUT: (295 "gpt_neox.layers.{bid}.attention.dense", # gptneox296 "transformer.h.{bid}.attn.c_proj", # gpt2 refact qwen jais297 "transformer.blocks.{bid}.attn.out_proj", # mpt298 "transformer.h.{bid}.self_attention.dense", # falcon299 "h.{bid}.self_attention.dense", # bloom300 "model.layers.{bid}.self_attn.o_proj", # llama-hf nemotron olmoe olmo2 phimoe301 "layers.{bid}.self_attn.o_proj", # embeddinggemma302 "model.layers.{bid}.self_attn.out_proj", # lfm2303 "model.layers.{bid}.self_attn.linear_attn", # deci304 "layers.{bid}.attention.wo", # llama-pth305 "encoder.layer.{bid}.attention.output.dense", # bert306 "layers.{bid}.attn.Wo", # modern-bert307 "transformer.layer.{bid}.attention.out_lin", # distillbert308 "transformer.h.{bid}.attn.out_proj", # gpt-j309 "language_model.encoder.layers.{bid}.self_attention.dense", # persimmon310 "model.layers.{bid}.self_attn.dense", # persimmon311 "model.layers.{bid}.attention.dense", # bailingmoe2312 "h.{bid}.attn.c_proj", # gpt2313 "transformer.h.{bid}.mixer.out_proj", # phi2314 "model.layers.layers.{bid}.self_attn.o_proj", # plamo315 "model.layers.layers.{bid}.mixer.o_proj", # plamo2316 "model.layers.{bid}.attention.wo", # internlm2317 "encoder.layers.{bid}.attn.out_proj", # nomic-bert318 "encoder.layers.{bid}.mixer.out_proj", # jina319 "transformer.decoder_layer.{bid}.multi_head_attention.linear", # Grok320 "transformer.blocks.{bid}.norm_attn_norm.attn.out_proj", # dbrx321 "encoder.layers.{bid}.self_attention.dense", # chatglm322 "transformer.layers.{bid}.attn.out_proj", # openelm323 "transformer.h.{bid}.attn.attention.out_proj", # exaone324 "model.layers.{bid}.self_attn.o_proj", # llama4325 "transformer_encoder.{bid}.wo", # neobert326 "model.transformer.blocks.{bid}.attn_out", # llada327 "layers.{bid}.self_attn.o_proj", # qwen3-embedding328 "backbone.layers.{bid}.mixer.o_proj", # nemotron-h329 "model.layers.{bid}.self_attn.language_expert_dense", # cogvlm330 ),331 332 # Attention output norm333 MODEL_TENSOR.ATTN_OUT_NORM: (334 "encoder.layer.{bid}.attention.output.LayerNorm", # bert335 "transformer.layer.{bid}.sa_layer_norm", # distillbert336 "encoder.layers.{bid}.norm1", # nomic-bert337 "transformer.decoder_layer.{bid}.rms_norm_1", # Grok338 "model.layers.{bid}.post_attn_norm", # grok-2339 "transformer.blocks.{bid}.norm_attn_norm.norm_2", # dbrx340 ),341 342 MODEL_TENSOR.ATTN_POST_NORM: (343 "model.layers.{bid}.post_attention_layernorm", # gemma2 olmo2 # ge344 "layers.{bid}.post_attention_layernorm", # embeddinggemma345 "model.layers.{bid}.post_self_attn_layernorm", # glm-4-0414346 "model.layers.layers.{bid}.post_mixer_norm.weight", # plamo2347 ),348 349 # Rotary embeddings350 MODEL_TENSOR.ATTN_ROT_EMBD: (351 "model.layers.{bid}.self_attn.rotary_emb.inv_freq", # llama-hf352 "layers.{bid}.attention.inner_attention.rope.freqs", # llama-pth353 "model.layers.layers.{bid}.self_attn.rotary_emb.inv_freq", # plamo354 "transformer.h.{bid}.attn.rotary_emb.inv_freq", # codeshell355 ),356 357 MODEL_TENSOR.ATTN_SINKS: (358 "model.layers.{bid}.self_attn.sinks", # openai-moe359 "model.layers.{bid}.self_attn.attention_sink_bias", # mimov2360 ),361 362 MODEL_TENSOR.ATTN_GATE: (363 "model.layers.{bid}.self_attn.gate_proj", # afmoe364 "model.layers.{bid}.linear_attn.in_proj_z", # qwen3.5365 "model.layers.{bid}.self_attn.g_proj", # step3.5 head-wise attention gate366 ),367 368 # Feed-forward norm369 MODEL_TENSOR.FFN_NORM: (370 "gpt_neox.layers.{bid}.post_attention_layernorm", # gptneox371 "transformer.h.{bid}.ln_2", # gpt2 refact qwen jais exaone372 "h.{bid}.post_attention_layernorm", # bloom373 "transformer.blocks.{bid}.norm_2", # mpt374 "model.layers.{bid}.post_attention_layernorm", # llama-hf nemotron olmoe phimoe375 "layers.{bid}.ffn_norm", # llama-pth376 "language_model.encoder.layers.{bid}.post_attention_layernorm", # persimmon377 "model.layers.{bid}.ln2", # yi378 "h.{bid}.ln_2", # gpt2379 "model.layers.{bid}.ffn_norm", # internlm2380 "transformer.decoder_layer.{bid}.rms_norm_2", # Grok381 "model.layers.{bid}.pre_moe_norm", # grok-2382 "encoder.layers.{bid}.post_attention_layernorm", # chatglm383 "transformer.layers.{bid}.ffn_norm", # openelm384 "model.layers.{bid}.pre_ff_layernorm", # jamba granite-hybrid385 "model.layers.{bid}.pre_moe_layernorm", # mini-jamba386 "model.layers.{bid}.post_attention_layernorm", # llama4387 "transformer_encoder.{bid}.ffn_norm", # neobert388 "model.layers.layers.{bid}.pre_mlp_norm", # plamo2389 "model.transformer.blocks.{bid}.ff_norm", # llada390 "layers.{bid}.post_attention_layernorm", # qwen3-embedding391 "model.layers.{bid}.feedforward_layernorm", # apertus392 "model.layers.{bid}.pre_mlp_layernorm", # kormo393 "layers.{bid}.mlp_norm" # modern-bert394 ),395 396 # Pre feed-forward norm397 MODEL_TENSOR.FFN_PRE_NORM: (398 "model.layers.{bid}.pre_feedforward_layernorm", # gemma2399 "layers.{bid}.pre_feedforward_layernorm", # embeddinggemma400 "model.layers.{bid}.pre_ff_layernorm.weight",401 "model.layers.{bid}.pre_mlp_layernorm", # afmoe402 ),403 404 MODEL_TENSOR.FFN_PRE_NORM_2: (405 "model.layers.{bid}.pre_feedforward_layernorm_2", # gemma4406 ),407 408 # Post feed-forward norm409 MODEL_TENSOR.FFN_POST_NORM: (410 "model.layers.{bid}.post_feedforward_layernorm", # gemma2 olmo2411 "layers.{bid}.post_feedforward_layernorm", # embeddinggemma412 "model.layers.{bid}.post_mlp_layernorm", # glm-4-0414413 "model.layers.layers.{bid}.post_mlp_norm.weight", # plamo2414 "model.layers.{bid}.feed_forward.up_proj",415 "model.layers.{bid}.post_moe_norm", # grok-2416 ),417 418 MODEL_TENSOR.FFN_POST_NORM_1: (419 "model.layers.{bid}.post_feedforward_layernorm_1", # gemma4420 ),421 422 MODEL_TENSOR.FFN_POST_NORM_2: (423 "model.layers.{bid}.post_feedforward_layernorm_2", # gemma4424 ),425 426 MODEL_TENSOR.FFN_GATE_INP: (427 "layers.{bid}.feed_forward.gate", # mixtral428 "model.layers.{bid}.block_sparse_moe.gate", # mixtral phimoe429 "model.layers.{bid}.mlp.gate", # qwen2moe olmoe430 "transformer.decoder_layer.{bid}.router", # Grok431 "transformer.blocks.{bid}.ffn.router.layer", # dbrx432 "model.layers.{bid}.block_sparse_moe.router.layer", # granitemoe433 "model.layers.{bid}.feed_forward.router", # llama4 jamba434 "encoder.layers.{bid}.mlp.router.layer", # nomic-bert-moe435 "model.layers.{bid}.mlp.router", # openai-moe436 "model.layers.{bid}.mlp.gate.wg", # hunyuan437 "model.layers.{bid}.block_sparse_moe.primary_router", # smallthinker438 "model.layers.{bid}.feed_forward.gate", # lfm2moe439 "model.layers.{bid}.mlp.router.gate", # afmoe440 "layers.{bid}.gate", # mistral-large441 "backbone.layers.{bid}.mixer.gate", # nemotron-h-moe442 "model.layers.{bid}.moe.gate", # step3.5443 "model.layers.{bid}.router.proj", # gemma4444 ),445 446 MODEL_TENSOR.FFN_GATE_INP_SHEXP: (447 "model.layers.{bid}.mlp.shared_expert_gate", # qwen2moe448 ),449 450 MODEL_TENSOR.FFN_EXP_PROBS_B: (451 "model.layers.{bid}.mlp.gate.e_score_correction", # deepseek-v3 dots1452 "model.layers.{bid}.mlp.moe_statics.e_score_correction", # ernie4.5-moe453 "model.layers.{bid}.mlp.gate.expert_bias", # bailingmoe2454 "model.layers.{bid}.mlp.expert_bias", # afmoe455 "model.layers.{bid}.feed_forward.expert_bias", # lfm2moe456 "model.layers.{bid}.block_sparse_moe.e_score_correction", # minimax-m2457 "backbone.layers.{bid}.mixer.gate.e_score_correction", # nemotron-h-moe458 "model.layers.{bid}.mlp.e_score_correction", # exaone-moe459 "model.layers.{bid}.block_sparse_moe.gate.e_score_correction", # kimi460 "model.layers.{bid}.moe.router_bias", # step3.5 expert selection bias461 ),462 463 # Feed-forward up464 MODEL_TENSOR.FFN_UP: (465 "gpt_neox.layers.{bid}.mlp.dense_h_to_4h", # gptneox466 "transformer.h.{bid}.mlp.c_fc", # gpt2 jais467 "transformer.blocks.{bid}.ffn.up_proj", # mpt468 "transformer.h.{bid}.mlp.dense_h_to_4h", # falcon469 "h.{bid}.mlp.dense_h_to_4h", # bloom470 "model.layers.{bid}.mlp.up_proj", # llama-hf refact nemotron olmo2471 "layers.{bid}.mlp.up_proj", # embeddinggemma472 "layers.{bid}.feed_forward.w3", # llama-pth473 "encoder.layer.{bid}.intermediate.dense", # bert474 "layers.{bid}.mlp.Wi", # modern-bert475 "transformer.layer.{bid}.ffn.lin1", # distillbert476 "transformer.h.{bid}.mlp.fc_in", # gpt-j477 "transformer.h.{bid}.mlp.linear_3", # refact478 "language_model.encoder.layers.{bid}.mlp.dense_h_to_4h", # persimmon479 "model.layers.{bid}.mlp.dense_h_to_4h", # persimmon480 "transformer.h.{bid}.mlp.w1", # qwen481 "h.{bid}.mlp.c_fc", # gpt2482 "transformer.h.{bid}.mlp.fc1", # phi2483 "model.layers.{bid}.mlp.fc1", # phi2484 "model.layers.{bid}.mlp.gate_up_proj", # phi3 glm-4-0414485 "model.layers.layers.{bid}.mlp.up_proj", # plamo486 "model.layers.layers.{bid}.mlp.gate_up_proj", # plamo2487 "model.layers.{bid}.feed_forward.w3", # internlm2488 "encoder.layers.{bid}.mlp.fc11", # nomic-bert489 "encoder.layers.{bid}.mlp.fc1", # nomic-bert-moe490 "model.layers.{bid}.mlp.c_fc", # starcoder2491 "encoder.layer.{bid}.mlp.gated_layers_v", # jina-bert-v2 (split up/gate, no longer used)492 "encoder.layer.{bid}.mlp.gated_layers", # jina-bert-v2 (GEGLU)493 "encoder.layer.{bid}.mlp.up_gated_layer", # jina-v2-code (GEGLU)494 "model.layers.{bid}.residual_mlp.w3", # arctic495 "encoder.layers.{bid}.mlp.dense_h_to_4h", # chatglm496 "transformer.h.{bid}.mlp.c_fc_1", # exaone497 "model.layers.{bid}.feed_forward.up_proj", # llama4 jamba granite-hybrid498 "transformer_encoder.{bid}.ffn.w12", # neobert499 "model.layers.{bid}.block_sparse_moe.up", # smallthinker500 "model.transformer.blocks.{bid}.up_proj", # llada501 "layers.{bid}.mlp.up_proj", # qwen3-embedding502 "backbone.layers.{bid}.mixer.up_proj", # nemotron-h503 "model.layers.{bid}.mlp.language_mlp.up_proj", # cogvlm504 ),505 506 MODEL_TENSOR.FFN_UP_EXP: (507 "layers.{bid}.feed_forward.experts.w3", # mixtral (merged)508 "transformer.decoder_layer.{bid}.moe.linear_v", # Grok (merged)509 "transformer.blocks.{bid}.ffn.experts.mlp.v1", # dbrx510 "model.layers.{bid}.mlp.experts.up_proj", # qwen2moe olmoe (merged) ernie4.5-moe, nemotron-h-moe (merged)511 "model.layers.{bid}.block_sparse_moe.experts.w3", # phimoe (merged)512 "model.layers.{bid}.feed_forward.experts.up_proj", # llama4513 "encoder.layers.{bid}.mlp.experts.mlp.w1", # nomic-bert-moe514 "model.layers.{bid}.block_sparse_moe.experts.up", # smallthinker515 "model.layers.{bid}.moe.up_proj", # step3.5516 ),517 518 MODEL_TENSOR.FFN_UP_SHEXP: (519 "model.layers.{bid}.mlp.shared_expert.up_proj", # qwen2moe520 "model.layers.{bid}.mlp.shared_experts.up_proj", # deepseek deepseek2521 "model.layers.{bid}.feed_forward.shared_expert.up_proj", # llama4522 "model.layers.{bid}.feed_forward.down_proj",523 "model.layers.{bid}.mlp.shared_mlp.up_proj", # hunyuan524 "layers.{bid}.shared_experts.w3", # mistral-large525 "backbone.layers.{bid}.mixer.shared_experts.up_proj", # nemotron-h-moe526 "model.layers.{bid}.block_sparse_moe.shared_experts.up_proj", # kimi527 "model.layers.{bid}.share_expert.up_proj", # step3.5528 ),529 530 MODEL_TENSOR.FFN_UP_CHEXP: (531 "model.layers.{bid}.mlp.chunk_experts.up_proj", # grovemoe532 ),533 534 # AWQ-activation gate535 MODEL_TENSOR.FFN_ACT: (536 "transformer.blocks.{bid}.ffn.act", # mpt537 ),538 539 # Feed-forward gate540 MODEL_TENSOR.FFN_GATE: (541 "model.layers.{bid}.mlp.gate_proj", # llama-hf refact olmo2542 "layers.{bid}.mlp.gate_proj", # embeddinggemma543 "layers.{bid}.feed_forward.w1", # llama-pth544 "transformer.h.{bid}.mlp.w2", # qwen545 "transformer.h.{bid}.mlp.c_fc2", # jais546 "model.layers.layers.{bid}.mlp.gate_proj", # plamo547 "model.layers.{bid}.feed_forward.w1", # internlm2548 "encoder.layers.{bid}.mlp.fc12", # nomic-bert549 "encoder.layer.{bid}.mlp.gated_layers_w", # jina-bert-v2 (split up/gate, no longer used)550 "transformer.h.{bid}.mlp.linear_1", # refact551 "model.layers.{bid}.residual_mlp.w1", # arctic552 "transformer.h.{bid}.mlp.c_fc_0", # exaone553 "model.layers.{bid}.feed_forward.gate_proj", # llama4 jamba granite-hybrid554 "model.transformer.blocks.{bid}.ff_proj", # llada555 "layers.{bid}.mlp.gate_proj", # qwen3-embedding556 "model.layers.{bid}.mlp.language_mlp.gate_proj", # cogvlm557 ),558 559 MODEL_TENSOR.FFN_GATE_EXP: (560 "layers.{bid}.feed_forward.experts.w1", # mixtral (merged)561 "transformer.decoder_layer.{bid}.moe.linear", # Grok (merged)562 "transformer.blocks.{bid}.ffn.experts.mlp.w1", # dbrx563 "model.layers.{bid}.mlp.experts.gate_proj", # qwen2moe olmoe (merged) ernie4.5-moe564 "model.layers.{bid}.block_sparse_moe.experts.w1", # phimoe (merged)565 "model.layers.{bid}.feed_forward.experts.gate_proj", # llama4566 "model.layers.{bid}.block_sparse_moe.experts.gate", # smallthinker567 "model.layers.{bid}.moe.gate_proj", # step3.5568 ),569 570 MODEL_TENSOR.FFN_GATE_SHEXP: (571 "model.layers.{bid}.mlp.shared_expert.gate_proj", # qwen2moe572 "model.layers.{bid}.mlp.shared_experts.gate_proj", # deepseek deepseek2573 "model.layers.{bid}.feed_forward.shared_expert.gate_proj", # llama4574 "model.layers.{bid}.mlp.shared_mlp.gate_proj", # hunyuan575 "layers.{bid}.shared_experts.w1", # mistral-large576 "model.layers.{bid}.block_sparse_moe.shared_experts.gate_proj", # kimi577 "model.layers.{bid}.share_expert.gate_proj", # step3.5578 ),579 580 MODEL_TENSOR.FFN_GATE_CHEXP: (581 "model.layers.{bid}.mlp.chunk_experts.gate_proj", # grovemoe582 ),583 584 MODEL_TENSOR.FFN_GATE_UP_EXP: (585 "model.layers.{bid}.mlp.experts.gate_up_proj",586 "model.layers.{bid}.experts.gate_up_proj", # gemma4587 ),588 589 MODEL_TENSOR.MOE_LATENT_DOWN: (590 "backbone.layers.{bid}.mixer.fc1_latent_proj", # nemotron 3 super591 ),592 593 MODEL_TENSOR.MOE_LATENT_UP: (594 "backbone.layers.{bid}.mixer.fc2_latent_proj", # nemotron 3 super595 ),596 597 # Feed-forward down598 MODEL_TENSOR.FFN_DOWN: (599 "gpt_neox.layers.{bid}.mlp.dense_4h_to_h", # gptneox600 "transformer.h.{bid}.mlp.c_proj", # gpt2 refact qwen jais601 "transformer.blocks.{bid}.ffn.down_proj", # mpt602 "transformer.h.{bid}.mlp.dense_4h_to_h", # falcon603 "h.{bid}.mlp.dense_4h_to_h", # bloom604 "model.layers.{bid}.mlp.down_proj", # llama-hf nemotron olmo2605 "layers.{bid}.mlp.down_proj", # embeddinggemma606 "layers.{bid}.feed_forward.w2", # llama-pth607 "encoder.layer.{bid}.output.dense", # bert608 "layers.{bid}.mlp.Wo", # modern-bert609 "transformer.layer.{bid}.ffn.lin2", # distillbert610 "transformer.h.{bid}.mlp.fc_out", # gpt-j611 "language_model.encoder.layers.{bid}.mlp.dense_4h_to_h", # persimmon612 "model.layers.{bid}.mlp.dense_4h_to_h", # persimmon613 "h.{bid}.mlp.c_proj", # gpt2614 "transformer.h.{bid}.mlp.fc2", # phi2615 "model.layers.{bid}.mlp.fc2", # phi2616 "model.layers.layers.{bid}.mlp.down_proj", # plamo617 "model.layers.{bid}.feed_forward.w2", # internlm2618 "encoder.layers.{bid}.mlp.fc2", # nomic-bert619 "model.layers.{bid}.mlp.c_proj", # starcoder2620 "encoder.layer.{bid}.mlp.wo", # jina-bert-v2621 "transformer.layers.{bid}.ffn.proj_2", # openelm622 "model.layers.{bid}.residual_mlp.w2", # arctic623 "encoder.layer.{bid}.mlp.down_layer", # jina-bert-v2624 "encoder.layers.{bid}.mlp.dense_4h_to_h", # chatglm625 "model.layers.h.{bid}.mlp.c_proj", # exaone626 "model.layers.{bid}.feed_forward.down_proj", # llama4 jamba granite-hybrid627 "transformer_encoder.{bid}.ffn.w3", # neobert628 "model.layers.{bid}.block_sparse_moe.down", # smallthinker629 "model.transformer.blocks.{bid}.ff_out", # llada630 "layers.{bid}.mlp.down_proj", # qwen3-embedding631 "backbone.layers.{bid}.mixer.down_proj", # nemotron-h632 "model.layers.{bid}.mlp.language_mlp.down_proj", # cogvlm633 ),634 635 MODEL_TENSOR.FFN_DOWN_EXP: (636 "layers.{bid}.feed_forward.experts.w2", # mixtral (merged)637 "transformer.decoder_layer.{bid}.moe.linear_1", # Grok (merged)638 "transformer.blocks.{bid}.ffn.experts.mlp.w2", # dbrx639 "model.layers.{bid}.mlp.experts.down_proj", # qwen2moe olmoe (merged) ernie4.5-moe nemotron-h-moe (merged)640 "model.layers.{bid}.block_sparse_moe.output_linear", # granitemoe641 "model.layers.{bid}.block_sparse_moe.experts.w2", # phimoe (merged)642 "model.layers.{bid}.feed_forward.experts.down_proj", # llama4643 "encoder.layers.{bid}.mlp.experts.mlp.w2", # nomic-bert-moe644 "model.layers.{bid}.block_sparse_moe.experts.down", # smallthinker645 "model.layers.{bid}.moe.down_proj", # step3.5646 "model.layers.{bid}.experts.down_proj", # gemma4647 ),648 649 MODEL_TENSOR.FFN_DOWN_SHEXP: (650 "model.layers.{bid}.mlp.shared_expert.down_proj", # qwen2moe651 "model.layers.{bid}.mlp.shared_experts.down_proj", # deepseek deepseek2652 "model.layers.{bid}.feed_forward.shared_expert.down_proj", # llama4653 "model.layers.{bid}.shared_mlp.output_linear", # granitemoe654 "model.layers.{bid}.mlp.shared_mlp.down_proj", # hunyuan655 "layers.{bid}.shared_experts.w2", # mistral-large656 "backbone.layers.{bid}.mixer.shared_experts.down_proj", # nemotron-h-moe657 "model.layers.{bid}.block_sparse_moe.shared_experts.down_proj", # kimi658 "model.layers.{bid}.share_expert.down_proj", # step3.5659 ),660 661 MODEL_TENSOR.FFN_DOWN_CHEXP: (662 "model.layers.{bid}.mlp.chunk_experts.down_proj", # grovemoe663 ),664 665 MODEL_TENSOR.ATTN_Q_NORM: (666 "language_model.encoder.layers.{bid}.self_attention.q_layernorm",667 "model.layers.{bid}.self_attn.q_layernorm", # persimmon668 "model.layers.{bid}.self_attn.query_layernorm", # hunyuan669 "model.layers.{bid}.attention.query_layernorm", # bailingmoe2670 "model.layers.{bid}.self_attn.q_norm", # cohere olmoe chameleon olmo2671 "layers.{bid}.self_attn.q_norm", # embeddinggemma672 "transformer.blocks.{bid}.attn.q_ln", # sea-lion673 "encoder.layer.{bid}.attention.self.layer_norm_q", # jina-bert-v2674 "transformer.layers.{bid}.attn.q_norm", # openelm675 "model.layers.layers.{bid}.mixer.q", # plamo2676 "model.layers.layers.{bid}.mixer.q_norm", # plamo3677 "layers.{bid}.self_attn.q_norm", # qwen3-embedding678 "model.layers.{bid}.attention.query_layernorm", # apertus679 ),680 681 MODEL_TENSOR.ATTN_K_NORM: (682 "language_model.encoder.layers.{bid}.self_attention.k_layernorm",683 "model.layers.{bid}.self_attn.k_layernorm", # persimmon684 "model.layers.{bid}.self_attn.key_layernorm", # hunyuan685 "model.layers.{bid}.attention.key_layernorm", # bailingmoe2686 "model.layers.{bid}.self_attn.k_norm", # cohere olmoe chameleon olmo2687 "layers.{bid}.self_attn.k_norm", # embeddinggemma688 "transformer.blocks.{bid}.attn.k_ln", # sea-lion689 "encoder.layer.{bid}.attention.self.layer_norm_k", # jina-bert-v2690 "transformer.layers.{bid}.attn.k_norm", # openelm691 "model.layers.layers.{bid}.mixer.k", # plamo2692 "model.layers.layers.{bid}.mixer.k_norm", # plamo3693 "layers.{bid}.self_attn.k_norm", # qwen3-embedding694 "model.layers.{bid}.attention.key_layernorm", # apertus695 ),696 697 MODEL_TENSOR.ROPE_FREQS: (698 "language_model.encoder.layers.{bid}.self_attention.rotary_emb.inv_freq", # persimmon699 ),700 701 MODEL_TENSOR.LAYER_OUT_NORM: (702 "encoder.layer.{bid}.output.LayerNorm", # bert703 "transformer.layer.{bid}.output_layer_norm", # distillbert704 "encoder.layers.{bid}.norm2", # nomic-bert705 "transformer.decoder_layer.{bid}.rms_norm_3", # Grok706 "encoder.layer.{bid}.mlp.layernorm", # jina-bert-v2707 "encoder.layer.{bid}.layer_norm_2", # jina-v2-code708 "model.layers.{bid}.final_layernorm", # bailingmoe2709 ),710 711 MODEL_TENSOR.LAYER_OUT_SCALE: (712 "model.layers.{bid}.layer_scalar", # gemma4713 ),714 715 MODEL_TENSOR.PER_LAYER_TOKEN_EMBD: (716 "model.embed_tokens_per_layer", # gemma3n717 ),718 719 MODEL_TENSOR.PER_LAYER_MODEL_PROJ: (720 "model.per_layer_model_projection", # gemma3n721 ),722 723 MODEL_TENSOR.PER_LAYER_PROJ_NORM: (724 "model.per_layer_projection_norm", # gemma3n725 ),726 727 MODEL_TENSOR.ALTUP_PROJ: (728 "model.altup_projections", # gemma3n729 ),730 731 MODEL_TENSOR.ALTUP_UNEMBD_PROJ: (732 "model.altup_unembed_projections", # gemma3n733 ),734 735 MODEL_TENSOR.PER_LAYER_INP_GATE: (736 "model.layers.{bid}.per_layer_input_gate", # gemma3n737 ),738 739 MODEL_TENSOR.PER_LAYER_PROJ: (740 "model.layers.{bid}.per_layer_projection", # gemma3n741 ),742 743 MODEL_TENSOR.PER_LAYER_POST_NORM: (744 "model.layers.{bid}.post_per_layer_input_norm", # gemma3n745 ),746 747 MODEL_TENSOR.ALTUP_CORRECT_COEF: (748 "model.layers.{bid}.altup.correction_coefs", # gemma3n749 ),750 751 MODEL_TENSOR.ALTUP_CORRECT_SCALE: (752 "model.layers.{bid}.altup.correct_output_scale", # gemma3n753 ),754 755 MODEL_TENSOR.ALTUP_PREDICT_COEF: (756 "model.layers.{bid}.altup.prediction_coefs", # gemma3n757 ),758 759 MODEL_TENSOR.ALTUP_ROUTER: (760 "model.layers.{bid}.altup.modality_router", # gemma3n761 ),762 763 MODEL_TENSOR.ALTUP_ROUTER_NORM: (764 "model.layers.{bid}.altup.router_norm", # gemma3n765 ),766 767 MODEL_TENSOR.LAUREL_L: (768 "model.layers.{bid}.laurel.linear_left", # gemma3n769 ),770 771 MODEL_TENSOR.LAUREL_R: (772 "model.layers.{bid}.laurel.linear_right", # gemma3n773 ),774 775 MODEL_TENSOR.LAUREL_POST_NORM: (776 "model.layers.{bid}.laurel.post_laurel_norm", # gemma3n777 ),778 779 MODEL_TENSOR.SSM_IN: (780 "model.layers.{bid}.in_proj", # mamba-hf781 "backbone.layers.{bid}.mixer.in_proj", # mamba782 "model.layers.{bid}.mamba.in_proj", # jamba falcon-h1 granite-hybrid783 "model.layers.layers.{bid}.mixer.in_proj", # plamo2784 "model.layers.{bid}.linear_attn.in_proj_qkvz", # qwen3next785 ),786 787 MODEL_TENSOR.SSM_CONV1D: (788 "model.layers.{bid}.conv1d", # mamba-hf789 "backbone.layers.{bid}.mixer.conv1d", # mamba790 "model.layers.{bid}.mamba.conv1d", # jamba falcon-h1 granite-hybrid791 "model.layers.layers.{bid}.mixer.conv1d", # plamo2792 "model.layers.{bid}.linear_attn.conv1d", # qwen3next793 ),794 795 MODEL_TENSOR.SSM_X: (796 "model.layers.{bid}.x_proj", # mamba-hf797 "backbone.layers.{bid}.mixer.x_proj", # mamba798 "model.layers.{bid}.mamba.x_proj", # jamba799 "model.layers.layers.{bid}.mixer.bcdt_proj", # plamo2800 ),801 802 MODEL_TENSOR.SSM_DT: (803 "model.layers.{bid}.dt_proj", # mamba-hf804 "backbone.layers.{bid}.mixer.dt_proj", # mamba805 "model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid806 "model.layers.layers.{bid}.mixer.dt_proj", # plamo2807 "model.layers.{bid}.linear_attn.dt_proj", # qwen3next808 "backbone.layers.{bid}.mixer.dt", # nemotron-h-moe809 "model.layers.{bid}.self_attn.dt_proj", # kimi810 ),811 812 MODEL_TENSOR.SSM_DT_NORM: (813 "model.layers.layers.{bid}.mixer.dt_norm.weight", # plamo2814 "model.layers.{bid}.mamba.dt_layernorm", # jamba815 ),816 817 MODEL_TENSOR.SSM_A: (818 "model.layers.{bid}.A_log", # mamba-hf819 "backbone.layers.{bid}.mixer.A_log", # mamba820 "model.layers.{bid}.mamba.A_log", # jamba falcon-h1 granite-hybrid821 "model.layers.layers.{bid}.mixer.A_log", # plamo2822 "model.layers.{bid}.linear_attn.A_log", # qwen3next823 "model.layers.{bid}.self_attn.A_log", # kimi824 ),825 826 MODEL_TENSOR.SSM_B_NORM: (827 "model.layers.{bid}.mamba.b_layernorm", # jamba828 "model.layers.{bid}.mamba.B_layernorm", # mini-jamba829 "model.layers.layers.{bid}.mixer.B_norm.weight", # plamo2830 ),831 832 MODEL_TENSOR.SSM_C_NORM: (833 "model.layers.{bid}.mamba.c_layernorm", # jamba834 "model.layers.{bid}.mamba.C_layernorm", # mini-jamba835 "model.layers.layers.{bid}.mixer.C_norm.weight", # plamo2836 ),837 838 MODEL_TENSOR.SSM_D: (839 "model.layers.{bid}.D", # mamba-hf840 "backbone.layers.{bid}.mixer.D", # mamba841 "model.layers.{bid}.mamba.D", # jamba falcon-h1 granite-hybrid842 "model.layers.layers.{bid}.mixer.D", # plamo2843 ),844 845 MODEL_TENSOR.SSM_NORM: (846 "model.layers.{bid}.mamba.norm", # falcon-h1 granite-hybrid847 "model.layers.{bid}.linear_attn.norm", # qwen3next848 "backbone.layers.{bid}.mixer.norm", # mamba2849 "model.layers.{bid}.self_attn.o_norm", # kimi850 ),851 852 MODEL_TENSOR.SSM_OUT: (853 "model.layers.{bid}.out_proj", # mamba-hf854 "backbone.layers.{bid}.mixer.out_proj", # mamba855 "model.layers.{bid}.mamba.out_proj", # jamba falcon-h1 granite-hybrid856 "model.layers.{bid}.linear_attn.out_proj", # qwen3next857 "model.layers.layers.{bid}.mixer.out_proj", # plamo2858 ),859 860 MODEL_TENSOR.SSM_ALPHA: (861 "model.layers.{bid}.linear_attn.in_proj_a", # qwen3.5862 ),863 864 MODEL_TENSOR.SSM_BETA_ALPHA: (865 "model.layers.{bid}.linear_attn.in_proj_ba", # qwen3next866 ),867 868 # Kimi Linear KDA (using SSM_ prefix for consistency)869 MODEL_TENSOR.SSM_CONV1D_Q: (870 "model.layers.{bid}.self_attn.q_conv1d",871 ),872 MODEL_TENSOR.SSM_CONV1D_K: (873 "model.layers.{bid}.self_attn.k_conv1d",874 ),875 MODEL_TENSOR.SSM_CONV1D_V: (876 "model.layers.{bid}.self_attn.v_conv1d",877 ),878 MODEL_TENSOR.SSM_F_A: (879 "model.layers.{bid}.self_attn.f_a_proj",880 ),881 MODEL_TENSOR.SSM_F_B: (882 "model.layers.{bid}.self_attn.f_b_proj",883 ),884 MODEL_TENSOR.SSM_BETA: (885 "model.layers.{bid}.linear_attn.in_proj_b", # qwen3.5886 "model.layers.{bid}.self_attn.b_proj", # Kimi Linear887 ),888 MODEL_TENSOR.SSM_G_A: (889 "model.layers.{bid}.self_attn.g_a_proj",890 ),891 MODEL_TENSOR.SSM_G_B: (892 "model.layers.{bid}.self_attn.g_b_proj",893 ),894 MODEL_TENSOR.TIME_MIX_W0: (895 "model.layers.{bid}.attention.w0", # rwkv7896 ),897 898 MODEL_TENSOR.TIME_MIX_W1: (899 "rwkv.blocks.{bid}.attention.time_maa_w1", # rwkv6900 "model.layers.{bid}.self_attn.time_maa_w1", # rwkv6qwen2901 "model.layers.{bid}.attention.w1", # rwkv7902 ),903 904 MODEL_TENSOR.TIME_MIX_W2: (905 "rwkv.blocks.{bid}.attention.time_maa_w2", # rwkv6906 "model.layers.{bid}.self_attn.time_maa_w2", # rwkv6qwen2907 "model.layers.{bid}.attention.w2", # rwkv7908 ),909 910 MODEL_TENSOR.TIME_MIX_A0: (911 "model.layers.{bid}.attention.a0", # rwkv7912 ),913 914 MODEL_TENSOR.TIME_MIX_A1: (915 "model.layers.{bid}.attention.a1", # rwkv7916 ),917 918 MODEL_TENSOR.TIME_MIX_A2: (919 "model.layers.{bid}.attention.a2", # rwkv7920 ),921 922 MODEL_TENSOR.TIME_MIX_V0: (923 "model.layers.{bid}.attention.v0", # rwkv7924 ),925 926 MODEL_TENSOR.TIME_MIX_V1: (927 "model.layers.{bid}.attention.v1", # rwkv7928 ),929 930 MODEL_TENSOR.TIME_MIX_V2: (931 "model.layers.{bid}.attention.v2", # rwkv7932 ),933 934 MODEL_TENSOR.TIME_MIX_G1: (935 "model.layers.{bid}.attention.g1", # rwkv7936 ),937 938 MODEL_TENSOR.TIME_MIX_G2: (939 "model.layers.{bid}.attention.g2", # rwkv7940 ),941 942 MODEL_TENSOR.TIME_MIX_K_K: (943 "model.layers.{bid}.attention.k_k", # rwkv7944 ),945 946 MODEL_TENSOR.TIME_MIX_K_A: (947 "model.layers.{bid}.attention.k_a", # rwkv7948 ),949 950 MODEL_TENSOR.TIME_MIX_R_K: (951 "model.layers.{bid}.attention.r_k", # rwkv7952 ),953 954 MODEL_TENSOR.TIME_MIX_LERP_X: (955 "rwkv.blocks.{bid}.attention.time_maa_x", # rwkv6956 "model.layers.{bid}.self_attn.time_maa_x", # rwkv6qwen2957 ),958 959 MODEL_TENSOR.TIME_MIX_LERP_K: (960 "rwkv.blocks.{bid}.attention.time_maa_k", # rwkv6961 "model.layers.{bid}.self_attn.time_maa_k", # rwkv6qwen2962 ),963 964 MODEL_TENSOR.TIME_MIX_LERP_V: (965 "rwkv.blocks.{bid}.attention.time_maa_v", # rwkv6966 "model.layers.{bid}.self_attn.time_maa_v", # rwkv6qwen2967 ),968 969 MODEL_TENSOR.TIME_MIX_LERP_R: (970 "rwkv.blocks.{bid}.attention.time_maa_r", # rwkv6971 "model.layers.{bid}.self_attn.time_maa_r", # rwkv6qwen2972 ),973 974 MODEL_TENSOR.TIME_MIX_LERP_G: (975 "rwkv.blocks.{bid}.attention.time_maa_g", # rwkv6976 "model.layers.{bid}.self_attn.time_maa_g", # rwkv6qwen2977 ),978 979 MODEL_TENSOR.TIME_MIX_LERP_W: (980 "rwkv.blocks.{bid}.attention.time_maa_w", # rwkv6981 "model.layers.{bid}.self_attn.time_maa_w", # rwkv6qwen2982 ),983 984 MODEL_TENSOR.TIME_MIX_FIRST: (985 "rwkv.blocks.{bid}.attention.time_faaaa", # rwkv6986 ),987 988 MODEL_TENSOR.TIME_MIX_DECAY: (989 "rwkv.blocks.{bid}.attention.time_decay", # rwkv6990 "model.layers.{bid}.self_attn.time_decay", # rwkv6qwen2991 ),992 993 MODEL_TENSOR.TIME_MIX_DECAY_W1: (994 "rwkv.blocks.{bid}.attention.time_decay_w1", # rwkv6995 "model.layers.{bid}.self_attn.time_decay_w1", # rwkv6qwen2996 ),997 998 MODEL_TENSOR.TIME_MIX_DECAY_W2: (999 "rwkv.blocks.{bid}.attention.time_decay_w2", # rwkv61000 "model.layers.{bid}.self_attn.time_decay_w2", # rwkv6qwen21001 ),1002 1003 MODEL_TENSOR.TIME_MIX_KEY: (1004 "rwkv.blocks.{bid}.attention.key", # rwkv61005 "model.layers.{bid}.self_attn.k_proj", # rwkv6qwen21006 "model.layers.{bid}.attention.key", # rwkv71007 "model.layers.{bid}.attention.k_proj", # rwkv71008 ),1009 1010 MODEL_TENSOR.TIME_MIX_VALUE: (1011 "rwkv.blocks.{bid}.attention.value", # rwkv61012 "model.layers.{bid}.self_attn.v_proj", # rwkv6qwen21013 "model.layers.{bid}.attention.value", # rwkv71014 "model.layers.{bid}.attention.v_proj", # rwkv71015 ),1016 1017 MODEL_TENSOR.TIME_MIX_RECEPTANCE: (1018 "rwkv.blocks.{bid}.attention.receptance", # rwkv61019 "model.layers.{bid}.self_attn.q_proj", # rwkv6qwen21020 "model.layers.{bid}.attention.receptance", # rwkv71021 "model.layers.{bid}.attention.r_proj", # rwkv71022 ),1023 1024 MODEL_TENSOR.TIME_MIX_GATE: (1025 "rwkv.blocks.{bid}.attention.gate", # rwkv61026 "model.layers.{bid}.self_attn.gate", # rwkv6qwen21027 ),1028 1029 MODEL_TENSOR.TIME_MIX_LN: (1030 "rwkv.blocks.{bid}.attention.ln_x", # rwkv61031 "model.layers.{bid}.attention.ln_x" # rwkv71032 ),1033 1034 MODEL_TENSOR.TIME_MIX_OUTPUT: (1035 "rwkv.blocks.{bid}.attention.output", # rwkv61036 "model.layers.{bid}.self_attn.o_proj", # rwkv6qwen21037 "model.layers.{bid}.attention.output", # rwkv71038 "model.layers.{bid}.attention.o_proj", # rwkv71039 ),1040 1041 MODEL_TENSOR.CHANNEL_MIX_LERP_K: (1042 "rwkv.blocks.{bid}.feed_forward.time_maa_k", # rwkv61043 "model.layers.{bid}.feed_forward.x_k", # rwkv71044 ),1045 1046 MODEL_TENSOR.CHANNEL_MIX_LERP_R: (1047 "rwkv.blocks.{bid}.feed_forward.time_maa_r", # rwkv61048 ),1049 1050 MODEL_TENSOR.CHANNEL_MIX_KEY: (1051 "rwkv.blocks.{bid}.feed_forward.key", # rwkv61052 "model.layers.{bid}.feed_forward.key", # rwkv71053 ),1054 1055 MODEL_TENSOR.CHANNEL_MIX_RECEPTANCE: (1056 "rwkv.blocks.{bid}.feed_forward.receptance", # rwkv61057 ),1058 1059 MODEL_TENSOR.CHANNEL_MIX_VALUE: (1060 "rwkv.blocks.{bid}.feed_forward.value", # rwkv61061 "model.layers.{bid}.feed_forward.value", # rwkv71062 ),1063 1064 MODEL_TENSOR.ATTN_Q_A: (1065 "model.layers.{bid}.self_attn.q_a_proj", # deepseek21066 "layers.{bid}.attention.wq_a", # mistral-large1067 ),1068 1069 MODEL_TENSOR.ATTN_Q_B: (1070 "model.layers.{bid}.self_attn.q_b_proj", # deepseek21071 "layers.{bid}.attention.wq_b", # mistral-large1072 ),1073 1074 MODEL_TENSOR.ATTN_KV_A_MQA: (1075 "model.layers.{bid}.self_attn.kv_a_proj_with_mqa", # deepseek21076 "layers.{bid}.attention.wkv_a_with_mqa", # mistral-large1077 ),1078 1079 MODEL_TENSOR.ATTN_KV_B: (1080 "model.layers.{bid}.self_attn.kv_b_proj", # deepseek21081 ),1082 1083 MODEL_TENSOR.ATTN_K_B: (1084 "model.layers.{bid}.self_attn.k_b_proj", # deepseek21085 "layers.{bid}.attention.k_b_proj", # mistral-large1086 ),1087 1088 MODEL_TENSOR.ATTN_V_B: (1089 "model.layers.{bid}.self_attn.v_b_proj", # deepseek21090 "layers.{bid}.attention.v_b_proj", # mistral-large1091 ),1092 1093 MODEL_TENSOR.ATTN_Q_A_NORM: (1094 "model.layers.{bid}.self_attn.q_a_layernorm", # deepseek21095 "layers.{bid}.attention.q_a_norm", # mistral-large1096 ),1097 1098 MODEL_TENSOR.ATTN_KV_A_NORM: (1099 "model.layers.{bid}.self_attn.kv_a_layernorm", # deepseek21100 "layers.{bid}.attention.kv_a_norm", # mistral-large1101 ),1102 1103 MODEL_TENSOR.ATTN_SUB_NORM: (1104 "model.layers.{bid}.self_attn.inner_attn_ln", # bitnet1105 ),1106 1107 MODEL_TENSOR.FFN_SUB_NORM: (1108 "model.layers.{bid}.mlp.ffn_layernorm", # bitnet1109 ),1110 1111 MODEL_TENSOR.DEC_ATTN_NORM: (1112 "decoder.block.{bid}.layer.0.layer_norm", # t51113 ),1114 1115 MODEL_TENSOR.DEC_ATTN_Q: (1116 "decoder.block.{bid}.layer.0.SelfAttention.q", # t51117 ),1118 1119 MODEL_TENSOR.DEC_ATTN_K: (1120 "decoder.block.{bid}.layer.0.SelfAttention.k", # t51121 ),1122 1123 MODEL_TENSOR.DEC_ATTN_V: (1124 "decoder.block.{bid}.layer.0.SelfAttention.v", # t51125 ),1126 1127 MODEL_TENSOR.DEC_ATTN_OUT: (1128 "decoder.block.{bid}.layer.0.SelfAttention.o", # t51129 ),1130 1131 MODEL_TENSOR.DEC_ATTN_REL_B: (1132 "decoder.block.{bid}.layer.0.SelfAttention.relative_attention_bias", # t51133 ),1134 1135 MODEL_TENSOR.DEC_CROSS_ATTN_NORM: (1136 "decoder.block.{bid}.layer.1.layer_norm", # t51137 ),1138 1139 MODEL_TENSOR.DEC_CROSS_ATTN_Q: (1140 "decoder.block.{bid}.layer.1.EncDecAttention.q", # t51141 ),1142 1143 MODEL_TENSOR.DEC_CROSS_ATTN_K: (1144 "decoder.block.{bid}.layer.1.EncDecAttention.k", # t51145 ),1146 1147 MODEL_TENSOR.DEC_CROSS_ATTN_V: (1148 "decoder.block.{bid}.layer.1.EncDecAttention.v", # t51149 ),1150 1151 MODEL_TENSOR.DEC_CROSS_ATTN_OUT: (1152 "decoder.block.{bid}.layer.1.EncDecAttention.o", # t51153 ),1154 1155 MODEL_TENSOR.DEC_CROSS_ATTN_REL_B: (1156 "decoder.block.{bid}.layer.1.EncDecAttention.relative_attention_bias", # t51157 ),1158 1159 MODEL_TENSOR.DEC_FFN_NORM: (1160 "decoder.block.{bid}.layer.2.layer_norm", # t51161 ),1162 1163 MODEL_TENSOR.DEC_FFN_GATE: (1164 "decoder.block.{bid}.layer.2.DenseReluDense.wi_0", # flan-t51165 ),1166 1167 MODEL_TENSOR.DEC_FFN_UP: (1168 "decoder.block.{bid}.layer.2.DenseReluDense.wi", # t51169 "decoder.block.{bid}.layer.2.DenseReluDense.wi_1", # flan-t51170 ),1171 1172 MODEL_TENSOR.DEC_FFN_DOWN: (1173 "decoder.block.{bid}.layer.2.DenseReluDense.wo", # t51174 ),1175 1176 MODEL_TENSOR.DEC_OUTPUT_NORM: (1177 "decoder.final_layer_norm", # t51178 ),1179 1180 MODEL_TENSOR.ENC_ATTN_NORM: (1181 "encoder.block.{bid}.layer.0.layer_norm", # t51182 ),1183 1184 MODEL_TENSOR.ENC_ATTN_Q: (1185 "encoder.block.{bid}.layer.0.SelfAttention.q", # t51186 ),1187 1188 MODEL_TENSOR.ENC_ATTN_K: (1189 "encoder.block.{bid}.layer.0.SelfAttention.k", # t51190 ),1191 1192 MODEL_TENSOR.ENC_ATTN_V: (1193 "encoder.block.{bid}.layer.0.SelfAttention.v", # t51194 ),1195 1196 MODEL_TENSOR.ENC_ATTN_OUT: (1197 "encoder.block.{bid}.layer.0.SelfAttention.o", # t51198 ),1199 1200 MODEL_TENSOR.ENC_ATTN_REL_B: (