Felipe97/llama-cpp-compiled

1#pragma once2 3#include "llama.h"4#include "llama-arch.h"5#include "llama-graph.h"6#include "llama-hparams.h"7#include "llama-memory.h"8#include "llama-vocab.h"9 10#include <map>11#include <memory>12#include <string>13#include <unordered_map>14#include <unordered_set>15#include <vector>16 17struct llama_cparams;18struct llama_ubatch;19struct llama_model_loader;20 21// available models22enum llm_type {23    LLM_TYPE_UNKNOWN,24    LLM_TYPE_14M,25    LLM_TYPE_17M,26    LLM_TYPE_22M,27    LLM_TYPE_33M,28    LLM_TYPE_47M,29    LLM_TYPE_60M,30    LLM_TYPE_70M,31    LLM_TYPE_80M,32    LLM_TYPE_109M,33    LLM_TYPE_137M,34    LLM_TYPE_140M,35    LLM_TYPE_149M,36    LLM_TYPE_160M,37    LLM_TYPE_190M,38    LLM_TYPE_220M,39    LLM_TYPE_230M,40    LLM_TYPE_250M,41    LLM_TYPE_256M,42    LLM_TYPE_270M,43    LLM_TYPE_335M,44    LLM_TYPE_350M,45    LLM_TYPE_360M,46    LLM_TYPE_395M,47    LLM_TYPE_410M,48    LLM_TYPE_450M,49    LLM_TYPE_475M,50    LLM_TYPE_558M,51    LLM_TYPE_700M,52    LLM_TYPE_770M,53    LLM_TYPE_780M,54    LLM_TYPE_950M,55    LLM_TYPE_0_3B,56    LLM_TYPE_0_5B,57    LLM_TYPE_0_6B,58    LLM_TYPE_0_8B,59    LLM_TYPE_1B,60    LLM_TYPE_1_2B,61    LLM_TYPE_1_3B,62    LLM_TYPE_1_4B,63    LLM_TYPE_1_5B,64    LLM_TYPE_1_6B,65    LLM_TYPE_1_7B,66    LLM_TYPE_1_8B,67    LLM_TYPE_2B,68    LLM_TYPE_2_6B,69    LLM_TYPE_2_8B,70    LLM_TYPE_2_9B,71    LLM_TYPE_3B,72    LLM_TYPE_4B,73    LLM_TYPE_6B,74    LLM_TYPE_6_9B,75    LLM_TYPE_7B,76    LLM_TYPE_8B,77    LLM_TYPE_9B,78    LLM_TYPE_11B,79    LLM_TYPE_12B,80    LLM_TYPE_13B,81    LLM_TYPE_14B,82    LLM_TYPE_15B,83    LLM_TYPE_16B,84    LLM_TYPE_20B,85    LLM_TYPE_26B,86    LLM_TYPE_27B,87    LLM_TYPE_30B,88    LLM_TYPE_31B,89    LLM_TYPE_32B,90    LLM_TYPE_34B,91    LLM_TYPE_35B,92    LLM_TYPE_36B,93    LLM_TYPE_40B,94    LLM_TYPE_65B,95    LLM_TYPE_70B,96    LLM_TYPE_120B,97    LLM_TYPE_142B,98    LLM_TYPE_236B,99    LLM_TYPE_290B,100    LLM_TYPE_314B,101    LLM_TYPE_405B,102    LLM_TYPE_456B,103    LLM_TYPE_671B,104    LLM_TYPE_SMALL,105    LLM_TYPE_MEDIUM,106    LLM_TYPE_LARGE,107    LLM_TYPE_XL,108    LLM_TYPE_A1_7B,109    LLM_TYPE_A2_7B,110    LLM_TYPE_8x7B,111    LLM_TYPE_8x22B,112    LLM_TYPE_16x12B,113    LLM_TYPE_16x3_8B,114    LLM_TYPE_10B_128x3_66B,115    LLM_TYPE_57B_A14B,116    LLM_TYPE_17B_16E, // llama4 Scout117    LLM_TYPE_17B_128E, // llama4 Maverick118    LLM_TYPE_A13B,119    LLM_TYPE_1B_A400M, // Granite3 MoE120    LLM_TYPE_3B_A800M, // Granite3 MoE121    LLM_TYPE_7B_A1B,122    LLM_TYPE_8B_A1B, // lfm2moe123    LLM_TYPE_7_9B_A1_3B, // Ling-3.0-tiny124    LLM_TYPE_12B_A2_5B,125    LLM_TYPE_16B_A1B,126    LLM_TYPE_21B_A3B, // Ernie MoE small127    LLM_TYPE_24B_A2B, // lfm2moe128    LLM_TYPE_26B_A4B, // Gemma4129    LLM_TYPE_30B_A3B,130    LLM_TYPE_31B_A3_5B,131    LLM_TYPE_32B_A9B, // Granite4 Hybrid132    LLM_TYPE_35B_A3B, // Qwen3.5133    LLM_TYPE_48B_A3B, // Kimi Linear134    LLM_TYPE_75B_A9B, // Nemotron 3 Puzzle135    LLM_TYPE_80B_A3B, // Qwen3 Next136    LLM_TYPE_A3B,     // Qwen3.8 Flash Next137    LLM_TYPE_100B_A6B,138    LLM_TYPE_102B_A12B, // Solar-Open139    LLM_TYPE_106B_A12B, // GLM-4.5-Air140    LLM_TYPE_118B_A8B,  // Laguna-S-2141    LLM_TYPE_120B_A12B, // Nemotron 3 Super142    LLM_TYPE_122B_A10B, // Qwen3.5143    LLM_TYPE_124B_A5_1B, // Ling-3.0-flash144    LLM_TYPE_196B_A11B, // Step3.5-Flash145    LLM_TYPE_230B_A10B, // Minimax M2146    LLM_TYPE_428B_A23B, // Minimax M3147    LLM_TYPE_235B_A22B,148    LLM_TYPE_288B_A19B, // dots3-note149    LLM_TYPE_300B_A47B, // Ernie MoE big150    LLM_TYPE_310B_A15B, // /MiMo-V2-Flash151    LLM_TYPE_355B_A32B, // GLM-4.5152    LLM_TYPE_397B_A17B, // Qwen3.5153    LLM_TYPE_685B_A37B, // DeepSeek V3.2154    LLM_TYPE_744B_A40B, // GLM-5155    LLM_TYPE_2_8T_A50B, // Kimi-K3156    LLM_TYPE_E2B,157    LLM_TYPE_E4B,158};159 160std::string llama_rope_scaling_type_name(llama_rope_scaling_type rope_scaling_type);161 162// Map a GGUF activation-name string to llm_ffn_op_type. Returns `fallback` if163// the string is empty or not recognized.164llm_ffn_op_type llm_ffn_op_type_from_string(const std::string & name, llm_ffn_op_type fallback);165 166struct llama_layer_posnet {167    // resnet168    struct ggml_tensor * norm1   = nullptr;169    struct ggml_tensor * norm1_b = nullptr;170 171    struct ggml_tensor * conv1   = nullptr;172    struct ggml_tensor * conv1_b = nullptr;173 174    struct ggml_tensor * norm2   = nullptr;175    struct ggml_tensor * norm2_b = nullptr;176 177    struct ggml_tensor * conv2   = nullptr;178    struct ggml_tensor * conv2_b = nullptr;179 180    // attention181    struct ggml_tensor * attn_norm   = nullptr;182    struct ggml_tensor * attn_norm_b = nullptr;183 184    struct ggml_tensor * attn_q   = nullptr;185    struct ggml_tensor * attn_q_b = nullptr;186 187    struct ggml_tensor * attn_k   = nullptr;188    struct ggml_tensor * attn_k_b = nullptr;189 190    struct ggml_tensor * attn_v   = nullptr;191    struct ggml_tensor * attn_v_b = nullptr;192 193    struct ggml_tensor * attn_o   = nullptr;194    struct ggml_tensor * attn_o_b = nullptr;195 196    // normalize197    struct ggml_tensor * norm   = nullptr;198    struct ggml_tensor * norm_b = nullptr;199};200 201struct llama_layer_convnext {202    struct ggml_tensor * dw   = nullptr;203    struct ggml_tensor * dw_b = nullptr;204 205    struct ggml_tensor * norm   = nullptr;206    struct ggml_tensor * norm_b = nullptr;207 208    struct ggml_tensor * pw1   = nullptr;209    struct ggml_tensor * pw1_b = nullptr;210 211    struct ggml_tensor * pw2   = nullptr;212    struct ggml_tensor * pw2_b = nullptr;213 214    struct ggml_tensor * gamma = nullptr;215};216 217struct llama_layer_shortconv {218    struct ggml_tensor * in_proj  = nullptr;219    struct ggml_tensor * conv     = nullptr;220    struct ggml_tensor * out_proj = nullptr;221};222 223struct llama_layer_nextn {224    struct ggml_tensor * eh_proj               = nullptr;225    struct ggml_tensor * eh_proj_s             = nullptr;226    struct ggml_tensor * eh_proj_in_s          = nullptr;227    struct ggml_tensor * embed_tokens          = nullptr;228    struct ggml_tensor * enorm                 = nullptr;229    struct ggml_tensor * hnorm                 = nullptr;230    struct ggml_tensor * shared_head_head      = nullptr;231    struct ggml_tensor * shared_head_head_s    = nullptr;232    struct ggml_tensor * shared_head_head_in_s = nullptr;233    struct ggml_tensor * shared_head_norm      = nullptr;234};235 236struct llama_layer_switch_lora {237    struct ggml_tensor * a_q    = nullptr;238    struct ggml_tensor * b_q    = nullptr;239    struct ggml_tensor * a_k    = nullptr;240    struct ggml_tensor * b_k    = nullptr;241    struct ggml_tensor * a_v    = nullptr;242    struct ggml_tensor * b_v    = nullptr;243    struct ggml_tensor * a_o    = nullptr;244    struct ggml_tensor * b_o    = nullptr;245 246    struct ggml_tensor * a_gate = nullptr;247    struct ggml_tensor * b_gate = nullptr;248    struct ggml_tensor * a_up   = nullptr;249    struct ggml_tensor * b_up   = nullptr;250    struct ggml_tensor * a_down = nullptr;251    struct ggml_tensor * b_down = nullptr;252};253 254struct llama_layer {255    // normalization256    struct ggml_tensor * attn_norm       = nullptr;257    struct ggml_tensor * attn_norm_b     = nullptr;258    struct ggml_tensor * attn_norm_2     = nullptr;259    struct ggml_tensor * attn_norm_2_b   = nullptr;260    struct ggml_tensor * attn_q_norm     = nullptr;261    struct ggml_tensor * attn_q_norm_b   = nullptr;262    struct ggml_tensor * attn_k_norm     = nullptr;263    struct ggml_tensor * attn_k_norm_b   = nullptr;264    struct ggml_tensor * attn_out_norm   = nullptr;265    struct ggml_tensor * attn_out_norm_b = nullptr;266    struct ggml_tensor * attn_q_a_norm   = nullptr;267    struct ggml_tensor * attn_kv_a_norm  = nullptr;268    struct ggml_tensor * attn_sub_norm   = nullptr;269    struct ggml_tensor * attn_post_norm  = nullptr;270    struct ggml_tensor * ffn_sub_norm    = nullptr;271    struct ggml_tensor * attn_norm_cross = nullptr;272    struct ggml_tensor * attn_norm_enc   = nullptr;273    struct ggml_tensor * ssm_norm        = nullptr;274    struct ggml_tensor * ssm_dt_norm     = nullptr;275    struct ggml_tensor * ssm_b_norm      = nullptr;276    struct ggml_tensor * ssm_c_norm      = nullptr;277 278    // attention279    struct ggml_tensor * wq        = nullptr;280    struct ggml_tensor * wk        = nullptr;281    struct ggml_tensor * wv        = nullptr;282    struct ggml_tensor * wo        = nullptr;283    struct ggml_tensor * wqkv      = nullptr;284    struct ggml_tensor * wg        = nullptr;285    struct ggml_tensor * wq_a      = nullptr;286    struct ggml_tensor * wq_b      = nullptr;287    struct ggml_tensor * wkv_a_mqa = nullptr;288    struct ggml_tensor * wkv_b     = nullptr;289    struct ggml_tensor * wkv       = nullptr;290    struct ggml_tensor * wk_b      = nullptr;291    struct ggml_tensor * wv_b      = nullptr;292    struct ggml_tensor * wqkv_b    = nullptr;293    struct ggml_tensor * wo_a      = nullptr;294    struct ggml_tensor * wo_b      = nullptr;295    struct ggml_tensor * wq_cross  = nullptr;296    struct ggml_tensor * wk_cross  = nullptr;297    struct ggml_tensor * wv_cross  = nullptr;298    struct ggml_tensor * wo_cross  = nullptr;299    struct ggml_tensor * wq_enc    = nullptr;300    struct ggml_tensor * wk_enc    = nullptr;301    struct ggml_tensor * wv_enc    = nullptr;302    struct ggml_tensor * wo_enc    = nullptr;303    struct ggml_tensor * wqkv_gate = nullptr;304 305    // relative position bias306    struct ggml_tensor * attn_rel_b       = nullptr;307    struct ggml_tensor * attn_rel_b_enc   = nullptr;308    struct ggml_tensor * attn_rel_b_cross = nullptr;309 310    // normalization311    struct ggml_tensor * ffn_norm         = nullptr;312    struct ggml_tensor * ffn_norm_b       = nullptr;313    struct ggml_tensor * ffn_post_norm    = nullptr;314    struct ggml_tensor * ffn_post_norm_1  = nullptr; // gemma4315    struct ggml_tensor * ffn_post_norm_2  = nullptr; // gemma4316    struct ggml_tensor * ffn_pre_norm_2   = nullptr; // gemma4317    struct ggml_tensor * layer_out_norm   = nullptr;318    struct ggml_tensor * layer_out_norm_b = nullptr;319    struct ggml_tensor * ffn_norm_exps    = nullptr;320    struct ggml_tensor * ffn_norm_enc     = nullptr;321 322    // ff323    struct ggml_tensor * ffn_gate     = nullptr; // w1324    struct ggml_tensor * ffn_down     = nullptr; // w2325    struct ggml_tensor * ffn_up       = nullptr; // w3326    struct ggml_tensor * ffn_gate_enc = nullptr;327    struct ggml_tensor * ffn_down_enc = nullptr;328    struct ggml_tensor * ffn_up_enc   = nullptr;329 330    // ff MoE331    struct ggml_tensor * ffn_gate_inp      = nullptr;332    struct ggml_tensor * ffn_gate_inp_s    = nullptr; // gemma4333    struct ggml_tensor * ffn_gate_exps     = nullptr;334    struct ggml_tensor * ffn_down_exps     = nullptr;335    struct ggml_tensor * ffn_up_exps       = nullptr;336    struct ggml_tensor * ffn_gate_up_exps  = nullptr;337    struct ggml_tensor * ffn_gate_inp_b    = nullptr;338    struct ggml_tensor * ffn_gate_exps_b   = nullptr;339    struct ggml_tensor * ffn_down_exps_b   = nullptr;340    struct ggml_tensor * ffn_up_exps_b     = nullptr;341    struct ggml_tensor * ffn_gate_up_exps_b = nullptr;342 343    // ff MoE per-expert scales (NVFP4 per-tensor scale2)344    struct ggml_tensor * ffn_gate_exps_s   = nullptr;345    struct ggml_tensor * ffn_down_exps_s   = nullptr;346    struct ggml_tensor * ffn_up_exps_s     = nullptr;347 348    // ff MoE latent proj349    struct ggml_tensor * ffn_latent_down = nullptr;350    struct ggml_tensor * ffn_latent_up   = nullptr;351 352    // ff shared expert (shexp)353    struct ggml_tensor * ffn_gate_inp_shexp = nullptr;354    struct ggml_tensor * ffn_gate_shexp     = nullptr;355    struct ggml_tensor * ffn_down_shexp     = nullptr;356    struct ggml_tensor * ffn_up_shexp       = nullptr;357 358    // ff adjugate experts (chexps)359    struct ggml_tensor * ffn_gate_chexps     = nullptr;360    struct ggml_tensor * ffn_down_chexps     = nullptr;361    struct ggml_tensor * ffn_up_chexps       = nullptr;362 363    // ff bias364    struct ggml_tensor * ffn_gate_b = nullptr;365    struct ggml_tensor * ffn_down_b = nullptr; // b2366    struct ggml_tensor * ffn_up_b   = nullptr; // b3367    struct ggml_tensor * ffn_act    = nullptr;368    struct ggml_tensor * ffn_exp_probs_b = nullptr;369    struct ggml_tensor * ffn_exp_probs_b_vl = nullptr; // deepseek4 vision (bias for image tokens)370    struct ggml_tensor * ffn_gate_tid2eid = nullptr;371 372    struct ggml_tensor * dflash_attn_conv_base = nullptr;373    struct ggml_tensor * dflash_attn_conv_proj = nullptr;374    struct ggml_tensor * dflash_ffn_conv_base  = nullptr;375    struct ggml_tensor * dflash_ffn_conv_proj  = nullptr;376 377    // mamba proj378    struct ggml_tensor * ssm_in  = nullptr;379    struct ggml_tensor * ssm_x   = nullptr;380    struct ggml_tensor * ssm_dt  = nullptr;381    struct ggml_tensor * ssm_out = nullptr;382 383    // mamba384    struct ggml_tensor * ssm_conv1d = nullptr;385    struct ggml_tensor * ssm_a      = nullptr;386    struct ggml_tensor * ssm_d      = nullptr;387 388    // mamba bias389    struct ggml_tensor * ssm_conv1d_b = nullptr;390    struct ggml_tensor * ssm_dt_b     = nullptr;391 392    // qwen3next393    struct ggml_tensor * ssm_beta_alpha = nullptr;394 395    // qwen3.5396    struct ggml_tensor * ssm_alpha = nullptr;397 398    // rwkv399    struct ggml_tensor * time_mix_w1         = nullptr;400    struct ggml_tensor * time_mix_w2         = nullptr;401    struct ggml_tensor * time_mix_lerp_x     = nullptr;402    struct ggml_tensor * time_mix_lerp_w     = nullptr;403    struct ggml_tensor * time_mix_lerp_k     = nullptr;404    struct ggml_tensor * time_mix_lerp_v     = nullptr;405    struct ggml_tensor * time_mix_lerp_r     = nullptr;406    struct ggml_tensor * time_mix_lerp_g     = nullptr;407    struct ggml_tensor * time_mix_lerp_fused = nullptr;408 409    struct ggml_tensor * time_mix_first        = nullptr;410    struct ggml_tensor * time_mix_decay        = nullptr;411    struct ggml_tensor * time_mix_decay_w1     = nullptr;412    struct ggml_tensor * time_mix_decay_w2     = nullptr;413    struct ggml_tensor * time_mix_key          = nullptr;414    struct ggml_tensor * time_mix_key_b        = nullptr;415    struct ggml_tensor * time_mix_value        = nullptr;416    struct ggml_tensor * time_mix_value_b      = nullptr;417    struct ggml_tensor * time_mix_receptance   = nullptr;418    struct ggml_tensor * time_mix_receptance_b = nullptr;419    struct ggml_tensor * time_mix_gate         = nullptr;420 421    // rwkv7422    struct ggml_tensor * time_mix_w0         = nullptr;423    struct ggml_tensor * time_mix_a0         = nullptr;424    struct ggml_tensor * time_mix_a1         = nullptr;425    struct ggml_tensor * time_mix_a2         = nullptr;426    struct ggml_tensor * time_mix_v0         = nullptr;427    struct ggml_tensor * time_mix_v1         = nullptr;428    struct ggml_tensor * time_mix_v2         = nullptr;429    struct ggml_tensor * time_mix_g1         = nullptr;430    struct ggml_tensor * time_mix_g2         = nullptr;431    struct ggml_tensor * time_mix_k_k        = nullptr;432    struct ggml_tensor * time_mix_k_a        = nullptr;433    struct ggml_tensor * time_mix_r_k        = nullptr;434 435    struct ggml_tensor * time_mix_ln     = nullptr;436    struct ggml_tensor * time_mix_ln_b   = nullptr;437    struct ggml_tensor * time_mix_output = nullptr;438 439    struct ggml_tensor * channel_mix_lerp_k = nullptr;440    struct ggml_tensor * channel_mix_lerp_r = nullptr;441 442    struct ggml_tensor * channel_mix_key        = nullptr;443    struct ggml_tensor * channel_mix_receptance = nullptr;444    struct ggml_tensor * channel_mix_value      = nullptr;445 446    // long rope factors447    struct ggml_tensor * rope_long  = nullptr;448    struct ggml_tensor * rope_short = nullptr;449    struct ggml_tensor * rope_freqs = nullptr;450 451    // bitnet scale452    struct ggml_tensor * wq_s       = nullptr;453    struct ggml_tensor * wk_s       = nullptr;454    struct ggml_tensor * wv_s       = nullptr;455    struct ggml_tensor * wo_s       = nullptr;456    struct ggml_tensor * wqkv_s     = nullptr;457    struct ggml_tensor * wqkv_gate_s = nullptr;458    struct ggml_tensor * ffn_gate_s = nullptr;459    struct ggml_tensor * ffn_up_s   = nullptr;460    struct ggml_tensor * ffn_down_s = nullptr;461    struct ggml_tensor * ffn_gate_shexp_s = nullptr;462    struct ggml_tensor * ffn_up_shexp_s   = nullptr;463    struct ggml_tensor * ffn_down_shexp_s = nullptr;464    struct ggml_tensor * ssm_in_s    = nullptr;465    struct ggml_tensor * ssm_out_s   = nullptr;466    struct ggml_tensor * ssm_alpha_s = nullptr;467    struct ggml_tensor * ssm_beta_s  = nullptr;468 469    // input scales470    struct ggml_tensor * wq_in_s            = nullptr;471    struct ggml_tensor * wk_in_s            = nullptr;472    struct ggml_tensor * wv_in_s            = nullptr;473    struct ggml_tensor * wo_in_s            = nullptr;474    struct ggml_tensor * wqkv_in_s          = nullptr;475    struct ggml_tensor * wqkv_gate_in_s     = nullptr;476    struct ggml_tensor * ffn_gate_in_s      = nullptr;477    struct ggml_tensor * ffn_up_in_s        = nullptr;478    struct ggml_tensor * ffn_down_in_s      = nullptr;479    struct ggml_tensor * ffn_gate_exps_in_s = nullptr;480    struct ggml_tensor * ffn_down_exps_in_s = nullptr;481    struct ggml_tensor * ffn_up_exps_in_s   = nullptr;482    struct ggml_tensor * ffn_gate_shexp_in_s= nullptr;483    struct ggml_tensor * ffn_up_shexp_in_s  = nullptr;484    struct ggml_tensor * ffn_down_shexp_in_s= nullptr;485    struct ggml_tensor * ssm_in_in_s        = nullptr;486    struct ggml_tensor * ssm_out_in_s       = nullptr;487    struct ggml_tensor * ssm_alpha_in_s     = nullptr;488    struct ggml_tensor * ssm_beta_in_s      = nullptr;489 490    // altup & laurel491    struct ggml_tensor * per_layer_inp_gate   = nullptr;492    struct ggml_tensor * per_layer_proj       = nullptr;493    struct ggml_tensor * per_layer_post_norm  = nullptr;494    struct ggml_tensor * altup_correct_coef   = nullptr;495    struct ggml_tensor * altup_correct_scale  = nullptr;496    struct ggml_tensor * altup_predict_coef   = nullptr;497    struct ggml_tensor * altup_router         = nullptr;498    struct ggml_tensor * altup_router_norm    = nullptr;499    struct ggml_tensor * laurel_l             = nullptr;500    struct ggml_tensor * laurel_r             = nullptr;501    struct ggml_tensor * laurel_post_norm     = nullptr;502 503    // openai-moe504    struct ggml_tensor * attn_sinks = nullptr;505 506    // DeepSeek-V4507    struct ggml_tensor * attn_kv_norm = nullptr;508    struct ggml_tensor * hc_attn_fn   = nullptr;509    struct ggml_tensor * hc_attn_base = nullptr;510    struct ggml_tensor * hc_attn_scale = nullptr;511    struct ggml_tensor * hc_ffn_fn    = nullptr;512    struct ggml_tensor * hc_ffn_base  = nullptr;513    struct ggml_tensor * hc_ffn_scale = nullptr;514    struct ggml_tensor * attn_comp_wkv   = nullptr;515    struct ggml_tensor * attn_comp_wgate = nullptr;516    struct ggml_tensor * attn_comp_ape   = nullptr;517    struct ggml_tensor * attn_comp_norm  = nullptr;518    struct ggml_tensor * indexer_comp_wkv   = nullptr;519    struct ggml_tensor * indexer_comp_wgate = nullptr;520    struct ggml_tensor * indexer_comp_ape   = nullptr;521    struct ggml_tensor * indexer_comp_norm  = nullptr;522 523    // cogvlm524    struct ggml_tensor * visexp_attn_wqkv = nullptr;525    struct ggml_tensor * visexp_attn_wo   = nullptr;526    struct ggml_tensor * visexp_ffn_gate  = nullptr;527    struct ggml_tensor * visexp_ffn_down  = nullptr;528    struct ggml_tensor * visexp_ffn_up    = nullptr;529 530    // xIELU activation parameters for Apertus531    struct ggml_tensor * ffn_act_alpha_n = nullptr;532    struct ggml_tensor * ffn_act_alpha_p = nullptr;533    struct ggml_tensor * ffn_act_beta    = nullptr;534    struct ggml_tensor * ffn_act_eps     = nullptr;535 536    // Kimi Linear KDA (using ssm_ prefix for consistency)537    // Note: ssm_dt_b already exists above (mamba bias), reused for Kimi dt_bias538    struct ggml_tensor * ssm_q_conv = nullptr;539    struct ggml_tensor * ssm_k_conv = nullptr;540    struct ggml_tensor * ssm_v_conv = nullptr;541    struct ggml_tensor * ssm_f_a    = nullptr;542    struct ggml_tensor * ssm_f_b    = nullptr;543    struct ggml_tensor * ssm_beta   = nullptr;544    struct ggml_tensor * ssm_g_a    = nullptr;545    struct ggml_tensor * ssm_g_b    = nullptr;546    struct ggml_tensor * ssm_o_norm = nullptr;547 548    // kimi-k3549    struct ggml_tensor * ssm_g           = nullptr; // full-rank KDA gate (replaces ssm_g_a/ssm_g_b)550    struct ggml_tensor * attn_res_score  = nullptr; // fused res_norm*res_proj, pre-attention551    struct ggml_tensor * ffn_res_score   = nullptr; // fused res_norm*res_proj, pre-FFN552    struct ggml_tensor * ffn_routed_down = nullptr; // latent MoE: n_embd -> n_expert_latent553    struct ggml_tensor * ffn_routed_up   = nullptr; // latent MoE: n_expert_latent -> n_embd554    struct ggml_tensor * ffn_routed_norm = nullptr;555 556    // DSA (deepseek sparse attention)557    struct ggml_tensor * indexer_k_norm   = nullptr;558    struct ggml_tensor * indexer_k_norm_b = nullptr;559    struct ggml_tensor * indexer_proj     = nullptr;560    struct ggml_tensor * indexer_attn_k   = nullptr;561    struct ggml_tensor * indexer_attn_q_b = nullptr; // note: for lora a/b, not bias562 563    // MSA564    struct ggml_tensor * index_q_proj = nullptr;565    struct ggml_tensor * index_k_proj = nullptr;566    struct ggml_tensor * index_q_norm = nullptr;567    struct ggml_tensor * index_k_norm = nullptr;568 569    struct ggml_tensor * hc_attn_norm   = nullptr;570    struct ggml_tensor * hc_attn_down   = nullptr;571    struct ggml_tensor * hc_attn_up     = nullptr;572    struct ggml_tensor * hc_attn_inject = nullptr;573    struct ggml_tensor * hc_ffn_norm    = nullptr;574    struct ggml_tensor * hc_ffn_down    = nullptr;575    struct ggml_tensor * hc_ffn_up      = nullptr;576    struct ggml_tensor * hc_ffn_inject  = nullptr;577 578    struct ggml_tensor * ple_key        = nullptr;579    struct ggml_tensor * ple_value      = nullptr;580    struct ggml_tensor * ple_norm_key   = nullptr;581    struct ggml_tensor * ple_norm_query = nullptr;582    struct ggml_tensor * ple_norm_conv  = nullptr;583    struct ggml_tensor * ple_conv1d     = nullptr;584 585    // gemma4 layer output scale, reused for talkie embedding skip scale586    struct ggml_tensor * out_scale = nullptr;587 588    struct llama_layer_posnet posnet;589 590    struct llama_layer_convnext convnext;591 592    struct llama_layer_shortconv shortconv;593 594    struct llama_layer_nextn nextn;595 596    struct llama_layer_switch_lora switch_lora;597};598 599struct llama_device {600    bool is_meta;601 602    ggml_backend_dev_t dev;603};604 605struct llama_meta_device_get_split_state_userdata {606    size_t                     n_devices;607    const struct llama_model * model;608};609 610struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata);611 612struct llama_model {613    llm_type type = LLM_TYPE_UNKNOWN;614    llm_arch arch = LLM_ARCH_UNKNOWN;615 616    std::string name = "n/a";617 618    llama_hparams hparams = {};619    llama_vocab   vocab;620 621    // for classifier models622    std::vector<std::string> classifier_labels;623 624    struct ggml_tensor * tok_embd   = nullptr;625    struct ggml_tensor * type_embd  = nullptr;626    struct ggml_tensor * pos_embd   = nullptr;627    struct ggml_tensor * tok_norm   = nullptr;628    struct ggml_tensor * tok_norm_b = nullptr;629 630    struct ggml_tensor * output_norm     = nullptr;631    struct ggml_tensor * output_res_score = nullptr; // kimi-k3: final cross-layer residual mix632    struct ggml_tensor * output_norm_b   = nullptr;633    struct ggml_tensor * output          = nullptr;634    struct ggml_tensor * output_b        = nullptr;635    struct ggml_tensor * output_norm_enc = nullptr;636 637 638    // NVFP4 per-tensor scale2, input_scale for LM head639    struct ggml_tensor * output_s    = nullptr;640    struct ggml_tensor * output_in_s = nullptr;641 642    // NextN/MTP model-level projections643    struct ggml_tensor * nextn_proj_pre  = nullptr;644    struct ggml_tensor * nextn_proj_post = nullptr;645 646    // hrm-text initial low-cycle state647    struct ggml_tensor * hrm_z_l_init = nullptr;648 649    // DeepSeek-V4650    struct ggml_tensor * hc_head_fn    = nullptr;651    struct ggml_tensor * hc_head_base  = nullptr;652    struct ggml_tensor * hc_head_scale = nullptr;653 654    // classifier655    struct ggml_tensor * cls       = nullptr;656    struct ggml_tensor * cls_b     = nullptr;657    struct ggml_tensor * cls_out   = nullptr;658    struct ggml_tensor * cls_out_b = nullptr;659    struct ggml_tensor * cls_norm  = nullptr;660 661    struct ggml_tensor * conv1d   = nullptr;662    struct ggml_tensor * conv1d_b = nullptr;663 664    // gemma3n altup665    struct ggml_tensor * altup_proj           = nullptr;666    struct ggml_tensor * altup_unembd_proj    = nullptr;667    struct ggml_tensor * per_layer_tok_embd   = nullptr;668 669    struct ggml_tensor * hc_head_norm = nullptr;670    struct ggml_tensor * hc_head_down = nullptr;671    struct ggml_tensor * hc_head_up   = nullptr;672    struct ggml_tensor * per_layer_model_proj = nullptr;673    struct ggml_tensor * per_layer_proj_norm  = nullptr;674 675    // eagle3 / dflash feature fusion layer676    struct ggml_tensor * fc   = nullptr;677    struct ggml_tensor * fc_s = nullptr;678    struct ggml_tensor * d2t = nullptr;  // draft to target vocabulary mapping679 680    // dspark681    struct ggml_tensor * dspark_markov_w1   = nullptr;682    struct ggml_tensor * dspark_markov_w2   = nullptr;683    struct ggml_tensor * dspark_markov_w2_s = nullptr;684    struct ggml_tensor * dspark_conf_proj   = nullptr;685    struct ggml_tensor * dspark_conf_proj_b = nullptr;686 687    struct ggml_tensor * dflash_selector_prev   = nullptr;688    struct ggml_tensor * dflash_selector_next   = nullptr;689    struct ggml_tensor * dflash_selector_hidden = nullptr;690 691    // unified vector to store target-model extracted layer ids in eagle3, dflash, etc.692    std::vector<int32_t> target_layer_ids;693 694    std::vector<llama_layer> layers;695 696    //Dense linear projections for SentenceTransformers models like embeddinggemma697    // For Sentence Transformers models structure see698    // https://sbert.net/docs/sentence_transformer/usage/custom_models.html#structure-of-sentence-transformer-models699    struct ggml_tensor * dense_2_out_layers   = nullptr;700    struct ggml_tensor * dense_2_out_layers_b = nullptr;701    struct ggml_tensor * dense_3_out_layers   = nullptr;702 703    // gguf metadata704    std::unordered_map<std::string, std::string> gguf_kv;705 706    // list of devices used in this model707    std::vector<llama_device> devices;708 709    // for quantize-stats only710    std::vector<std::pair<std::string, struct ggml_tensor *>> tensors_by_name;711 712    // for keeping track of associated LoRA adapters713    std::unordered_set<llama_adapter_lora *> loras;714 715    // statically allocated context for assigning716    struct llama_meta_device_get_split_state_userdata get_split_state_ud;717 718    int64_t t_load_us  = 0;719    int64_t t_start_us = 0;720 721    explicit llama_model(const llama_model_params & params);722    virtual ~llama_model();723 724    std::string arch_name() const;725    std::string type_name() const;726 727    std::string desc() const;728 729    llama_ftype ftype() const;730 731    size_t size() const; // file size732    size_t n_tensors() const;733    size_t n_devices() const;734    const float * tensor_split() const;735 736    uint32_t n_gpu_layers() const;737    llama_split_mode split_mode() const;738 739    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;740 741    // total number of parameters in the model742    uint64_t n_elements() const;743 744    void print_info() const;745 746    ggml_backend_dev_t dev_layer(int il) const;747    ggml_backend_dev_t dev_output() const;748 749    ggml_backend_buffer_type_t select_buft(int il) const;750 751    bool has_tensor_overrides() const;752 753    const struct ggml_tensor * get_tensor(const char * name) const;754 755    float get_rope_freq_base (const llama_cparams & cparams, int il) const;756    float get_rope_freq_scale(const llama_cparams & cparams, int il) const;757 758    ggml_tensor * get_rope_factors(const llama_cparams & cparams, int il) const;759 760    llama_memory_i * create_memory(const llama_memory_params & params, const llama_cparams & cparams) const;761 762    ggml_cgraph * build_graph(const llm_graph_params & params) const;763 764    virtual void load_stats  (llama_model_loader & ml) = 0;765    virtual void load_hparams(llama_model_loader & ml) = 0;766    virtual void load_vocab  (llama_model_loader & ml) = 0;767    virtual bool load_tensors(llama_model_loader & ml) = 0; // returns false if cancelled by progress_callback768 769    // model must define these770    virtual void load_arch_hparams(llama_model_loader & ml) = 0;771    virtual void load_arch_tensors(llama_model_loader & ml) = 0;772    virtual std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const = 0;773 774protected:775    llama_model_params params;776 777    struct impl;778    std::unique_ptr<impl> pimpl;779};780 781llama_model * llama_model_create(llm_arch arch, const llama_model_params & params);782llama_model * llama_model_create(llama_model_loader & ml, const llama_model_params & params);783 784// model must inherit from this785struct llama_model_base : public llama_model {786    friend struct llama_model;787 788    llama_model * model;789    llama_model_loader * ml = nullptr;790    const LLM_TN tn;791 792    // llama_model_loader is not yet defined at this point, so we will set it after construction793    const int TENSOR_DUPLICATED;794    const int TENSOR_NOT_REQUIRED;795    const int TENSOR_SKIP;796    const int TENSOR_SKIP_IF_VIRTUAL;797    const int TENSOR_ALLOW_RESHAPE;798    const int TENSOR_READ_LAZY;799 800    explicit llama_model_base(const llama_model_params & params);801    virtual ~llama_model_base() = default;802 803    ggml_tensor * create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags);804 805    // convenience overload of create_tensor that doesn't require llama_model_loader806    ggml_tensor * create_tensor(const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags);807 808    // helper: try merged gate_up_exps first, fall back to separate gate and up809    void create_tensor_gate_up_exps(llama_layer & layer, int bid, int64_t n_embd_,810                int64_t n_ff_, int64_t n_expert_, int flags);811 812    // helper: try to load merged qkv first, fall back to separate q, k, v813    void create_tensor_qkv(llama_layer & layer, int bid,814                int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,815                int flags);816 817    // helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern818    void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false);819 820    void load_stats  (llama_model_loader & ml) override;821    void load_hparams(llama_model_loader & ml) override;822    void load_vocab  (llama_model_loader & ml) override;823    bool load_tensors(llama_model_loader & ml) override;824 825    // model must define these826    void load_arch_hparams(llama_model_loader & ml) override = 0;827    void load_arch_tensors(llama_model_loader & ml) override = 0;828    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override = 0;829};830 831const char * llm_type_name(llm_type type);832 833// convenience macro for loading local variables for load_tensors() in llama_model_base834// note: cast to int64_t since we will use these for the tensor dimensions835#define LLAMA_LOAD_LOCALS \836    const int     n_layer        = hparams.n_layer();        GGML_UNUSED(n_layer); \837    const int     n_layer_all    = hparams.n_layer_all;      GGML_UNUSED(n_layer_all); \838    const int     n_layer_nextn  = hparams.n_layer_nextn;    GGML_UNUSED(n_layer_nextn); \839    const int64_t n_head         = hparams.n_head();         GGML_UNUSED(n_head); \840    const int64_t n_head_kv      = hparams.n_head_kv();      GGML_UNUSED(n_head_kv); \841    const int64_t n_embd         = hparams.n_embd;           GGML_UNUSED(n_embd); \842    const int64_t n_embd_k_gqa   = hparams.n_embd_k_gqa();   GGML_UNUSED(n_embd_k_gqa); \843    const int64_t n_embd_v_gqa   = hparams.n_embd_v_gqa();   GGML_UNUSED(n_embd_v_gqa); \844    const int64_t n_embd_head_k  = hparams.n_embd_head_k();  GGML_UNUSED(n_embd_head_k); \845    const int64_t n_embd_head_v  = hparams.n_embd_head_v();  GGML_UNUSED(n_embd_head_v); \846    const int64_t n_ff           = hparams.n_ff();           GGML_UNUSED(n_ff); \847    const int64_t n_embd_gqa     = n_embd_v_gqa;             GGML_UNUSED(n_embd_gqa); \848    const int64_t n_vocab        = vocab.n_tokens();         GGML_UNUSED(n_vocab); \849    const int64_t n_token_types  = vocab.n_token_types();    GGML_UNUSED(n_token_types); \850    const int64_t n_rot          = hparams.n_rot();          GGML_UNUSED(n_rot); \851    const int64_t n_expert       = hparams.n_expert;         GGML_UNUSED(n_expert); \852    const int64_t n_expert_used  = hparams.n_expert_used();  GGML_UNUSED(n_expert_used); \853    const int64_t n_ctx_train    = hparams.n_ctx_train;      GGML_UNUSED(n_ctx_train);854 855// For internal test use856// TODO: remove857const std::vector<std::pair<std::string, ggml_tensor *>> & llama_internal_get_tensor_map(const llama_model * model);858