Felipe97/llama-cpp-compiled
01.1k
1#pragma once2 3#include "llama.h"4 5#include <array>6#include <bitset>7#include <cassert>8#include <cmath>9 10// bump if necessary11#define LLAMA_MAX_LAYERS 51212#define LLAMA_MAX_EXPERTS 1024 // Kimi K313#define LLAMA_MAX_PLE_NGRAM 8 // qwen4exp14#define LLAMA_MAX_PLE_HEADS 64 // qwen4exp15 16enum llama_expert_gating_func_type {17 LLAMA_EXPERT_GATING_FUNC_TYPE_NONE = 0,18 LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX = 1,19 LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID = 2,20 LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT = 3, // applied to the router weights instead of the logits21 LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS = 4,22};23 24enum llama_swa_type {25 LLAMA_SWA_TYPE_NONE = 0,26 LLAMA_SWA_TYPE_STANDARD = 1,27 LLAMA_SWA_TYPE_CHUNKED = 2,28 LLAMA_SWA_TYPE_SYMMETRIC = 3,29};30 31// how the non-causal mask should be constructed with llama_set_causal_attn(ctx, false)32// (e.g. mtmd decoding image tokens)33enum llama_non_causal_type {34 LLAMA_NON_CAUSAL_TYPE_ALL = 0, // all layers non-causal, SWA still applied (gemma 3, qwen-vl, ...)35 LLAMA_NON_CAUSAL_TYPE_SWA_ONLY = 1, // SWA layers non-causal, dense layers stay causal (gemma 4)36 LLAMA_NON_CAUSAL_TYPE_SWA_FULL = 2, // all layers non-causal, SWA not applied between tokens of the current ubatch (deepseek 4)37};38 39// forward declaration; full definition in llama-graph.h40enum llm_ffn_op_type : int;41 42struct llama_hparams_posnet {43 uint32_t n_embd;44 uint32_t n_layer;45};46 47struct llama_hparams_convnext {48 uint32_t n_embd;49 uint32_t n_layer;50};51 52struct llama_hparams {53 // note: use the `_impl` suffix to avoid name conflict between members and getters54 // for example: n_embd_out() vs n_embd_out_impl55 56 bool vocab_only;57 bool no_alloc;58 bool rope_finetuned;59 bool use_par_res;60 bool swin_norm;61 bool norm_before_residual = false;62 bool norm_before_fc = false;63 64 uint32_t n_ctx_train; // context size the model was trained on65 uint32_t n_embd;66 uint32_t n_layer_all;67 uint32_t n_layer_nextn = 0;68 69 // granite-switch: index of the single-head "router" KV layer that encodes70 // per-token adapter selection. -1 when the model has no such layer.71 int32_t router_layer = -1;72 uint32_t n_expert = 0;73 uint32_t n_rel_attn_bkts = 0;74 75 // TODO: this needs to be reworked76 int32_t n_layer_kv_from_start = -1; // if non-negative, the first n_layer_kv_from_start layers have KV cache77 78 // different head size for full_attention and SWA layers79 uint32_t n_embd_head_k_full; // dimension of keys (d_k). d_q is assumed to be the same, but there are n_head q heads, and only n_head_kv k-v heads80 uint32_t n_embd_head_v_full; // dimension of values (d_v) aka n_embd_head81 uint32_t n_embd_head_k_swa;82 uint32_t n_embd_head_v_swa;83 84 // different RoPE dimensions for full_attention and SWA layers85 uint32_t n_rot_full;86 uint32_t n_rot_swa;87 88 // note: deepseek2 using MLA converts into MQA with larger heads, then decompresses to MHA89 uint32_t n_embd_head_k_mla_impl = 0;90 uint32_t n_embd_head_v_mla_impl = 0;91 92 // for WavTokenizer93 struct llama_hparams_posnet posnet;94 struct llama_hparams_convnext convnext;95 96 uint32_t n_shortconv_l_cache = 0;97 98 std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_arr;99 std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;100 std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;101 102 // per-layer expert feed-forward size103 std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_exp_arr;104 // per-layer top-k expert routing count105 std::array<uint32_t, LLAMA_MAX_LAYERS> n_expert_used_arr;106 107 uint32_t n_layer_dense_lead = 0;108 uint32_t n_lora_q = 0;109 uint32_t n_lora_kv = 0;110 uint32_t n_ff_shexp = 0;111 uint32_t n_ff_chexp = 0;112 uint32_t n_expert_shared = 0;113 uint32_t n_norm_groups = 0;114 uint32_t n_expert_groups = 0;115 uint32_t n_group_used = 0;116 uint32_t n_group_experts = 0;117 118 // MLA + SWA (i.e. dots3note)119 uint32_t n_lora_kv_swa = 0;120 uint32_t n_embd_head_k_mla_swa = 0;121 uint32_t n_embd_head_v_mla_swa = 0;122 123 float expert_group_scale = 0.05f;124 float expert_weights_scale = 0.0f;125 bool expert_weights_norm = false;126 uint32_t expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_NONE;127 uint32_t moe_every_n_layers = 0;128 uint32_t moe_latent_size = 0;129 130 float f_norm_eps;131 float f_norm_rms_eps;132 float f_norm_group_eps;133 134 float f_attn_logit_softcapping = 50.0f;135 float f_router_logit_softcapping = 30.0f;136 float f_final_logit_softcapping = 30.0f;137 138 // for RWKV139 uint32_t rescale_every_n_layers = 0;140 uint32_t time_mix_extra_dim = 0;141 uint32_t time_decay_extra_dim = 0;142 uint32_t wkv_head_size = 0;143 uint32_t token_shift_count = 2;144 uint32_t n_lora_decay = 0;145 uint32_t n_lora_iclr = 0;146 uint32_t n_lora_value_res_mix = 0;147 uint32_t n_lora_gate = 0;148 149 float rope_attn_factor = 1.0f;150 float rope_freq_base_train;151 float rope_freq_base_train_swa = 10000.0f;152 float rope_freq_scale_train;153 float rope_freq_scale_train_swa = 1.0f;154 float rope_scaling_alpha = 0.0f; // NTK-aware alpha for XDRoPE155 156 uint32_t n_ctx_orig_yarn;157 float rope_yarn_log_mul = 0.0f;158 159 float yarn_ext_factor = -1.0f;160 float yarn_attn_factor = 1.0f;161 float yarn_beta_fast = 32.0f;162 float yarn_beta_slow = 1.0f;163 164 std::array<int, 4> rope_sections;165 166 // Per-layer RoPE enable flags (1 = use RoPE, 0 = NoPE)167 // by default, all layers use RoPE (controlled by rope_finetuned)168 std::array<uint32_t, LLAMA_MAX_LAYERS> rope_pattern;169 170 // Sliding Window Attention (SWA)171 llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;172 // the size of the sliding window (0 - no SWA)173 uint32_t n_swa = 0;174 175 // see llama_non_causal_type176 // note: for SWA_FULL, older tokens (outside the current ubatch) are still window-clipped177 llama_non_causal_type non_causal_type = LLAMA_NON_CAUSAL_TYPE_ALL;178 179 // if is_swa_impl[il] == 1, then layer il is SWA180 // if is_swa_impl[il] == 0, then layer il is dense (i.e. non-SWA)181 // by default, all layers are dense182 // note: using uint32_t type for compatibility reason183 std::array<uint32_t, LLAMA_MAX_LAYERS> is_swa_impl;184 185 // for hybrid state space models186 std::array<uint32_t, LLAMA_MAX_LAYERS> is_recr_impl;187 188 // for State Space Models189 uint32_t ssm_d_conv = 0;190 uint32_t ssm_d_inner = 0;191 uint32_t ssm_d_state = 0;192 uint32_t ssm_dt_rank = 0;193 uint32_t ssm_n_group = 0;194 195 // for MiniMax-Text-01 linear attention196 uint32_t n_embd_head_la = 0;197 198 // for Kimi Linear KDA199 uint32_t n_embd_head_kda = 0;200 bool kda_safe_gate = false;201 202 // kimi-k3203 uint32_t n_expert_latent = 0; // routed_expert_hidden_size (0 = experts run at n_embd)204 uint32_t attn_res_block_size = 0; // 0 = no cross-layer attention residuals205 float kda_gate_lower_bound = -INFINITY;206 float situ_beta = 1.0f;207 float situ_linear_beta = 0.0f; // 0 = no linear-beta transform on the up branch208 209 // hrm-text (looped H/L stacks)210 uint32_t n_hrm_layers_per_stack = 0;211 uint32_t n_hrm_h_cycles = 0;212 uint32_t n_hrm_l_cycles = 0;213 bool hrm_prefix_lm = false;214 215 bool ssm_dt_b_c_rms = false;216 217 float f_clamp_kqv = 0.0f;218 float f_max_alibi_bias = 0.0f;219 float f_logit_scale = 0.0f;220 221 // Additional scale factors (Granite/Granite MoE)222 float f_residual_scale = 0.0f;223 float f_embedding_scale = 0.0f;224 float f_attention_scale = 0.0f;225 226 // grok-2227 float f_attn_out_scale = 0.0f;228 uint32_t attn_temp_length = 0;229 230 float f_attn_value_scale = 0.0f;231 232 bool causal_attn = true;233 bool use_alibi = false;234 bool attn_soft_cap = false;235 bool use_kq_norm = false;236 237 // for Classifiers238 uint32_t n_cls_out = 1;239 240 // input embedding dimension (0 = use n_embd)241 uint32_t n_embd_inp_impl = 0;242 243 // encoder input embedding dimension (0 = use n_embd_inp())244 // e.g. the eagle3 encoder fuses target_layers * target_hidden features245 uint32_t n_embd_inp_enc_impl = 0;246 247 // output embedding dimension (0 = use n_embd)248 uint32_t n_embd_out_impl = 0;249 250 uint32_t dflash_block_size = 0;251 uint32_t dflash_conv_kernel_size = 0;252 uint32_t dflash_conv_group_size = 0;253 uint32_t dflash_selector_rank = 0;254 uint32_t dflash_selector_top_k = 0;255 256 // llama4 smallthinker257 uint32_t n_moe_layer_step = 0;258 uint32_t n_no_rope_layer_step = 4;259 uint32_t n_attn_temp_floor_scale = 0;260 float f_attn_temp_scale = 0.0f;261 float f_attn_temp_offset = 0.0f; // offset position index262 263 // gemma3n altup264 uint32_t n_altup = 4; // altup_num_inputs265 uint32_t i_altup_act = 0; // altup_active_idx266 uint32_t laurel_rank = 64;267 uint32_t n_embd_altup = 256;268 269 // needed for sentence-transformers dense layers270 uint32_t dense_2_feat_in = 0; // in_features of the 2_Dense271 uint32_t dense_2_feat_out = 0; // out_features of the 2_Dense272 uint32_t dense_3_feat_in = 0; // in_features of the 3_Dense273 uint32_t dense_3_feat_out = 0; // out_features of the 3_Dense274 275 // xIELU276 std::array<float, LLAMA_MAX_LAYERS> xielu_alpha_n;277 std::array<float, LLAMA_MAX_LAYERS> xielu_alpha_p;278 std::array<float, LLAMA_MAX_LAYERS> xielu_beta;279 std::array<float, LLAMA_MAX_LAYERS> xielu_eps;280 281 // DSA (deepseek sparse attention)282 uint32_t indexer_n_head = 0;283 uint32_t indexer_head_size = 0;284 uint32_t indexer_top_k = 0;285 // MSA286 uint32_t indexer_block_size = 0;287 uint32_t indexer_local_blocks = 0;288 289 // Indexer is "full" (1) or "shared" (0)290 // Shared indexers reuse top-k from previous full layer291 std::array<uint32_t, LLAMA_MAX_LAYERS> is_indexer_full_impl;292 293 // DeepSeek-V4294 uint32_t dsv4_o_group_count = 0;295 uint32_t dsv4_o_lora_rank = 0;296 uint32_t dsv4_hc_mult = 0;297 uint32_t dsv4_hc_sinkhorn_iters = 0;298 uint32_t dsv4_hash_layer_count = 0;299 float dsv4_compress_rope_base = 0.0f;300 float dsv4_hc_eps = 0.0f;301 std::array<uint32_t, LLAMA_MAX_LAYERS> dsv4_compress_ratios;302 303 // 0 = full rank (DeepSeek-V4)304 uint32_t hc_low_rank = 0;305 306 // scale of the hyper-connection post gate (DeepSeek-V4 hardcodes 2.0)307 float hc_magnitude = 0.0f;308 309 uint32_t ple_ngram_size = 0;310 uint32_t ple_heads_per_ngram = 0;311 uint32_t ple_conv_kernel = 0;312 uint32_t ple_n_heads = 0; // (ngram_size - 1) * heads_per_ngram313 uint32_t ple_head_dim = 0;314 uint32_t ple_eos_token_id = 0;315 // the id the PLE hash stands in at image positions; 0 makes the loader fall back to EOS316 uint32_t ple_image_token_id = 0;317 // the file lists PLE layer indices, so this is never a per-layer gguf array and can hold one bit per layer318 std::bitset<LLAMA_MAX_LAYERS> is_ple_impl;319 // the hash multipliers reach ~2e13 and have to stay 64-bit320 std::array<uint64_t, LLAMA_MAX_PLE_NGRAM> ple_layer_multipliers;321 // head offsets and vocab sizes are token-space indices; the gather truncates them to int32 anyway322 std::array<uint32_t, LLAMA_MAX_PLE_HEADS> ple_head_offsets;323 std::array<uint32_t, LLAMA_MAX_PLE_HEADS> ple_head_vocab_sizes;324 325 bool is_ple(uint32_t il) const;326 327 // PLE conv history rows: (kernel - 1) * ngram_size; 0 without a PLE module328 uint32_t ple_conv_state() const;329 330 // qwen3vl deepstack331 // When parsed from GGUF, this implies the first N layers consume the first332 // N deepstack embeddings. Use deepstack_mapping_arr if you need a more333 // complex mapping. If using deepstack_mapping_arr, also make sure to set334 // n_deepstack_layers to the number of unique deepstack layers so that335 // n_embd_imp is accurate (see granite.cpp).336 // TODO: can be expressed via the `new n_embd_inp_impl` and remove this param337 uint32_t n_deepstack_layers = 0;338 339 // deepstack layer array (Granite4 Vision)340 // -1 => no deepstack341 // >=0 => input embedding index for deepstack injection342 std::array<int32_t, LLAMA_MAX_LAYERS> deepstack_mapping_arr;343 344 // gemma4 per-layer embedding345 uint32_t n_embd_per_layer = 0;346 347 // needed by encoder-decoder models (e.g. T5, FLAN-T5)348 // ref: https://github.com/ggml-org/llama.cpp/pull/8141349 llama_token dec_start_token_id = LLAMA_TOKEN_NULL;350 uint32_t dec_n_layer = 0;351 352 enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE;353 enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE;354 enum llama_rope_scaling_type rope_scaling_type_train = LLAMA_ROPE_SCALING_TYPE_NONE;355 356 357 // Resolved FFN gated activation flavor for archs that read358 // `<arch>.hidden_activation` from the GGUF (e.g. ModernBert derivatives).359 // Defaults to LLM_FFN_NONE (sentinel = 0); the mapping from the GGUF360 // string to a real op is done at hparam-load time via361 // llm_ffn_op_type_from_string() in llama-model.cpp, mirroring how362 // rope_scaling_type_train is handled.363 enum llm_ffn_op_type llm_ffn_op;364 365 // Step35: optional per-layer clamps for (Swi)GLU366 std::array<float, LLAMA_MAX_LAYERS> swiglu_clamp_exp; // clamping for expert FFN367 std::array<float, LLAMA_MAX_LAYERS> swiglu_clamp_shexp; // shared expert368 369 // this value n_pattern means that every nth layer is dense (i.e. non-SWA)370 // dense_first means whether the pattern is start with a dense layer371 // note that if n_pattern == 0, all layers are SWA372 // if n_pattern == 1, all layers are dense373 // example 1: n_pattern = 3, dense_first = false374 // il == 0: swa375 // il == 1: swa376 // il == 2: dense377 // il == 3: swa378 // il == 4: swa379 // il == 5: dense380 // il == 6: swa381 // etc ...382 // example 2: n_pattern = 2, dense_first = true383 // il == 0: dense384 // il == 1: swa385 // il == 2: dense386 // il == 3: swa387 // etc ...388 void set_swa_pattern(uint32_t n_pattern, bool dense_first = false);389 390 // return true if one of the layers is SWA391 bool is_swa_any() const;392 393 bool is_swa(uint32_t il) const;394 395 bool is_indexer_full(uint32_t il) const;396 397 void set_recr_pattern(uint32_t n_pattern, bool dense_first = false);398 399 // whether or not the given layer is recurrent (for hybrid models)400 bool is_recr(uint32_t il) const;401 402 uint32_t n_head(uint32_t il = 0) const;403 404 uint32_t n_head_kv(uint32_t il = 0) const;405 406 uint32_t n_ff(uint32_t il = 0) const;407 408 uint32_t n_ff_exp(uint32_t il = 0) const;409 410 uint32_t n_expert_used(uint32_t il = 0) const;411 412 // return the maximum n_expert_used across all layers413 uint32_t n_expert_used_max() const;414 415 uint32_t n_gqa(uint32_t il = 0) const;416 417 uint32_t n_rot(uint32_t il = 0) const;418 419 // dimension of main + auxiliary input embeddings420 uint32_t n_embd_inp() const;421 422 // dimension of the encoder input embeddings423 uint32_t n_embd_inp_enc() const;424 425 // dimension of output embeddings426 uint32_t n_embd_out() const;427 428 // dimension of key/value embeddings for each head (per layer)429 uint32_t n_embd_head_k(uint32_t il = 0) const;430 uint32_t n_embd_head_v(uint32_t il = 0) const;431 432 // dimension of key embeddings across all k-v heads433 uint32_t n_embd_k_gqa(uint32_t il = 0) const;434 435 // dimension of value embeddings across all k-v heads436 uint32_t n_embd_v_gqa(uint32_t il = 0) const;437 438 // true if any layer has a different n_embd_k_gqa/n_embd_v_gqa439 bool is_n_embd_k_gqa_variable() const;440 bool is_n_embd_v_gqa_variable() const;441 442 // return the maximum n_embd_k_gqa/n_embd_v_gqa across all layers443 uint32_t n_embd_k_gqa_max() const;444 uint32_t n_embd_v_gqa_max() const;445 446 // dimension of the rolling state embeddings447 // corresponds to Mamba's conv_states size or RWKV's token_shift states size448 uint32_t n_embd_r() const;449 450 // dimension of the recurrent state embeddings451 uint32_t n_embd_s() const;452 453 uint32_t n_pos_per_embd() const;454 455 // note: currently only support if either all or none of the layers are MLA456 bool is_mla() const;457 458 uint32_t n_embd_head_k_mla() const;459 uint32_t n_embd_head_v_mla() const;460 461 bool has_kv(uint32_t il) const;462 463 bool has_rope(uint32_t il) const;464 465 // number of effective layers (excludes nextn layers)466 uint32_t n_layer() const;467 468 // note that this function uses different SWA parameters from those in the hparams469 // note: inlined on purpose for performance reasons470 // TODO: think of a better place for this function471 // TODO: pack the SWA params in a struct?472 static bool is_masked_swa(uint32_t n_swa, llama_swa_type swa_type, llama_pos p0, llama_pos p1) {473 assert(p0 >= 0 && p1 >= 0);474 475 switch (swa_type) {476 case LLAMA_SWA_TYPE_NONE:477 {478 } break;479 case LLAMA_SWA_TYPE_STANDARD:480 {481 if (p1 - p0 >= (int32_t) n_swa) {482 return true;483 }484 } break;485 case LLAMA_SWA_TYPE_CHUNKED:486 {487 const llama_pos pos_chunk_start = (p1 / n_swa) * n_swa;488 489 if (p0 < pos_chunk_start) {490 return true;491 }492 } break;493 case LLAMA_SWA_TYPE_SYMMETRIC:494 {495 const int32_t half_n_swa = (int32_t) n_swa / 2;496 const int32_t pos_diff = p1 - p0;497 498 // Mask if outside the symmetric window499 if (pos_diff < -half_n_swa || pos_diff > half_n_swa) {500 return true;501 }502 } break;503 }504 505 return false;506 }507 508 509 bool use_mrope() const;510};511 512static_assert(std::is_trivially_copyable<llama_hparams>::value, "llama_hparams must be trivially copyable");513