Felipe97/llama-cpp-compiled
01.1k
1#pragma once2 3#include "llama-model.h"4#include "llama-graph.h"5#include "llama-model-loader.h"6 7// note: almost all graphs require at least sqrtf, so include cmath globally8#include <cmath>9#include <map>10 11class llama_memory_hybrid_idx_context;12 13// ref: https://github.com/ggml-org/llama.cpp/pull/2806814static inline ggml_tensor * build_gdn_l2_norm(ggml_context * ctx, ggml_tensor * x, float eps) {15 const float n = x->ne[0];16 17 return ggml_scale(ctx, ggml_rms_norm(ctx, x, eps/n), 1.0f/sqrtf(n));18}19 20//21// base classes22//23 24struct llm_build_mamba_base : public llm_graph_context {25 llm_build_mamba_base(const llm_graph_params & params);26 27 virtual ~llm_build_mamba_base() = default;28 29 ggml_tensor * build_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_model & model, const llama_ubatch & ubatch, int il);30 ggml_tensor * build_mamba2_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_model & model, const llama_ubatch & ubatch, int il) const;31 32};33 34struct llm_build_delta_net_base : public llm_graph_context {35 llm_build_delta_net_base(const llm_graph_params & params);36 37 virtual ~llm_build_delta_net_base() = default;38 39 // returns pair of output and new state40 std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_chunking(41 ggml_tensor * q,42 ggml_tensor * k,43 ggml_tensor * v,44 ggml_tensor * g,45 ggml_tensor * b,46 ggml_tensor * s,47 int il);48 49 // returns pair of output and new state50 std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_autoregressive(51 ggml_tensor * q,52 ggml_tensor * k,53 ggml_tensor * v,54 ggml_tensor * g,55 ggml_tensor * b,56 ggml_tensor * s,57 int il);58 59 // use the ggml_gated_delta_net fused operator (K=1; state has shape [S_v, S_v, H_v, n_seqs])60 std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_fused(61 ggml_tensor * q,62 ggml_tensor * k,63 ggml_tensor * v,64 ggml_tensor * g,65 ggml_tensor * b,66 ggml_tensor * s,67 int il);68 69 // choose one of two implementations above based on the number of tokens70 std::pair<ggml_tensor *, ggml_tensor *> build_delta_net(71 ggml_tensor * q,72 ggml_tensor * k,73 ggml_tensor * v,74 ggml_tensor * g,75 ggml_tensor * b,76 ggml_tensor * s,77 int il);78 79 // read conv state from cache, concat with qkv_mixed, write back (single slot or per-token)80 // qkv_mixed: (qkv_dim, n_seq_tokens, n_seqs); returns conv_input: (kernel_size + n_seq_tokens - 1, channels, n_seqs)81 ggml_tensor * build_conv_state(82 llm_graph_input_rs * inp,83 ggml_tensor * conv_states_all,84 ggml_tensor * qkv_mixed,85 int64_t conv_kernel_size,86 int64_t conv_channels,87 int il);88 89 // run delta-net attention and write the new recurrent state(s) back to ssm_states_all90 // s: (head_v_dim, head_v_dim, num_v_heads, n_seqs); returns output: (head_v_dim, num_v_heads, n_seq_tokens, n_seqs)91 ggml_tensor * build_recurrent_attn(92 llm_graph_input_rs * inp,93 ggml_tensor * ssm_states_all,94 ggml_tensor * q,95 ggml_tensor * k,96 ggml_tensor * v,97 ggml_tensor * g,98 ggml_tensor * b,99 ggml_tensor * s,100 int il);101};102 103struct llm_build_rwkv6_base : public llm_graph_context {104 const llama_model & model;105 106 llm_build_rwkv6_base(const llama_model & model, const llm_graph_params & params);107 108 virtual ~llm_build_rwkv6_base() = default;109 110 ggml_tensor * build_rwkv6_channel_mix(const llama_layer * layer,111 ggml_tensor * cur,112 ggml_tensor * x_prev,113 llm_arch arch) const;114 115 ggml_tensor * build_rwkv6_time_mix(llm_graph_input_rs * inp,116 ggml_tensor * cur,117 ggml_tensor * x_prev,118 const llama_ubatch & ubatch,119 int il) const;120};121 122// Base class for RWKV7-related models123struct llm_build_rwkv7_base : public llm_graph_context {124 const llama_model & model;125 126 llm_build_rwkv7_base(const llama_model & model, const llm_graph_params & params);127 128 virtual ~llm_build_rwkv7_base() = default;129 130 // RWKV7-specific graph building methods131 ggml_tensor * build_rwkv7_channel_mix(const llama_layer * layer,132 ggml_tensor * cur,133 ggml_tensor * x_prev,134 llm_arch arch) const;135 ggml_tensor * build_rwkv7_time_mix(llm_graph_input_rs * inp,136 ggml_tensor * cur,137 ggml_tensor * x_prev,138 ggml_tensor *& first_layer_value,139 const llama_ubatch & ubatch,140 int il) const;141};142 143//144// models145//146 147struct llama_model_llama : public llama_model_base {148 llama_model_llama(const struct llama_model_params & params) : llama_model_base(params) {}149 void load_arch_hparams(llama_model_loader & ml) override;150 void load_arch_tensors(llama_model_loader & ml) override;151 152 template <bool embed>153 struct graph : public llm_graph_context {154 graph(const llama_model & model, const llm_graph_params & params);155 };156 157 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;158};159 160 161struct llama_model_llama4 : public llama_model_base {162 llama_model_llama4(const struct llama_model_params & params) : llama_model_base(params) {}163 void load_arch_hparams(llama_model_loader & ml) override;164 void load_arch_tensors(llama_model_loader & ml) override;165 166 template <bool iswa>167 struct graph : public llm_graph_context {168 graph(const llama_model & model, const llm_graph_params & params);169 };170 171 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;172};173 174 175struct llama_model_llama_embed : public llama_model_llama {176 llama_model_llama_embed(const struct llama_model_params & params) : llama_model_llama(params) {}177 // reuse load_arch_hparams and load_arch_tensors from llama_model_llama178 179 template <bool embed>180 using graph = llama_model_llama::graph<embed>;181 182 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;183};184 185 186struct llama_model_maincoder : public llama_model_base {187 llama_model_maincoder(const struct llama_model_params & params) : llama_model_base(params) {}188 void load_arch_hparams(llama_model_loader & ml) override;189 void load_arch_tensors(llama_model_loader & ml) override;190 191 struct graph : public llm_graph_context {192 graph(const llama_model & model, const llm_graph_params & params);193 };194 195 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;196};197 198 199struct llama_model_talkie : public llama_model_base {200 llama_model_talkie(const struct llama_model_params & params) : llama_model_base(params) {}201 void load_arch_hparams(llama_model_loader & ml) override;202 void load_arch_tensors(llama_model_loader & ml) override;203 204 struct graph : public llm_graph_context {205 graph(const llama_model & model, const llm_graph_params & params);206 };207 208 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;209};210 211 212struct llama_model_deci : public llama_model_base {213 llama_model_deci(const struct llama_model_params & params) : llama_model_base(params) {}214 void load_arch_hparams(llama_model_loader & ml) override;215 void load_arch_tensors(llama_model_loader & ml) override;216 217 struct graph : public llm_graph_context {218 graph(const llama_model & model, const llm_graph_params & params);219 };220 221 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;222};223 224 225struct llama_model_baichuan : public llama_model_base {226 llama_model_baichuan(const struct llama_model_params & params) : llama_model_base(params) {}227 void load_arch_hparams(llama_model_loader & ml) override;228 void load_arch_tensors(llama_model_loader & ml) override;229 230 struct graph : public llm_graph_context {231 graph(const llama_model & model, const llm_graph_params & params);232 };233 234 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;235};236 237 238struct llama_model_falcon : public llama_model_base {239 llama_model_falcon(const struct llama_model_params & params) : llama_model_base(params) {}240 void load_arch_hparams(llama_model_loader & ml) override;241 void load_arch_tensors(llama_model_loader & ml) override;242 243 struct graph : public llm_graph_context {244 graph(const llama_model & model, const llm_graph_params & params);245 };246 247 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;248};249 250 251struct llama_model_grok : public llama_model_base {252 llama_model_grok(const struct llama_model_params & params) : llama_model_base(params) {}253 void load_arch_hparams(llama_model_loader & ml) override;254 void load_arch_tensors(llama_model_loader & ml) override;255 256 struct graph : public llm_graph_context {257 graph(const llama_model & model, const llm_graph_params & params);258 };259 260 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;261};262 263 264struct llama_model_starcoder : public llama_model_base {265 llama_model_starcoder(const struct llama_model_params & params) : llama_model_base(params) {}266 void load_arch_hparams(llama_model_loader & ml) override;267 void load_arch_tensors(llama_model_loader & ml) override;268 269 struct graph : public llm_graph_context {270 graph(const llama_model & model, const llm_graph_params & params);271 };272 273 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;274};275 276 277struct llama_model_refact : public llama_model_base {278 llama_model_refact(const struct llama_model_params & params) : llama_model_base(params) {}279 void load_arch_hparams(llama_model_loader & ml) override;280 void load_arch_tensors(llama_model_loader & ml) override;281 282 struct graph : public llm_graph_context {283 graph(const llama_model & model, const llm_graph_params & params);284 };285 286 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;287};288 289 290struct llama_model_bert : public llama_model_base {291 llama_model_bert(const struct llama_model_params & params) : llama_model_base(params) {}292 void load_arch_hparams(llama_model_loader & ml) override;293 void load_arch_tensors(llama_model_loader & ml) override;294 295 struct graph : public llm_graph_context {296 graph(const llama_model & model, const llm_graph_params & params);297 };298 299 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;300};301 302 303struct llama_model_jina_bert_v2 : public llama_model_base {304 llama_model_jina_bert_v2(const struct llama_model_params & params) : llama_model_base(params) {}305 void load_arch_hparams(llama_model_loader & ml) override;306 void load_arch_tensors(llama_model_loader & ml) override;307 308 using graph = llama_model_bert::graph;309 310 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;311};312 313 314struct llama_model_jina_bert_v3 : public llama_model_base {315 llama_model_jina_bert_v3(const struct llama_model_params & params) : llama_model_base(params) {}316 void load_arch_hparams(llama_model_loader & ml) override;317 void load_arch_tensors(llama_model_loader & ml) override;318 319 using graph = llama_model_bert::graph;320 321 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;322};323 324 325struct llama_model_nomic_bert : public llama_model_base {326 llama_model_nomic_bert(const struct llama_model_params & params) : llama_model_base(params) {}327 void load_arch_hparams(llama_model_loader & ml) override;328 void load_arch_tensors(llama_model_loader & ml) override;329 330 using graph = llama_model_bert::graph;331 332 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;333};334 335 336struct llama_model_nomic_bert_moe : public llama_model_base {337 llama_model_nomic_bert_moe(const struct llama_model_params & params) : llama_model_base(params) {}338 void load_arch_hparams(llama_model_loader & ml) override;339 void load_arch_tensors(llama_model_loader & ml) override;340 341 using graph = llama_model_bert::graph;342 343 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;344};345 346 347struct llama_model_modern_bert : public llama_model_base {348 llama_model_modern_bert(const struct llama_model_params & params) : llama_model_base(params) {}349 void load_arch_hparams(llama_model_loader & ml) override;350 void load_arch_tensors(llama_model_loader & ml) override;351 352 struct graph : public llm_graph_context {353 graph(const llama_model & model, const llm_graph_params & params);354 };355 356 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;357};358 359 360struct llama_model_neo_bert : public llama_model_base {361 llama_model_neo_bert(const struct llama_model_params & params) : llama_model_base(params) {}362 void load_arch_hparams(llama_model_loader & ml) override;363 void load_arch_tensors(llama_model_loader & ml) override;364 365 struct graph : public llm_graph_context {366 graph(const llama_model & model, const llm_graph_params & params);367 };368 369 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;370};371 372 373struct llama_model_eurobert : public llama_model_base {374 llama_model_eurobert(const struct llama_model_params & params) : llama_model_base(params) {}375 void load_arch_hparams(llama_model_loader & ml) override;376 void load_arch_tensors(llama_model_loader & ml) override;377 378 struct graph : public llm_graph_context {379 graph(const llama_model & model, const llm_graph_params & params);380 };381 382 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;383};384 385 386struct llama_model_bloom : public llama_model_base {387 llama_model_bloom(const struct llama_model_params & params) : llama_model_base(params) {}388 void load_arch_hparams(llama_model_loader & ml) override;389 void load_arch_tensors(llama_model_loader & ml) override;390 391 struct graph : public llm_graph_context {392 graph(const llama_model & model, const llm_graph_params & params);393 };394 395 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;396};397 398 399// Quant-only stub for mmproj GGUFs400// none of these are ever called, they only exist to satisfy the llama_model_base interface401struct llama_model_clip : public llama_model_base {402 llama_model_clip(const struct llama_model_params & params) : llama_model_base(params) {}403 404 [[noreturn]]405 void load_arch_hparams(llama_model_loader & ml) override;406 407 [[noreturn]]408 void load_arch_tensors(llama_model_loader & ml) override;409 410 [[noreturn]]411 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;412};413 414 415struct llama_model_mpt : public llama_model_base {416 llama_model_mpt(const struct llama_model_params & params) : llama_model_base(params) {}417 void load_arch_hparams(llama_model_loader & ml) override;418 void load_arch_tensors(llama_model_loader & ml) override;419 420 struct graph : public llm_graph_context {421 graph(const llama_model & model, const llm_graph_params & params);422 };423 424 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;425};426 427 428struct llama_model_stablelm : public llama_model_base {429 llama_model_stablelm(const struct llama_model_params & params) : llama_model_base(params) {}430 void load_arch_hparams(llama_model_loader & ml) override;431 void load_arch_tensors(llama_model_loader & ml) override;432 433 struct graph : public llm_graph_context {434 graph(const llama_model & model, const llm_graph_params & params);435 };436 437 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;438};439 440struct llama_model_mellum : public llama_model_base {441 llama_model_mellum(const struct llama_model_params & params) : llama_model_base(params) {}442 void load_arch_hparams(llama_model_loader & ml) override;443 void load_arch_tensors(llama_model_loader & ml) override;444 445 template <bool iswa>446 struct graph : public llm_graph_context {447 graph(const llama_model & model, const llm_graph_params & params);448 };449 450 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;451};452 453struct llama_model_nanbeige : public llama_model_base {454 llama_model_nanbeige(const struct llama_model_params & params) : llama_model_base(params) {}455 void load_arch_hparams(llama_model_loader & ml) override;456 void load_arch_tensors(llama_model_loader & ml) override;457 458 int n_loops = 1;459 int n_layer_phys = 0;460 bool skip_loop_final_norm = false;461 462 struct graph : public llm_graph_context {463 graph(const llama_model & model, const llm_graph_params & params);464 };465 466 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;467};468 469struct llama_model_qwen : public llama_model_base {470 llama_model_qwen(const struct llama_model_params & params) : llama_model_base(params) {}471 void load_arch_hparams(llama_model_loader & ml) override;472 void load_arch_tensors(llama_model_loader & ml) override;473 474 struct graph : public llm_graph_context {475 graph(const llama_model & model, const llm_graph_params & params);476 };477 478 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;479};480 481 482struct llama_model_qwen2 : public llama_model_base {483 llama_model_qwen2(const struct llama_model_params & params) : llama_model_base(params) {}484 void load_arch_hparams(llama_model_loader & ml) override;485 void load_arch_tensors(llama_model_loader & ml) override;486 487 struct graph : public llm_graph_context {488 graph(const llama_model & model, const llm_graph_params & params);489 };490 491 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;492};493 494 495struct llama_model_dream : public llama_model_base {496 llama_model_dream(const struct llama_model_params & params) : llama_model_base(params) {}497 void load_arch_hparams(llama_model_loader & ml) override;498 void load_arch_tensors(llama_model_loader & ml) override;499 500 struct graph : public llm_graph_context {501 graph(const llama_model & model, const llm_graph_params & params);502 };503 504 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;505};506 507 508struct llama_model_llada : public llama_model_base {509 llama_model_llada(const struct llama_model_params & params) : llama_model_base(params) {}510 void load_arch_hparams(llama_model_loader & ml) override;511 void load_arch_tensors(llama_model_loader & ml) override;512 513 struct graph : public llm_graph_context {514 graph(const llama_model & model, const llm_graph_params & params);515 };516 517 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;518};519 520 521struct llama_model_llada_moe : public llama_model_base {522 llama_model_llada_moe(const struct llama_model_params & params) : llama_model_base(params) {}523 void load_arch_hparams(llama_model_loader & ml) override;524 void load_arch_tensors(llama_model_loader & ml) override;525 526 struct graph : public llm_graph_context {527 graph(const llama_model & model, const llm_graph_params & params);528 };529 530 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;531};532 533 534struct llama_model_rnd1 : public llama_model_base {535 llama_model_rnd1(const struct llama_model_params & params) : llama_model_base(params) {}536 void load_arch_hparams(llama_model_loader & ml) override;537 void load_arch_tensors(llama_model_loader & ml) override;538 539 struct graph : public llm_graph_context {540 graph(const llama_model & model, const llm_graph_params & params);541 };542 543 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;544};545 546 547struct llama_model_qwen2vl : public llama_model_base {548 llama_model_qwen2vl(const struct llama_model_params & params) : llama_model_base(params) {}549 void load_arch_hparams(llama_model_loader & ml) override;550 void load_arch_tensors(llama_model_loader & ml) override;551 552 struct graph : public llm_graph_context {553 graph(const llama_model & model, const llm_graph_params & params);554 };555 556 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;557};558 559 560struct llama_model_qwen2moe : public llama_model_base {561 llama_model_qwen2moe(const struct llama_model_params & params) : llama_model_base(params) {}562 void load_arch_hparams(llama_model_loader & ml) override;563 void load_arch_tensors(llama_model_loader & ml) override;564 565 struct graph : public llm_graph_context {566 graph(const llama_model & model, const llm_graph_params & params);567 };568 569 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;570};571 572 573struct llama_model_qwen3 : public llama_model_base {574 llama_model_qwen3(const struct llama_model_params & params) : llama_model_base(params) {}575 void load_arch_hparams(llama_model_loader & ml) override;576 void load_arch_tensors(llama_model_loader & ml) override;577 578 struct graph : public llm_graph_context {579 graph(const llama_model & model, const llm_graph_params & params);580 };581 582 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;583};584 585 586struct llama_model_qwen3moe : public llama_model_base {587 llama_model_qwen3moe(const struct llama_model_params & params) : llama_model_base(params) {}588 void load_arch_hparams(llama_model_loader & ml) override;589 void load_arch_tensors(llama_model_loader & ml) override;590 591 struct graph : public llm_graph_context {592 graph(const llama_model & model, const llm_graph_params & params);593 };594 595 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;596};597 598 599struct llama_model_qwen3vl : public llama_model_base {600 llama_model_qwen3vl(const struct llama_model_params & params) : llama_model_base(params) {}601 void load_arch_hparams(llama_model_loader & ml) override;602 void load_arch_tensors(llama_model_loader & ml) override;603 604 struct graph : public llm_graph_context {605 graph(const llama_model & model, const llm_graph_params & params);606 };607 608 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;609};610 611 612struct llama_model_qwen3vlmoe : public llama_model_base {613 llama_model_qwen3vlmoe(const struct llama_model_params & params) : llama_model_base(params) {}614 void load_arch_hparams(llama_model_loader & ml) override;615 void load_arch_tensors(llama_model_loader & ml) override;616 617 struct graph : public llm_graph_context {618 graph(const llama_model & model, const llm_graph_params & params);619 };620 621 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;622};623 624 625struct llama_model_qwen3tts : public llama_model_qwen3vl {626 llama_model_qwen3tts(const struct llama_model_params & params) : llama_model_qwen3vl(params) {}627};628 629 630struct llama_model_phi2 : public llama_model_base {631 llama_model_phi2(const struct llama_model_params & params) : llama_model_base(params) {}632 void load_arch_hparams(llama_model_loader & ml) override;633 void load_arch_tensors(llama_model_loader & ml) override;634 635 struct graph : public llm_graph_context {636 graph(const llama_model & model, const llm_graph_params & params);637 };638 639 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;640};641 642 643struct llama_model_phi3 : public llama_model_base {644 llama_model_phi3(const struct llama_model_params & params) : llama_model_base(params) {}645 void load_arch_hparams(llama_model_loader & ml) override;646 void load_arch_tensors(llama_model_loader & ml) override;647 648 template <bool iswa>649 struct graph : public llm_graph_context {650 graph(const llama_model & model, const llm_graph_params & params);651 };652 653 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;654};655 656 657struct llama_model_phimoe : public llama_model_base {658 llama_model_phimoe(const struct llama_model_params & params) : llama_model_base(params) {}659 void load_arch_hparams(llama_model_loader & ml) override;660 void load_arch_tensors(llama_model_loader & ml) override;661 662 template <bool iswa>663 using graph = llama_model_phi3::graph<iswa>;664 665 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;666};667 668 669struct llama_model_plamo : public llama_model_base {670 llama_model_plamo(const struct llama_model_params & params) : llama_model_base(params) {}671 void load_arch_hparams(llama_model_loader & ml) override;672 void load_arch_tensors(llama_model_loader & ml) override;673 674 struct graph : public llm_graph_context {675 graph(const llama_model & model, const llm_graph_params & params);676 };677 678 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;679};680 681 682struct llama_model_plamo2 : public llama_model_base {683 llama_model_plamo2(const struct llama_model_params & params) : llama_model_base(params) {}684 void load_arch_hparams(llama_model_loader & ml) override;685 void load_arch_tensors(llama_model_loader & ml) override;686 687 struct graph : public llm_build_mamba_base {688 graph(const llama_model & model, const llm_graph_params & params);689 private:690 ggml_tensor * build_plamo2_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_model & model, const llama_ubatch & ubatch, int il);691 ggml_tensor * build_plamo2_attn_layer(llm_graph_input_attn_kv * inp, ggml_tensor * inp_pos, ggml_tensor * cur,692 const llama_model & model, int il);693 };694 695 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;696};697 698 699struct llama_model_plamo3 : public llama_model_base {700 llama_model_plamo3(const struct llama_model_params & params) : llama_model_base(params) {}701 void load_arch_hparams(llama_model_loader & ml) override;702 void load_arch_tensors(llama_model_loader & ml) override;703 704 template <bool iswa>705 struct graph : public llm_graph_context {706 graph(const llama_model & model, const llm_graph_params & params);707 };708 709 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;710};711 712 713struct llama_model_gpt2 : public llama_model_base {714 llama_model_gpt2(const struct llama_model_params & params) : llama_model_base(params) {}715 void load_arch_hparams(llama_model_loader & ml) override;716 void load_arch_tensors(llama_model_loader & ml) override;717 718 struct graph : public llm_graph_context {719 graph(const llama_model & model, const llm_graph_params & params);720 };721 722 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;723};724 725 726struct llama_model_pockettts : public llama_model_base {727 llama_model_pockettts(const struct llama_model_params & params) : llama_model_base(params) {}728 void load_arch_hparams(llama_model_loader & ml) override;729 void load_arch_tensors(llama_model_loader & ml) override;730 731 struct graph : public llm_graph_context {732 graph(const llama_model & model, const llm_graph_params & params);733 };734 735 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;736};737 738 739struct llama_model_codeshell : public llama_model_base {740 llama_model_codeshell(const struct llama_model_params & params) : llama_model_base(params) {}741 void load_arch_hparams(llama_model_loader & ml) override;742 void load_arch_tensors(llama_model_loader & ml) override;743 744 struct graph : public llm_graph_context {745 graph(const llama_model & model, const llm_graph_params & params);746 };747 748 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;749};750 751 752struct llama_model_orion : public llama_model_base {753 llama_model_orion(const struct llama_model_params & params) : llama_model_base(params) {}754 void load_arch_hparams(llama_model_loader & ml) override;755 void load_arch_tensors(llama_model_loader & ml) override;756 757 struct graph : public llm_graph_context {758 graph(const llama_model & model, const llm_graph_params & params);759 };760 761 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;762};763 764 765struct llama_model_internlm2 : public llama_model_base {766 llama_model_internlm2(const struct llama_model_params & params) : llama_model_base(params) {}767 void load_arch_hparams(llama_model_loader & ml) override;768 void load_arch_tensors(llama_model_loader & ml) override;769 770 struct graph : public llm_graph_context {771 graph(const llama_model & model, const llm_graph_params & params);772 };773 774 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;775};776 777 778struct llama_model_minicpm3 : public llama_model_base {779 llama_model_minicpm3(const struct llama_model_params & params) : llama_model_base(params) {}780 void load_arch_hparams(llama_model_loader & ml) override;781 void load_arch_tensors(llama_model_loader & ml) override;782 783 struct graph : public llm_graph_context {784 graph(const llama_model & model, const llm_graph_params & params);785 };786 787 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;788};789 790 791struct llama_model_gemma : public llama_model_base {792 llama_model_gemma(const struct llama_model_params & params) : llama_model_base(params) {}793 void load_arch_hparams(llama_model_loader & ml) override;794 void load_arch_tensors(llama_model_loader & ml) override;795 796 struct graph : public llm_graph_context {797 graph(const llama_model & model, const llm_graph_params & params);798 };799 800 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;801};802 803 804struct llama_model_gemma2 : public llama_model_base {805 llama_model_gemma2(const struct llama_model_params & params) : llama_model_base(params) {}806 void load_arch_hparams(llama_model_loader & ml) override;807 void load_arch_tensors(llama_model_loader & ml) override;808 809 struct graph : public llm_graph_context {810 graph(const llama_model & model, const llm_graph_params & params);811 };812 813 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;814};815 816 817struct llama_model_gemma3 : public llama_model_base {818 llama_model_gemma3(const struct llama_model_params & params) : llama_model_base(params) {}819 void load_arch_hparams(llama_model_loader & ml) override;820 void load_arch_tensors(llama_model_loader & ml) override;821 822 template <bool iswa>823 struct graph : public llm_graph_context {824 graph(const llama_model & model, const llm_graph_params & params);825 };826 827 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;828};829 830 831struct llama_model_gemma3n : public llama_model_base {832 llama_model_gemma3n(const struct llama_model_params & params) : llama_model_base(params) {}833 void load_arch_hparams(llama_model_loader & ml) override;834 void load_arch_tensors(llama_model_loader & ml) override;835 836 struct graph : public llm_graph_context {837 const llama_model & model;838 839 const int64_t n_embd_head;840 const int64_t n_embd_altup;841 const int64_t n_altup;842 const int i_altup_act;843 const int n_layer_sparsity = 10; // number of layers using activation sparsity844 const float f_sparsity_std_mul = 1.6448533535003662f; // std_multiplier = normal_dist.icdf(0.95)845 846 graph(const llama_model & model, const llm_graph_params & params);847 ggml_tensor * calc_magnitude(ggml_tensor * x);848 849 // TODO: refactor in common "per-layer" functionality [TAG_PER_LAYER]850 ggml_tensor * build_inp_per_layer();851 ggml_tensor * project_per_layer_inputs(ggml_tensor * inp_batch, ggml_tensor * inp_per_layer);852 853 ggml_tensor * gaussian_topk(ggml_tensor * x);854 ggml_tensor * altup_compute_router_modalities(ggml_tensor * x, int il);855 ggml_tensor * altup_predict(ggml_tensor * cur, int il);856 ggml_tensor * laurel(ggml_tensor * cur, int il);857 ggml_tensor * altup_correct(ggml_tensor * predictions, ggml_tensor * activated, int il);858 };859 860 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;861};862 863 864struct llama_model_gemma4 : public llama_model_base {865 llama_model_gemma4(const struct llama_model_params & params) : llama_model_base(params) {}866 void load_arch_hparams(llama_model_loader & ml) override;867 void load_arch_tensors(llama_model_loader & ml) override;868 869 struct graph : public llm_graph_context {870 const llama_model & model;871 872 const int64_t n_embd_per_layer;873 874 graph(const llama_model & model, const llm_graph_params & params);875 876 // TODO: refactor in common "per-layer" functionality [TAG_PER_LAYER]877 ggml_tensor * build_inp_per_layer();878 ggml_tensor * project_per_layer_inputs(ggml_tensor * inp_batch, ggml_tensor * inp_per_layer);879 };880 881 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;882};883 884 885struct llama_model_gemma4_assistant : public llama_model_base {886 llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {}887 void load_arch_hparams(llama_model_loader & ml) override;888 void load_arch_tensors(llama_model_loader & ml) override;889 890 struct graph : public llm_graph_context {891 graph(const llama_model & model, const llm_graph_params & params);892 };893 894 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;895};896 897 898struct llama_model_gemma_embedding : public llama_model_base {899 llama_model_gemma_embedding(const struct llama_model_params & params) : llama_model_base(params) {}900 void load_arch_hparams(llama_model_loader & ml) override;901 void load_arch_tensors(llama_model_loader & ml) override;902 903 struct graph : public llm_graph_context {904 graph(const llama_model & model, const llm_graph_params & params);905 };906 907 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;908};909 910 911struct llama_model_starcoder2 : public llama_model_base {912 llama_model_starcoder2(const struct llama_model_params & params) : llama_model_base(params) {}913 void load_arch_hparams(llama_model_loader & ml) override;914 void load_arch_tensors(llama_model_loader & ml) override;915 916 struct graph : public llm_graph_context {917 graph(const llama_model & model, const llm_graph_params & params);918 };919 920 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;921};922 923 924struct llama_model_mamba : public llama_model_base {925 llama_model_mamba(const struct llama_model_params & params) : llama_model_base(params) {}926 void load_arch_hparams(llama_model_loader & ml) override;927 void load_arch_tensors(llama_model_loader & ml) override;928 929 struct graph : public llm_build_mamba_base {930 graph(const llama_model & model, const llm_graph_params & params);931 };932 933 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;934};935 936 937struct llama_model_mamba2 : public llama_model_base {938 llama_model_mamba2(const struct llama_model_params & params) : llama_model_base(params) {}939 void load_arch_hparams(llama_model_loader & ml) override;940 void load_arch_tensors(llama_model_loader & ml) override;941 942 using graph = llama_model_mamba::graph;943 944 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;945};946 947 948struct llama_model_maple : public llama_model_base {949 llama_model_maple(const struct llama_model_params & params) : llama_model_base(params) {}950 void load_arch_hparams(llama_model_loader & ml) override;951 void load_arch_tensors(llama_model_loader & ml) override;952 953 struct graph : public llm_graph_context {954 graph(const llama_model & model, const llm_graph_params & params);955 };956 957 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;958};959 960 961struct llama_model_jamba : public llama_model_base {962 llama_model_jamba(const struct llama_model_params & params) : llama_model_base(params) {}963 void load_arch_hparams(llama_model_loader & ml) override;964 void load_arch_tensors(llama_model_loader & ml) override;965 966 struct graph : public llm_build_mamba_base {967 graph(const llama_model & model, const llm_graph_params & params);968 };969 970 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;971};972 973 974struct llama_model_xverse : public llama_model_base {975 llama_model_xverse(const struct llama_model_params & params) : llama_model_base(params) {}976 void load_arch_hparams(llama_model_loader & ml) override;977 void load_arch_tensors(llama_model_loader & ml) override;978 979 struct graph : public llm_graph_context {980 graph(const llama_model & model, const llm_graph_params & params);981 };982 983 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;984};985 986 987struct llama_model_command_r : public llama_model_base {988 llama_model_command_r(const struct llama_model_params & params) : llama_model_base(params) {}989 void load_arch_hparams(llama_model_loader & ml) override;990 void load_arch_tensors(llama_model_loader & ml) override;991 992 struct graph : public llm_graph_context {993 graph(const llama_model & model, const llm_graph_params & params);994 };995 996 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;997};998 999 1000struct llama_model_cohere2 : public llama_model_base {1001 llama_model_cohere2(const struct llama_model_params & params) : llama_model_base(params) {}1002 void load_arch_hparams(llama_model_loader & ml) override;1003 void load_arch_tensors(llama_model_loader & ml) override;1004 1005 struct graph : public llm_graph_context {1006 graph(const llama_model & model, const llm_graph_params & params);1007 };1008 1009 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1010};1011 1012 1013struct llama_model_cohere2moe : public llama_model_base {1014 llama_model_cohere2moe(const struct llama_model_params & params) : llama_model_base(params) {}1015 void load_arch_hparams(llama_model_loader & ml) override;1016 void load_arch_tensors(llama_model_loader & ml) override;1017 1018 struct graph : public llm_graph_context {1019 graph(const llama_model & model, const llm_graph_params & params);1020 };1021 1022 struct graph_mtp : public llm_graph_context {1023 graph_mtp(const llama_model & model, const llm_graph_params & params);1024 };1025 1026 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1027};1028 1029 1030struct llama_model_dbrx : public llama_model_base {1031 llama_model_dbrx(const struct llama_model_params & params) : llama_model_base(params) {}1032 void load_arch_hparams(llama_model_loader & ml) override;1033 void load_arch_tensors(llama_model_loader & ml) override;1034 1035 struct graph : public llm_graph_context {1036 graph(const llama_model & model, const llm_graph_params & params);1037 };1038 1039 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1040};1041 1042 1043struct llama_model_olmo : public llama_model_base {1044 llama_model_olmo(const struct llama_model_params & params) : llama_model_base(params) {}1045 void load_arch_hparams(llama_model_loader & ml) override;1046 void load_arch_tensors(llama_model_loader & ml) override;1047 1048 struct graph : public llm_graph_context {1049 graph(const llama_model & model, const llm_graph_params & params);1050 };1051 1052 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1053};1054 1055 1056struct llama_model_olmo2 : public llama_model_base {1057 llama_model_olmo2(const struct llama_model_params & params) : llama_model_base(params) {}1058 void load_arch_hparams(llama_model_loader & ml) override;1059 void load_arch_tensors(llama_model_loader & ml) override;1060 1061 template <bool iswa>1062 struct graph : public llm_graph_context {1063 graph(const llama_model & model, const llm_graph_params & params);1064 };1065 1066 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1067};1068 1069 1070struct llama_model_olmoe : public llama_model_base {1071 llama_model_olmoe(const struct llama_model_params & params) : llama_model_base(params) {}1072 void load_arch_hparams(llama_model_loader & ml) override;1073 void load_arch_tensors(llama_model_loader & ml) override;1074 1075 struct graph : public llm_graph_context {1076 graph(const llama_model & model, const llm_graph_params & params);1077 };1078 1079 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1080};1081 1082 1083struct llama_model_muse_glimmer : public llama_model_base {1084 llama_model_muse_glimmer(const struct llama_model_params & params) : llama_model_base(params) {}1085 void load_arch_hparams(llama_model_loader & ml) override;1086 void load_arch_tensors(llama_model_loader & ml) override;1087 1088 struct graph : public llm_graph_context {1089 graph(const llama_model & model, const llm_graph_params & params);1090 };1091 1092 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1093};1094 1095 1096struct llama_model_openelm : public llama_model_base {1097 llama_model_openelm(const struct llama_model_params & params) : llama_model_base(params) {}1098 void load_arch_hparams(llama_model_loader & ml) override;1099 void load_arch_tensors(llama_model_loader & ml) override;1100 1101 struct graph : public llm_graph_context {1102 graph(const llama_model & model, const llm_graph_params & params);1103 };1104 1105 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1106};1107 1108 1109struct llama_model_gptneox : public llama_model_base {1110 llama_model_gptneox(const struct llama_model_params & params) : llama_model_base(params) {}1111 void load_arch_hparams(llama_model_loader & ml) override;1112 void load_arch_tensors(llama_model_loader & ml) override;1113 1114 struct graph : public llm_graph_context {1115 graph(const llama_model & model, const llm_graph_params & params);1116 };1117 1118 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1119};1120 1121 1122struct llama_model_arctic : public llama_model_base {1123 llama_model_arctic(const struct llama_model_params & params) : llama_model_base(params) {}1124 void load_arch_hparams(llama_model_loader & ml) override;1125 void load_arch_tensors(llama_model_loader & ml) override;1126 1127 struct graph : public llm_graph_context {1128 graph(const llama_model & model, const llm_graph_params & params);1129 };1130 1131 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1132};1133 1134 1135struct llama_model_deepseek : public llama_model_base {1136 llama_model_deepseek(const struct llama_model_params & params) : llama_model_base(params) {}1137 void load_arch_hparams(llama_model_loader & ml) override;1138 void load_arch_tensors(llama_model_loader & ml) override;1139 1140 struct graph : public llm_graph_context {1141 graph(const llama_model & model, const llm_graph_params & params);1142 };1143 1144 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1145};1146 1147 1148struct llama_model_deepseek2 : public llama_model_base {1149 llama_model_deepseek2(const struct llama_model_params & params) : llama_model_base(params) {}1150 void load_arch_hparams(llama_model_loader & ml) override;1151 void load_arch_tensors(llama_model_loader & ml) override;1152 1153 struct graph : public llm_graph_context {1154 graph(const llama_model & model, const llm_graph_params & params);1155 };1156 1157 struct graph_mtp : public llm_graph_context {1158 graph_mtp(const llama_model & model, const llm_graph_params & params);1159 };1160 1161 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1162};1163 1164 1165struct llama_model_deepseek32 : public llama_model_base {1166 llama_model_deepseek32(const struct llama_model_params & params) : llama_model_base(params) {}1167 void load_arch_hparams(llama_model_loader & ml) override;1168 void load_arch_tensors(llama_model_loader & ml) override;1169 1170 struct graph : public llm_graph_context {1171 graph(const llama_model & model, const llm_graph_params & params);1172 };1173 1174 struct graph_mtp : public llm_graph_context {1175 graph_mtp(const llama_model & model, const llm_graph_params & params);1176 };1177 1178 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1179};1180 1181 1182struct llama_model_dots3note : public llama_model_base {1183 llama_model_dots3note(const struct llama_model_params & params) : llama_model_base(params) {}1184 void load_arch_hparams(llama_model_loader & ml) override;1185 void load_arch_tensors(llama_model_loader & ml) override;1186 1187 struct graph : public llm_graph_context {1188 graph(const llama_model & model, const llm_graph_params & params);1189 };1190 1191 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;1192};1193 1194struct llama_model_deepseek4 : public llama_model_base {1195 llama_model_deepseek4(const struct llama_model_params & params) : llama_model_base(params) {}1196 void load_arch_hparams(llama_model_loader & ml) override;1197 void load_arch_tensors(llama_model_loader & ml) override;1198 1199 struct graph : public llm_graph_context {1200 graph(const llm_graph_params & params) : llm_graph_context(params) {}