Felipe97/llama-cpp-compiled
01.1k
1#include "llama-model.h"2 3#include "llama-arch.h"4#include "llama-ext.h"5#include "llama-hparams.h"6#include "llama-impl.h"7#include "llama-mmap.h"8#include "llama-cparams.h"9#include "llama-model-loader.h"10 11#include "llama-kv-cache.h"12#include "llama-kv-cache-iswa.h"13#include "llama-kv-cache-dsa.h"14#include "llama-kv-cache-dsa-iswa.h"15#include "llama-kv-cache-msa.h"16#include "llama-kv-cache-dsv4.h"17#include "llama-memory-hybrid.h"18#include "llama-memory-hybrid-iswa.h"19#include "llama-memory-hybrid-idx.h"20#include "llama-memory-recurrent.h"21 22#include "llama.h"23#include "models/models.h"24 25#include "ggml.h"26#include "ggml-cpp.h"27 28#include <algorithm>29#include <cassert>30#include <cfloat>31#include <cstdint>32#include <cstring>33#include <cmath>34#include <functional>35#include <map>36#include <numeric>37#include <regex>38#include <sstream>39#include <stdexcept>40#include <string>41#include <vector>42 43static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params & params) {44 switch (arch) {45 case LLM_ARCH_CLIP:46 return new llama_model_clip(params);47 case LLM_ARCH_LLAMA:48 return new llama_model_llama(params);49 case LLM_ARCH_LLAMA4:50 return new llama_model_llama4(params);51 case LLM_ARCH_LLAMA_EMBED:52 return new llama_model_llama_embed(params);53 case LLM_ARCH_MAINCODER:54 return new llama_model_maincoder(params);55 case LLM_ARCH_TALKIE:56 return new llama_model_talkie(params);57 case LLM_ARCH_DECI:58 return new llama_model_deci(params);59 case LLM_ARCH_BAICHUAN:60 return new llama_model_baichuan(params);61 case LLM_ARCH_FALCON:62 return new llama_model_falcon(params);63 case LLM_ARCH_GROK:64 return new llama_model_grok(params);65 case LLM_ARCH_STARCODER:66 return new llama_model_starcoder(params);67 case LLM_ARCH_REFACT:68 return new llama_model_refact(params);69 case LLM_ARCH_BERT:70 return new llama_model_bert(params);71 case LLM_ARCH_JINA_BERT_V2:72 return new llama_model_jina_bert_v2(params);73 case LLM_ARCH_JINA_BERT_V3:74 return new llama_model_jina_bert_v3(params);75 case LLM_ARCH_NOMIC_BERT:76 return new llama_model_nomic_bert(params);77 case LLM_ARCH_NOMIC_BERT_MOE:78 return new llama_model_nomic_bert_moe(params);79 case LLM_ARCH_MODERN_BERT:80 return new llama_model_modern_bert(params);81 case LLM_ARCH_NEO_BERT:82 return new llama_model_neo_bert(params);83 case LLM_ARCH_EUROBERT:84 return new llama_model_eurobert(params);85 case LLM_ARCH_BLOOM:86 return new llama_model_bloom(params);87 case LLM_ARCH_MPT:88 return new llama_model_mpt(params);89 case LLM_ARCH_STABLELM:90 return new llama_model_stablelm(params);91 case LLM_ARCH_MELLUM:92 return new llama_model_mellum(params);93 case LLM_ARCH_NANBEIGE:94 return new llama_model_nanbeige(params);95 case LLM_ARCH_QWEN:96 return new llama_model_qwen(params);97 case LLM_ARCH_QWEN2:98 return new llama_model_qwen2(params);99 case LLM_ARCH_DREAM:100 return new llama_model_dream(params);101 case LLM_ARCH_LLADA:102 return new llama_model_llada(params);103 case LLM_ARCH_LLADA_MOE:104 return new llama_model_llada_moe(params);105 case LLM_ARCH_RND1:106 return new llama_model_rnd1(params);107 case LLM_ARCH_QWEN2VL:108 return new llama_model_qwen2vl(params);109 case LLM_ARCH_QWEN2MOE:110 return new llama_model_qwen2moe(params);111 case LLM_ARCH_QWEN3:112 return new llama_model_qwen3(params);113 case LLM_ARCH_QWEN3MOE:114 return new llama_model_qwen3moe(params);115 case LLM_ARCH_QWEN3VL:116 return new llama_model_qwen3vl(params);117 case LLM_ARCH_QWEN3VLMOE:118 return new llama_model_qwen3vlmoe(params);119 case LLM_ARCH_QWEN3TTS:120 return new llama_model_qwen3tts(params);121 case LLM_ARCH_POCKETTTS:122 return new llama_model_pockettts(params);123 case LLM_ARCH_PHI2:124 return new llama_model_phi2(params);125 case LLM_ARCH_PHI3:126 return new llama_model_phi3(params);127 case LLM_ARCH_PHIMOE:128 return new llama_model_phimoe(params);129 case LLM_ARCH_PLAMO:130 return new llama_model_plamo(params);131 case LLM_ARCH_PLAMO2:132 return new llama_model_plamo2(params);133 case LLM_ARCH_PLAMO3:134 return new llama_model_plamo3(params);135 case LLM_ARCH_GPT2:136 return new llama_model_gpt2(params);137 case LLM_ARCH_CODESHELL:138 return new llama_model_codeshell(params);139 case LLM_ARCH_ORION:140 return new llama_model_orion(params);141 case LLM_ARCH_INTERNLM2:142 return new llama_model_internlm2(params);143 case LLM_ARCH_MINICPM3:144 return new llama_model_minicpm3(params);145 case LLM_ARCH_GEMMA:146 return new llama_model_gemma(params);147 case LLM_ARCH_GEMMA2:148 return new llama_model_gemma2(params);149 case LLM_ARCH_GEMMA3:150 return new llama_model_gemma3(params);151 case LLM_ARCH_GEMMA3N:152 return new llama_model_gemma3n(params);153 case LLM_ARCH_GEMMA4:154 return new llama_model_gemma4(params);155 case LLM_ARCH_GEMMA4_ASSISTANT:156 return new llama_model_gemma4_assistant(params);157 case LLM_ARCH_GEMMA_EMBEDDING:158 return new llama_model_gemma_embedding(params);159 case LLM_ARCH_STARCODER2:160 return new llama_model_starcoder2(params);161 case LLM_ARCH_MAMBA:162 return new llama_model_mamba(params);163 case LLM_ARCH_MAMBA2:164 return new llama_model_mamba2(params);165 case LLM_ARCH_MAPLE:166 return new llama_model_maple(params);167 case LLM_ARCH_JAMBA:168 return new llama_model_jamba(params);169 case LLM_ARCH_XVERSE:170 return new llama_model_xverse(params);171 case LLM_ARCH_COMMAND_R:172 return new llama_model_command_r(params);173 case LLM_ARCH_COHERE2:174 return new llama_model_cohere2(params);175 case LLM_ARCH_COHERE2MOE:176 return new llama_model_cohere2moe(params);177 case LLM_ARCH_DBRX:178 return new llama_model_dbrx(params);179 case LLM_ARCH_OLMO:180 return new llama_model_olmo(params);181 case LLM_ARCH_OLMO2:182 return new llama_model_olmo2(params);183 case LLM_ARCH_OLMOE:184 return new llama_model_olmoe(params);185 case LLM_ARCH_MUSE_GLIMMER:186 return new llama_model_muse_glimmer(params);187 case LLM_ARCH_OPENELM:188 return new llama_model_openelm(params);189 case LLM_ARCH_GPTNEOX:190 return new llama_model_gptneox(params);191 case LLM_ARCH_ARCTIC:192 return new llama_model_arctic(params);193 case LLM_ARCH_DEEPSEEK:194 return new llama_model_deepseek(params);195 case LLM_ARCH_DEEPSEEK2:196 return new llama_model_deepseek2(params);197 case LLM_ARCH_DEEPSEEK2OCR:198 return new llama_model_deepseek2ocr(params);199 case LLM_ARCH_DEEPSEEK32:200 return new llama_model_deepseek32(params);201 case LLM_ARCH_DOTS3NOTE:202 return new llama_model_dots3note(params);203 case LLM_ARCH_DEEPSEEK4:204 return new llama_model_deepseek4(params);205 case LLM_ARCH_GLM_DSA:206 return new llama_model_glm_dsa(params);207 case LLM_ARCH_MISTRAL4:208 return new llama_model_mistral4(params);209 case LLM_ARCH_CHATGLM:210 return new llama_model_chatglm(params);211 case LLM_ARCH_GLM4:212 return new llama_model_glm4(params);213 case LLM_ARCH_GLM4_MOE:214 return new llama_model_glm4_moe(params);215 case LLM_ARCH_BITNET:216 return new llama_model_bitnet(params);217 case LLM_ARCH_T5:218 return new llama_model_t5(params);219 case LLM_ARCH_T5ENCODER:220 return new llama_model_t5encoder(params);221 case LLM_ARCH_JAIS:222 return new llama_model_jais(params);223 case LLM_ARCH_JAIS2:224 return new llama_model_jais2(params);225 case LLM_ARCH_NEMOTRON:226 return new llama_model_nemotron(params);227 case LLM_ARCH_NEMOTRON_H:228 return new llama_model_nemotron_h(params);229 case LLM_ARCH_NEMOTRON_H_MOE:230 return new llama_model_nemotron_h_moe(params);231 case LLM_ARCH_EXAONE:232 return new llama_model_exaone(params);233 case LLM_ARCH_EXAONE4:234 return new llama_model_exaone4(params);235 case LLM_ARCH_EXAONE_MOE:236 return new llama_model_exaone_moe(params);237 case LLM_ARCH_RWKV6:238 return new llama_model_rwkv6(params);239 case LLM_ARCH_RWKV6QWEN2:240 return new llama_model_rwkv6qwen2(params);241 case LLM_ARCH_RWKV7:242 return new llama_model_rwkv7(params);243 case LLM_ARCH_ARWKV7:244 return new llama_model_arwkv7(params);245 case LLM_ARCH_GRANITE:246 return new llama_model_granite(params);247 case LLM_ARCH_GRANITE_MOE:248 return new llama_model_granite_moe(params);249 case LLM_ARCH_GRANITE_SWITCH:250 return new llama_model_granite_switch(params);251 case LLM_ARCH_MINICPM:252 return new llama_model_minicpm(params);253 case LLM_ARCH_GRANITE_HYBRID:254 return new llama_model_granite_hybrid(params);255 case LLM_ARCH_GRANITE_SWA:256 return new llama_model_granite_swa(params);257 case LLM_ARCH_CHAMELEON:258 return new llama_model_chameleon(params);259 case LLM_ARCH_WAVTOKENIZER_DEC:260 return new llama_model_wavtokenizer_dec(params);261 case LLM_ARCH_PLM:262 return new llama_model_plm(params);263 case LLM_ARCH_BAILINGMOE:264 return new llama_model_bailingmoe(params);265 case LLM_ARCH_BAILINGMOE2:266 return new llama_model_bailingmoe2(params);267 case LLM_ARCH_BAILINGMOE3:268 return new llama_model_bailingmoe3(params);269 case LLM_ARCH_SEED_OSS:270 return new llama_model_seed_oss(params);271 case LLM_ARCH_DOTS1:272 return new llama_model_dots1(params);273 case LLM_ARCH_ARCEE:274 return new llama_model_arcee(params);275 case LLM_ARCH_AFMOE:276 return new llama_model_afmoe(params);277 case LLM_ARCH_LAGUNA:278 return new llama_model_laguna(params);279 case LLM_ARCH_ERNIE4_5:280 return new llama_model_ernie4_5(params);281 case LLM_ARCH_ERNIE4_5_MOE:282 return new llama_model_ernie4_5_moe(params);283 case LLM_ARCH_PADDLEOCR:284 return new llama_model_paddleocr(params);285 case LLM_ARCH_HUNYUAN_MOE:286 return new llama_model_hunyuan_moe(params);287 case LLM_ARCH_HUNYUAN_VL:288 return new llama_model_hunyuan_vl(params);289 case LLM_ARCH_HUNYUAN_DENSE:290 return new llama_model_hunyuan_dense(params);291 case LLM_ARCH_HY_V3:292 return new llama_model_hy_v3(params);293 case LLM_ARCH_HY_V4:294 return new llama_model_hy_v4(params);295 case LLM_ARCH_SMOLLM3:296 return new llama_model_smollm3(params);297 case LLM_ARCH_OPENAI_MOE:298 return new llama_model_openai_moe(params);299 case LLM_ARCH_FALCON_H1:300 return new llama_model_falcon_h1(params);301 case LLM_ARCH_LFM2:302 return new llama_model_lfm2(params);303 case LLM_ARCH_LFM2MOE:304 return new llama_model_lfm2moe(params);305 case LLM_ARCH_SMALLTHINKER:306 return new llama_model_smallthinker(params);307 case LLM_ARCH_GROVEMOE:308 return new llama_model_grovemoe(params);309 case LLM_ARCH_APERTUS:310 return new llama_model_apertus(params);311 case LLM_ARCH_MINIMAX_01:312 return new llama_model_minimax_01(params);313 case LLM_ARCH_MINIMAX_M2:314 return new llama_model_minimax_m2(params);315 case LLM_ARCH_MINIMAX_M3:316 return new llama_model_minimax_m3(params);317 case LLM_ARCH_HRM_TEXT:318 return new llama_model_hrm_text(params);319 case LLM_ARCH_COGVLM:320 return new llama_model_cogvlm(params);321 case LLM_ARCH_PANGU_EMBED:322 return new llama_model_pangu_embed(params);323 case LLM_ARCH_QWEN3NEXT:324 return new llama_model_qwen3next(params);325 case LLM_ARCH_QWEN35:326 return new llama_model_qwen35(params);327 case LLM_ARCH_QWEN35MOE:328 return new llama_model_qwen35moe(params);329 case LLM_ARCH_QWEN4EXP:330 return new llama_model_qwen4exp(params);331 case LLM_ARCH_MISTRAL3:332 return new llama_model_mistral3(params);333 case LLM_ARCH_EAGLE3:334 return new llama_model_eagle3(params);335 case LLM_ARCH_DFLASH:336 return new llama_model_dflash(params);337 case LLM_ARCH_MIMO2:338 return new llama_model_mimo2(params);339 case LLM_ARCH_KIMI_LINEAR:340 return new llama_model_kimi_linear(params);341 case LLM_ARCH_KIMI_K3:342 return new llama_model_kimi_k3(params);343 case LLM_ARCH_STEP35:344 return new llama_model_step35(params);345 case LLM_ARCH_SPARK2_5:346 return new llama_model_spark2_5(params);347 default:348 throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'");349 }350 351}352 353llama_model * llama_model_create(llm_arch arch, const llama_model_params & params) {354 llama_model * model = llama_model_mapping(arch, params);355 356 if (model != nullptr) {357 model->arch = arch;358 if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR && !llm_arch_supports_sm_tensor(arch)) {359 throw std::runtime_error(std::string("LLAMA_SPLIT_MODE_TENSOR not implemented for architecture '") + llm_arch_name(arch) + "'");360 }361 }362 363 return model;364}365 366llama_model * llama_model_create(llama_model_loader & ml, const llama_model_params & params) {367 llm_arch arch = ml.get_arch();368 if (arch == LLM_ARCH_UNKNOWN) {369 throw std::runtime_error("unknown model architecture: '" + ml.get_arch_name() + "'");370 }371 372 return llama_model_create(arch, params);373}374 375struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata) {376 const llama_meta_device_get_split_state_userdata * ud = (const llama_meta_device_get_split_state_userdata *) userdata;377 const llama_hparams & hparams = ud->model->hparams;378 const std::string tensor_name = tensor->name;379 const bool is_dsv4 = ud->model->arch == LLM_ARCH_DEEPSEEK4 ||380 (ud->model->arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0);381 382 static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");383 static const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");384 static const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");385 static const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");386 static const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");387 static const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");388 static const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");389 static const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");390 static const std::regex pattern_idx_cache ("cache_idx_(k|v)_l\\d*");391 static const std::regex pattern_dsv4_state ("dsv4_(csa|hca|lid)_state_(kv|score)_l\\d*");392 static const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");393 static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");394 static const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");395 static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight");396 static const std::regex pattern_attn_out_b_weight("blk\\.\\d*\\.attn_output_b\\.weight");397 static const std::regex pattern_attn_q_b_weight ("blk\\.\\d*\\.attn_q_b\\.weight");398 static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");399 400 static const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");401 static const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");402 static const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");403 static const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");404 static const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");405 static const std::regex pattern_r_cache ("cache_r_l\\d*");406 static const std::regex pattern_ple_r_cache ("cache_ple_r_l\\d*");407 static const std::regex pattern_s_cache ("cache_s_l\\d*");408 static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");409 static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");410 411 static const std::regex pattern_ffn_up_weight ("blk\\.\\d*\\.ffn_up(_exps)?.weight");412 static const std::regex pattern_ffn_up_bias ("blk\\.\\d*\\.ffn_up(_exps)?.bias");413 static const std::regex pattern_ffn_gate_weight ("blk\\.\\d*\\.ffn_gate(_exps)?.weight");414 static const std::regex pattern_ffn_gate_bias ("blk\\.\\d*\\.ffn_gate(_exps)?.bias");415 static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");416 static const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");417 static const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");418 static const std::regex pattern_ffn_down_exps_bias ("blk\\.\\d*\\.ffn_down_exps.bias");419 static const std::regex pattern_ffn_up_shexp_weight ("blk\\.\\d*\\.ffn_up_shexp.weight");420 static const std::regex pattern_ffn_gate_shexp_weight ("blk\\.\\d*\\.ffn_gate_shexp.weight");421 static const std::regex pattern_ffn_down_shexp_weight ("blk\\.\\d*\\.ffn_down_shexp.weight");422 423 static const std::regex pattern_output_weight("output\\.weight");424 static const std::regex pattern_output_bias ("output\\.bias");425 426 struct tensor_config {427 ggml_backend_meta_split_axis axis;428 429 const ggml_tensor * tensor_axis_0;430 431 uint32_t il;432 size_t rotation; // when assigning tensor slices, rotate how the rounding is done for more even allocation433 };434 435 auto get_tensor_config_impl = [&](436 const ggml_backend_meta_split_axis axis, const std::string & suffix = "", const std::string & suffix_fallback = "") -> tensor_config {437 // the layers in a tensor can be inhomogeneous, if the pattern is cleanly divided by the number of GPUs there can be aliasing effects,438 // count only the same type of previous layers to avoid this439 auto get_il_eff = [&](const size_t il){440 size_t ret = 0;441 const bool il_is_recr = hparams.is_recr(il);442 const bool il_is_swa = hparams.is_swa(il);443 for (size_t il_prev = 0; il_prev < il; il_prev++) {444 ret += hparams.is_recr(il_prev) == il_is_recr && hparams.is_swa(il_prev) == il_is_swa;445 }446 return ret;447 };448 449 uint32_t il;450 std::string prefix;451 size_t rotation;452 if (tensor_name.substr(0, 4) == "blk.") {453 const size_t length_prefix = tensor_name.find('.', 4);454 GGML_ASSERT(length_prefix != std::string::npos);455 prefix = tensor_name.substr(0, length_prefix + 1);456 il = std::stoull(tensor_name.substr(4, length_prefix));457 rotation = get_il_eff(il) % ud->n_devices;458 } else if (tensor_name.substr(0, 6) == "cache_") {459 const size_t layer_index_start = tensor_name.find("_l", 6);460 GGML_ASSERT(layer_index_start != std::string::npos);461 il = std::stoull(tensor_name.substr(layer_index_start + 2));462 prefix = "blk." + std::to_string(il) + ".";463 rotation = get_il_eff(il) % ud->n_devices;464 } else {465 il = 0;466 rotation = hparams.n_layer() % ud->n_devices;467 }468 const ggml_tensor * tensor_axis_0 = suffix.empty() ? tensor : ud->model->get_tensor((prefix + suffix).c_str());469 if (tensor_axis_0 == nullptr) {470 GGML_ASSERT(!suffix_fallback.empty());471 tensor_axis_0 = ud->model->get_tensor((prefix + suffix_fallback).c_str());472 }473 GGML_ASSERT(tensor_axis_0 != nullptr);474 return {axis, tensor_axis_0, il, rotation};475 };476 477 auto get_tensor_config = [&]() -> tensor_config {478 if (ud->model->arch == LLM_ARCH_HRM_TEXT) {479 // aliased cache slots cannot satisfy the meta-split invariants, so replicate all tensors480 return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, tensor, 0, 0};481 }482 if (is_dsv4) {483 if (std::regex_match(tensor_name, pattern_kv_cache) ||484 std::regex_match(tensor_name, pattern_dsv4_state)) {485 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);486 }487 if (std::regex_match(tensor_name, pattern_attn_sinks)) {488 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output_a.weight");489 }490 if (std::regex_match(tensor_name, pattern_attn_q_b_weight)) {491 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output_a.weight");492 }493 if (std::regex_match(tensor_name, pattern_attn_out_a_weight)) {494 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_2);495 }496 if (std::regex_match(tensor_name, pattern_attn_out_b_weight)) {497 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);498 }499 if (std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) ||500 std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight)) {501 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down_shexp.weight");502 }503 if (std::regex_match(tensor_name, pattern_ffn_down_shexp_weight)) {504 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down_shexp.weight");505 }506 }507 508 // the qsa indexer has one key head and its projections are mirrored, so its cache cannot be split509 if (std::regex_match(tensor_name, pattern_idx_cache)) {510 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);511 }512 513 // the PLE table is model-level and its conv is mirrored, so every device runs the whole conv and needs the whole history514 if (std::regex_match(tensor_name, pattern_ple_r_cache)) {515 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);516 }517 518 // standard attention519 if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight)) {520 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");521 }522 if (std::regex_match(tensor_name, pattern_q_bias) || std::regex_match(tensor_name, pattern_kv_bias)) {523 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight");524 }525 if (std::regex_match(tensor_name, pattern_qkv_weight)) {526 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");527 }528 if ( std::regex_match(tensor_name, pattern_qkv_bias)) {529 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight");530 }531 if (std::regex_match(tensor_name, pattern_qk_norm)) {532 return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");533 }534 if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks)) {535 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");536 }537 if (std::regex_match(tensor_name, pattern_attn_out_weight)) {538 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);539 }540 if (std::regex_match(tensor_name, pattern_attn_out_bias)) {541 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);542 }543 544 if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {545 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");546 }547 if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a)) {548 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");549 }550 if (std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta) ||551 std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {552 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");553 }554 if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {555 if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) {556 // the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too557 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, "");558 }559 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");560 }561 if (std::regex_match(tensor_name, pattern_ssm_conv1d)) {562 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");563 }564 if (std::regex_match(tensor_name, pattern_ssm_out_weight)) {565 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);566 }567 568 // FFN569 if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_gate_weight)) {570 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");571 }572 if (std::regex_match(tensor_name, pattern_ffn_up_bias) || std::regex_match(tensor_name, pattern_ffn_gate_bias)) {573 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");574 }575 if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {576 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");577 }578 if (std::regex_match(tensor_name, pattern_ffn_down_weight)) {579 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");580 }581 if (std::regex_match(tensor_name, pattern_ffn_down_bias)) {582 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);583 }584 if (std::regex_match(tensor_name, pattern_ffn_down_exps_bias)) {585 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_PARTIAL, "ffn_down_exps.weight");586 }587 588 // output589 if (std::regex_match(tensor_name, pattern_output_weight)) {590 if (is_dsv4) {591 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);592 }593 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);594 }595 if (std::regex_match(tensor_name, pattern_output_bias)) {596 const ggml_tensor * output_weight = ud->model->get_tensor("output.weight");597 GGML_ASSERT(output_weight != nullptr);598 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);599 }600 601 // everything else602 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);603 };604 605 auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<std::pair<int64_t, uint32_t>> {606 // TODO: clarify why this is necessary specifically for these models607 // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]608 if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||609 ud->model->arch == LLM_ARCH_QWEN4EXP) {610 611 // fused full attention layers with Q gate tensors that need n_embd doubled:612 if (!hparams.is_recr(il) && (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias))) {613 const int64_t n_embd = hparams.n_head(il) * hparams.n_embd_head_k(il) * 2;614 const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il);615 GGML_ASSERT(hparams.n_embd_k_gqa(il) == n_embd_gqa);616 GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa);617 return {{n_embd, 1}, {n_embd_gqa, 2}};618 }619 620 const int64_t head_k_dim = hparams.ssm_d_state;621 const int64_t head_v_dim = hparams.ssm_d_state;622 const int64_t n_k_heads = hparams.ssm_n_group;623 const int64_t n_v_heads = hparams.ssm_dt_rank;624 const int64_t key_dim = head_k_dim * n_k_heads;625 const int64_t value_dim = head_v_dim * n_v_heads;626 627 // both Qwen 3 Next and Qwen 3.5 support n_v_heads > n_k_heads but the broadcasting pattern is different:628 // - Qwen 3 Next: [k0_v0, k0_v1, k1_v2, k1_v3] (this is the default split pattern)629 // - Qwen 3.5: [k0_v0, k1_v1, k0_v2, k1_v3] (needs segmenting of V on the scale of K to get the correct pattern)630 if (ud->model->arch == LLM_ARCH_QWEN3NEXT) {631 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {632 GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);633 return {{key_dim, 2}, {value_dim, 1}};634 }635 if (std::regex_match(tensor_name, pattern_r_cache)) {636 return {{key_dim * (hparams.ssm_d_conv - 1), 2}, {value_dim * (hparams.ssm_d_conv - 1), 1}};637 }638 } else {639 const int64_t head_ratio = n_v_heads / n_k_heads;640 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {641 GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);642 return {{key_dim, 2 + head_ratio}};643 }644 if (std::regex_match(tensor_name, pattern_attn_gate_weight) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {645 return {{key_dim, head_ratio}};646 }647 if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||648 std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {649 return {{n_k_heads, head_ratio}};650 }651 if (std::regex_match(tensor_name, pattern_r_cache)) {652 return {{key_dim * (hparams.ssm_d_conv - 1), 2 + head_ratio}};653 }654 if (std::regex_match(tensor_name, pattern_s_cache)) {655 return {{n_k_heads * head_v_dim * head_v_dim, head_ratio}};656 }657 }658 659 // the FFN is the same for Qwen 3 Next and Qwen 3.5:660 if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {661 const int64_t n_ff_exp = hparams.n_ff_exp(il);662 GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);663 return {{n_ff_exp, 2}};664 }665 return {{tensor->ne[axis], 1}};666 }667 668 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {669 const int64_t n_embd = hparams.n_head(il) * hparams.n_embd_head_k(il);670 const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il);671 GGML_ASSERT(hparams.n_embd_k_gqa(il) == n_embd_gqa);672 GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa);673 return {{n_embd, 1}, {n_embd_gqa, 2}};674 }675 if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias)) {676 const int64_t n_ff = hparams.n_ff(il);677 // some models such as Phi 3 have fused up + gate tensors named "up" tensors, which need to be segmented678 if (tensor->ne[axis] == 2*n_ff) {679 return {{n_ff, 2}};680 }681 return {{tensor->ne[axis], 1}};682 }683 if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {684 const int64_t n_ff_exp = hparams.n_ff_exp(il);685 GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);686 return {{n_ff_exp, 2}};687 }688 return {{tensor->ne[axis], 1}};689 };690 691 auto get_split_granularity = [&](int64_t blck_size, uint32_t il, const std::vector<std::pair<int64_t, uint32_t>> & segments) -> std::vector<int64_t> {692 // for better performance it may make sense to round up blck_size to a higher power of 2 so that more efficient kernels can be used693 if (hparams.is_recr(il)) {694 // linear attention695 const int64_t head_dim = hparams.ssm_d_state;696 const int64_t blck_size_perf = std::lcm(blck_size, 128);697 const int64_t granularity_qkv = std::lcm(blck_size_perf, head_dim);698 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_attn_gate_weight) ||699 std::regex_match(tensor_name, pattern_ssm_conv1d) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {700 return std::vector<int64_t>(segments.size(), granularity_qkv);701 }702 if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||703 std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {704 return std::vector<int64_t>(segments.size(), granularity_qkv / head_dim);705 }706 if (std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {707 return std::vector<int64_t>(segments.size(), 2 * (granularity_qkv / head_dim));708 }709 if (std::regex_match(tensor_name, pattern_r_cache)) {710 return std::vector<int64_t>(segments.size(), granularity_qkv * (hparams.ssm_d_conv - 1));711 }712 if (std::regex_match(tensor_name, pattern_s_cache)) {713 return std::vector<int64_t>(segments.size(), granularity_qkv * head_dim);714 }715 } else {716 // regular attention717 const uint32_t n_gqa = hparams.n_gqa(il);718 const uint32_t n_embd_q = n_gqa * hparams.n_embd_head_k(il);719 720 // to handle head sizes like 80, only increase granularity while it doesn't cause underutilization721 int64_t blck_size_perf = blck_size;722 while (blck_size_perf < 128 && blck_size_perf*ud->n_devices < n_embd_q) {723 blck_size_perf *= 2;724 }725 726 const int64_t granularity_q = std::lcm(n_embd_q, blck_size_perf);727 const int64_t granularity_head = granularity_q / hparams.n_embd_head_k(il); // for tensors with one value per head728 if (std::regex_match(tensor_name, pattern_attn_sinks)) {729 GGML_ASSERT(segments.size() == 1);730 if (is_dsv4) {731 return {hparams.n_head(il) / hparams.dsv4_o_group_count};732 }733 return {granularity_head};734 }735 736 if (is_dsv4) {737 if (std::regex_match(tensor_name, pattern_attn_q_b_weight)) {738 GGML_ASSERT(segments.size() == 1);739 // the grouped output projection requires each device to hold whole groups of heads740 const int64_t n_head_group = hparams.n_head(il) / hparams.dsv4_o_group_count;741 return {n_head_group * hparams.n_embd_head_k(il)};742 }743 if (std::regex_match(tensor_name, pattern_attn_out_a_weight)) {744 GGML_ASSERT(segments.size() == 1);745 return {1};746 }747 if (std::regex_match(tensor_name, pattern_attn_out_b_weight)) {748 GGML_ASSERT(segments.size() == 1);749 // the boundaries must align with wo_a's per-group split, so quant blocks must not straddle groups750 GGML_ASSERT(hparams.dsv4_o_lora_rank % blck_size == 0);751 return {hparams.dsv4_o_lora_rank};752 }753 }754 if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_q_bias)) {755 GGML_ASSERT(segments.size() == 1);756 // some models have Q gate tensors, for those cases the granularity needs to be doubled:757 // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]758 if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||759 ud->model->arch == LLM_ARCH_QWEN4EXP) {760 return {std::lcm(2*n_embd_q, blck_size_perf)};761 }762 return {granularity_q};763 }764 if (std::regex_match(tensor_name, pattern_attn_out_weight)) {765 GGML_ASSERT(segments.size() == 1);766 return {granularity_q};767 }768 if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {769 GGML_ASSERT(segments.size() == 1);770 if (tensor->ne[1] == hparams.n_head(il)) {771 return {granularity_head};772 }773 return {granularity_q};774 }775 776 const int64_t granularity_kv = granularity_q / n_gqa;777 if (std::regex_match(tensor_name, pattern_kv_weight) ||778 std::regex_match(tensor_name, pattern_kv_bias) ||779 std::regex_match(tensor_name, pattern_kv_cache)) {780 GGML_ASSERT(segments.size() == 1);781 return {granularity_kv};782 }783 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {784 GGML_ASSERT(segments.size() == 2);785 // fused full attention layers need Q gate tensors handled like above:786 // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]787 if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||788 ud->model->arch == LLM_ARCH_QWEN4EXP) {789 return {std::lcm(2*n_embd_q, blck_size_perf), granularity_kv};790 }791 return {granularity_q, granularity_kv};792 }793 }794 795 // FFN796 if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias) ||797 std::regex_match(tensor_name, pattern_ffn_gate_weight) || std::regex_match(tensor_name, pattern_ffn_gate_bias) ||798 std::regex_match(tensor_name, pattern_ffn_gate_up_weight) ||799 std::regex_match(tensor_name, pattern_ffn_down_weight) ||800 std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) ||801 std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight) ||802 std::regex_match(tensor_name, pattern_ffn_down_shexp_weight)) {803 const int64_t blck_size_perf = std::lcm(blck_size, 128);804 GGML_ASSERT(segments.size() == 1);805 return {blck_size_perf};806 }807 808 // everything else809 GGML_ASSERT(segments.size() == 1);810 return {1};811 };812 813 ggml_backend_meta_split_state split_state;814 memset(&split_state, 0, sizeof(split_state));815 tensor_config tc = get_tensor_config();816 split_state.axis = tc.axis;817 if (split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS) {818 const int64_t blck_size = ggml_blck_size(tc.tensor_axis_0->type);819 const float * tensor_split = ud->model->tensor_split();820 std::vector<float> tensor_split_scan;821 tensor_split_scan.reserve(ud->n_devices);822 for (size_t j = 0; j < ud->n_devices; j++) {823 tensor_split_scan.push_back(tensor_split == nullptr ? 0.0f : tensor_split[(j + tc.rotation) % ud->n_devices]);824 if (j > 0) {825 tensor_split_scan[j] += tensor_split_scan[j - 1];826 }827 }828 const std::vector<std::pair<int64_t, uint32_t>> segments = get_split_segments(split_state.axis, tc.il);829 const std::vector<int64_t> granularity = get_split_granularity(blck_size, tc.il, segments);830 for (size_t is = 0; is < segments.size(); is++) {831 const int64_t ne_s = segments[is].first;832 const uint32_t nr_s = segments[is].second;833 const int64_t g_s = granularity[is];834 int64_t low = 0;835 size_t j = 0;836 for (; j < ud->n_devices - 1; j++) {837 int64_t high = tensor_split_scan.back() == 0.0f ?838 ne_s * (j+1)/ud->n_devices : ne_s * tensor_split_scan[j]/tensor_split_scan.back();839 if (high % g_s != 0) {840 high -= high % g_s;841 }842 split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = high - low;843 low = high;844 }845 split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = ne_s - low;846 split_state.nr[is] = nr_s;847 }848 split_state.n_segments = segments.size();849 } else {850 memset(split_state.ne, 0, sizeof(split_state.ne));851 split_state.nr[0] = 1;852 split_state.n_segments = 1;853 if (split_state.axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) {854 GGML_ASSERT(tc.tensor_axis_0 != tensor);855 const ggml_backend_meta_split_state source_split_state = llama_meta_device_get_split_state(tc.tensor_axis_0, userdata);856 GGML_ASSERT(source_split_state.axis >= 0 && source_split_state.axis < GGML_MAX_DIMS);857 for (size_t j = 0; j < ud->n_devices; j++) {858 for (size_t is = 0; is < source_split_state.n_segments; is++) {859 split_state.ne[j] += source_split_state.ne[is*ud->n_devices + j] * source_split_state.nr[is];860 }861 }862 }863 }864 return split_state;865 GGML_UNUSED(userdata);866}867 868const char * llm_type_name(llm_type type) {869 switch (type) {870 case LLM_TYPE_14M: return "14M";871 case LLM_TYPE_17M: return "17M";872 case LLM_TYPE_22M: return "22M";873 case LLM_TYPE_33M: return "33M";874 case LLM_TYPE_47M: return "47M";875 case LLM_TYPE_60M: return "60M";876 case LLM_TYPE_70M: return "70M";877 case LLM_TYPE_80M: return "80M";878 case LLM_TYPE_109M: return "109M";879 case LLM_TYPE_137M: return "137M";880 case LLM_TYPE_140M: return "140M";881 case LLM_TYPE_149M: return "149M";882 case LLM_TYPE_160M: return "160M";883 case LLM_TYPE_190M: return "190M";884 case LLM_TYPE_220M: return "220M";885 case LLM_TYPE_230M: return "230M";886 case LLM_TYPE_250M: return "250M";887 case LLM_TYPE_256M: return "256M";888 case LLM_TYPE_270M: return "270M";889 case LLM_TYPE_335M: return "335M";890 case LLM_TYPE_350M: return "350M";891 case LLM_TYPE_360M: return "360M";892 case LLM_TYPE_395M: return "395M";893 case LLM_TYPE_410M: return "410M";894 case LLM_TYPE_450M: return "450M";895 case LLM_TYPE_475M: return "475M";896 case LLM_TYPE_558M: return "558M";897 case LLM_TYPE_700M: return "700M";898 case LLM_TYPE_770M: return "770M";899 case LLM_TYPE_780M: return "780M";900 case LLM_TYPE_950M: return "950M";901 case LLM_TYPE_0_3B: return "0.3B";902 case LLM_TYPE_0_5B: return "0.5B";903 case LLM_TYPE_0_6B: return "0.6B";904 case LLM_TYPE_0_8B: return "0.8B";905 case LLM_TYPE_1B: return "1B";906 case LLM_TYPE_1_2B: return "1.2B";907 case LLM_TYPE_1_3B: return "1.3B";908 case LLM_TYPE_1_4B: return "1.4B";909 case LLM_TYPE_1_5B: return "1.5B";910 case LLM_TYPE_1_6B: return "1.6B";911 case LLM_TYPE_1_7B: return "1.7B";912 case LLM_TYPE_1_8B: return "1.8B";913 case LLM_TYPE_2B: return "2B";914 case LLM_TYPE_2_6B: return "2.6B";915 case LLM_TYPE_2_8B: return "2.8B";916 case LLM_TYPE_2_9B: return "2.9B";917 case LLM_TYPE_3B: return "3B";918 case LLM_TYPE_4B: return "4B";919 case LLM_TYPE_6B: return "6B";920 case LLM_TYPE_6_9B: return "6.9B";921 case LLM_TYPE_7B: return "7B";922 case LLM_TYPE_8B: return "8B";923 case LLM_TYPE_9B: return "9B";924 case LLM_TYPE_11B: return "11B";925 case LLM_TYPE_12B: return "12B";926 case LLM_TYPE_13B: return "13B";927 case LLM_TYPE_14B: return "14B";928 case LLM_TYPE_15B: return "15B";929 case LLM_TYPE_16B: return "16B";930 case LLM_TYPE_20B: return "20B";931 case LLM_TYPE_26B: return "26B";932 case LLM_TYPE_27B: return "27B";933 case LLM_TYPE_30B: return "30B";934 case LLM_TYPE_31B: return "31B";935 case LLM_TYPE_32B: return "32B";936 case LLM_TYPE_34B: return "34B";937 case LLM_TYPE_35B: return "35B";938 case LLM_TYPE_36B: return "36B";939 case LLM_TYPE_40B: return "40B";940 case LLM_TYPE_65B: return "65B";941 case LLM_TYPE_70B: return "70B";942 case LLM_TYPE_120B: return "120B";943 case LLM_TYPE_142B: return "142B";944 case LLM_TYPE_236B: return "236B";945 case LLM_TYPE_290B: return "290B";946 case LLM_TYPE_314B: return "314B";947 case LLM_TYPE_405B: return "405B";948 case LLM_TYPE_456B: return "456B";949 case LLM_TYPE_671B: return "671B";950 case LLM_TYPE_SMALL: return "0.1B";951 case LLM_TYPE_MEDIUM: return "0.4B";952 case LLM_TYPE_LARGE: return "0.8B";953 case LLM_TYPE_XL: return "1.5B";954 case LLM_TYPE_A1_7B: return "A1.7B";955 case LLM_TYPE_A2_7B: return "A2.7B";956 case LLM_TYPE_8x7B: return "8x7B";957 case LLM_TYPE_8x22B: return "8x22B";958 case LLM_TYPE_16x12B: return "16x12B";959 case LLM_TYPE_16x3_8B: return "16x3.8B";960 case LLM_TYPE_10B_128x3_66B: return "10B+128x3.66B";961 case LLM_TYPE_57B_A14B: return "57B.A14B";962 case LLM_TYPE_17B_16E: return "17Bx16E (Scout)";963 case LLM_TYPE_17B_128E: return "17Bx128E (Maverick)";964 case LLM_TYPE_A13B: return "A13B";965 case LLM_TYPE_1B_A400M: return "1B.A400M";966 case LLM_TYPE_3B_A800M: return "3B.A800M";967 case LLM_TYPE_7B_A1B: return "7B.A1B";968 case LLM_TYPE_8B_A1B: return "8B.A1B";969 case LLM_TYPE_7_9B_A1_3B: return "7.9B.A1.3B";970 case LLM_TYPE_12B_A2_5B: return "12B.A2.5B";971 case LLM_TYPE_16B_A1B: return "16B.A1B";972 case LLM_TYPE_21B_A3B: return "21B.A3B";973 case LLM_TYPE_24B_A2B: return "24B.A2B";974 case LLM_TYPE_26B_A4B: return "26B.A4B";975 case LLM_TYPE_30B_A3B: return "30B.A3B";976 case LLM_TYPE_31B_A3_5B: return "31B.A3.5B";977 case LLM_TYPE_32B_A9B: return "32B.A9B";978 case LLM_TYPE_35B_A3B: return "35B.A3B";979 case LLM_TYPE_48B_A3B: return "48B.A3B";980 case LLM_TYPE_75B_A9B: return "75B.A9B";981 case LLM_TYPE_80B_A3B: return "80B.A3B";982 case LLM_TYPE_A3B: return "A3B";983 case LLM_TYPE_100B_A6B: return "100B.A6B";984 case LLM_TYPE_102B_A12B: return "102B.A12B";985 case LLM_TYPE_106B_A12B: return "106B.A12B";986 case LLM_TYPE_118B_A8B: return "118B.A8B";987 case LLM_TYPE_120B_A12B: return "120B.A12B";988 case LLM_TYPE_122B_A10B: return "122B.A10B";989 case LLM_TYPE_124B_A5_1B: return "124B.A5.1B";990 case LLM_TYPE_196B_A11B: return "196B.A11B";991 case LLM_TYPE_230B_A10B: return "230B.A10B";992 case LLM_TYPE_428B_A23B: return "428B.A23B";993 case LLM_TYPE_235B_A22B: return "235B.A22B";994 case LLM_TYPE_288B_A19B: return "288B.A19B";995 case LLM_TYPE_300B_A47B: return "300B.A47B";996 case LLM_TYPE_310B_A15B: return "310B.A15B";997 case LLM_TYPE_355B_A32B: return "355B.A32B";998 case LLM_TYPE_397B_A17B: return "397B.A17B";999 case LLM_TYPE_685B_A37B: return "685B.A37B";1000 case LLM_TYPE_744B_A40B: return "744B.A40B";1001 case LLM_TYPE_2_8T_A50B: return "2.8T.A50B";1002 case LLM_TYPE_E2B: return "E2B";1003 case LLM_TYPE_E4B: return "E4B";1004 default: return "?B";1005 }1006}1007 1008static const char * llama_expert_gating_func_name(llama_expert_gating_func_type type) {1009 switch (type) {1010 case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: return "softmax";1011 case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: return "sigmoid";1012 case LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS: return "sqrtsoftplus";1013 default: return "unknown";1014 }1015}1016 1017static const std::map<llama_rope_scaling_type, const char *> LLAMA_ROPE_SCALING_TYPES = {1018 { LLAMA_ROPE_SCALING_TYPE_NONE, "none" },1019 { LLAMA_ROPE_SCALING_TYPE_LINEAR, "linear" },1020 { LLAMA_ROPE_SCALING_TYPE_YARN, "yarn" },1021 { LLAMA_ROPE_SCALING_TYPE_LONGROPE, "longrope" },1022};1023 1024std::string llama_rope_scaling_type_name(llama_rope_scaling_type rope_scaling_type) {1025 return LLAMA_ROPE_SCALING_TYPES.at(rope_scaling_type);1026}1027 1028static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::string & name) {1029 for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {1030 if (kv.second == name) {1031 return (llama_rope_scaling_type) kv.first;1032 }1033 }1034 1035 return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;1036}1037 1038// Maps the GGUF `<arch>.hidden_activation` string to the FFN op type used by the1039// graph builders. Only gated activations that map cleanly to llm_ffn_op_type are1040// listed; unrecognized values fall back to GeGLU, which matches the historical1041// default for ModernBert-style architectures.1042static const std::map<std::string, llm_ffn_op_type> LLM_FFN_OP_TYPES_FROM_STRING = {1043 { "gelu", LLM_FFN_GEGLU },1044 { "geglu", LLM_FFN_GEGLU },1045 { "silu", LLM_FFN_SWIGLU },1046 { "swish", LLM_FFN_SWIGLU },1047 { "swiglu", LLM_FFN_SWIGLU },1048 { "relu", LLM_FFN_RELU },1049 { "reglu", LLM_FFN_REGLU },1050};1051 1052llm_ffn_op_type llm_ffn_op_type_from_string(const std::string & name, llm_ffn_op_type fallback) {1053 const auto it = LLM_FFN_OP_TYPES_FROM_STRING.find(name);1054 if (it != LLM_FFN_OP_TYPES_FROM_STRING.end()) {1055 return it->second;1056 }1057 return fallback;1058}1059 1060// CPU: ACCEL -> GPU host -> CPU extra -> CPU1061static buft_list_t make_cpu_buft_list(const std::vector<llama_device> & devices, bool use_extra_bufts, bool no_host) {1062 buft_list_t buft_list;1063 1064 // add ACCEL buffer types1065 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {1066 ggml_backend_dev_t dev = ggml_backend_dev_get(i);1067 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {1068 auto * buft = ggml_backend_dev_buffer_type(dev);1069 // skip1070 if (buft != ggml_backend_cpu_buffer_type()) {1071 buft_list.emplace_back(dev, buft);1072 }1073 }1074 }1075 1076 // add a host buffer type1077 // storing the tensors in a host buffer is useful when the processing of large batches1078 // is offloaded to a GPU device, since it reduces the time spent on data transfers1079 // generally, this will be done using the first device in the list1080 // a better approach would be to handle this on a weight-by-weight basis using the offload_op1081 // function of the device to determine if it would benefit from being stored in a host buffer1082 if (!no_host) {1083 for (const auto & dev : devices) {1084 ggml_backend_buffer_type_t buft = ggml_backend_dev_host_buffer_type(dev.dev);1085 if (buft) {1086 buft_list.emplace_back(dev.dev, buft);1087 break;1088 }1089 }1090 }1091 1092 // add extra buffer types1093 if (use_extra_bufts) {1094 auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);1095 if (cpu_dev == nullptr) {1096 throw std::runtime_error(format("%s: no CPU backend found", __func__));1097 }1098 1099 auto * cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);1100 auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)1101 ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_dev_get_extra_bufts");1102 if (ggml_backend_dev_get_extra_bufts_fn) {1103 ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(cpu_dev);1104 while (extra_bufts && *extra_bufts) {1105 buft_list.emplace_back(cpu_dev, *extra_bufts);1106 ++extra_bufts;1107 }1108 }1109 }1110 1111 // add the CPU buffer type1112 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {1113 ggml_backend_dev_t dev = ggml_backend_dev_get(i);1114 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {1115 buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));1116 }1117 }1118 1119 return buft_list;1120}1121 1122// GPU: split if LLAMA_SPLIT_MODE_ROW -> GPU1123static buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, llama_split_mode split_mode, const float * tensor_split) {1124 buft_list_t buft_list;1125 1126 // add the device split buffer type if requested and available1127 if (split_mode == LLAMA_SPLIT_MODE_ROW) {1128 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);1129 auto ggml_backend_split_buffer_type_fn = (ggml_backend_split_buffer_type_t)1130 ggml_backend_reg_get_proc_address(reg, "ggml_backend_split_buffer_type");1131 if (ggml_backend_split_buffer_type_fn) {1132 size_t dev_index = [&]() {1133 auto * reg = ggml_backend_dev_backend_reg(dev);1134 for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); ++i) {1135 if (ggml_backend_reg_dev_get(reg, i) == dev) {1136 return i;1137 }1138 }1139 throw std::runtime_error(format("device %s not found in its backend reg", ggml_backend_dev_name(dev)));1140 }();1141 auto * buft = ggml_backend_split_buffer_type_fn(dev_index, tensor_split);1142 if (buft != nullptr) {1143 buft_list.emplace_back(dev, buft);1144 }1145 } else {1146 throw std::runtime_error(format("device %s does not support split buffers", ggml_backend_dev_name(dev)));1147 }1148 }1149 1150 // add the device default buffer type1151 buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));1152 1153 // add the device extra buffer type (if any)1154 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);1155 if (reg) {1156 auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)1157 ggml_backend_reg_get_proc_address(reg, "ggml_backend_dev_get_extra_bufts");1158 1159 if (ggml_backend_dev_get_extra_bufts_fn) {1160 ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(dev);1161 while (extra_bufts && *extra_bufts) {1162 buft_list.emplace_back(dev, *extra_bufts);1163 ++extra_bufts;1164 }1165 }1166 }1167 1168 return buft_list;1169}1170 1171struct llama_model::impl {1172 impl() = default;1173 ~impl() = default;1174 1175 uint64_t n_elements = 0;1176 1177 size_t n_bytes = 0;1178 1179 std::string desc_str;1180 1181 llama_ftype ftype = LLAMA_FTYPE_ALL_F32;1182 1183 // model memory mapped files1184 llama_mmaps mappings;1185 1186 // objects representing data potentially being locked in memory1187 llama_mlocks mlock_bufs;1188 llama_mlocks mlock_mmaps;1189 1190 // contexts where the model tensors metadata is stored as well as the corresponding buffers:1191 std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;1192 1193 buft_list_t cpu_buft_list;1194 std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;1195 1196 struct layer_dev {1197 ggml_backend_dev_t dev;1198 buft_list_t * buft_list;1199 };1200