Felipe97/llama-cpp-compiled
01.1k
1#include "llama-model-loader.h"2 3#include "ggml-alloc.h"4#include "ggml.h"5#include "gguf.h"6#include "llama-hparams.h"7#include "llama.h"8 9#include <algorithm>10#include <array>11#include <cinttypes>12#include <cstdint>13#include <cstring>14#include <future>15#include <regex>16 17static const size_t kiB = 1024;18static const size_t MiB = 1024*kiB;19static const size_t GiB = 1024*MiB;20 21const char * llama_file_version_name(llama_fver version) {22 switch (version) {23 case GGUF_FILE_VERSION_V1: return "GGUF V1 (support until nov 2023)";24 case GGUF_FILE_VERSION_V2: return "GGUF V2";25 case GGUF_FILE_VERSION_V3: return "GGUF V3 (latest)";26 }27 28 return "unknown";29}30 31#define LLAMA_FTYPE_PREFIX "(guessed) "32 33const char * llama_ftype_name(llama_ftype ftype) {34 static constexpr size_t guessed_prefix_len = sizeof(LLAMA_FTYPE_PREFIX) - 1;35 const char * name;36 switch ((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) {37 case LLAMA_FTYPE_ALL_F32: name = LLAMA_FTYPE_PREFIX "all F32"; break;38 case LLAMA_FTYPE_MOSTLY_F16: name = LLAMA_FTYPE_PREFIX "F16"; break;39 case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break;40 case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break;41 case LLAMA_FTYPE_MOSTLY_Q2_0: name = LLAMA_FTYPE_PREFIX "Q2_0"; break;42 case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break;43 case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break;44 case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break;45 case LLAMA_FTYPE_MOSTLY_Q5_1: name = LLAMA_FTYPE_PREFIX "Q5_1"; break;46 case LLAMA_FTYPE_MOSTLY_Q8_0: name = LLAMA_FTYPE_PREFIX "Q8_0"; break;47 case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: name = LLAMA_FTYPE_PREFIX "MXFP4 MoE"; break;48 case LLAMA_FTYPE_MOSTLY_NVFP4: name = LLAMA_FTYPE_PREFIX "NVFP4"; break;49 case LLAMA_FTYPE_MOSTLY_Q2_K: name = LLAMA_FTYPE_PREFIX "Q2_K - Medium"; break;50 case LLAMA_FTYPE_MOSTLY_Q2_K_S: name = LLAMA_FTYPE_PREFIX "Q2_K - Small"; break;51 case LLAMA_FTYPE_MOSTLY_Q3_K_S: name = LLAMA_FTYPE_PREFIX "Q3_K - Small"; break;52 case LLAMA_FTYPE_MOSTLY_Q3_K_M: name = LLAMA_FTYPE_PREFIX "Q3_K - Medium"; break;53 case LLAMA_FTYPE_MOSTLY_Q3_K_L: name = LLAMA_FTYPE_PREFIX "Q3_K - Large"; break;54 case LLAMA_FTYPE_MOSTLY_Q4_K_S: name = LLAMA_FTYPE_PREFIX "Q4_K - Small"; break;55 case LLAMA_FTYPE_MOSTLY_Q4_K_M: name = LLAMA_FTYPE_PREFIX "Q4_K - Medium"; break;56 case LLAMA_FTYPE_MOSTLY_Q5_K_S: name = LLAMA_FTYPE_PREFIX "Q5_K - Small"; break;57 case LLAMA_FTYPE_MOSTLY_Q5_K_M: name = LLAMA_FTYPE_PREFIX "Q5_K - Medium"; break;58 case LLAMA_FTYPE_MOSTLY_Q6_K: name = LLAMA_FTYPE_PREFIX "Q6_K"; break;59 case LLAMA_FTYPE_MOSTLY_TQ1_0: name = LLAMA_FTYPE_PREFIX "TQ1_0 - 1.69 bpw ternary"; break;60 case LLAMA_FTYPE_MOSTLY_TQ2_0: name = LLAMA_FTYPE_PREFIX "TQ2_0 - 2.06 bpw ternary"; break;61 case LLAMA_FTYPE_MOSTLY_IQ2_XXS: name = LLAMA_FTYPE_PREFIX "IQ2_XXS - 2.0625 bpw"; break;62 case LLAMA_FTYPE_MOSTLY_IQ2_XS: name = LLAMA_FTYPE_PREFIX "IQ2_XS - 2.3125 bpw"; break;63 case LLAMA_FTYPE_MOSTLY_IQ2_S: name = LLAMA_FTYPE_PREFIX "IQ2_S - 2.5 bpw"; break;64 case LLAMA_FTYPE_MOSTLY_IQ2_M: name = LLAMA_FTYPE_PREFIX "IQ2_M - 2.7 bpw"; break;65 case LLAMA_FTYPE_MOSTLY_IQ3_XS: name = LLAMA_FTYPE_PREFIX "IQ3_XS - 3.3 bpw"; break;66 case LLAMA_FTYPE_MOSTLY_IQ3_XXS: name = LLAMA_FTYPE_PREFIX "IQ3_XXS - 3.0625 bpw"; break;67 case LLAMA_FTYPE_MOSTLY_IQ1_S: name = LLAMA_FTYPE_PREFIX "IQ1_S - 1.5625 bpw"; break;68 case LLAMA_FTYPE_MOSTLY_IQ1_M: name = LLAMA_FTYPE_PREFIX "IQ1_M - 1.75 bpw"; break;69 case LLAMA_FTYPE_MOSTLY_IQ4_NL: name = LLAMA_FTYPE_PREFIX "IQ4_NL - 4.5 bpw"; break;70 case LLAMA_FTYPE_MOSTLY_IQ4_XS: name = LLAMA_FTYPE_PREFIX "IQ4_XS - 4.25 bpw"; break;71 case LLAMA_FTYPE_MOSTLY_IQ3_S: name = LLAMA_FTYPE_PREFIX "IQ3_S - 3.4375 bpw"; break;72 case LLAMA_FTYPE_MOSTLY_IQ3_M: name = LLAMA_FTYPE_PREFIX "IQ3_S mix - 3.66 bpw"; break;73 default: name = LLAMA_FTYPE_PREFIX "unknown, may not work"; break;74 }75 return (ftype & LLAMA_FTYPE_GUESSED) ? name : name + guessed_prefix_len;76}77 78#undef LLAMA_FTYPE_PREFIX79 80// return a list of splits for a given path81// for example, given "<name>-00002-of-00004.gguf", returns list of all 4 splits82static std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {83 std::vector<std::string> paths;84 std::string split_prefix;85 std::vector<char> buf(llama_path_max(), 0);86 87 {88 int ret = llama_split_prefix(buf.data(), buf.size(), path.c_str(), idx, n_split);89 if (!ret) {90 throw std::runtime_error(format("invalid split file name: %s", path.c_str()));91 }92 split_prefix = std::string(buf.data(), ret);93 }94 95 if (split_prefix.empty()) {96 throw std::runtime_error(format("invalid split file: %s", path.c_str()));97 }98 99 for (int idx = 0; idx < n_split; ++idx) {100 int ret = llama_split_path(buf.data(), buf.size(), split_prefix.c_str(), idx, n_split);101 paths.push_back(std::string(buf.data(), ret));102 }103 104 return paths;105}106 107namespace GGUFMeta {108 template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int64_t)>109 struct GKV_Base_Type {110 static constexpr gguf_type gt = gt_;111 112 static T getter(const gguf_context * ctx, const int kid) {113 return gfun(ctx, kid);114 }115 };116 117 template<typename T> struct GKV_Base;118 119 template<> struct GKV_Base<bool >: GKV_Base_Type<bool, GGUF_TYPE_BOOL, gguf_get_val_bool> {};120 template<> struct GKV_Base<uint8_t >: GKV_Base_Type<uint8_t, GGUF_TYPE_UINT8, gguf_get_val_u8 > {};121 template<> struct GKV_Base<uint16_t >: GKV_Base_Type<uint16_t, GGUF_TYPE_UINT16, gguf_get_val_u16 > {};122 template<> struct GKV_Base<uint32_t >: GKV_Base_Type<uint32_t, GGUF_TYPE_UINT32, gguf_get_val_u32 > {};123 template<> struct GKV_Base<uint64_t >: GKV_Base_Type<uint64_t, GGUF_TYPE_UINT64, gguf_get_val_u64 > {};124 template<> struct GKV_Base<int8_t >: GKV_Base_Type<int8_t, GGUF_TYPE_INT8, gguf_get_val_i8 > {};125 template<> struct GKV_Base<int16_t >: GKV_Base_Type<int16_t, GGUF_TYPE_INT16, gguf_get_val_i16 > {};126 template<> struct GKV_Base<int32_t >: GKV_Base_Type<int32_t, GGUF_TYPE_INT32, gguf_get_val_i32 > {};127 template<> struct GKV_Base<int64_t >: GKV_Base_Type<int64_t, GGUF_TYPE_INT64, gguf_get_val_i64 > {};128 template<> struct GKV_Base<float >: GKV_Base_Type<float, GGUF_TYPE_FLOAT32, gguf_get_val_f32 > {};129 template<> struct GKV_Base<double >: GKV_Base_Type<double, GGUF_TYPE_FLOAT64, gguf_get_val_f64 > {};130 template<> struct GKV_Base<const char *>: GKV_Base_Type<const char *, GGUF_TYPE_STRING, gguf_get_val_str > {};131 132 template<> struct GKV_Base<std::string> {133 static constexpr gguf_type gt = GGUF_TYPE_STRING;134 135 static std::string getter(const gguf_context * ctx, const int kid) {136 return gguf_get_val_str(ctx, kid);137 }138 };139 140 struct ArrayInfo {141 const gguf_type gt;142 const size_t length;143 const void * data;144 };145 146 template<> struct GKV_Base<ArrayInfo> {147 public:148 static constexpr gguf_type gt = GGUF_TYPE_ARRAY;149 static ArrayInfo getter(const gguf_context *ctx, const int k) {150 const enum gguf_type arr_type = gguf_get_arr_type(ctx, k);151 return ArrayInfo {152 arr_type,153 gguf_get_arr_n(ctx, k),154 arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx, k),155 };156 }157 };158 159 template<typename T>160 class GKV : public GKV_Base<T> {161 GKV() = delete;162 163 public:164 static T get_kv(const gguf_context * ctx, const int k) {165 const enum gguf_type kt = gguf_get_kv_type(ctx, k);166 167 if (kt != GKV::gt) {168 throw std::runtime_error(format("key %s has wrong type %s but expected type %s",169 gguf_get_key(ctx, k), gguf_type_name(kt), gguf_type_name(GKV::gt)));170 }171 return GKV::getter(ctx, k);172 }173 174 static const char * override_type_to_str(const llama_model_kv_override_type ty) {175 switch (ty) {176 case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool";177 case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";178 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";179 case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";180 }181 return "unknown";182 }183 184 static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override * ovrd) {185 if (!ovrd) { return false; }186 if (ovrd->tag == expected_type) {187 LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ",188 __func__, override_type_to_str(ovrd->tag), ovrd->key);189 switch (ovrd->tag) {190 case LLAMA_KV_OVERRIDE_TYPE_BOOL: {191 LLAMA_LOG_INFO("%s\n", ovrd->val_bool ? "true" : "false");192 } break;193 case LLAMA_KV_OVERRIDE_TYPE_INT: {194 LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64);195 } break;196 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {197 LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64);198 } break;199 case LLAMA_KV_OVERRIDE_TYPE_STR: {200 LLAMA_LOG_INFO("%s\n", ovrd->val_str);201 } break;202 default:203 // Shouldn't be possible to end up here, but just in case...204 throw std::runtime_error(205 format("Unsupported attempt to override %s type for metadata key %s\n",206 override_type_to_str(ovrd->tag), ovrd->key));207 }208 return true;209 }210 LLAMA_LOG_WARN("%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n",211 __func__, ovrd->key, override_type_to_str(expected_type), override_type_to_str(ovrd->tag));212 return false;213 }214 215 template<typename OT>216 static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type217 try_override(OT & target, const struct llama_model_kv_override * ovrd) {218 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {219 target = ovrd->val_bool;220 return true;221 }222 return false;223 }224 225 template<typename OT>226 static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type227 try_override(OT & target, const struct llama_model_kv_override * ovrd) {228 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {229 target = ovrd->val_i64;230 return true;231 }232 return false;233 }234 235 template<typename OT>236 static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type237 try_override(T & target, const struct llama_model_kv_override * ovrd) {238 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {239 target = ovrd->val_f64;240 return true;241 }242 return false;243 }244 245 template<typename OT>246 static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type247 try_override(T & target, const struct llama_model_kv_override * ovrd) {248 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {249 target = ovrd->val_str;250 return true;251 }252 return false;253 }254 255 static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {256 if (try_override<T>(target, ovrd)) {257 return true;258 }259 if (k < 0) { return false; }260 target = get_kv(ctx, k);261 return true;262 }263 264 static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {265 return set(ctx, gguf_find_key(ctx, key), target, ovrd);266 }267 268 static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {269 return set(ctx, key.c_str(), target, ovrd);270 }271 };272}273 274 template<typename T>275 typename std::enable_if<std::is_integral<T>::value, bool>::type276 llama_model_loader::get_arr_n(const std::string & key, T & result, bool required) {277 const int kid = gguf_find_key(metadata, key.c_str());278 279 if (kid < 0) {280 if (required) {281 throw std::runtime_error(format("key not found in model: %s", key.c_str()));282 }283 return false;284 }285 286 struct GGUFMeta::ArrayInfo arr_info =287 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);288 289 290 result = arr_info.length;291 return true;292 }293 294 template<typename T>295 typename std::enable_if<std::is_integral<T>::value, bool>::type296 llama_model_loader::get_arr_n(enum llm_kv kid, T & result, bool required) {297 return get_arr_n(llm_kv(kid), result, required);298 }299 300 template bool llama_model_loader::get_arr_n(enum llm_kv kid, uint32_t & result, bool required);301 template std::enable_if<std::is_integral<uint32_t>::value, bool>::type302 llama_model_loader::get_arr_n<uint32_t>(const std::string & key, uint32_t & result, bool required);303 304 template<typename T>305 bool llama_model_loader::get_arr(const std::string & key, std::vector<T> & result, bool required) {306 const gguf_context * ctx = metadata;307 const int kid = gguf_find_key(ctx, key.c_str());308 309 if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {310 if (required) {311 throw std::runtime_error(format("array key not found in model: %s", key.c_str()));312 }313 return false;314 }315 316 struct GGUFMeta::ArrayInfo arr_info =317 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);318 319 bool type_ok = false;320 switch (arr_info.gt) {321 case GGUF_TYPE_UINT32:322 case GGUF_TYPE_INT32: type_ok = (std::is_same<T, int32_t>::value) ||323 (std::is_same<T, uint32_t>::value); break;324 case GGUF_TYPE_UINT64: type_ok = (std::is_same<T, uint64_t>::value); break;325 case GGUF_TYPE_FLOAT32: type_ok = (std::is_same<T, float>::value); break;326 case GGUF_TYPE_STRING: type_ok = (std::is_same<T, std::string>::value); break;327 default:328 throw std::runtime_error(format("%s is not a string/float32/uint32/int32/uint64 array", key.c_str()));329 }330 if (!type_ok) {331 throw std::runtime_error(format("%s has wrong array element type %s", key.c_str(), gguf_type_name(arr_info.gt)));332 }333 334 if constexpr (std::is_same<T, std::string>::value) {335 const size_t n_items = gguf_get_arr_n(ctx, kid);336 result.clear();337 338 for (size_t i = 0; i < n_items; i++) {339 const T value = gguf_get_arr_str(ctx, kid, i);340 result.emplace_back(value);341 }342 } else {343 result.resize(arr_info.length);344 result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);345 }346 347 return true;348 }349 350 template<typename T, size_t N_MAX>351 bool llama_model_loader::get_arr(const std::string & key, std::array<T, N_MAX> & result, bool required) {352 const gguf_context * ctx = metadata;353 const int kid = gguf_find_key(ctx, key.c_str());354 355 if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {356 if (required) {357 throw std::runtime_error(format("array key not found in model: %s", key.c_str()));358 }359 return false;360 }361 362 struct GGUFMeta::ArrayInfo arr_info =363 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);364 365 bool type_ok = false;366 switch (arr_info.gt) {367 case GGUF_TYPE_BOOL:368 case GGUF_TYPE_UINT32:369 case GGUF_TYPE_INT32: type_ok = (std::is_same<T, int32_t>::value) ||370 (std::is_same<T, uint32_t>::value); break;371 case GGUF_TYPE_UINT64: type_ok = (std::is_same<T, uint64_t>::value); break;372 case GGUF_TYPE_FLOAT32: type_ok = (std::is_same<T, float>::value); break;373 case GGUF_TYPE_STRING: type_ok = (std::is_same<T, std::string>::value); break;374 default:375 throw std::runtime_error(format("%s is not a string/float32/uint32/int32/uint64 array", key.c_str()));376 }377 if (!type_ok) {378 throw std::runtime_error(format("%s has wrong array element type %s", key.c_str(), gguf_type_name(arr_info.gt)));379 }380 381 if (arr_info.length > N_MAX) {382 throw std::runtime_error(format("array length %u for key %s exceeds max %u", (uint32_t) arr_info.length, key.c_str(), (uint32_t) N_MAX));383 }384 385 if constexpr (std::is_same<T, std::string>::value) {386 const size_t n_items = gguf_get_arr_n(ctx, kid);387 388 for (size_t i = 0; i < n_items; i++) {389 const T value = gguf_get_arr_str(ctx, kid, i);390 result[i] = value;391 }392 } else {393 if (arr_info.gt == GGUF_TYPE_BOOL) {394 const int8_t * values = (const int8_t *) arr_info.data;395 std::transform(values, values + arr_info.length, result.begin(), [](int8_t x) {396 return static_cast<T>(x != 0);397 });398 } else {399 std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());400 }401 }402 403 return true;404 }405 406 template<typename T>407 bool llama_model_loader::get_arr(enum llm_kv kid, T & result, bool required) {408 return get_arr(llm_kv(kid), result, required);409 }410 411 template bool llama_model_loader::get_arr<std::vector<std::string>>(enum llm_kv kid, std::vector<std::string> & result, bool required);412 template bool llama_model_loader::get_arr<std::array<int32_t, 512>>(enum llm_kv kid, std::array<int32_t, 512> & result, bool required);413 template bool llama_model_loader::get_arr<std::vector<int32_t>>(enum llm_kv kid, std::vector<int32_t> & result, bool required);414 template bool llama_model_loader::get_arr<std::array<uint32_t, LLAMA_MAX_LAYERS>>(enum llm_kv kid, std::array<uint32_t, LLAMA_MAX_LAYERS> & result, bool required);415 template bool llama_model_loader::get_arr<std::vector<uint32_t>>(enum llm_kv kid, std::vector<uint32_t> & result, bool required);416 template bool llama_model_loader::get_arr<std::array<uint64_t, LLAMA_MAX_PLE_NGRAM>>(enum llm_kv kid, std::array<uint64_t, LLAMA_MAX_PLE_NGRAM> & result, bool required);417 template bool llama_model_loader::get_arr<std::array<uint64_t, LLAMA_MAX_PLE_HEADS>>(enum llm_kv kid, std::array<uint64_t, LLAMA_MAX_PLE_HEADS> & result, bool required);418 419 template<typename T>420 bool llama_model_loader::get_key(const std::string & key, T & result, bool required) {421 auto it = kv_overrides.find(key);422 423 const struct llama_model_kv_override * override =424 it != kv_overrides.end() ? &it->second : nullptr;425 426 const bool found = GGUFMeta::GKV<T>::set(metadata, key, result, override);427 428 if (required && !found) {429 throw std::runtime_error(format("key not found in model: %s", key.c_str()));430 }431 432 return found;433 }434 435 template<typename T>436 bool llama_model_loader::get_key(enum llm_kv kid, T & result, bool required) {437 return get_key(llm_kv(kid), result, required);438 }439 440 template bool llama_model_loader::get_key<bool> (enum llm_kv kid, bool & result, bool required);441 template bool llama_model_loader::get_key<float> (enum llm_kv kid, float & result, bool required);442 template bool llama_model_loader::get_key<uint32_t> (enum llm_kv kid, uint32_t & result, bool required);443 template bool llama_model_loader::get_key<std::string>(enum llm_kv kid, std::string & result, bool required);444 445 template<>446 bool llama_model_loader::get_key(enum llm_kv kid, enum llama_pooling_type & result, bool required) {447 uint32_t tmp;448 const bool found = get_key(kid, tmp, required);449 if (found) {450 result = (enum llama_pooling_type) tmp;451 } else {452 result = LLAMA_POOLING_TYPE_UNSPECIFIED;453 }454 return found;455 }456 457 // get array of n <= N_MAX elements, or a single element repeated n times458 template<typename T, size_t N_MAX>459 bool llama_model_loader::get_key_or_arr(const std::string & key, std::array<T, N_MAX> & result, uint32_t n, bool required) {460 const int kid = gguf_find_key(metadata, key.c_str());461 462 if (kid < 0) {463 if (required) {464 throw std::runtime_error(format("key not found in model: %s", key.c_str()));465 }466 return false;467 }468 469 if (n > N_MAX) {470 throw std::runtime_error(format("n > N_MAX: %u > %u for key %s", n, (uint32_t) N_MAX, key.c_str()));471 }472 473 if (gguf_get_kv_type(metadata, kid) == GGUF_TYPE_ARRAY) {474 struct GGUFMeta::ArrayInfo arr_info =475 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);476 477 if (n != arr_info.length) {478 throw std::runtime_error(format("key %s has wrong array length; expected %u, got %u", key.c_str(), n, (uint32_t) arr_info.length));479 }480 481 return get_arr(key, result, required);482 }483 484 T value;485 486 bool ok = get_key(key, value, required);487 if (!ok) {488 return false;489 }490 491 for (uint32_t i = 0; i < n; i++) {492 result[i] = value;493 }494 495 return true;496 }497 498 template<typename T>499 bool llama_model_loader::get_key_or_arr(enum llm_kv kid, T & result, uint32_t n, bool required) {500 return get_key_or_arr(llm_kv(kid), result, n, required);501 }502 503 bool llama_model_loader::get_key_or_arr(enum llm_kv kid, uint32_t & result, bool required) {504 const std::string key = llm_kv(kid);505 506 const int id = gguf_find_key(metadata, key.c_str());507 508 if (id < 0) {509 if (required) {510 throw std::runtime_error(format("key not found in model: %s", key.c_str()));511 }512 return false;513 }514 515 // throw and error if type is an array516 if (gguf_get_kv_type(metadata, id) == GGUF_TYPE_ARRAY) {517 if (required) {518 throw std::runtime_error(format("expected scalar, found array for key: %s", key.c_str()));519 }520 return false;521 }522 523 return get_key(key, result, required);524 }525 526 // TODO: this is not very clever - figure out something better527 template bool llama_model_loader::get_key_or_arr<std::array<int, 4>> (enum llm_kv kid, std::array<int, 4> & result, uint32_t n, bool required);528 template bool llama_model_loader::get_key_or_arr<std::array<uint32_t, 512>>(enum llm_kv kid, std::array<uint32_t, 512> & result, uint32_t n, bool required);529 template bool llama_model_loader::get_key_or_arr<std::array<float, 512>>(enum llm_kv kid, std::array<float, 512> & result, uint32_t n, bool required);530 531 532llama_model_loader::llama_model_loader(533 struct gguf_context * meta,534 llama_model_set_tensor_data_t set_tensor_data,535 void * set_tensor_data_ud,536 const std::string & fname,537 std::vector<std::string> & splits,538 FILE * file,539 llama_load_mode load_mode,540 bool check_tensors,541 bool no_alloc,542 bool load_mtp,543 const llama_model_kv_override * param_overrides_p,544 const llama_model_tensor_buft_override * param_tensor_buft_overrides_p)545 : metadata(meta), set_tensor_data(set_tensor_data), set_tensor_data_ud(set_tensor_data_ud) {546 int trace = 0;547 if (getenv("LLAMA_TRACE")) {548 trace = atoi(getenv("LLAMA_TRACE"));549 }550 551 if (param_overrides_p != nullptr) {552 for (const struct llama_model_kv_override * p = param_overrides_p; p->key[0] != 0; p++) {553 kv_overrides.insert({std::string(p->key), *p});554 }555 }556 557 tensor_buft_overrides = param_tensor_buft_overrides_p;558 559 this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK || load_mode == LLAMA_LOAD_MODE_AUTO;560 this->use_direct_io = load_mode == LLAMA_LOAD_MODE_DIRECT_IO;561 562 if (!fname.empty()) {563 // Load the main GGUF564 struct ggml_context * ctx = NULL;565 struct gguf_init_params params = {566 /*.no_alloc = */ true,567 /*.ctx = */ &ctx,568 };569 570 metadata_ptr.reset(gguf_init_from_file(fname.c_str(), params));571 metadata = metadata_ptr.get();572 if (metadata == nullptr) {573 throw std::runtime_error(format("%s: failed to load model from %s", __func__, fname.c_str()));574 }575 576 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);577 llm_kv = LLM_KV(llm_arch_from_string(arch_name));578 579 files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));580 contexts.emplace_back(ctx);581 582 // Save tensors data offset of the main file.583 // For subsidiary files, `meta` tensor data offset must not be used,584 // so we build a unified tensors index for weights.585 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {586 std::string tensor_name = std::string(cur->name);587 // make sure there is no duplicated tensor names588 if (weights_map.find(tensor_name) != weights_map.end()) {589 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));590 }591 n_elements += ggml_nelements(cur);592 n_bytes += ggml_nbytes(cur);593 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));594 }595 uint16_t n_split = 0;596 get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false);597 598 // Load additional GGML contexts599 if (n_split > 1) {600 // make sure the main file is loaded first601 uint16_t idx = 0;602 const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);603 get_key(kv_split_no, idx);604 if (idx != 0) {605 throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str()));606 }607 608 // generate list of splits if needed609 if (splits.empty()) {610 splits = llama_get_list_splits(fname, idx, n_split);611 }612 613 // in case user give a custom list of splits, check if it matches the expected number614 if (n_split != (uint16_t)splits.size()) {615 throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split));616 }617 618 if (trace > 0) {619 LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split);620 }621 622 // load other splits623 for (idx = 1; idx < n_split; idx++) {624 const char * fname_split = splits[idx].c_str();625 626 struct gguf_init_params split_params = {627 /*.no_alloc = */ true,628 /*.ctx = */ &ctx,629 };630 gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };631 if (!ctx_gguf) {632 throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split));633 }634 635 // check idx636 {637 const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());638 if (kid < 0) {639 throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split));640 }641 int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);642 if (idx_gguf != idx) {643 throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx));644 }645 }646 647 files.emplace_back(new llama_file(fname_split, "rb", use_direct_io));648 contexts.emplace_back(ctx);649 650 // Save tensors data offset info of the shard.651 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {652 std::string tensor_name = std::string(cur->name);653 // make sure there is no duplicated tensor names654 if (weights_map.find(tensor_name) != weights_map.end()) {655 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));656 }657 n_elements += ggml_nelements(cur);658 n_bytes += ggml_nbytes(cur);659 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur));660 }661 }662 663 get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors);664 665 // sanity check666 {667 const int n_tensors_loaded = (int) weights_map.size();668 if (n_tensors != n_tensors_loaded) {669 throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded));670 }671 }672 673 LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1);674 }675 } else if (file != nullptr) {676 struct ggml_context * ctx = NULL;677 struct gguf_init_params params = {678 /*.no_alloc = */ true,679 /*.ctx = */ &ctx,680 };681 682 metadata_ptr.reset(gguf_init_from_file_ptr(file, params));683 metadata = metadata_ptr.get();684 if (metadata == nullptr) {685 throw std::runtime_error(format("%s: failed to load model from file pointer", __func__));686 }687 688 // mmap places tensors at their file offsets, so an embedded GGUF must be aligned in the file too689 const size_t tensor_align = ggml_backend_buft_get_alignment(ggml_backend_cpu_buffer_type());690 if (use_mmap && gguf_get_data_offset(metadata) % tensor_align != 0) {691 throw std::runtime_error(format("%s: GGUF data section at file offset %zu is not %zu byte aligned, cannot mmap",692 __func__, gguf_get_data_offset(metadata), tensor_align));693 }694 695 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);696 llm_kv = LLM_KV(llm_arch_from_string(arch_name));697 698 files.emplace_back(new llama_file(file));699 contexts.emplace_back(ctx);700 701 // Save tensors data offset info of the main file.702 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {703 std::string tensor_name = std::string(cur->name);704 // make sure there is no duplicated tensor names705 if (weights_map.find(tensor_name) != weights_map.end()) {706 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));707 }708 n_elements += ggml_nelements(cur);709 n_bytes += ggml_nbytes(cur);710 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));711 }712 } else {713 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);714 llm_kv = LLM_KV(llm_arch_from_string(arch_name));715 }716 717 n_kv = gguf_get_n_kv(metadata);718 n_tensors = weights_map.size();719 720 fver = (enum llama_fver) gguf_get_version(metadata);721 722 LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",723 __func__, n_kv, n_tensors, fname.empty() ? "(file*)" : fname.c_str(), llama_file_version_name(fver));724 725 // determine file type based on the number of tensors for each quantization and print meta data726 // TODO: make optional727 {728 std::map<enum ggml_type, uint32_t> n_type;729 730 uint32_t n_type_max = 0;731 enum ggml_type type_max = GGML_TYPE_F32;732 733 for (const auto & it : weights_map) {734 const llama_tensor_weight & w = it.second;735 const ggml_tensor * tensor = w.tensor;736 737 enum ggml_type type = tensor->type;738 739 n_type[type]++;740 741 if (n_type_max < n_type[type]) {742 n_type_max = n_type[type];743 type_max = type;744 }745 746 if (trace > 0) {747 const uint16_t sid = w.idx;748 LLAMA_LOG_INFO("%s: - tensor split %2d: %32s %-8s [ %s ] %8.2f MiB\n", __func__,749 sid, ggml_get_name(tensor), ggml_type_name(type), llama_format_tensor_shape(tensor).c_str(),750 ggml_nbytes(tensor)/1024.0f/1024.0f);751 }752 }753 754 switch (type_max) {755 case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;756 case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;757 case GGML_TYPE_BF16: ftype = LLAMA_FTYPE_MOSTLY_BF16; break;758 case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;759 case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;760 case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;761 case GGML_TYPE_Q5_1: ftype = LLAMA_FTYPE_MOSTLY_Q5_1; break;762 case GGML_TYPE_Q8_0: ftype = LLAMA_FTYPE_MOSTLY_Q8_0; break;763 case GGML_TYPE_Q2_K: ftype = LLAMA_FTYPE_MOSTLY_Q2_K; break;764 case GGML_TYPE_Q3_K: ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M; break;765 case GGML_TYPE_Q4_K: ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M; break;766 case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;767 case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;768 case GGML_TYPE_TQ1_0: ftype = LLAMA_FTYPE_MOSTLY_TQ1_0; break;769 case GGML_TYPE_TQ2_0: ftype = LLAMA_FTYPE_MOSTLY_TQ2_0; break;770 case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;771 case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break;772 case GGML_TYPE_IQ2_S: ftype = LLAMA_FTYPE_MOSTLY_IQ2_S; break;773 case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break;774 case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break;775 case GGML_TYPE_IQ1_M: ftype = LLAMA_FTYPE_MOSTLY_IQ1_M; break;776 case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break;777 case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break;778 case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break;779 case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break;780 case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break;781 case GGML_TYPE_Q2_0: ftype = LLAMA_FTYPE_MOSTLY_Q2_0; break;782 default:783 {784 LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));785 ftype = LLAMA_FTYPE_ALL_F32;786 } break;787 }788 789 // this is a way to mark that we have "guessed" the file type790 ftype = (llama_ftype) (ftype | LLAMA_FTYPE_GUESSED);791 792 {793 uint32_t ftype_val = 0;794 if (get_key(LLM_KV_GENERAL_FILE_TYPE, ftype_val, false)) {795 ftype = (llama_ftype) ftype_val;796 }797 }798 799 LLAMA_LOG_INFO("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__);800 801 for (int i = 0; i < n_kv; i++) {802 const char * name = gguf_get_key(metadata, i);803 const enum gguf_type type = gguf_get_kv_type(metadata, i);804 const std::string type_name =805 type == GGUF_TYPE_ARRAY806 ? format("%s[%s,%zu]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(metadata, i)), gguf_get_arr_n(metadata, i))807 : gguf_type_name(type);808 809 std::string value = gguf_kv_to_str(metadata, i);810 const size_t MAX_VALUE_LEN = 40;811 if (value.size() > MAX_VALUE_LEN) {812 value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());813 }814 replace_all(value, "\n", "\\n");815 816 LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str());817 }818 819 // print type counts820 for (auto & kv : n_type) {821 if (kv.second == 0) {822 continue;823 }824 825 LLAMA_LOG_INFO("%s: - type %4s: %4d tensors\n", __func__, ggml_type_name(kv.first), kv.second);826 }827 }828 829 if (this->use_mmap && !llama_mmap::SUPPORTED) {830 LLAMA_LOG_WARN("%s: mmap is not supported on this platform\n", __func__);831 this->use_mmap = false;832 }833 834 this->check_tensors = check_tensors;835 this->no_alloc = no_alloc;836 this->load_mtp = load_mtp;837}838 839std::string llama_model_loader::get_arch_name() const {840 return arch_name;841}842 843enum llm_arch llama_model_loader::get_arch() const {844 return llm_kv.arch;845}846 847const llama_model_loader::llama_tensor_weight * llama_model_loader::get_weight(const char * name) const {848 auto pos = weights_map.find(name);849 if (pos != weights_map.end()) {850 return &pos->second;851 }852 853 return nullptr;854}855 856const llama_model_loader::llama_tensor_weight & llama_model_loader::require_weight(const char * name) const {857 const llama_tensor_weight * weight = get_weight(name);858 if (!weight) {859 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name));860 }861 return *weight;862}863 864struct ggml_tensor * llama_model_loader::get_tensor_meta(const char * name) const {865 const auto * weight = get_weight(name);866 if (!weight) {867 return nullptr;868 }869 return weight->tensor;870}871 872struct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string & name) const {873 struct ggml_tensor * tensor = get_tensor_meta(name.c_str());874 if (!tensor) {875 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));876 }877 return tensor;878}879 880const struct ggml_tensor * llama_model_loader::check_tensor_dims(881 const std::string & name,882 const std::vector<int64_t> & ne,883 bool required,884 bool allow_reshape) const {885 const struct ggml_tensor * cur = get_tensor_meta(name.c_str());886 887 if (cur == NULL) {888 if (!required) {889 return NULL;890 }891 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));892 }893 894 bool is_ok = true;895 896 if (allow_reshape) {897 // check total number of elements only898 const int64_t ncur = ggml_nelements(cur);899 int64_t nexp = 1;900 for (size_t i = 0; i < ne.size(); ++i) {901 nexp *= ne[i];902 }903 if (ncur != nexp) {904 is_ok = false;905 }906 } else {907 for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {908 if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) {909 is_ok = false;910 break;911 }912 }913 }914 915 if (!is_ok) {916 throw std::runtime_error(917 format("%s: tensor '%s' has wrong shape; expected %s, got %s",918 __func__, name.c_str(),919 llama_format_tensor_shape(ne).c_str(),920 llama_format_tensor_shape(cur).c_str()));921 }922 923 return cur;924}925 926// checks if the weight tensor can be used with the specified buffer type and device927static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w, ggml_op op, ggml_backend_buffer_type_t buft, ggml_backend_dev_t dev) {928 GGML_ASSERT(w != nullptr);929 930 if (op == GGML_OP_NONE) {931 return true;932 }933 934 ggml_init_params params = {935 /*.mem_size =*/ ggml_tensor_overhead()*8,936 /*.mem_buffer =*/ NULL,937 /*.no_alloc =*/ true,938 };939 ggml_context_ptr ctx_ptr { ggml_init(params) };940 if (!ctx_ptr) {941 throw std::runtime_error(format("failed to create ggml context"));942 }943 ggml_context * ctx = ctx_ptr.get();944 945 ggml_tensor * op_tensor = nullptr;946 947 switch (op) {948 case GGML_OP_GET_ROWS:949 {950 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);951 op_tensor = ggml_get_rows(ctx, w, b);952 } break;953 case GGML_OP_MUL_MAT:954 {955 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]);956 op_tensor = ggml_mul_mat(ctx, w, b);957 } break;958 case GGML_OP_MUL_MAT_ID:959 {960 // Used for either MoE expert routing or embedded adapter routing961 const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used_max();962 GGML_ASSERT(n_ids_used > 0);963 ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_ids_used, 512);964 ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_ids_used, 512);965 op_tensor = ggml_mul_mat_id(ctx, w, b, ids);966 } break;967 case GGML_OP_ADD:968 {969 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);970 op_tensor = ggml_add(ctx, a, w);971 } break;972 case GGML_OP_ADD_ID:973 {974 const int n_expert_used = hparams.n_expert_used_max();975 GGML_ASSERT(n_expert_used > 0);976 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);977 ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);978 op_tensor = ggml_add_id(ctx, a, w, c);979 } break;980 case GGML_OP_MUL:981 {982 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);983 op_tensor = ggml_mul(ctx, a, w);984 } break;985 case GGML_OP_DIV:986 {987 ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, w->ne[0]);988 op_tensor = ggml_div(ctx, a, w);989 } break;990 case GGML_OP_ROPE:991 {992 const int n_embd_head = hparams.n_embd_head_v();993 const int n_head = hparams.n_head();994 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_head, n_head, 512);995 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);996 op_tensor = ggml_rope_ext(997 ctx, a, b, w,998 0, 0, 0, 0, 0,999 0, 0, 0, 01000 );1001 1002 } break;1003 case GGML_OP_SSM_CONV:1004 {1005 const int64_t n_seq_tokens = 512;1006 const int64_t n_seqs = 3;1007 ggml_tensor * conv_x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0] - 1 + n_seq_tokens, w->ne[1], n_seqs);1008 op_tensor = ggml_ssm_conv(ctx, conv_x, w);1009 } break;1010 case GGML_OP_SSM_SCAN:1011 {1012 // w is ssm_a, which is used to distinguish Mamba-1 and Mamba-21013 const int64_t d_state = w->ne[0] == 1 ? hparams.ssm_d_state : w->ne[0];1014 const int64_t n_head = w->ne[1];1015 const int64_t head_dim = hparams.ssm_d_inner / n_head;1016 const int64_t n_group = hparams.ssm_n_group ? hparams.ssm_n_group : 1;1017 const int64_t n_seq_tokens = 512;1018 const int64_t n_seqs = 3;1019 ggml_tensor * s = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, head_dim, n_head, n_seqs);1020 ggml_tensor * x = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, head_dim, n_head, n_seq_tokens, n_seqs);1021 ggml_tensor * dt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_head, n_seq_tokens, n_seqs);1022 ggml_tensor * B = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);1023 ggml_tensor * C = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);1024 ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_seqs);1025 op_tensor = ggml_ssm_scan(ctx, s, x, dt, w, B, C, ids, /*K=*/1);1026 } break;1027 case GGML_OP_RWKV_WKV6:1028 {1029 // FIXME1030 const int64_t S = 123;1031 const int64_t H = 123;1032 const int64_t n_tokens = 123;1033 const int64_t n_seqs = 123;1034 ggml_tensor * k = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1035 ggml_tensor * v = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1036 ggml_tensor * r = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1037 ggml_tensor * tf = w;1038 ggml_tensor * td = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1039 ggml_tensor * state = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, S, n_seqs, S, H);1040 op_tensor = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, state);1041 } break;1042 case GGML_OP_IM2COL:1043 {1044 const int n_embd_inp = hparams.n_embd_inp();1045 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_embd_inp, w->ne[1], 1, 1);1046 op_tensor = ggml_im2col(ctx, w, b, 1, 0, 0, 0, 1, 0, false, GGML_TYPE_F16);1047 } break;1048 case GGML_OP_SCALE:1049 {1050 op_tensor = ggml_scale(ctx, w, 1.0f);1051 } break;1052 default:1053 GGML_ABORT("%s: missing test for op %s for tensor %s", __func__, ggml_op_name(op), w->name);1054 }1055 1056 // create a temporary dummy buffer for the weight so that supports_op can check the buffer type1057 GGML_ASSERT(w->buffer == nullptr);1058 w->buffer = ggml_backend_buft_alloc_buffer(buft, 0);1059 bool op_supported = ggml_backend_dev_supports_op(dev, op_tensor);1060 ggml_backend_buffer_free(w->buffer);1061 w->buffer = nullptr;1062 1063 return op_supported;1064}1065 1066// find the first buffer type in the list that can use the tensor1067static ggml_backend_buffer_type_t select_weight_buft(const llama_hparams & hparams, ggml_tensor * tensor, ggml_op op, const buft_list_t * buft_list) {1068 GGML_ASSERT(!buft_list->empty());1069 for (const auto & cur : *buft_list) {1070 ggml_backend_dev_t cur_dev = cur.first;1071 ggml_backend_buffer_type_t cur_buft = cur.second;1072 if (weight_buft_supported(hparams, tensor, op, cur_buft, cur_dev)) {1073 return cur_buft;1074 }1075 }1076 1077 return nullptr;1078}1079 1080ggml_backend_buffer_type_t llama_model_loader::lazy_read::buft() {1081 auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);1082 if (!cpu_dev) {1083 throw std::runtime_error("no CPU backend found");1084 }1085 return ggml_backend_dev_buffer_type(cpu_dev);1086}1087 1088bool llama_model_loader::lazy_read::add(const std::string & name, const ggml_tensor * t, const llama_tensor_weight * w) {1089 if (mode == LLAMA_LAZY_MODE_OFF) {1090 return false;1091 }1092 1093 // do not lazy-read small tensors, it has significant overhead and is not worth it1094 constexpr size_t auto_min_size = 4ull * 1024 * 1024 * 1024;1095 if (mode != LLAMA_LAZY_MODE_ON && ggml_nbytes(t) <= auto_min_size) {1096 return false;1097 }1098 1099 if (!llama_mmap::SUPPORTED) {1100 LLAMA_LOG_WARN("%s: mmap is not available, so tensor %s (size = %zu MiB) is loaded into RAM in full\n",1101 __func__, name.c_str(), ggml_nbytes(t)/1024/1024);1102 return false;1103 }1104 1105 if (w) {1106 ranges[w->idx].emplace_back(w->offs, w->offs + ggml_nbytes(t));1107 tensors.insert(name);1108 1109 LLAMA_LOG_INFO("%s: tensor %s (size = %zu MiB) lazy read enabled\n",1110 __func__, name.c_str(), ggml_nbytes(t)/1024/1024);1111 }1112 1113 return true;1114}1115 1116struct ggml_tensor * llama_model_loader::create_tensor(1117 const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output,1118 const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {1119 // set below, before buft_for_tensor() runs1120 bool is_lazy = false;1121 1122 auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {1123 const ctx_key key { buft, is_lazy };1124 1125 auto it = ctx_map.find(key);1126 if (it == ctx_map.end()) {1127 // one ggml context per buffer type1128 int max_n_tensors = n_tensors;1129 max_n_tensors += 1; // duplicated output tensor1130 max_n_tensors += hparams.n_layer()*2; // duplicated rope freq tensors1131 if (files.empty()) {1132 max_n_tensors += hparams.n_layer()*256; // this should be well above what any model actually uses1133 }1134 const size_t ctx_size = ggml_tensor_overhead()*max_n_tensors;1135 1136 ggml_init_params params = {1137 /*.mem_size =*/ ctx_size,1138 /*.mem_buffer =*/ NULL,1139 /*.no_alloc =*/ true,1140 };1141 1142 ggml_context * ctx = ggml_init(params);1143 if (!ctx) {1144 throw std::runtime_error(format("failed to create ggml context"));1145 }1146 1147 ctx_map.emplace(key, ctx);1148 1149 return ctx;1150 }1151 return it->second.get();1152 };1153 1154 auto buft_for_tensor = [&](ggml_tensor * t_meta) -> ggml_backend_buffer_type_t {1155 if (!t_meta) {1156 if (flags & TENSOR_NOT_REQUIRED) {1157 return nullptr;1158 }1159 throw std::runtime_error(format("missing tensor '%s'", tn.str().c_str()));1160 }1161 1162 // some models use the token embedding tensor as the output, but since these are used in different layers and with different ops1163 // the tensor is duplicated1164 // to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor1165 llm_tensor tn_tensor = tn.tensor;1166 if (tn.tensor == LLM_TENSOR_TOKEN_EMBD && (flags & TENSOR_DUPLICATED)) {1167 tn_tensor = LLM_TENSOR_OUTPUT;1168 }1169 1170 llm_tensor_info info;1171 try {1172 info = llm_tensor_info_for(tn_tensor);1173 } catch (const std::out_of_range & e) {1174 throw std::runtime_error(format("missing tensor info mapping for %s", tn.str().c_str()));1175 }1176 1177 // skip unused tensors1178 if (info.op == GGML_OP_NONE || (flags & TENSOR_SKIP)) {1179 const size_t nbytes = ggml_nbytes(t_meta);1180 LLAMA_LOG_WARN("model has unused tensor %s (size = %zu bytes) -- ignoring\n", tn.str().c_str(), nbytes);1181 1182 size_data -= nbytes;1183 n_created++;1184 1185 return nullptr;1186 }1187 1188 // tensors with "bias" suffix are always used with GGML_OP_ADD or GGML_OP_ADD_ID;1189 // embedded-adapter ".lora_a"/".lora_b" tensors are always used with GGML_OP_MUL_MAT_ID1190 ggml_op op;1191 if (tn.suffix != nullptr && strcmp(tn.suffix, "bias") == 0) {1192 op = info.op == GGML_OP_MUL_MAT_ID ? GGML_OP_ADD_ID : GGML_OP_ADD;1193 } else if (hparams.router_layer >= 0 && tn.suffix != nullptr &&1194 (strcmp(tn.suffix, "lora_a") == 0 || strcmp(tn.suffix, "lora_b") == 0)) {1195 op = GGML_OP_MUL_MAT_ID;1196 } else {1197 op = info.op;1198 }1199 1200 // sanity checks