Felipe97/llama-cpp-compiled
01.1k
1#include "llama.h"2 3#include "llama-impl.h"4#include "llama-version.h"5 6#include "llama-chat.h"7#include "llama-context.h"8#include "llama-mmap.h"9#include "llama-vocab.h"10#include "llama-model-loader.h"11#include "llama-model-saver.h"12#include "llama-model.h"13 14#include "ggml.h"15#include "ggml-cpp.h"16#include "ggml-backend.h"17#include "gguf.h"18 19#include <algorithm>20#include <cassert>21#include <cinttypes>22#include <cstddef>23#include <cstdint>24#include <cstdio>25#include <cstring>26#include <ctime>27#include <stdexcept>28#include <vector>29 30#if defined(_MSC_VER)31#pragma warning(disable: 4244 4267) // possible loss of data32#endif33 34//35// interface implementation36//37 38const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type) {39 switch (flash_attn_type) {40 case LLAMA_FLASH_ATTN_TYPE_AUTO:41 return "auto";42 case LLAMA_FLASH_ATTN_TYPE_DISABLED:43 return "disabled";44 case LLAMA_FLASH_ATTN_TYPE_ENABLED:45 return "enabled";46 }47 GGML_ABORT("fatal error");48}49 50const char * llama_load_mode_name(enum llama_load_mode load_mode) {51 switch (load_mode) {52 case LLAMA_LOAD_MODE_AUTO:53 return "auto";54 case LLAMA_LOAD_MODE_NONE:55 return "none";56 case LLAMA_LOAD_MODE_MMAP:57 return "mmap";58 case LLAMA_LOAD_MODE_MLOCK:59 return "mlock";60 case LLAMA_LOAD_MODE_MMAP_MLOCK:61 return "mmap+mlock";62 case LLAMA_LOAD_MODE_DIRECT_IO:63 return "dio";64 }65 GGML_ABORT("fatal error");66}67 68enum llama_load_mode llama_load_mode_from_str(const char * str) {69 if (std::strcmp(str, "auto") == 0) { return LLAMA_LOAD_MODE_AUTO; }70 if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; }71 if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; }72 if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; }73 if (std::strcmp(str, "mmap+mlock") == 0) { return LLAMA_LOAD_MODE_MMAP_MLOCK; }74 if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; }75 throw std::invalid_argument(std::string("unknown load mode: ") + str);76}77 78struct llama_sampler_chain_params llama_sampler_chain_default_params() {79 struct llama_sampler_chain_params result = {80 /*.no_perf =*/ true,81 };82 83 return result;84}85 86size_t llama_max_devices(void) {87 return 16;88}89 90size_t llama_max_tensor_buft_overrides() {91 return 4096;92}93 94bool llama_supports_mmap(void) {95 return llama_mmap::SUPPORTED;96}97 98bool llama_supports_mlock(void) {99 return llama_mlock::SUPPORTED;100}101 102bool llama_supports_gpu_offload(void) {103 if (!ggml_backend_reg_count()) {104 ggml_backend_load_all();105 }106 return ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU) != nullptr ||107 ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU) != nullptr ||108 llama_supports_rpc();109}110 111bool llama_supports_rpc(void) {112 if (!ggml_backend_reg_count()) {113 ggml_backend_load_all();114 }115 return ggml_backend_reg_by_name("RPC") != nullptr;116}117 118const char * llama_version(void) {119 return LLAMA_VERSION;120}121 122void llama_backend_init(void) {123 ggml_time_init();124 125 // needed to initialize f16 tables126 {127 struct ggml_init_params params = { 0, NULL, false };128 struct ggml_context * ctx = ggml_init(params);129 ggml_free(ctx);130 }131 132 if (!ggml_backend_reg_count()) {133 ggml_backend_load_all();134 }135}136 137void llama_numa_init(enum ggml_numa_strategy numa) {138 if (numa != GGML_NUMA_STRATEGY_DISABLED) {139 auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);140 GGML_ASSERT(dev && "CPU backend is not loaded");141 auto * reg = ggml_backend_dev_backend_reg(dev);142 auto * numa_init_fn = (decltype(ggml_numa_init) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_numa_init");143 if (numa_init_fn) {144 numa_init_fn(numa);145 }146 }147}148 149void llama_backend_free(void) {150 ggml_quantize_free();151}152 153int64_t llama_time_us(void) {154 return ggml_time_us();155}156 157// returns true on success158static bool llama_prepare_model_devices(const llama_model_params & params, llama_model * model) {159 // create list of devices to use with this model160 if (params.devices) {161 if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR) {162 size_t n_devs = 0;163 while (params.devices[n_devs]) {164 n_devs++;165 }166 if (n_devs == 0) {167 LLAMA_LOG_ERROR("%s: LLAMA_SPLIT_MODE_TENSOR needs >= 1 devices\n", __func__);168 return false;169 }170 LLAMA_LOG_INFO("%s: creating a Meta device with %zu devices\n", __func__, n_devs);171 for (size_t i = 0; i < n_devs; ++i) {172 LLAMA_LOG_INFO("%s: - device %zu: %s\n", __func__, i, ggml_backend_dev_name(params.devices[i]));173 }174 model->get_split_state_ud.n_devices = n_devs;175 model->get_split_state_ud.model = model;176 model->devices.push_back({177 true, ggml_backend_meta_device(178 params.devices, n_devs, llama_meta_device_get_split_state, &model->get_split_state_ud)179 });180 } else {181 for (ggml_backend_dev_t * dev = params.devices; *dev; ++dev) {182 model->devices.push_back({false, *dev});183 }184 }185 } else {186 // default device selection187 188 // build list of available devices189 std::vector<llama_device> gpus;190 std::vector<llama_device> igpus;191 std::vector<llama_device> rpc_servers;192 193 if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR) {194 std::vector<ggml_backend_dev_t> devs;195 devs.reserve(ggml_backend_dev_count());196 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {197 auto * dev = ggml_backend_dev_get(i);198 if (ggml_backend_dev_buffer_type(dev) == ggml_backend_cpu_buffer_type()) {199 LLAMA_LOG_INFO("%s: skipping %s (%s) for tensor parallelism\n", __func__, ggml_backend_dev_name(dev), ggml_backend_dev_description(dev));200 continue;201 }202 devs.push_back(dev);203 }204 if (devs.empty()) {205 LLAMA_LOG_ERROR("%s: LLAMA_SPLIT_MODE_TENSOR needs >= 1 devices\n", __func__);206 return false;207 }208 209 LLAMA_LOG_INFO("%s: creating a Meta device for tensor parallelism from %zu devices:\n", __func__, devs.size());210 for (size_t i = 0; i < devs.size(); ++i) {211 LLAMA_LOG_INFO("%s: - device %zu: %s (%s)\n", __func__, i, ggml_backend_dev_name(devs[i]), ggml_backend_dev_description(devs[i]));212 }213 214 GGML_ASSERT(!devs.empty());215 model->get_split_state_ud.n_devices = devs.size();216 model->get_split_state_ud.model = model;217 gpus.push_back({218 true, ggml_backend_meta_device(219 devs.data(), devs.size(), llama_meta_device_get_split_state, &model->get_split_state_ud)220 });221 } else {222 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {223 ggml_backend_dev_t dev = ggml_backend_dev_get(i);224 switch (ggml_backend_dev_type(dev)) {225 case GGML_BACKEND_DEVICE_TYPE_CPU:226 case GGML_BACKEND_DEVICE_TYPE_ACCEL:227 // skip CPU backends since they are handled separately228 break;229 230 case GGML_BACKEND_DEVICE_TYPE_GPU: {231 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);232 if (ggml_backend_reg_name(reg) == std::string("RPC")) {233 rpc_servers.push_back({false, dev});234 } else {235 // check if there is already a GPU with the same device id236 ggml_backend_dev_props props;237 ggml_backend_dev_get_props(dev, &props);238 auto it = std::find_if(gpus.begin(), gpus.end(), [&props](const llama_device & d) {239 ggml_backend_dev_props d_props;240 ggml_backend_dev_get_props(d.dev, &d_props);241 if (props.device_id && d_props.device_id) {242 return strcmp(props.device_id, d_props.device_id) == 0;243 }244 return false;245 });246 247 if (it != gpus.end()) {248 LLAMA_LOG_INFO("%s: skipping device %s (%s) with id %s - already using device %s (%s) with the same id\n",249 __func__,250 ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),251 props.device_id ? props.device_id : "unknown id",252 ggml_backend_dev_name(it->dev), ggml_backend_dev_description(it->dev));253 } else {254 gpus.push_back({false, dev});255 }256 }257 break;258 }259 260 case GGML_BACKEND_DEVICE_TYPE_IGPU:261 // igpus.empty() - workaround for integrated devices seen by multiple backends262 // ref: https://github.com/ggml-org/llama.cpp/pull/23897263 // ggml_backend_dev_backend_reg - allow devices of the same backend regardless if integrated264 // ref: https://github.com/ggml-org/llama.cpp/pull/23897#issuecomment-5264222997265 if (igpus.empty() || ggml_backend_dev_backend_reg(dev) == ggml_backend_dev_backend_reg(igpus.back().dev)) {266 igpus.push_back({false, dev});267 }268 break;269 case GGML_BACKEND_DEVICE_TYPE_META:270 GGML_ABORT("fatal error");271 }272 }273 }274 275 // add RPC servers at the front of the list to minimize network transfers276 model->devices.insert(model->devices.begin(), rpc_servers.begin(), rpc_servers.end());277 278 // add GPUs279 model->devices.insert(model->devices.end(), gpus.begin(), gpus.end());280 281 // add integrated GPUs only if no discrete GPUs were found282 // (RPC servers do not count, otherwise the local iGPU would be dropped on iGPU+RPC setups)283 if (gpus.empty()) {284 model->devices.insert(model->devices.end(), igpus.begin(), igpus.end());285 }286 }287 288 // if using single GPU mode, remove all except the main GPU289 if (params.split_mode == LLAMA_SPLIT_MODE_NONE && !model->devices.empty()) {290 if (params.main_gpu < 0) {291 model->devices.clear();292 } else {293 if (params.main_gpu >= (int)model->devices.size()) {294 LLAMA_LOG_ERROR("%s: invalid value for main_gpu: %d (available devices: %zu)\n", __func__, params.main_gpu, model->devices.size());295 return false;296 }297 llama_device main_gpu = model->devices[params.main_gpu];298 model->devices.clear();299 model->devices.push_back(main_gpu);300 }301 }302 303 for (const auto & dev : model->devices) {304 ggml_backend_dev_props props;305 ggml_backend_dev_get_props(dev.dev, &props);306 LLAMA_LOG_INFO("%s: using device %s (%s) (%s) - %zu MiB free\n", __func__,307 ggml_backend_dev_name(dev.dev), ggml_backend_dev_description(dev.dev),308 props.device_id ? props.device_id : "unknown id",309 props.memory_free/1024/1024);310 }311 312 return true;313}314 315// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback316static std::pair<int, llama_model *> llama_model_load(struct gguf_context * metadata, llama_model_set_tensor_data_t set_tensor_data, void * set_tensor_data_ud,317 const std::string & fname, std::vector<std::string> & splits, FILE * file, llama_model_params & params) {318 try {319 llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,320 params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides);321 322 ml.lazy.mode = params.lazy_mode;323 324 ml.print_info();325 std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, params));326 327 bool ok = llama_prepare_model_devices(params, model_ptr.get());328 if (!ok) {329 return {-1, nullptr};330 }331 332 auto * model = dynamic_cast<llama_model_base *>(model_ptr.get());333 if (model == nullptr) {334 GGML_ABORT("fatal error: model does not implement llama_model_base");335 }336 337 // loading time will be recalculated after the first eval, so338 // we take page faults deferred by mmap() into consideration339 model->t_load_us = 0;340 time_meas tm(model->t_load_us);341 342 model->t_start_us = tm.t_start_us;343 344 model->hparams.vocab_only = params.vocab_only;345 model->hparams.no_alloc = params.no_alloc;346 347 try {348 model->load_hparams(ml);349 } catch(const std::exception & e) {350 throw std::runtime_error("error loading model hyperparameters: " + std::string(e.what()));351 }352 if (model->arch == LLM_ARCH_CLIP) {353 throw std::runtime_error("CLIP cannot be used as main model, use it with --mmproj instead");354 }355 try {356 model->load_vocab(ml);357 } catch(const std::exception & e) {358 throw std::runtime_error("error loading model vocabulary: " + std::string(e.what()));359 }360 361 model->load_stats(ml);362 model->print_info();363 364 if (params.vocab_only) {365 LLAMA_LOG_INFO("%s: vocab only - skipping tensors\n", __func__);366 return {0, model_ptr.release()};367 }368 369 if (!model->load_tensors(ml)) {370 return {-2, nullptr};371 }372 373 return {0, model_ptr.release()};374 } catch (const std::exception & err) {375 LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what());376 return {-1, nullptr};377 }378}379 380static struct llama_model * llama_model_load_from_file_impl(381 struct gguf_context * metadata,382 llama_model_set_tensor_data_t set_tensor_data,383 void * set_tensor_data_ud,384 const std::string & path_model,385 std::vector<std::string> & splits,386 FILE * file,387 struct llama_model_params params) {388 {389 int n_sources_defined = 0;390 if (metadata != nullptr) {391 n_sources_defined++;392 }393 if (!path_model.empty()) {394 n_sources_defined++;395 }396 if (file != nullptr) {397 n_sources_defined++;398 }399 if (n_sources_defined != 1) {400 LLAMA_LOG_ERROR("%s: exactly one out metadata, path_model, and file must be defined\n", __func__);401 return nullptr;402 }403 }404 ggml_time_init();405 406 if (!params.vocab_only && ggml_backend_reg_count() == 0) {407 LLAMA_LOG_ERROR("%s: no backends are loaded. hint: use ggml_backend_load() or ggml_backend_load_all() to load a backend before calling this function\n", __func__);408 return nullptr;409 }410 411 unsigned cur_percentage = 0;412 if (params.progress_callback == NULL) {413 params.progress_callback_user_data = &cur_percentage;414 params.progress_callback = [](float progress, void * ctx) {415 unsigned * cur_percentage_p = (unsigned *) ctx;416 unsigned percentage = (unsigned) (100 * progress);417 while (percentage > *cur_percentage_p) {418 *cur_percentage_p = percentage;419 LLAMA_LOG_CONT(".");420 if (percentage >= 100) {421 LLAMA_LOG_CONT("\n");422 }423 }424 return true;425 };426 }427 428 const auto [status, model] = llama_model_load(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, file, params);429 GGML_ASSERT(status <= 0);430 if (status < 0) {431 if (status == -1) {432 LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);433 } else if (status == -2) {434 LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);435 }436 437 if (model) {438 llama_model_free(model);439 }440 return nullptr;441 }442 443 return model;444}445 446struct llama_model * llama_model_init_from_user(447 struct gguf_context * metadata,448 llama_model_set_tensor_data_t set_tensor_data,449 void * set_tensor_data_ud,450 struct llama_model_params params) {451 GGML_ASSERT(metadata != nullptr);452 std::string path_model;453 std::vector<std::string> splits = {};454 params.load_mode = LLAMA_LOAD_MODE_NONE;455 params.use_extra_bufts = false;456 return llama_model_load_from_file_impl(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, /*file*/ nullptr, params);457}458// deprecated459struct llama_model * llama_load_model_from_file(460 const char * path_model,461 struct llama_model_params params) {462 return llama_model_load_from_file(path_model, params);463}464 465struct llama_model * llama_model_load_from_file(466 const char * path_model,467 struct llama_model_params params) {468 std::vector<std::string> splits = {};469 return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, /*file*/ nullptr, params);470}471 472struct llama_model * llama_model_load_from_splits(473 const char ** paths,474 size_t n_paths,475 struct llama_model_params params) {476 std::vector<std::string> splits;477 if (n_paths == 0) {478 LLAMA_LOG_ERROR("%s: list of splits is empty\n", __func__);479 return nullptr;480 }481 splits.reserve(n_paths);482 for (size_t i = 0; i < n_paths; ++i) {483 splits.push_back(paths[i]);484 }485 return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, splits.front(), splits, /*file*/ nullptr, params);486}487 488struct llama_model * llama_model_load_from_file_ptr(FILE * file, struct llama_model_params params) {489 if (!file) {490 LLAMA_LOG_ERROR("%s: file is NULL\n", __func__);491 return nullptr;492 }493 std::string path_model;494 std::vector<std::string> splits = {};495 return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, file, params);496}497 498void llama_model_save_to_file(const struct llama_model * model, const char * path_model) {499 llama_model_saver ms(model);500 ms.add_kv_from_model();501 ms.add_tensors_from_model();502 ms.save(path_model);503}504 505//506// chat templates507//508 509int32_t llama_chat_apply_template(510 const char * tmpl,511 const struct llama_chat_message * chat,512 size_t n_msg,513 bool add_ass,514 char * buf,515 int32_t length) {516 const std::string curr_tmpl(tmpl == nullptr ? "chatml" : tmpl);517 518 // format the chat to string519 std::vector<const llama_chat_message *> chat_vec;520 chat_vec.resize(n_msg);521 for (size_t i = 0; i < n_msg; i++) {522 chat_vec[i] = &chat[i];523 }524 525 std::string formatted_chat;526 llm_chat_template detected_tmpl = llm_chat_detect_template(curr_tmpl);527 if (detected_tmpl == LLM_CHAT_TEMPLATE_UNKNOWN) {528 return -1;529 }530 int32_t res = llm_chat_apply_template(detected_tmpl, chat_vec, formatted_chat, add_ass);531 if (res < 0) {532 return res;533 }534 if (buf && length > 0) {535 strncpy(buf, formatted_chat.c_str(), length);536 }537 return res;538}539 540//541// model split542//543 544int32_t llama_split_path(545 char * split_path,546 size_t maxlen,547 const char * path_prefix,548 int32_t split_no,549 int32_t split_count) {550 551 static const char * const SPLIT_PATH_FORMAT = "%s-%05d-of-%05d.gguf";552 553 const int written = snprintf(554 split_path,555 maxlen,556 SPLIT_PATH_FORMAT,557 path_prefix,558 split_no + 1,559 split_count560 );561 562 if (written < 0 || (size_t) written >= maxlen) {563 return 0;564 }565 566 return (int32_t) written;567}568 569int32_t llama_split_prefix(570 char * split_prefix,571 size_t maxlen,572 const char * split_path,573 int32_t split_no,574 int32_t split_count) {575 576 const std::string str_split_path(split_path);577 578 char postfix[32];579 snprintf(postfix, sizeof(postfix), "-%05d-of-%05d.gguf", split_no + 1, split_count);580 581 const std::string str_postfix(postfix);582 if (str_split_path.size() <= str_postfix.size()) {583 return 0;584 }585 586 const size_t size_prefix = str_split_path.size() - str_postfix.size();587 588 if (str_split_path.compare(size_prefix, std::string::npos, str_postfix) == 0) {589 const size_t copy_len = std::min(size_prefix + 1, maxlen);590 snprintf(split_prefix, copy_len, "%s", split_path);591 592 return (int32_t) size_prefix;593 }594 595 return 0;596}597 598const char * llama_print_system_info(void) {599 static std::string s;600 s.clear(); // Clear the string, since it's static, otherwise it will accumulate data from previous calls.601 602 for (size_t i = 0; i < ggml_backend_reg_count(); i++) {603 auto * reg = ggml_backend_reg_get(i);604 auto * get_features_fn = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features");605 if (get_features_fn) {606 ggml_backend_feature * features = get_features_fn(reg);607 s += ggml_backend_reg_name(reg);608 s += " : ";609 for (; features->name; features++) {610 s += features->name;611 s += " = ";612 s += features->value;613 s += " | ";614 }615 }616 }617 618 return s.c_str();619}620 621