Felipe97/llama-cpp-compiled
01.1k
1#include "fit.h"2 3#include "json.h"4#include "log.h"5 6#include "../src/llama-ext.h"7 8#include <array>9#include <cassert>10#include <stdexcept>11#include <cinttypes>12#include <set>13#include <string>14#include <vector>15 16// this enum is only used in llama_params_fit_impl but needs to be defined outside of it to fix a Windows compilation issue17// enum to identify part of a layer for distributing its tensors:18enum common_layer_fraction_t {19 LAYER_FRACTION_NONE = 0, // nothing20 LAYER_FRACTION_ATTN = 1, // attention21 LAYER_FRACTION_UP = 2, // attention + up22 LAYER_FRACTION_GATE = 3, // attention + up + gate23 LAYER_FRACTION_MOE = 4, // everything but sparse MoE weights24};25 26class common_params_fit_exception : public std::runtime_error {27 using std::runtime_error::runtime_error;28};29 30static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(31 const char * path_model,32 const llama_model_params * mparams,33 const llama_context_params * cparams,34 std::vector<ggml_backend_dev_t> & devs,35 uint32_t & hp_ngl,36 uint32_t & hp_n_ctx_train,37 uint32_t & hp_n_expert,38 ggml_log_level log_level) {39 struct user_data_t {40 struct {41 ggml_log_callback callback;42 void * user_data;43 } original_logger;44 ggml_log_level min_level; // prints below this log level go to debug log45 };46 user_data_t ud;47 llama_log_get(&ud.original_logger.callback, &ud.original_logger.user_data);48 ud.min_level = log_level;49 50 llama_log_set([](ggml_log_level level, const char * text, void * user_data) {51 const user_data_t * ud = (const user_data_t *) user_data;52 const ggml_log_level level_eff = level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG;53 ud->original_logger.callback(level_eff, text, ud->original_logger.user_data);54 }, &ud);55 56 llama_model_params mparams_copy = *mparams;57 mparams_copy.no_alloc = true;58 mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;59 60 llama_model * model = llama_model_load_from_file(path_model, mparams_copy);61 if (model == nullptr) {62 llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);63 throw std::runtime_error("failed to load model");64 }65 66 llama_context * ctx = llama_init_from_model(model, *cparams);67 if (ctx == nullptr) {68 llama_model_free(model);69 llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);70 throw std::runtime_error("failed to create llama_context from model");71 }72 73 const size_t nd = llama_model_n_devices(model);74 std::vector<llama_device_memory_data> ret(nd + 1);75 76 llama_memory_breakdown memory_breakdown = llama_get_memory_breakdown(ctx);77 78 for (const auto & [buft, mb] : memory_breakdown) {79 if (ggml_backend_buft_is_host(buft)) {80 ret.back().mb.model += mb.model;81 ret.back().mb.context += mb.context;82 ret.back().mb.compute += mb.compute;83 continue;84 }85 86 ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);87 if (!dev) {88 continue;89 }90 for (size_t i = 0; i < nd; i++) {91 if (dev == llama_model_get_device(model, i)) {92 ret[i].mb.model += mb.model;93 ret[i].mb.context += mb.context;94 ret[i].mb.compute += mb.compute;95 break;96 }97 }98 }99 100 {101 ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);102 if (cpu_dev == nullptr) {103 throw std::runtime_error("no CPU backend found");104 }105 size_t free;106 size_t total;107 ggml_backend_dev_memory(cpu_dev, &free, &total);108 ret.back().free = free;109 ret.back().total = total;110 }111 for (size_t i = 0; i < nd; i++) {112 ggml_backend_dev_t dev = llama_model_get_device(model, i);113 114 size_t free;115 size_t total;116 ggml_backend_dev_memory(dev, &free, &total);117 118 // Some non-GPU accelerator backends, such as BLAS, report 0/0 and rely on119 // the host-memory fallback. For GPU-like backends, keep 0/0 so --fit does120 // not assign anything to a device with an unknown memory budget.121 if (free == 0 && total == 0) {122 const enum ggml_backend_dev_type type = ggml_backend_dev_type(dev);123 if (type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU) {124 LOG_WRN("%s: device %s did not report memory; --fit will not use it\n",125 __func__, ggml_backend_dev_name(dev));126 } else {127 free = ret.back().free;128 total = ret.back().total;129 }130 }131 ret[i].free = free;132 ret[i].total = total;133 }134 135 devs.clear();136 for (int i = 0; i < llama_model_n_devices(model); i++) {137 devs.push_back(llama_model_get_device(model, i));138 }139 140 hp_ngl = llama_model_n_layer(model);141 if (mparams->load_mtp) {142 hp_ngl += llama_model_n_layer_nextn(model);143 }144 hp_n_ctx_train = llama_model_n_ctx_train(model);145 hp_n_expert = llama_model_n_expert(model);146 147 common_memory_breakdown_print(ctx);148 149 llama_free(ctx);150 llama_model_free(model);151 llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);152 153 return ret;154}155 156common_device_memory_data_vec common_get_device_memory_data(157 const char * path_model,158 const llama_model_params * mparams,159 const llama_context_params * cparams,160 std::vector<ggml_backend_dev_t> & devs,161 uint32_t & hp_ngl,162 uint32_t & hp_n_ctx_train,163 uint32_t & hp_n_expert,164 ggml_log_level log_level) {165 std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(166 path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);167 168 common_device_memory_data_vec ret(impl.size());169 for (size_t i = 0; i < impl.size(); i++) {170 ret[i].total = impl[i].total;171 ret[i].free = impl[i].free;172 ret[i].model = impl[i].mb.model;173 ret[i].context = impl[i].mb.context;174 ret[i].compute = impl[i].mb.compute;175 }176 return ret;177}178 179static void common_params_fit_impl(180 const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,181 float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,182 size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) {183 if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) {184 throw common_params_fit_exception("llama_params_fit is not implemented for SPLIT_MODE_TENSOR, abort");185 }186 constexpr int64_t MiB = 1024*1024;187 typedef std::vector<llama_device_memory_data> dmds_t;188 const llama_model_params default_mparams = llama_model_default_params();189 190 std::vector<ggml_backend_dev_t> devs;191 uint32_t hp_ngl = 0; // hparams.n_gpu_layers192 uint32_t hp_nct = 0; // hparams.n_ctx_train193 uint32_t hp_nex = 0; // hparams.n_expert194 195 // size the context for all sequences, but keep minimums and alignment per KV stream196 const uint32_t n_seq_max = std::max<uint32_t>(1, cparams->n_seq_max);197 const uint32_t n_streams = cparams->kv_unified ? 1 : n_seq_max;198 const bool n_ctx_auto = cparams->n_ctx == 0;199 200 dmds_t dmds_extra; // memory of the extra model, laid out on the devices of the main model201 uint32_t n_ctx_extra = 0; // context that memory was measured at202 203 // the extra model competes for the same memory as the main model, add it to every measurement204 // its memory is measured again whenever the context it follows changes205 auto add_extra_memory = [&](dmds_t & dmds) {206 if (extra == nullptr) {207 return;208 }209 210 if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) {211 std::vector<ggml_backend_dev_t> devs_extra;212 uint32_t ngl_extra = 0;213 uint32_t nct_extra = 0;214 uint32_t nex_extra = 0;215 216 extra->cparams->n_ctx = cparams->n_ctx;217 218 LOG_TRC("%s: getting device memory data for the extra model at a context size of %" PRIu32 ":\n",219 __func__, cparams->n_ctx);220 221 dmds_t measured;222 try {223 measured = common_get_device_memory_data_impl(224 extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level);225 } catch (const std::runtime_error & e) {226 // the extra model is optional, fit the main model alone rather than giving up227 LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());228 dmds_extra = dmds_t(devs.size() + 1);229 n_ctx_extra = cparams->n_ctx;230 return;231 }232 233 dmds_extra = dmds_t(devs.size() + 1);234 dmds_extra.back().mb = measured.back().mb;235 for (size_t je = 0; je < devs_extra.size(); je++) {236 for (size_t id = 0; id < devs.size(); id++) {237 if (devs_extra[je] == devs[id]) {238 dmds_extra[id].mb.model += measured[je].mb.model;239 dmds_extra[id].mb.context += measured[je].mb.context;240 dmds_extra[id].mb.compute += measured[je].mb.compute;241 break;242 }243 }244 }245 if (extra->shares_model) {246 for (llama_device_memory_data & dmd : dmds_extra) {247 dmd.mb.model = 0;248 }249 }250 251 n_ctx_extra = cparams->n_ctx;252 }253 254 for (size_t id = 0; id < dmds.size(); id++) {255 dmds[id].mb.model += dmds_extra[id].mb.model;256 dmds[id].mb.context += dmds_extra[id].mb.context;257 dmds[id].mb.compute += dmds_extra[id].mb.compute;258 }259 };260 261 // step 1: get data for default parameters and check whether any changes are necessary in the first place262 263 LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);264 dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);265 266 // saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:267 const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_seq_max, UINT32_MAX);268 const uint32_t n_ctx_min_total = (uint32_t) std::min<uint64_t>(uint64_t(n_ctx_min) * n_streams, UINT32_MAX);269 270 // llama_context would use only hp_nct in total for n_ctx == 0, resolve the context before measuring anything else:271 if (n_ctx_auto) {272 cparams->n_ctx = n_ctx_max;273 if (n_seq_max > 1) {274 LOG_TRC("%s: context size unset -> using %" PRIu32 " for %" PRIu32 " sequences:\n",275 __func__, n_ctx_max, n_seq_max);276 dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);277 }278 }279 add_extra_memory(dmds_full);280 281 const size_t nd = devs.size(); // number of devices282 283 std::vector<int64_t> margins; // this function uses int64_t rather than size_t for memory sizes to more conveniently handle deficits284 margins.reserve(nd);285 if (nd == 0) {286 margins.push_back(margins_s[0]);287 } else {288 for (size_t id = 0; id < nd; id++) {289 margins.push_back(margins_s[id]);290 }291 }292 293 std::vector<std::string> dev_names;294 {295 dev_names.reserve(nd);296 size_t max_length = 0;297 for (const auto & dev : devs) {298 std::string name = ggml_backend_dev_name(dev);299 name += " (";300 name += ggml_backend_dev_description(dev);301 name += ")";302 dev_names.push_back(name);303 max_length = std::max(max_length, name.length());304 }305 for (std::string & dn : dev_names) {306 dn.insert(dn.end(), max_length - dn.length(), ' ');307 }308 }309 310 int64_t sum_free = 0;311 int64_t sum_projected_free = 0;312 int64_t sum_projected_used = 0;313 int64_t sum_projected_model = 0;314 std::vector<int64_t> projected_free_per_device;315 projected_free_per_device.reserve(nd);316 317 if (nd == 0) {318 sum_projected_used = dmds_full.back().mb.total();319 sum_free = dmds_full.back().total;320 sum_projected_free = sum_free - sum_projected_used;321 LOG_TRC("%s: projected to use %" PRId64 " MiB of host memory vs. %" PRId64 " MiB of total host memory\n",322 __func__, sum_projected_used/MiB, sum_free/MiB);323 if (sum_projected_free >= margins[0]) {324 LOG_TRC("%s: will leave %" PRId64 " >= %" PRId64 " MiB of system memory, no changes needed\n",325 __func__, sum_projected_free/MiB, margins[0]/MiB);326 return;327 }328 } else {329 if (nd > 1) {330 LOG_TRC("%s: projected memory use with initial parameters [MiB]:\n", __func__);331 }332 for (size_t id = 0; id < nd; id++) {333 const llama_device_memory_data & dmd = dmds_full[id];334 335 const int64_t projected_used = dmd.mb.total();336 const int64_t projected_free = dmd.free - projected_used;337 projected_free_per_device.push_back(projected_free);338 339 sum_free += dmd.free;340 sum_projected_used += projected_used;341 sum_projected_free += projected_free;342 sum_projected_model += dmd.mb.model;343 344 if (nd > 1) {345 LOG_TRC("%s: - %s: %6" PRId64 " total, %6" PRId64 " used, %6" PRId64 " free vs. target of %6" PRId64 "\n",346 __func__, dev_names[id].c_str(), dmd.total/MiB, projected_used/MiB, projected_free/MiB, margins[id]/MiB);347 }348 }349 assert(sum_free >= 0 && sum_projected_used >= 0);350 LOG_TRC("%s: projected to use %" PRId64 " MiB of device memory vs. %" PRId64 " MiB of free device memory\n",351 __func__, sum_projected_used/MiB, sum_free/MiB);352 if (nd == 1) {353 if (projected_free_per_device[0] >= margins[0]) {354 LOG_TRC("%s: will leave %" PRId64 " >= %" PRId64 " MiB of free device memory, no changes needed\n",355 __func__, projected_free_per_device[0]/MiB, margins[0]/MiB);356 return;357 }358 } else {359 bool changes_needed = false;360 for (size_t id = 0; id < nd; id++) {361 if (projected_free_per_device[id] < margins[id]) {362 changes_needed = true;363 break;364 }365 }366 if (!changes_needed) {367 LOG_TRC("%s: targets for free memory can be met on all devices, no changes needed\n", __func__);368 return;369 }370 }371 }372 373 // step 2: try reducing memory use by reducing the context size374 375 {376 int64_t global_surplus = sum_projected_free;377 if (nd == 0) {378 global_surplus -= margins[0];379 } else {380 for (size_t id = 0; id < nd; id++) {381 global_surplus -= margins[id];382 }383 }384 if (global_surplus < 0) {385 if (nd <= 1) {386 LOG_TRC("%s: cannot meet free memory target of %" PRId64 " MiB, need to reduce device memory by %" PRId64 " MiB\n",387 __func__, margins[0]/MiB, -global_surplus/MiB);388 } else {389 LOG_TRC(390 "%s: cannot meet free memory targets on all devices, need to use %" PRId64 " MiB less in total\n",391 __func__, -global_surplus/MiB);392 }393 if (n_ctx_auto) {394 if (n_ctx_max > n_ctx_min_total) {395 int64_t sum_used_target = sum_free;396 if (nd == 0) {397 sum_used_target -= margins[0];398 } else {399 for (size_t id = 0; id < nd; id++) {400 sum_used_target -= margins[id];401 }402 }403 if (nd > 1) {404 // for multiple devices we need to be more conservative in terms of how much context we think can fit:405 // - for dense models only whole layers can be assigned to devices406 // - for MoE models only whole tensors can be assigned to devices, which we estimate to be <= 1/3 of a layer407 // - on average we expect a waste of 0.5 layers/tensors per device408 // - use slightly more than the expected average for nd devices to be safe409 const int64_t model_per_layer = sum_projected_model / std::min(uint32_t(mparams->n_gpu_layers), hp_ngl);410 sum_used_target -= (nd + 1) * model_per_layer / (hp_nex == 0 ? 2 : 6);411 }412 413 int64_t sum_projected_used_min_ctx = 0;414 cparams->n_ctx = n_ctx_min_total;415 dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);416 add_extra_memory(dmds_min_ctx);417 if (nd == 0) {418 sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();419 } else {420 for (size_t id = 0; id < nd; id++) {421 sum_projected_used_min_ctx += dmds_min_ctx[id].mb.total();422 }423 }424 if (sum_used_target > sum_projected_used_min_ctx) {425 // linear interpolation between minimum and maximum context size:426 cparams->n_ctx += (n_ctx_max - n_ctx_min_total) * (sum_used_target - sum_projected_used_min_ctx)427 / (sum_projected_used - sum_projected_used_min_ctx);428 // round down context for CUDA backend, keep it divisible by the number of streams:429 const uint32_t align = 256 * n_streams;430 cparams->n_ctx = std::max(cparams->n_ctx - cparams->n_ctx % align, n_ctx_min_total);431 432 const int64_t bytes_per_ctx = (sum_projected_used - sum_projected_used_min_ctx) / (n_ctx_max - n_ctx_min_total);433 const int64_t memory_reduction = (n_ctx_max - cparams->n_ctx) * bytes_per_ctx;434 LOG_TRC("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",435 __func__, n_ctx_max, cparams->n_ctx, memory_reduction/MiB);436 if (nd <= 1) {437 LOG_TRC("%s: entire model can be fit by reducing context\n", __func__);438 return;439 }440 LOG_TRC("%s: entire model should be fit across devices by reducing context\n", __func__);441 } else {442 const int64_t memory_reduction = sum_projected_used - sum_projected_used_min_ctx;443 LOG_TRC("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",444 __func__, n_ctx_max, cparams->n_ctx, memory_reduction/MiB);445 }446 } else {447 if (n_ctx_min == UINT32_MAX) {448 LOG_TRC("%s: user has requested full context size of %" PRIu32 " -> no change\n", __func__, n_ctx_max);449 } else {450 LOG_TRC("%s: default model context size is %" PRIu32 " which is <= the min. context size of %" PRIu32 " -> no change\n",451 __func__, n_ctx_max, n_ctx_min_total);452 }453 }454 } else {455 LOG_TRC("%s: context size set by user to %" PRIu32 " -> no change\n", __func__, cparams->n_ctx);456 }457 }458 }459 if (nd == 0) {460 throw common_params_fit_exception("was unable to fit model into system memory by reducing context, abort");461 }462 463 if (mparams->n_gpu_layers != default_mparams.n_gpu_layers) {464 throw common_params_fit_exception("n_gpu_layers already set by user to " + std::to_string(mparams->n_gpu_layers) + ", abort");465 }466 if (nd > 1) {467 if (!tensor_split) {468 throw common_params_fit_exception("did not provide a buffer to write the tensor_split to, abort");469 }470 if (mparams->tensor_split) {471 for (size_t id = 0; id < nd; id++) {472 if (mparams->tensor_split[id] != 0.0f) {473 throw common_params_fit_exception("model_params::tensor_split already set by user, abort");474 }475 }476 }477 if (mparams->split_mode == LLAMA_SPLIT_MODE_ROW) {478 throw common_params_fit_exception("changing weight allocation for LLAMA_SPLIT_MODE_ROW not implemented, abort");479 }480 }481 if (!tensor_buft_overrides) {482 throw common_params_fit_exception("did not provide buffer to set tensor_buft_overrides, abort");483 }484 if (mparams->tensor_buft_overrides && (mparams->tensor_buft_overrides->pattern || mparams->tensor_buft_overrides->buft)) {485 throw common_params_fit_exception("model_params::tensor_buft_overrides already set by user, abort");486 }487 488 // step 3: iteratively fill the back to front with "dense" layers489 // - for a dense model simply fill full layers, giving each device a contiguous slice of the model490 // - for a MoE model, same as dense model but with all MoE tensors in system memory491 492 // utility function that returns a static C string matching the tensors for a specific layer index and layer fraction:493 auto get_overflow_pattern = [&](const size_t il, const common_layer_fraction_t lf) -> const char * {494 constexpr size_t n_strings = 1000;495 if (il >= n_strings) {496 throw std::runtime_error("at most " + std::to_string(n_strings) + " model layers are supported");497 }498 switch (lf) {499 case LAYER_FRACTION_ATTN: {500 static std::array<std::string, n_strings> patterns;501 if (patterns[il].empty()) {502 patterns[il] = "blk\\." + std::to_string(il) + "\\.ffn_(gate|up|gate_up|down).*";503 }504 return patterns[il].c_str();505 }506 case LAYER_FRACTION_UP: {507 static std::array<std::string, n_strings> patterns;508 if (patterns[il].empty()) {509 patterns[il] = "blk\\." + std::to_string(il) + "\\.ffn_(gate|gate_up|down).*";510 }511 return patterns[il].c_str();512 }513 case LAYER_FRACTION_GATE: {514 static std::array<std::string, n_strings> patterns;515 if (patterns[il].empty()) {516 patterns[il] = "blk\\." + std::to_string(il) + "\\.ffn_down.*";517 }518 return patterns[il].c_str();519 }520 case LAYER_FRACTION_MOE: {521 static std::array<std::string, n_strings> patterns;522 if (patterns[il].empty()) {523 patterns[il] = "blk\\." + std::to_string(il) + "\\.ffn_(up|down|gate_up|gate)_(ch|)exps";524 }525 return patterns[il].c_str();526 }527 default:528 GGML_ABORT("fatal error");529 }530 };531 532 struct ngl_t {533 uint32_t n_layer = 0; // number of total layers534 uint32_t n_part = 0; // number of partial layers, <= n_layer535 536 // for the first partial layer varying parts can overflow, all further layers use LAYER_FRACTION_MOE:537 common_layer_fraction_t overflow_type = LAYER_FRACTION_MOE;538 539 uint32_t n_full() const {540 assert(n_layer >= n_part);541 return n_layer - n_part;542 }543 };544 545 const size_t ntbo = llama_max_tensor_buft_overrides();546 547 // utility function to set n_gpu_layers and tensor_split548 auto set_ngl_tensor_split_tbo = [&](549 const std::vector<ngl_t> & ngl_per_device,550 const std::vector<ggml_backend_buffer_type_t> & overflow_bufts,551 llama_model_params & mparams) {552 mparams.n_gpu_layers = 0;553 for (size_t id = 0; id < nd; id++) {554 mparams.n_gpu_layers += ngl_per_device[id].n_layer;555 if (nd > 1) {556 tensor_split[id] = ngl_per_device[id].n_layer;557 }558 }559 assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1);560 uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides561 562 mparams.tensor_split = tensor_split;563 564 size_t itbo = 0;565 for (size_t id = 0; id < nd; id++) {566 il0 += ngl_per_device[id].n_full();567 for (uint32_t il = il0; il < il0 + ngl_per_device[id].n_part; il++) {568 if (itbo + 1 >= ntbo) {569 tensor_buft_overrides[itbo].pattern = nullptr;570 tensor_buft_overrides[itbo].buft = nullptr;571 itbo++;572 mparams.tensor_buft_overrides = tensor_buft_overrides;573 throw common_params_fit_exception("llama_max_tensor_buft_overrides() == "574 + std::to_string(ntbo) + " is insufficient for model");575 }576 tensor_buft_overrides[itbo].pattern = get_overflow_pattern(il, il == il0 ? ngl_per_device[id].overflow_type : LAYER_FRACTION_MOE);577 tensor_buft_overrides[itbo].buft = il == il0 ? overflow_bufts[id] : ggml_backend_cpu_buffer_type();578 itbo++;579 }580 il0 += ngl_per_device[id].n_part;581 }582 tensor_buft_overrides[itbo].pattern = nullptr;583 tensor_buft_overrides[itbo].buft = nullptr;584 itbo++;585 mparams.tensor_buft_overrides = tensor_buft_overrides;586 };587 588 // utility function that returns the memory use per device for given numbers of layers per device589 auto get_memory_for_layers = [&](590 const char * func_name,591 const std::vector<ngl_t> & ngl_per_device,592 const std::vector<ggml_backend_buffer_type_t> & overflow_bufts) -> std::vector<int64_t> {593 llama_model_params mparams_copy = *mparams;594 set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);595 596 dmds_t dmd_nl = common_get_device_memory_data_impl(597 path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);598 add_extra_memory(dmd_nl);599 600 LOG_TRC("%s: memory for test allocation by device:\n", func_name);601 for (size_t id = 0; id < nd; id++) {602 const ngl_t & n = ngl_per_device[id];603 LOG_TRC(604 "%s: id=%zu, n_layer=%2" PRIu32 ", n_part=%2" PRIu32 ", overflow_type=%d, mem=%6" PRId64 " MiB\n",605 func_name, id, n.n_layer, n.n_part, int(n.overflow_type), dmd_nl[id].mb.total()/MiB);606 }607 608 std::vector<int64_t> ret;609 ret.reserve(nd);610 for (size_t id = 0; id < nd; id++) {611 ret.push_back(dmd_nl[id].mb.total());612 }613 return ret;614 };615 616 int64_t global_surplus_cpu_moe = 0;617 if (hp_nex > 0) {618 const static std::string pattern_moe_all = "blk\\.\\d+\\.ffn_(up|down|gate_up|gate)_(ch|)exps"; // matches all MoE tensors619 ggml_backend_buffer_type_t cpu_buft = ggml_backend_cpu_buffer_type();620 tensor_buft_overrides[0] = {pattern_moe_all.c_str(), cpu_buft};621 tensor_buft_overrides[1] = {nullptr, nullptr};622 mparams->tensor_buft_overrides = tensor_buft_overrides;623 624 LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);625 dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(626 path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);627 add_extra_memory(dmds_cpu_moe);628 629 for (size_t id = 0; id < nd; id++) {630 global_surplus_cpu_moe += dmds_cpu_moe[id].free;631 global_surplus_cpu_moe -= int64_t(dmds_cpu_moe[id].mb.total()) + margins[id];632 }633 634 if (global_surplus_cpu_moe > 0) {635 LOG_TRC("%s: with only dense weights in device memory there is a total surplus of %" PRId64 " MiB\n",636 __func__, global_surplus_cpu_moe/MiB);637 } else {638 LOG_TRC("%s: with only dense weights in device memory there is still a total deficit of %" PRId64 " MiB\n",639 __func__, -global_surplus_cpu_moe/MiB);640 }641 642 // reset643 tensor_buft_overrides[0] = {nullptr, nullptr};644 mparams->tensor_buft_overrides = tensor_buft_overrides;645 }646 647 std::vector<int64_t> targets; // maximum acceptable memory use per device648 targets.reserve(nd);649 for (size_t id = 0; id < nd; id++) {650 targets.push_back(dmds_full[id].free - margins[id]);651 LOG_TRC("%s: id=%zu, target=%" PRId64 " MiB\n", __func__, id, targets[id]/MiB);652 }653 654 std::vector<ggml_backend_buffer_type_t> overflow_bufts; // which bufts the first partial layer of a device overflows to:655 overflow_bufts.reserve(nd);656 for (size_t id = 0; id < nd; id++) {657 overflow_bufts.push_back(ggml_backend_cpu_buffer_type());658 }659 660 std::vector<ngl_t> ngl_per_device(nd);661 std::vector<int64_t> mem = get_memory_for_layers(__func__, ngl_per_device, overflow_bufts);662 663 // optimize the number of layers per device using the method of false position:664 // - ngl_per_device has 0 layers for each device, lower bound665 // - try a "high" configuration where a device is given all unassigned layers666 // - interpolate the memory use / layer between low and high linearly to get a guess where it meets our target667 // - check memory use of our guess, replace either the low or high bound668 // - once we only have a difference of a single layer, stop and return the lower bound that just barely still fits669 // - the last device has the output layer, which cannot be a partial layer670 if (hp_nex == 0) {671 LOG_TRC("%s: filling dense layers back-to-front:\n", __func__);672 } else {673 LOG_TRC("%s: filling dense-only layers back-to-front:\n", __func__);674 }675 for (int id = nd - 1; id >= 0; id--) {676 uint32_t n_unassigned = hp_ngl + 1;677 for (size_t jd = id + 1; jd < nd; ++jd) {678 assert(n_unassigned >= ngl_per_device[jd].n_layer);679 n_unassigned -= ngl_per_device[jd].n_layer;680 }681 682 std::vector<ngl_t> ngl_per_device_high = ngl_per_device;683 ngl_per_device_high[id].n_layer = n_unassigned;684 if (hp_nex > 0) {685 ngl_per_device_high[id].n_part = size_t(id) < nd - 1 ? ngl_per_device_high[id].n_layer : ngl_per_device_high[id].n_layer - 1;686 }687 if (ngl_per_device_high[id].n_layer > 0) {688 std::vector<int64_t> mem_high = get_memory_for_layers(__func__, ngl_per_device_high, overflow_bufts);689 if (mem_high[id] > targets[id]) {690 assert(ngl_per_device_high[id].n_layer > ngl_per_device[id].n_layer);691 uint32_t delta = ngl_per_device_high[id].n_layer - ngl_per_device[id].n_layer;692 LOG_TRC("%s: start filling device %" PRIu32 ", delta=%" PRIu32 "\n", __func__, id, delta);693 while (delta > 1) {694 uint32_t step_size = int64_t(delta) * (targets[id] - mem[id]) / (mem_high[id] - mem[id]);695 step_size = std::max(step_size, uint32_t(1));696 step_size = std::min(step_size, delta - 1);697 698 std::vector<ngl_t> ngl_per_device_test = ngl_per_device;699 ngl_per_device_test[id].n_layer += step_size;700 if (hp_nex) {701 ngl_per_device_test[id].n_part += size_t(id) == nd - 1 && ngl_per_device_test[id].n_part == 0 ?702 step_size - 1 : step_size; // the first layer is the output layer which must always be full703 }704 const std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts);705 706 if (mem_test[id] <= targets[id]) {707 ngl_per_device = ngl_per_device_test;708 mem = mem_test;709 LOG_TRC("%s: set ngl_per_device[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device[id].n_layer);710 } else {711 ngl_per_device_high = ngl_per_device_test;712 mem_high = mem_test;713 LOG_TRC("%s: set ngl_per_device_high[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device_high[id].n_layer);714 }715 delta = ngl_per_device_high[id].n_layer - ngl_per_device[id].n_layer;716 }717 } else {718 assert(ngl_per_device_high[id].n_layer == n_unassigned);719 ngl_per_device = ngl_per_device_high;720 mem = mem_high;721 LOG_TRC("%s: set ngl_per_device[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device[id].n_layer);722 }723 }724 725 const int64_t projected_margin = dmds_full[id].free - mem[id];726 LOG_TRC(727 "%s: - %s: %2" PRIu32 " layers, %6" PRId64 " MiB used, %6" PRId64 " MiB free\n",728 __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, mem[id]/MiB, projected_margin/MiB);729 }730 if (hp_nex == 0 || global_surplus_cpu_moe <= 0) {731 set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, *mparams);732 return;733 }734 735 // step 4: for a MoE model where all dense tensors fit,736 // convert the dense-only layers in the back to full layers in the front until all devices are full737 // essentially the same procedure as for the dense-only layers except front-to-back738 // also, try fitting at least part of one more layer to reduce waste for "small" GPUs with e.g. 24 GiB VRAM739 740 size_t id_dense_start = nd;741 for (int id = nd - 1; id >= 0; id--) {742 if (ngl_per_device[id].n_layer > 0) {743 id_dense_start = id;744 continue;745 }746 break;747 }748 assert(id_dense_start < nd);749 750 LOG_TRC("%s: converting dense-only layers to full layers and filling them front-to-back with overflow to next device/system memory:\n", __func__);751 for (size_t id = 0; id <= id_dense_start && id_dense_start < nd; id++) {752 std::vector<ngl_t> ngl_per_device_high = ngl_per_device;753 for (size_t jd = id_dense_start; jd < nd; jd++) {754 const uint32_t n_layer_move = jd < nd - 1 ? ngl_per_device_high[jd].n_layer : ngl_per_device_high[jd].n_layer - 1;755 ngl_per_device_high[id].n_layer += n_layer_move;756 ngl_per_device_high[jd].n_layer -= n_layer_move;757 ngl_per_device_high[jd].n_part = 0;758 }759 size_t id_dense_start_high = nd - 1;760 std::vector<int64_t> mem_high = get_memory_for_layers(__func__, ngl_per_device_high, overflow_bufts);761 762 if (mem_high[id] > targets[id]) {763 assert(ngl_per_device_high[id].n_full() >= ngl_per_device[id].n_full());764 uint32_t delta = ngl_per_device_high[id].n_full() - ngl_per_device[id].n_full();765 while (delta > 1) {766 uint32_t step_size = int64_t(delta) * (targets[id] - mem[id]) / (mem_high[id] - mem[id]);767 step_size = std::max(step_size, uint32_t(1));768 step_size = std::min(step_size, delta - 1);769 770 std::vector<ngl_t> ngl_per_device_test = ngl_per_device;771 size_t id_dense_start_test = id_dense_start;772 uint32_t n_converted_test = 0;773 for (;id_dense_start_test < nd; id_dense_start_test++) {774 const uint32_t n_convert_jd = std::min(step_size - n_converted_test, ngl_per_device_test[id_dense_start_test].n_part);775 ngl_per_device_test[id_dense_start_test].n_layer -= n_convert_jd;776 ngl_per_device_test[id_dense_start_test].n_part -= n_convert_jd;777 ngl_per_device_test[id].n_layer += n_convert_jd;778 n_converted_test += n_convert_jd;779 780 if (ngl_per_device_test[id_dense_start_test].n_part > 0) {781 break;782 }783 }784 const std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts);785 786 if (mem_test[id] <= targets[id]) {787 ngl_per_device = ngl_per_device_test;788 mem = mem_test;789 id_dense_start = id_dense_start_test;790 LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start=%zu\n",791 __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);792 } else {793 ngl_per_device_high = ngl_per_device_test;794 mem_high = mem_test;795 id_dense_start_high = id_dense_start_test;796 LOG_TRC("%s: set ngl_per_device_high[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start_high=%zu\n",797 __func__, id, ngl_per_device_high[id].n_layer, ngl_per_device_high[id].n_part, id_dense_start_high);798 }799 assert(ngl_per_device_high[id].n_full() >= ngl_per_device[id].n_full());800 delta = ngl_per_device_high[id].n_full() - ngl_per_device[id].n_full();801 }802 } else {803 ngl_per_device = ngl_per_device_high;804 mem = mem_high;805 id_dense_start = id_dense_start_high;806 LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start=%zu\n",807 __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);808 }809 810 // try to fit at least part of one more layer811 if (ngl_per_device[id_dense_start].n_layer > (id < nd - 1 ? 0 : 1)) {812 std::vector<ngl_t> ngl_per_device_test = ngl_per_device;813 size_t id_dense_start_test = id_dense_start;814 ngl_per_device_test[id_dense_start_test].n_layer--;815 ngl_per_device_test[id_dense_start_test].n_part--;816 ngl_per_device_test[id].n_layer++;817 ngl_per_device_test[id].n_part++;818 if (ngl_per_device_test[id_dense_start_test].n_part == 0) {819 id_dense_start_test++;820 }821 ngl_per_device_test[id].overflow_type = LAYER_FRACTION_UP;822 std::vector<ggml_backend_buffer_type_t> overflow_bufts_test = overflow_bufts;823 if (id < nd - 1) {824 overflow_bufts_test[id] = ggml_backend_dev_buffer_type(devs[id + 1]);825 }826 LOG_TRC("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_UP\n", __func__);827 std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);828 if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {829 ngl_per_device = ngl_per_device_test;830 overflow_bufts = overflow_bufts_test;831 mem = mem_test;832 id_dense_start = id_dense_start_test;833 LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", UP), id_dense_start=%zu\n",834 __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);835 836 ngl_per_device_test[id].overflow_type = LAYER_FRACTION_GATE;837 LOG_TRC("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_GATE\n", __func__);838 mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);839 if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {840 ngl_per_device = ngl_per_device_test;841 overflow_bufts = overflow_bufts_test;842 mem = mem_test;843 id_dense_start = id_dense_start_test;844 LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", GATE), id_dense_start=%zu\n",845 __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);846 }847 } else {848 ngl_per_device_test[id].overflow_type = LAYER_FRACTION_ATTN;849 LOG_TRC("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_ATTN\n", __func__);850 mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);851 if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {852 ngl_per_device = ngl_per_device_test;853 overflow_bufts = overflow_bufts_test;854 mem = mem_test;855 id_dense_start = id_dense_start_test;856 LOG_TRC("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", ATTN), id_dense_start=%zu\n",857 __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);858 }859 }860 }861 862 const int64_t projected_margin = dmds_full[id].free - mem[id];863 LOG_TRC(864 "%s: - %s: %2" PRIu32 " layers (%2" PRIu32 " overflowing), %6" PRId64 " MiB used, %6" PRId64 " MiB free\n",865 __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, ngl_per_device[id].n_part, mem[id]/MiB, projected_margin/MiB);866 }867 868 // print info for devices that were not changed during the conversion from dense only to full layers:869 for (size_t id = id_dense_start + 1; id < nd; id++) {870 const int64_t projected_margin = dmds_full[id].free - mem[id];871 LOG_TRC(872 "%s: - %s: %2" PRIu32 " layers (%2" PRIu32 " overflowing), %6" PRId64 " MiB used, %6" PRId64 " MiB free\n",873 __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, ngl_per_device[id].n_part, mem[id]/MiB, projected_margin/MiB);874 }875 876 set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, *mparams);877}878 879enum common_params_fit_status common_fit_params(880 const char * path_model,881 llama_model_params * mparams,882 llama_context_params * cparams,883 float * tensor_split,884 llama_model_tensor_buft_override * tensor_buft_overrides,885 size_t * margins,886 uint32_t n_ctx_min,887 const common_fit_extra_model * extra,888 ggml_log_level log_level) {889 const int64_t t0_us = llama_time_us();890 common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS;891 try {892 common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);893 LOG_TRC("%s: successfully fit params to free device memory\n", __func__);894 } catch (const common_params_fit_exception & e) {895 LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what());896 status = COMMON_PARAMS_FIT_STATUS_FAILURE;897 } catch (const std::runtime_error & e) {898 LOG_ERR("%s: encountered an error while trying to fit params to free device memory: %s\n", __func__, e.what());899 status = COMMON_PARAMS_FIT_STATUS_ERROR;900 }901 const int64_t t1_us = llama_time_us();902 LOG_TRC("%s: fitting params to free memory took %.2f seconds\n", __func__, (t1_us - t0_us) * 1e-6);903 return status;904}905 906void common_memory_breakdown_print(const struct llama_context * ctx) {907 //const auto & devices = ctx->get_model().devices;908 const auto * model = llama_get_model(ctx);909 910 std::vector<ggml_backend_dev_t> devices;911 for (int i = 0; i < llama_model_n_devices(model); i++) {912 devices.push_back(llama_model_get_device(model, i));913 }914 915 llama_memory_breakdown memory_breakdown = llama_get_memory_breakdown(ctx);916 917 std::vector<std::array<std::string, 9>> table_data;918 table_data.reserve(devices.size());919 920 // same data as the table below, for --log-jsonl consumers921 common_json rows = common_json::array();922 const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n";923 const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";924 const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n";925 926 table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});927 928 constexpr size_t MiB = 1024 * 1024;929 const std::vector<std::string> desc_prefixes_strip = {"NVIDIA ", "GeForce ", "Tesla ", "AMD ", "Radeon ", "Instinct "};930 931 // track seen buffer types to avoid double counting:932 std::set<ggml_backend_buffer_type_t> seen_buffer_types;933 934 // accumulative memory breakdown for each device and for host:935 std::vector<llama_memory_breakdown_data> mb_dev(devices.size());936 llama_memory_breakdown_data mb_host;937 938 for (const auto & buft_mb : memory_breakdown) {939 ggml_backend_buffer_type_t buft = buft_mb.first;940 const llama_memory_breakdown_data & mb = buft_mb.second;941 if (ggml_backend_buft_is_host(buft)) {942 mb_host.model += mb.model;943 mb_host.context += mb.context;944 mb_host.compute += mb.compute;945 seen_buffer_types.insert(buft);946 continue;947 }948 ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);949 if (dev) {950 int i_dev = -1;951 for (size_t i = 0; i < devices.size(); i++) {952 if (devices[i] == dev) {953 i_dev = i;954 break;955 }956 }957 if (i_dev != -1) {958 mb_dev[i_dev].model += mb.model;959 mb_dev[i_dev].context += mb.context;960 mb_dev[i_dev].compute += mb.compute;961 seen_buffer_types.insert(buft);962 continue;963 }964 }965 }966 967 // print memory breakdown for each device:968 for (size_t i = 0; i < devices.size(); i++) {969 ggml_backend_dev_t dev = devices[i];970 llama_memory_breakdown_data mb = mb_dev[i];971 972 const std::string name = ggml_backend_dev_name(dev);973 std::string desc = ggml_backend_dev_description(dev);974 for (const std::string & prefix : desc_prefixes_strip) {975 if (desc.length() >= prefix.length() && desc.substr(0, prefix.length()) == prefix) {976 desc = desc.substr(prefix.length());977 }978 }979 980 size_t free, total;981 ggml_backend_dev_memory(dev, &free, &total);982 983 const size_t self = mb.model + mb.context + mb.compute;984 const int64_t unaccounted = static_cast<int64_t>(total) - static_cast<int64_t>(free) - static_cast<int64_t>(self);985 986 table_data.push_back({987 template_gpu,988 " - " + name + " (" + desc + ")",989 std::to_string(total / MiB),990 std::to_string(free / MiB),991 std::to_string(self / MiB),992 std::to_string(mb.model / MiB),993 std::to_string(mb.context / MiB),994 std::to_string(mb.compute / MiB),995 std::to_string(unaccounted / static_cast<int64_t>(MiB))});996 997 rows.push_back({998 {"kind", "device"},999 {"name", name},1000 {"description", desc},1001 {"total", total / MiB},1002 {"free", free / MiB},1003 {"self", self / MiB},1004 {"model", mb.model / MiB},1005 {"context", mb.context / MiB},1006 {"compute", mb.compute / MiB},1007 {"unaccounted", unaccounted / static_cast<int64_t>(MiB)},1008 });1009 }1010 1011 // print memory breakdown for host:1012 {1013 const size_t self = mb_host.model + mb_host.context + mb_host.compute;1014 table_data.push_back({1015 template_other,1016 " - Host",1017 "", // total1018 "", // free1019 std::to_string(self / MiB),1020 std::to_string(mb_host.model / MiB),1021 std::to_string(mb_host.context / MiB),1022 std::to_string(mb_host.compute / MiB),1023 ""}); // unaccounted1024 1025 rows.push_back({1026 {"kind", "host"},1027 {"name", "Host"},1028 {"self", self / MiB},1029 {"model", mb_host.model / MiB},1030 {"context", mb_host.context / MiB},1031 {"compute", mb_host.compute / MiB},1032 });1033 }1034 1035 // print memory breakdown for all remaining buffer types:1036 for (const auto & buft_mb : memory_breakdown) {1037 ggml_backend_buffer_type_t buft = buft_mb.first;1038 const llama_memory_breakdown_data & mb = buft_mb.second;1039 if (seen_buffer_types.count(buft) == 1) {1040 continue;1041 }1042 const std::string name = ggml_backend_buft_name(buft);1043 const size_t self = mb.model + mb.context + mb.compute;1044 table_data.push_back({1045 template_other,1046 " - " + name,1047 "", // total1048 "", // free1049 std::to_string(self / MiB),1050 std::to_string(mb.model / MiB),1051 std::to_string(mb.context / MiB),1052 std::to_string(mb.compute / MiB),1053 ""}); // unaccounted1054 1055 rows.push_back({1056 {"kind", "buffer_type"},1057 {"name", name},1058 {"self", self / MiB},1059 {"model", mb.model / MiB},1060 {"context", mb.context / MiB},1061 {"compute", mb.compute / MiB},1062 });1063 1064 seen_buffer_types.insert(buft);1065 }1066 1067 for (size_t j = 1; j < table_data[0].size(); j++) {1068 size_t max_len = 0;1069 for (const auto & td : table_data) {1070 max_len = std::max(max_len, td[j].length());1071 }1072 for (auto & td : table_data) {1073 td[j].insert(j == 1 ? td[j].length() : 0, max_len - td[j].length(), ' ');1074 }1075 }1076 for (const auto & td : table_data) {1077 LOG_TRC(td[0].c_str(),1078 __func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(),1079 td[6].c_str(), td[7].c_str(), td[8].c_str());1080 }1081 1082 LOG_JSON("fit_memory_breakdown", common_json({1083 {"unit", "MiB"},1084 {"rows", rows},1085 }));1086}1087 1088void common_fit_print(1089 const char * path_model,1090 llama_model_params * mparams,1091 llama_context_params * cparams) {1092 std::vector<ggml_backend_dev_t> devs;1093 uint32_t hp_ngl = 0; // hparams.n_gpu_layers1094 uint32_t hp_nct = 0; // hparams.n_ctx_train1095 uint32_t hp_nex = 0; // hparams.n_expert1096 1097 auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);1098 GGML_ASSERT(dmd.size() == devs.size() + 1);1099 1100 for (size_t id = 0; id < devs.size(); id++) {1101 printf("%s ", ggml_backend_dev_name(devs[id]));1102 printf("%zu ", dmd[id].mb.model/1024/1024);1103 printf("%zu ", dmd[id].mb.context/1024/1024);1104 printf("%zu ", dmd[id].mb.compute/1024/1024);1105 printf("\n");1106 }1107 1108 printf("Host ");1109 printf("%zu ", dmd.back().mb.model/1024/1024);1110 printf("%zu ", dmd.back().mb.context/1024/1024);1111 printf("%zu ", dmd.back().mb.compute/1024/1024);1112 printf("\n");1113}1114