Felipe97/llama-cpp-compiled

1#pragma once2 3#include "ggml.h"4#include "llama.h"5 6#include <vector>7 8enum common_params_fit_status {9    COMMON_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit10    COMMON_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit11    COMMON_PARAMS_FIT_STATUS_ERROR   = 2, // a hard error occurred, e.g. because no model could be found at the specified path12};13 14// a second model that shares the devices of the main model, e.g. a draft model15//   - its context follows the context of the main model, so its memory is measured again whenever that context changes16//   - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context17struct common_fit_extra_model {18    const char * path_model;19    llama_model_params * mparams;20    llama_context_params * cparams;21    bool shares_model;22};23 24// fits mparams and cparams to free device memory (assumes system memory is unlimited)25//   - returns true if the parameters could be successfully modified to fit device memory26//   - this function is NOT thread safe because it modifies the global llama logger state27//   - only parameters that have the same value as in llama_default_model_params are modified28//     with the exception of the context size which is modified if and only if equal to 029common_params_fit_status common_fit_params(30                         const char * path_model,31                 llama_model_params * mparams,32               llama_context_params * cparams,33                              float * tensor_split,          // writable buffer for tensor split, needs at least llama_max_devices elements34   llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements35                             size_t * margins,               // margins of memory to leave per device in bytes36                           uint32_t   n_ctx_min,             // minimum context size to set when trying to reduce memory use37      const common_fit_extra_model * extra,                  // model to fit alongside the main one, nullptr if there is none38                     ggml_log_level   log_level);            // minimum log level to print during fitting, lower levels go to debug log39 40// print estimated memory to stdout41void common_fit_print(42                         const char * path_model,43                 llama_model_params * mparams,44               llama_context_params * cparams);45 46void common_memory_breakdown_print(const llama_context * ctx);47 48struct common_device_memory_data {49    int64_t total;50    int64_t free;51    size_t  model;52    size_t  context;53    size_t  compute;54};55 56using common_device_memory_data_vec = std::vector<common_device_memory_data>;57 58// Load a model + context with no_alloc and return the per-device memory breakdown.59common_device_memory_data_vec common_get_device_memory_data(60                         const char * path_model,61           const llama_model_params * mparams,62         const llama_context_params * cparams,63    std::vector<ggml_backend_dev_t> & devs,64                           uint32_t & hp_ngl,65                           uint32_t & hp_n_ctx_train,66                           uint32_t & hp_n_expert,67                     ggml_log_level   log_level);68