echodict/llama.cpp

1#include "llama.h"2#include "../src/llama-ext.h"3 4#include "arg.h"5#include "common.h"6#include "fit.h"7#include "log.h"8 9#include <cinttypes>10 11#if defined(_MSC_VER)12#pragma warning(disable: 4244 4267) // possible loss of data13#endif14 15int main(int argc, char ** argv) {16    common_params params;17 18    common_init();19 20    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_FIT_PARAMS)) {21        return 1;22    }23 24    llama_backend_init();25    llama_numa_init(params.numa);26 27    auto mparams = common_model_params_to_llama(params);28    auto cparams = common_context_params_to_llama(params);29 30    if (!params.fit_params_print) {31        const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams,32                params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx,33                params.verbosity >= 4 ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);34        if (status != COMMON_PARAMS_FIT_STATUS_SUCCESS) {35            LOG_ERR("%s: failed to fit CLI arguments to free memory, exiting...\n", __func__);36            exit(1);37        }38 39        LOG_INF("%s: printing fitted CLI arguments to stdout...\n", __func__);40        common_log_flush(common_log_main());41        printf("-c %" PRIu32 " -ngl %" PRIi32, cparams.n_ctx, mparams.n_gpu_layers);42 43        size_t nd = llama_max_devices();44        while (nd > 1 && mparams.tensor_split[nd - 1] == 0.0f) {45            nd--;46        }47        if (nd > 1) {48            for (size_t id = 0; id < nd; id++) {49                if (id == 0) {50                    printf(" -ts ");51                }52                printf("%s%" PRIu32, id > 0 ? "," : "", uint32_t(mparams.tensor_split[id]));53            }54        }55 56        const size_t ntbo = llama_max_tensor_buft_overrides();57        bool any_tbo = false;58        for (size_t itbo = 0; itbo < ntbo && mparams.tensor_buft_overrides[itbo].pattern != nullptr; itbo++) {59            if (itbo == 0) {60                printf(" -ot \"");61            }62            printf("%s%s=%s", itbo > 0 ? "," : "", mparams.tensor_buft_overrides[itbo].pattern, ggml_backend_buft_name(mparams.tensor_buft_overrides[itbo].buft));63            any_tbo = true;64        }65        printf("%s\n", any_tbo ? "\"" : "");66    } else {67        LOG_INF("%s: printing estimated memory in MiB to stdout (device, model, context, compute) ...\n", __func__);68        common_log_flush(common_log_main());69 70        common_fit_print(params.model.path.c_str(), &mparams, &cparams);71    }72 73    return 0;74}75