Felipe97/llama-cpp-compiled
01.1k
1#include "arg.h"2 3#include "build-info.h"4#include "chat.h"5#include "common.h"6#include "download.h"7#include "json-schema-to-grammar.h"8#include "json.h"9#include "llama.h"10#include "log.h"11#include "sampling.h"12#include "speculative.h"13#include "preset.h"14 15// fix problem with std::min and std::max16#if defined(_WIN32)17#define WIN32_LEAN_AND_MEAN18#ifndef NOMINMAX19# define NOMINMAX20#endif21#include <windows.h>22#include <shellapi.h>23#endif24 25#include <algorithm>26#include <cinttypes>27#include <climits>28#include <cmath>29#include <cstdarg>30#include <filesystem>31#include <fstream>32#include <list>33#include <numeric>34#include <regex>35#include <set>36#include <string>37#include <system_error>38#include <thread> // for hardware_concurrency39#include <vector>40 41#ifndef __EMSCRIPTEN__42#ifdef __linux__43#include <linux/limits.h>44#elif defined(_WIN32)45# if !defined(PATH_MAX)46# define PATH_MAX MAX_PATH47# endif48#elif defined(_AIX)49#include <sys/limits.h>50#else51#include <sys/syslimits.h>52#endif53#endif54 55#define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 208356 57using json = common_json;58using namespace common_arg_utils;59 60static std::initializer_list<enum llama_example> mmproj_examples = {61 LLAMA_EXAMPLE_MTMD,62 LLAMA_EXAMPLE_SERVER,63 LLAMA_EXAMPLE_CLI,64 LLAMA_EXAMPLE_TTS,65};66 67static std::string read_file(const std::string & fname) {68 std::ifstream file(fname);69 if (!file) {70 throw std::runtime_error(string_format("error: failed to open file '%s'\n", fname.c_str()));71 }72 std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());73 file.close();74 return content;75}76 77static const std::vector<common_arg> & get_common_arg_defs() {78 static const std::vector<common_arg> options = [] {79 common_params params;80 auto ctx = common_params_parser_init(params, LLAMA_EXAMPLE_SERVER, nullptr);81 return ctx.options;82 }();83 return options;84}85 86common_arg & common_arg::set_examples(std::initializer_list<enum llama_example> examples) {87 this->examples = examples;88 return *this;89}90 91common_arg & common_arg::set_excludes(std::initializer_list<enum llama_example> excludes) {92 this->excludes = excludes;93 return *this;94}95 96common_arg & common_arg::set_env(const char * env) {97 help = help + "\n(env: " + env + ")";98 this->env = env;99 return *this;100}101 102common_arg & common_arg::set_sampling() {103 is_sampling = true;104 return *this;105}106 107common_arg & common_arg::set_spec() {108 is_spec = true;109 return *this;110}111 112common_arg & common_arg::set_preset_only() {113 is_preset_only = true;114 return *this;115}116 117bool common_arg::in_example(enum llama_example ex) {118 return examples.find(ex) != examples.end();119}120 121bool common_arg::is_exclude(enum llama_example ex) {122 return excludes.find(ex) != excludes.end();123}124 125bool common_arg::get_value_from_env(std::string & output) const {126 if (env == nullptr) return false;127 if (!args_neg.empty()) {128 // for compatibility, we need to check LLAMA_ARG_NO_ env as well129 std::string neg_env = env;130 string_replace_all(neg_env, "LLAMA_ARG_", "LLAMA_ARG_NO_");131 char * neg_value = std::getenv(neg_env.c_str());132 if (neg_value) {133 output = "0"; // falsey134 return true;135 }136 }137 char * value = std::getenv(env);138 if (value) {139 output = value;140 return true;141 }142 return false;143}144 145bool common_arg::has_value_from_env() const {146 if (env != nullptr && !args_neg.empty()) {147 // for compatibility, we need to check LLAMA_ARG_NO_ env as well148 std::string neg_env = env;149 string_replace_all(neg_env, "LLAMA_ARG_", "LLAMA_ARG_NO_");150 if (std::getenv(neg_env.c_str())) {151 return true;152 }153 }154 return env != nullptr && std::getenv(env);155}156 157static std::vector<std::string> break_str_into_lines(std::string input, size_t max_char_per_line) {158 std::vector<std::string> result;159 std::istringstream iss(input);160 std::string line;161 auto add_line = [&](const std::string& l) {162 if (l.length() <= max_char_per_line) {163 result.push_back(l);164 } else {165 std::istringstream line_stream(l);166 std::string word, current_line;167 while (line_stream >> word) {168 if (current_line.length() + !current_line.empty() + word.length() > max_char_per_line) {169 if (!current_line.empty()) result.push_back(current_line);170 current_line = word;171 } else {172 current_line += (!current_line.empty() ? " " : "") + word;173 }174 }175 if (!current_line.empty()) result.push_back(current_line);176 }177 };178 while (std::getline(iss, line)) {179 add_line(line);180 }181 return result;182}183 184std::string common_arg::to_string() const {185 // params for printing to console186 const static int n_leading_spaces = 40;187 const static int n_char_per_line_help = 70; // TODO: detect this based on current console188 std::string leading_spaces(n_leading_spaces, ' ');189 190 std::ostringstream ss;191 auto all_args = get_args(); // also contains args_neg192 for (const auto & arg : all_args) {193 if (arg == all_args.front()) {194 if (all_args.size() == 1) {195 ss << arg;196 } else {197 // first arg is usually abbreviation, we need padding to make it more beautiful198 auto tmp = std::string(arg) + ", ";199 auto spaces = std::string(std::max(0, 7 - (int)tmp.size()), ' ');200 ss << tmp << spaces;201 }202 } else {203 ss << arg << (arg != all_args.back() ? ", " : "");204 }205 }206 if (value_hint) ss << " " << value_hint;207 if (value_hint_2) ss << " " << value_hint_2;208 if (ss.tellp() > n_leading_spaces - 3) {209 // current line is too long, add new line210 ss << "\n" << leading_spaces;211 } else {212 // padding between arg and help, same line213 ss << std::string(leading_spaces.size() - ss.tellp(), ' ');214 }215 const auto help_lines = break_str_into_lines(help, n_char_per_line_help);216 for (const auto & line : help_lines) {217 ss << (&line == &help_lines.front() ? "" : leading_spaces) << line << "\n";218 }219 return ss.str();220}221 222std::vector<std::string> common_arg::get_args() const {223 std::vector<std::string> result;224 for (const auto & arg : args) {225 result.push_back(std::string(arg));226 }227 for (const auto & arg : args_neg) {228 result.push_back(std::string(arg));229 }230 return result;231}232 233std::vector<std::string> common_arg::get_env() const {234 std::vector<std::string> result;235 if (env) {236 result.push_back(std::string(env));237 }238 if (!args_neg.empty() && env) {239 // for compatibility, we need to add LLAMA_ARG_NO_ variant240 std::string neg_env = env;241 string_replace_all(neg_env, "LLAMA_ARG_", "LLAMA_ARG_NO_");242 result.push_back(neg_env);243 }244 return result;245}246 247//248// utils249//250 251// Helper function to parse tensor buffer override strings252static void parse_tensor_buffer_overrides(const std::string & value, std::vector<llama_model_tensor_buft_override> & overrides) {253 ggml_backend_load_all();254 255 std::map<std::string, ggml_backend_buffer_type_t> buft_list;256 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {257 auto * dev = ggml_backend_dev_get(i);258 auto * buft = ggml_backend_dev_buffer_type(dev);259 if (buft) {260 buft_list[ggml_backend_buft_name(buft)] = buft;261 }262 }263 264 for (const auto & override : string_split<std::string>(value, ',')) {265 std::string::size_type pos = override.find('=');266 if (pos == std::string::npos) {267 throw std::invalid_argument("invalid value");268 }269 std::string tensor_name = override.substr(0, pos);270 std::string buffer_type = override.substr(pos + 1);271 272 if (buft_list.find(buffer_type) == buft_list.end()) {273 printf("Available buffer types:\n");274 for (const auto & it : buft_list) {275 printf(" %s\n", ggml_backend_buft_name(it.second));276 }277 throw std::invalid_argument("unknown buffer type");278 }279 // keep strings alive and avoid leaking memory by storing them in a static vector280 static std::list<std::string> buft_overrides;281 buft_overrides.push_back(tensor_name);282 overrides.push_back({buft_overrides.back().c_str(), buft_list.at(buffer_type)});283 }284}285 286static std::string clean_file_name(const std::string & fname) {287 std::string clean_fname = fname;288 string_replace_all(clean_fname, "\\", "_");289 string_replace_all(clean_fname, "/", "_");290 return clean_fname;291}292 293struct handle_model_result {294 bool found_mmproj = false;295 common_params_model mmproj;296 297 bool found_mtp = false;298 common_params_model mtp;299 300 bool found_preset = false;301 std::string preset_path;302};303 304const std::vector<ggml_type> kv_cache_types = {305 GGML_TYPE_F32,306 GGML_TYPE_F16,307 GGML_TYPE_BF16,308 GGML_TYPE_Q8_0,309 GGML_TYPE_Q4_0,310 GGML_TYPE_Q4_1,311 GGML_TYPE_IQ4_NL,312 GGML_TYPE_Q5_0,313 GGML_TYPE_Q5_1,314};315 316static ggml_type kv_cache_type_from_str(const std::string & s) {317 for (const auto & type : kv_cache_types) {318 if (ggml_type_name(type) == s) {319 return type;320 }321 }322 throw std::runtime_error("Unsupported cache type: " + s);323}324 325static std::string get_all_kv_cache_types() {326 std::ostringstream msg;327 for (const auto & type : kv_cache_types) {328 msg << ggml_type_name(type) << (&type == &kv_cache_types.back() ? "" : ", ");329 }330 return msg.str();331}332 333static bool parse_bool_value(const std::string & value) {334 if (is_truthy(value)) {335 return true;336 } else if (is_falsey(value)) {337 return false;338 } else {339 throw std::invalid_argument("invalid boolean value");340 }341}342 343[[noreturn]] static void arg_removed(const std::string & msg) {344 throw std::invalid_argument("the argument has been removed. " + msg);345}346 347//348// common_models_handler349//350 351static std::string get_default_local_path(const std::string & url) {352 auto f = string_split<std::string>(url, '#').front();353 f = string_split<std::string>(f, '?').front();354 return fs_get_cache_file(string_split<std::string>(f, '/').back());355}356 357static bool spec_types_is_default(const common_params & params) {358 return params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_NONE};359}360 361common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) {362 common_download_hf_plan plan;363 common_download_hf_plan plan_spec;364 common_download_opts opts;365 366 const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(),367 params.speculative.types.end(),368 COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();369 370 const bool spec_type_draft_dflash = std::find(params.speculative.types.begin(),371 params.speculative.types.end(),372 COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) != params.speculative.types.end();373 374 const bool spec_type_draft_eagle3 = std::find(params.speculative.types.begin(),375 params.speculative.types.end(),376 COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end();377 378 const bool spec_type_draft_dspark = std::find(params.speculative.types.begin(),379 params.speculative.types.end(),380 COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) != params.speculative.types.end();381 382 // only download mmproj if the current example is using it383 bool use_mmproj = false;384 for (const auto & ex : mmproj_examples) {385 if (curr_ex == ex) {386 use_mmproj = true;387 break;388 }389 }390 391 opts.bearer_token = params.hf_token;392 opts.offline = params.offline;393 opts.download_mtp = spec_type_draft_mtp;394 opts.download_eagle3 = spec_type_draft_eagle3;395 opts.download_dflash = spec_type_draft_dflash;396 opts.download_dspark = spec_type_draft_dspark;397 opts.download_mmproj = use_mmproj && !params.no_mmproj398 && params.mmproj.path.empty() && params.mmproj.url.empty();399 400 if (!params.model.hf_repo.empty()) {401 plan = common_download_get_hf_plan(params.model, opts);402 }403 404 if (!params.speculative.draft.mparams.hf_repo.empty()) {405 // without a requested type, discover every sidecar the draft repo ships to infer the type later406 auto opts_spec = opts;407 if (spec_types_is_default(params)) {408 opts_spec.download_mtp = true;409 opts_spec.download_dflash = true;410 opts_spec.download_eagle3 = true;411 opts_spec.download_dspark = true;412 }413 plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec);414 }415 416 return common_models_handler{plan, plan_spec, opts};417}418 419bool common_models_handler_is_preset_repo(const common_models_handler & handler) {420 return !handler.plan.preset.url.empty();421}422 423static std::vector<common_download_task> build_url_tasks(const common_params_model & model, common_download_opts opts) {424 auto parts = common_download_get_all_parts(model.url);425 std::vector<common_download_task> tasks;426 427 // single-part: download straight to model.path if the user gave one (-m), else the cache default428 if (parts.size() == 1) {429 common_download_task task;430 task.url = parts[0];431 task.local_path = model.path.empty() ? get_default_local_path(parts[0]) : model.path;432 task.opts = opts;433 tasks.push_back(std::move(task));434 return tasks;435 }436 437 // multi-part: place each part under the user's -m directory (if given), else the cache default438 std::string base_dir;439 if (!model.path.empty()) {440 auto pos = model.path.rfind('/');441 base_dir = pos == std::string::npos ? std::string(".") : model.path.substr(0, pos);442 }443 444 for (const auto & part : parts) {445 common_download_task task;446 task.url = part;447 task.opts = opts;448 449 std::string local = get_default_local_path(part);450 if (!base_dir.empty()) {451 auto pos = local.rfind('/');452 std::string name = pos == std::string::npos ? local : local.substr(pos + 1);453 local = base_dir + "/" + name;454 }455 task.local_path = local;456 tasks.push_back(std::move(task));457 }458 return tasks;459}460 461void common_models_handler_apply(common_models_handler & handler, common_params & params, common_download_callback * callback) {462 std::vector<common_download_task> tasks;463 464 auto & plan = handler.plan;465 auto & plan_spec = handler.plan_spec;466 467 auto opts = handler.opts; // copy468 opts.callback = callback;469 470 // handle plain "url" if needed471 auto handle_url = [&](common_params_model & model) {472 if (!model.url.empty()) {473 if (model.path.empty()) {474 model.path = get_default_local_path(model.url);475 }476 }477 };478 handle_url(params.model);479 handle_url(params.mmproj);480 handle_url(params.speculative.draft.mparams);481 482 // optionally, if docker repo is set, resolve it483 if (!params.model.docker_repo.empty()) {484 params.model.url = common_docker_resolve_model(params.model.docker_repo);485 params.model.path = get_default_local_path(params.model.url);486 }487 488 // handle plain "url" tasks (non-hf)489 if (!params.model.url.empty()) {490 auto url_tasks = build_url_tasks(params.model, opts);491 // the first part is what gets loaded, so point params.model.path at it492 if (!url_tasks.empty()) {493 std::string first_path = url_tasks.front().local_path;494 url_tasks.front().on_done = [&, first_path]() { params.model.path = first_path; };495 }496 for (auto & task : url_tasks) {497 tasks.push_back(std::move(task));498 }499 }500 if (!params.mmproj.url.empty()) {501 common_download_task task;502 task.url = params.mmproj.url;503 task.local_path = params.mmproj.path;504 task.opts = opts;505 tasks.push_back(task);506 }507 bool had_spec_url = false;508 if (!params.speculative.draft.mparams.url.empty()) {509 common_download_task task;510 task.url = params.speculative.draft.mparams.url;511 task.local_path = params.speculative.draft.mparams.path;512 task.opts = opts;513 tasks.push_back(task);514 had_spec_url = true;515 }516 517 // handle hf_plan tasks518 auto add_tasks = [&opts, &tasks](const hf_cache::hf_files & model_files,519 const hf_cache::hf_file & primary,520 common_params_model & model) {521 for (size_t i = 0; i < model_files.size(); ++i) {522 auto & model_file = model_files[i];523 bool is_primary = (model_file.path == primary.path);524 tasks.emplace_back(model_file, opts, [&, is_primary]() {525 if (is_primary) {526 // the primary file is the first split (00001-of), use it as model path527 model.path = hf_cache::finalize_file(model_file);528 } else {529 hf_cache::finalize_file(model_file);530 }531 });532 }533 };534 535 // an explicit draft file selection (e.g. -md with -hfd) disables the sidecar resolution of the draft repo536 if (!params.speculative.draft.mparams.hf_file.empty()) {537 plan_spec.mtp = {};538 plan_spec.dflash = {};539 plan_spec.eagle3 = {};540 plan_spec.dspark = {};541 }542 543 // infer the speculative type from the sidecar shipped by the draft repo when none is requested544 if (spec_types_is_default(params)) {545 if (!plan_spec.mtp.local_path.empty()) {546 params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };547 plan_spec.dspark = {};548 plan_spec.dflash = {};549 plan_spec.eagle3 = {};550 } else if (!plan_spec.dspark.local_path.empty()) {551 // dspark outranks dflash, its sidecar carries the extra Markov head552 params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK };553 plan_spec.dflash = {};554 plan_spec.eagle3 = {};555 } else if (!plan_spec.dflash.local_path.empty()) {556 params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH };557 plan_spec.eagle3 = {};558 } else if (!plan_spec.eagle3.local_path.empty()) {559 params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 };560 }561 }562 563 // infer the speculative type from the draft GGUF metadata when none is requested564 // note: reads only the first split - sharded drafts need an explicit --spec-type565 if (spec_types_is_default(params) && !params.speculative.draft.mparams.path.empty()) {566 const auto types_gguf = common_speculative_types_from_gguf(params.speculative.draft.mparams.path);567 if (!types_gguf.empty()) {568 params.speculative.types = types_gguf;569 }570 }571 572 // when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model573 const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||574 !plan_spec.dflash.local_path.empty() ||575 !plan_spec.eagle3.local_path.empty() ||576 !plan_spec.dspark.local_path.empty();577 if (!plan_spec.mtp.local_path.empty() && !had_spec_url) {578 tasks.emplace_back(plan_spec.mtp, opts, [&]() {579 // only use the discovered MTP head when no draft path is set yet580 if (params.speculative.draft.mparams.path.empty()) {581 params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.mtp);582 } else {583 hf_cache::finalize_file(plan_spec.mtp);584 }585 });586 }587 if (!plan_spec.dflash.local_path.empty() && !had_spec_url) {588 tasks.emplace_back(plan_spec.dflash, opts, [&]() {589 // only use the discovered DFlash sidecar when no draft path is set yet590 if (params.speculative.draft.mparams.path.empty()) {591 params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dflash);592 } else {593 hf_cache::finalize_file(plan_spec.dflash);594 }595 });596 }597 if (!plan_spec.eagle3.local_path.empty() && !had_spec_url) {598 tasks.emplace_back(plan_spec.eagle3, opts, [&]() {599 // only use the discovered Eagle3 sidecar when no draft path is set yet600 if (params.speculative.draft.mparams.path.empty()) {601 params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.eagle3);602 } else {603 hf_cache::finalize_file(plan_spec.eagle3);604 }605 });606 }607 if (!plan_spec.dspark.local_path.empty() && !had_spec_url) {608 tasks.emplace_back(plan_spec.dspark, opts, [&]() {609 // only use the discovered DSpark sidecar when no draft path is set yet610 if (params.speculative.draft.mparams.path.empty()) {611 params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dspark);612 } else {613 hf_cache::finalize_file(plan_spec.dspark);614 }615 });616 }617 618 // a wired draft sidecar counts as an explicit draft for the main plan fallback below619 if (spec_sidecar_found) {620 had_spec_url = true;621 }622 623 // handle plan_spec (e.g. --spec-draft-hf)624 if (!plan_spec.model_files.empty() && !had_spec_url && !spec_sidecar_found) {625 add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams);626 had_spec_url = true;627 }628 629 if (!plan.model_files.empty()) {630 add_tasks(plan.model_files, plan.primary, params.model);631 }632 if (!plan.mmproj.local_path.empty()) {633 tasks.emplace_back(plan.mmproj, opts, [&]() {634 params.mmproj.path = hf_cache::finalize_file(plan.mmproj);635 });636 }637 if (!plan.mtp.local_path.empty() && !had_spec_url) {638 tasks.emplace_back(plan.mtp, opts, [&]() {639 // only fall back to the discovered MTP head when no draft was explicitly provided640 if (params.speculative.draft.mparams.empty()) {641 params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.mtp);642 } else {643 hf_cache::finalize_file(plan.mtp);644 }645 });646 }647 if (!plan.dflash.local_path.empty() && !had_spec_url) {648 tasks.emplace_back(plan.dflash, opts, [&]() {649 // only fall back to the discovered DFlash sidecar when no draft was explicitly provided650 if (params.speculative.draft.mparams.empty()) {651 params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dflash);652 } else {653 hf_cache::finalize_file(plan.dflash);654 }655 });656 }657 if (!plan.eagle3.local_path.empty() && !had_spec_url) {658 tasks.emplace_back(plan.eagle3, opts, [&]() {659 // only fall back to the discovered Eagle3 sidecar when no draft was explicitly provided660 if (params.speculative.draft.mparams.empty()) {661 params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.eagle3);662 } else {663 hf_cache::finalize_file(plan.eagle3);664 }665 });666 }667 if (!plan.dspark.local_path.empty() && !had_spec_url) {668 tasks.emplace_back(plan.dspark, opts, [&]() {669 // only fall back to the discovered DSpark sidecar when no draft was explicitly provided670 if (params.speculative.draft.mparams.empty()) {671 params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dspark);672 } else {673 hf_cache::finalize_file(plan.dspark);674 }675 });676 }677 if (!plan.preset.local_path.empty()) {678 tasks.emplace_back(plan.preset, opts, [&]() {679 // if HF repo is a preset repo, we simply run server in router mode with the preset.ini file680 params.models_preset_hf = params.model.hf_repo; // only for showing a warning681 params.models_preset = hf_cache::finalize_file(plan.preset);682 params.model = common_params_model{}; // make sure to clear model, so server starts in router mode683 });684 }685 686 // run all tasks in parallel687 if (!params.offline) {688 // if duplicated files are found, only download once (but still call on_done for each task)689 std::unordered_map<std::string, common_download_task *> unique_tasks;690 for (auto & task : tasks) {691 auto it = unique_tasks.find(task.local_path);692 if (it == unique_tasks.end()) {693 unique_tasks[task.local_path] = &task;694 }695 }696 std::vector<common_download_task> unique_tasks_vec;697 for (auto & pair : unique_tasks) {698 LOG_DBG("download task: %s -> %s\n", pair.second->url.c_str(), pair.second->local_path.c_str());699 unique_tasks_vec.push_back(*pair.second);700 }701 common_download_run_tasks(unique_tasks_vec);702 }703 704 // download successful, update params with the downloaded paths705 for (const auto & task : tasks) {706 if (task.on_done) {707 task.on_done();708 }709 }710}711 712//713// CLI argument parsing functions714//715 716// apply config files (if present), a later file overrides an earlier one:717// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows)718// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows)719static void common_params_apply_system_config(common_params & params, llama_example ex) {720 std::vector<std::string> paths;721 722#if defined(_WIN32)723 const std::string program_data = common_get_env("PROGRAMDATA");724 if (!program_data.empty()) {725 paths.push_back(program_data + "\\llama.cpp\\config.ini");726 }727#else728 paths.push_back("/etc/llama.cpp/config.ini");729#endif730 731 try {732 paths.push_back(fs_get_config_directory() + "config.ini");733 } catch (const std::exception & e) {734 LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what());735 }736 737 std::vector<std::string> found;738 for (const auto & path : paths) {739 std::error_code ec;740 if (std::filesystem::exists(path, ec)) {741 found.push_back(path);742 }743 }744 if (found.empty()) {745 return;746 }747 748 common_preset_context ctx(ex);749 ctx.ignore_unknown_keys = true; // the same config file is shared by all programs750 for (const auto & path : found) {751 LOG_INF("using config file: %s\n", path.c_str());752 common_preset global;753 common_presets presets = ctx.load_from_ini(path, global);754 global.apply_to_params(params);755 auto it = presets.find(COMMON_PRESET_DEFAULT_NAME);756 if (it != presets.end()) {757 it->second.apply_to_params(params);758 }759 }760}761 762static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {763 common_params & params = ctx_arg.params;764 765 // setup log directly from params.verbosity: see tools/cli/cli.cpp766 common_log_set_verbosity_thold(params.verbosity);767 768 // config file applies first, so env variables and CLI arguments override it769 common_params_apply_system_config(params, ctx_arg.ex);770 771 std::unordered_map<std::string, std::pair<common_arg *, bool>> arg_to_options;772 for (auto & opt : ctx_arg.options) {773 for (const auto & arg : opt.args) {774 arg_to_options[arg] = {&opt, /* is_positive */ true};775 }776 for (const auto & arg : opt.args_neg) {777 arg_to_options[arg] = {&opt, /* is_positive */ false};778 }779 }780 781 // handle environment variables782 for (auto & opt : ctx_arg.options) {783 std::string value;784 if (opt.get_value_from_env(value)) {785 try {786 if (opt.handler_void && is_truthy(value)) {787 opt.handler_void(params);788 }789 if (opt.handler_int) {790 opt.handler_int(params, std::stoi(value));791 }792 if (opt.handler_bool) {793 opt.handler_bool(params, parse_bool_value(value));794 }795 if (opt.handler_string) {796 opt.handler_string(params, value);797 continue;798 }799 } catch (std::exception & e) {800 throw std::invalid_argument(string_format(801 "error while handling environment variable \"%s\": %s\n\n", opt.env, e.what()));802 }803 }804 }805 806 // handle command line arguments807 auto check_arg = [&](int i) {808 if (i+1 >= argc) {809 throw std::invalid_argument("expected value for argument");810 }811 };812 813 auto parse_cli_args = [&]() {814 std::set<std::string> seen_args;815 816 for (int i = 1; i < argc; i++) {817 const std::string arg_prefix = "--";818 819 std::string arg = argv[i];820 if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {821 std::replace(arg.begin(), arg.end(), '_', '-');822 }823 if (arg_to_options.find(arg) == arg_to_options.end()) {824 throw std::invalid_argument(string_format("error: invalid argument: %s", arg.c_str()));825 }826 if (!seen_args.insert(arg).second) {827 const bool skip = (arg == "--spec-type");828 829 if (!skip) {830 LOG_WRN("DEPRECATED: argument '%s' specified multiple times, use comma-separated values instead (only last value will be used)\n", arg.c_str());831 }832 }833 auto & tmp = arg_to_options[arg];834 auto opt = *tmp.first;835 bool is_positive = tmp.second;836 if (opt.has_value_from_env()) {837 fprintf(stderr, "warn: %s environment variable is set, but will be overwritten by command line argument %s\n", opt.env, arg.c_str());838 }839 try {840 if (opt.handler_void) {841 opt.handler_void(params);842 continue;843 }844 if (opt.handler_bool) {845 opt.handler_bool(params, is_positive);846 continue;847 }848 849 // arg with single value850 check_arg(i);851 std::string val = argv[++i];852 if (opt.handler_int) {853 opt.handler_int(params, std::stoi(val));854 continue;855 }856 if (opt.handler_string) {857 opt.handler_string(params, val);858 continue;859 }860 861 // arg with 2 values862 check_arg(i);863 std::string val2 = argv[++i];864 if (opt.handler_str_str) {865 opt.handler_str_str(params, val, val2);866 continue;867 }868 } catch (std::exception & e) {869 throw std::invalid_argument(string_format(870 "error while handling argument \"%s\": %s\n\n"871 "usage:\n%s\n\nto show complete usage, run with -h",872 arg.c_str(), e.what(), opt.to_string().c_str()));873 }874 }875 };876 877 // parse all CLI args now, so that -hf is available below for remote preset resolution878 parse_cli_args();879 880 postprocess_cpu_params(params.cpuparams, nullptr);881 postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams);882 883 postprocess_cpu_params(params.speculative.draft.cpuparams, ¶ms.cpuparams);884 postprocess_cpu_params(params.speculative.draft.cpuparams_batch, ¶ms.cpuparams_batch);885 886 // default the mmproj device to the global device selection if not set explicitly with -mmdev887 if (params.mmproj_use_gpu && params.mmproj_device == nullptr && !params.devices.empty()) {888 params.mmproj_device = params.devices.front();889 params.mmproj_use_gpu = params.mmproj_device != nullptr;890 }891 892 if (params.prompt_cache_all && (params.interactive || params.interactive_first)) {893 throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");894 }895 896 const bool skip_model_download =897 // server will call common_params_handle_models() later, so we skip it here898 ctx_arg.ex == LLAMA_EXAMPLE_SERVER ||899 // download calls common_params_handle_models() itself and prints the paths900 ctx_arg.ex == LLAMA_EXAMPLE_DOWNLOAD ||901 // export_graph_ops loads only metadata902 ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS;903 904 if (!skip_model_download) {905 // handle model and download906 common_models_handler handler = common_models_handler_init(params, ctx_arg.ex);907 common_models_handler_apply(handler, params);908 909 // model is required (except for server)910 // TODO @ngxson : maybe show a list of available models in CLI in this case911 bool can_skip_model = params.usage || params.completion || !params.server_base.empty();912 if (!can_skip_model && params.model.path.empty()) {913 throw std::invalid_argument("error: --model is required\n");914 }915 }916 917 if (params.escape) {918 string_process_escapes(params.prompt);919 string_process_escapes(params.input_prefix);920 string_process_escapes(params.input_suffix);921 for (auto & antiprompt : params.antiprompt) {922 string_process_escapes(antiprompt);923 }924 for (auto & seq_breaker : params.sampling.dry_sequence_breakers) {925 string_process_escapes(seq_breaker);926 }927 }928 929 if (!params.kv_overrides.empty()) {930 params.kv_overrides.emplace_back();931 params.kv_overrides.back().key[0] = 0;932 }933 934 const bool mcp_enabled = !params.mcp_servers_config.empty() || !params.mcp_servers_json.empty();935 if ((!params.server_tools.empty() || mcp_enabled) && !params.cors_origins_explicit) {936 LOG_WRN("server tools or MCP servers are enabled, using localhost as default CORS origin (change via --cors-origins)\n");937 params.cors_origins = "localhost";938 }939 940 // pad tensor_buft_overrides for llama_params_fit:941 const size_t ntbo = llama_max_tensor_buft_overrides();942 while (params.tensor_buft_overrides.size() < ntbo) {943 params.tensor_buft_overrides.push_back({nullptr, nullptr});944 }945 946 if (!params.speculative.draft.tensor_buft_overrides.empty()) {947 params.speculative.draft.tensor_buft_overrides.push_back({nullptr, nullptr});948 }949 950 if (!params.chat_template.empty() && !common_chat_verify_template(params.chat_template, params.use_jinja)) {951 throw std::runtime_error(string_format(952 "error: the supplied chat template is not supported: %s%s\n",953 params.chat_template.c_str(),954 params.use_jinja ? "" : "\nnote: llama.cpp was started without --jinja, we only support commonly used templates"955 ));956 }957 958 // if the preserve_reasoning kwarg was not specified explicitly, enable it by default959 if (!params.default_template_kwargs.count("preserve_reasoning")) {960 params.default_template_kwargs["preserve_reasoning"] = "true";961 }962 963 return true;964}965 966static void common_params_print_usage(common_params_context & ctx_arg) {967 auto print_options = [](std::vector<common_arg *> & options) {968 for (common_arg * opt : options) {969 printf("%s", opt->to_string().c_str());970 }971 };972 973 std::vector<common_arg *> common_options;974 std::vector<common_arg *> sampling_options;975 std::vector<common_arg *> spec_options;976 std::vector<common_arg *> specific_options;977 for (auto & opt : ctx_arg.options) {978 // in case multiple LLAMA_EXAMPLE_* are set, we prioritize the LLAMA_EXAMPLE_* matching current example979 if (opt.is_sampling) {980 sampling_options.push_back(&opt);981 } else if (opt.is_spec) {982 spec_options.push_back(&opt);983 } else if (opt.in_example(ctx_arg.ex)) {984 specific_options.push_back(&opt);985 } else {986 common_options.push_back(&opt);987 }988 }989 bool first = true;990 auto print_section = [&](const char * header, std::vector<common_arg *> & options) {991 if (options.empty()) {992 return;993 }994 printf("%s----- %s -----\n\n", first ? "" : "\n\n", header);995 first = false;996 print_options(options);997 };998 print_section("common params", common_options);999 print_section("sampling params", sampling_options);1000 print_section("speculative params", spec_options);1001 print_section("example-specific params", specific_options);1002}1003 1004static void common_params_print_completion(common_params_context & ctx_arg) {1005 std::vector<common_arg *> common_options;1006 std::vector<common_arg *> sampling_options;1007 std::vector<common_arg *> spec_options;1008 std::vector<common_arg *> specific_options;1009 1010 for (auto & opt : ctx_arg.options) {1011 if (opt.is_sampling) {1012 sampling_options.push_back(&opt);1013 } else if (opt.is_spec) {1014 spec_options.push_back(&opt);1015 } else if (opt.in_example(ctx_arg.ex)) {1016 specific_options.push_back(&opt);1017 } else {1018 common_options.push_back(&opt);1019 }1020 }1021 1022 printf("_llama_completions() {\n");1023 printf(" local cur prev opts\n");1024 printf(" COMPREPLY=()\n");1025 printf(" cur=\"${COMP_WORDS[COMP_CWORD]}\"\n");1026 printf(" prev=\"${COMP_WORDS[COMP_CWORD-1]}\"\n\n");1027 1028 printf(" opts=\"");1029 auto print_options = [](const std::vector<common_arg *> & options) {1030 for (const common_arg * opt : options) {1031 for (const char * arg : opt->args) {1032 printf("%s ", arg);1033 }1034 }1035 };1036 1037 print_options(common_options);1038 print_options(sampling_options);1039 print_options(spec_options);1040 print_options(specific_options);1041 printf("\"\n\n");1042 1043 printf(" case \"$prev\" in\n");1044 printf(" --model|-m)\n");1045 printf(" COMPREPLY=( $(compgen -f -X '!*.gguf' -- \"$cur\") $(compgen -d -- \"$cur\") )\n");1046 printf(" return 0\n");1047 printf(" ;;\n");1048 printf(" --grammar-file)\n");1049 printf(" COMPREPLY=( $(compgen -f -X '!*.gbnf' -- \"$cur\") $(compgen -d -- \"$cur\") )\n");1050 printf(" return 0\n");1051 printf(" ;;\n");1052 printf(" --chat-template-file)\n");1053 printf(" COMPREPLY=( $(compgen -f -X '!*.jinja' -- \"$cur\") $(compgen -d -- \"$cur\") )\n");1054 printf(" return 0\n");1055 printf(" ;;\n");1056 printf(" *)\n");1057 printf(" COMPREPLY=( $(compgen -W \"${opts}\" -- \"$cur\") )\n");1058 printf(" return 0\n");1059 printf(" ;;\n");1060 printf(" esac\n");1061 printf("}\n\n");1062 1063 std::set<std::string> executables = {1064 "llama-batched",1065 "llama-batched-bench",1066 "llama-bench",1067 "llama-cli",1068 "llama-completion",1069 "llama-convert-llama2c-to-ggml",1070 "llama-cvector-generator",1071 "llama-debug",1072 "llama-diffusion-cli",1073 "llama-embedding",1074 "llama-eval-callback",1075 "llama-export-lora",1076 "llama-finetune",1077 "llama-fit-params",1078 "llama-gemma3-cli",1079 "llama-gen-docs",1080 "llama-gguf",1081 "llama-gguf-hash",1082 "llama-gguf-split",1083 "llama-idle",1084 "llama-imatrix",1085 "llama-llava-cli",1086 "llama-lookahead",1087 "llama-lookup",1088 "llama-lookup-create",1089 "llama-lookup-merge",1090 "llama-lookup-stats",1091 "llama-minicpmv-cli",1092 "llama-mtmd-cli",1093 "llama-parallel",1094 "llama-passkey",1095 "llama-perplexity",1096 "llama-q8dot",1097 "llama-quantize",1098 "llama-qwen2vl-cli",1099 "llama-retrieval",1100 "llama-save-load-state",1101 "llama-server",1102 "llama-simple",1103 "llama-simple-chat",1104 "llama-speculative",1105 "llama-speculative-simple",1106 "llama-tokenize",1107 "llama-tts",1108 "llama-vdot"1109 };1110 1111 for (const auto& exe : executables) {1112 printf("complete -F _llama_completions %s\n", exe.c_str());1113 }1114}1115 1116static std::vector<ggml_backend_dev_t> parse_device_list(const std::string & value) {1117 std::vector<ggml_backend_dev_t> devices;1118 auto dev_names = string_split<std::string>(value, ',');1119 if (dev_names.empty()) {1120 throw std::invalid_argument("no devices specified");1121 }1122 if (dev_names.size() == 1 && dev_names[0] == "none") {1123 devices.push_back(nullptr);1124 } else {1125 ggml_backend_load_all();1126 for (const auto & device : dev_names) {1127 auto * dev = ggml_backend_dev_by_name(device.c_str());1128 if (!dev || ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {1129 throw std::invalid_argument(string_format("invalid device: %s", device.c_str()));1130 }1131 devices.push_back(dev);1132 }1133 devices.push_back(nullptr);1134 }1135 return devices;1136}1137 1138void common_print_available_devices() {1139 constexpr size_t MiB = 1024 * 1024;1140 std::vector<ggml_backend_dev_t> devices;1141 1142 ggml_backend_load_all();1143 1144 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {1145 auto * dev = ggml_backend_dev_get(i);1146 if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) {1147 devices.push_back(dev);1148 }1149 }1150 printf("Available devices:\n");1151 1152 if (devices.empty()) {1153 printf(" (none)\n");1154 return;1155 }1156 for (auto * dev : devices) {1157 size_t free, total;1158 ggml_backend_dev_memory(dev, &free, &total);1159 printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / MiB, free / MiB);1160 }1161}1162 1163static void add_rpc_devices(const std::string & servers) {1164 auto rpc_servers = string_split<std::string>(servers, ',');1165 if (rpc_servers.empty()) {1166 throw std::invalid_argument("no RPC servers specified");1167 }1168 ggml_backend_load_all();1169 ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC");1170 if (!rpc_reg) {1171 throw std::invalid_argument("failed to find RPC backend");1172 }1173 typedef ggml_backend_reg_t (*ggml_backend_rpc_add_server_t)(const char * endpoint);1174 ggml_backend_rpc_add_server_t ggml_backend_rpc_add_server_fn = (ggml_backend_rpc_add_server_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server");1175 if (!ggml_backend_rpc_add_server_fn) {1176 throw std::invalid_argument("failed to find RPC add server function");1177 }1178 for (const auto & server : rpc_servers) {1179 auto reg = ggml_backend_rpc_add_server_fn(server.c_str());1180 ggml_backend_register(reg);1181 }1182}1183 1184bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<common_arg, std::string> & out_map) {1185 common_params dummy_params;1186 common_params_context ctx_arg = common_params_parser_init(dummy_params, ex, nullptr);1187 1188 std::unordered_map<std::string, common_arg *> arg_to_options;1189 for (auto & opt : ctx_arg.options) {1190 for (const auto & arg : opt.args) {1191 arg_to_options[arg] = &opt;1192 }1193 for (const auto & arg : opt.args_neg) {1194 arg_to_options[arg] = &opt;1195 }1196 }1197 1198 // TODO @ngxson : find a way to deduplicate this code1199 1200 // handle command line arguments