Codeprocastinator/optimized-tinyllama-covalent
0119
1#include "gguf.h" // for reading GGUF splits2#include "arg.h"3 4#include "common.h"5#include "log.h"6#include "sampling.h"7#include "chat.h"8 9// fix problem with std::min and std::max10#if defined(_WIN32)11#define WIN32_LEAN_AND_MEAN12#ifndef NOMINMAX13# define NOMINMAX14#endif15#include <windows.h>16#endif17 18#include <algorithm>19#include <climits>20#include <cstdarg>21#include <filesystem>22#include <fstream>23#include <regex>24#include <set>25#include <string>26#include <thread>27#include <vector>28 29//#define LLAMA_USE_CURL30 31#if defined(LLAMA_USE_CURL)32#include <curl/curl.h>33#include <curl/easy.h>34#include <future>35#endif36 37#include "json-schema-to-grammar.h"38 39using json = nlohmann::ordered_json;40 41common_arg & common_arg::set_examples(std::initializer_list<enum llama_example> examples) {42 this->examples = std::move(examples);43 return *this;44}45 46common_arg & common_arg::set_excludes(std::initializer_list<enum llama_example> excludes) {47 this->excludes = std::move(excludes);48 return *this;49}50 51common_arg & common_arg::set_env(const char * env) {52 help = help + "\n(env: " + env + ")";53 this->env = env;54 return *this;55}56 57common_arg & common_arg::set_sparam() {58 is_sparam = true;59 return *this;60}61 62bool common_arg::in_example(enum llama_example ex) {63 return examples.find(ex) != examples.end();64}65 66bool common_arg::is_exclude(enum llama_example ex) {67 return excludes.find(ex) != excludes.end();68}69 70bool common_arg::get_value_from_env(std::string & output) {71 if (env == nullptr) return false;72 char * value = std::getenv(env);73 if (value) {74 output = value;75 return true;76 }77 return false;78}79 80bool common_arg::has_value_from_env() {81 return env != nullptr && std::getenv(env);82}83 84static std::vector<std::string> break_str_into_lines(std::string input, size_t max_char_per_line) {85 std::vector<std::string> result;86 std::istringstream iss(input);87 std::string line;88 auto add_line = [&](const std::string& l) {89 if (l.length() <= max_char_per_line) {90 result.push_back(l);91 } else {92 std::istringstream line_stream(l);93 std::string word, current_line;94 while (line_stream >> word) {95 if (current_line.length() + !current_line.empty() + word.length() > max_char_per_line) {96 if (!current_line.empty()) result.push_back(current_line);97 current_line = word;98 } else {99 current_line += (!current_line.empty() ? " " : "") + word;100 }101 }102 if (!current_line.empty()) result.push_back(current_line);103 }104 };105 while (std::getline(iss, line)) {106 add_line(line);107 }108 return result;109}110 111std::string common_arg::to_string() {112 // params for printing to console113 const static int n_leading_spaces = 40;114 const static int n_char_per_line_help = 70; // TODO: detect this based on current console115 std::string leading_spaces(n_leading_spaces, ' ');116 117 std::ostringstream ss;118 for (const auto arg : args) {119 if (arg == args.front()) {120 if (args.size() == 1) {121 ss << arg;122 } else {123 // first arg is usually abbreviation, we need padding to make it more beautiful124 auto tmp = std::string(arg) + ", ";125 auto spaces = std::string(std::max(0, 7 - (int)tmp.size()), ' ');126 ss << tmp << spaces;127 }128 } else {129 ss << arg << (arg != args.back() ? ", " : "");130 }131 }132 if (value_hint) ss << " " << value_hint;133 if (value_hint_2) ss << " " << value_hint_2;134 if (ss.tellp() > n_leading_spaces - 3) {135 // current line is too long, add new line136 ss << "\n" << leading_spaces;137 } else {138 // padding between arg and help, same line139 ss << std::string(leading_spaces.size() - ss.tellp(), ' ');140 }141 const auto help_lines = break_str_into_lines(help, n_char_per_line_help);142 for (const auto & line : help_lines) {143 ss << (&line == &help_lines.front() ? "" : leading_spaces) << line << "\n";144 }145 return ss.str();146}147 148//149// downloader150//151 152struct common_hf_file_res {153 std::string repo; // repo name with ":tag" removed154 std::string ggufFile;155 std::string mmprojFile;156};157 158#ifdef LLAMA_USE_CURL159 160#ifdef __linux__161#include <linux/limits.h>162#elif defined(_WIN32)163# if !defined(PATH_MAX)164# define PATH_MAX MAX_PATH165# endif166#elif defined(_AIX)167#include <sys/limits.h>168#else169#include <sys/syslimits.h>170#endif171#define LLAMA_CURL_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083172 173//174// CURL utils175//176 177using curl_ptr = std::unique_ptr<CURL, decltype(&curl_easy_cleanup)>;178 179// cannot use unique_ptr for curl_slist, because we cannot update without destroying the old one180struct curl_slist_ptr {181 struct curl_slist * ptr = nullptr;182 ~curl_slist_ptr() {183 if (ptr) {184 curl_slist_free_all(ptr);185 }186 }187};188 189#define CURL_MAX_RETRY 3190#define CURL_RETRY_DELAY_SECONDS 2191 192static bool curl_perform_with_retry(const std::string & url, CURL * curl, int max_attempts, int retry_delay_seconds) {193 int remaining_attempts = max_attempts;194 195 while (remaining_attempts > 0) {196 LOG_INF("%s: Trying to download from %s (attempt %d of %d)...\n", __func__ , url.c_str(), max_attempts - remaining_attempts + 1, max_attempts);197 198 CURLcode res = curl_easy_perform(curl);199 if (res == CURLE_OK) {200 return true;201 }202 203 int exponential_backoff_delay = std::pow(retry_delay_seconds, max_attempts - remaining_attempts) * 1000;204 LOG_WRN("%s: curl_easy_perform() failed: %s, retrying after %d milliseconds...\n", __func__, curl_easy_strerror(res), exponential_backoff_delay);205 206 remaining_attempts--;207 std::this_thread::sleep_for(std::chrono::milliseconds(exponential_backoff_delay));208 }209 210 LOG_ERR("%s: curl_easy_perform() failed after %d attempts\n", __func__, max_attempts);211 212 return false;213}214 215// download one single file from remote URL to local path216static bool common_download_file_single(const std::string & url, const std::string & path, const std::string & bearer_token) {217 // Initialize libcurl218 curl_ptr curl(curl_easy_init(), &curl_easy_cleanup);219 curl_slist_ptr http_headers;220 if (!curl) {221 LOG_ERR("%s: error initializing libcurl\n", __func__);222 return false;223 }224 225 bool force_download = false;226 227 // Set the URL, allow to follow http redirection228 curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());229 curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);230 231 http_headers.ptr = curl_slist_append(http_headers.ptr, "User-Agent: llama-cpp");232 // Check if hf-token or bearer-token was specified233 if (!bearer_token.empty()) {234 std::string auth_header = "Authorization: Bearer " + bearer_token;235 http_headers.ptr = curl_slist_append(http_headers.ptr, auth_header.c_str());236 }237 curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, http_headers.ptr);238 239#if defined(_WIN32)240 // CURLSSLOPT_NATIVE_CA tells libcurl to use standard certificate store of241 // operating system. Currently implemented under MS-Windows.242 curl_easy_setopt(curl.get(), CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA);243#endif244 245 // Check if the file already exists locally246 auto file_exists = std::filesystem::exists(path);247 248 // If the file exists, check its JSON metadata companion file.249 std::string metadata_path = path + ".json";250 nlohmann::json metadata;251 std::string etag;252 std::string last_modified;253 254 if (file_exists) {255 // Try and read the JSON metadata file (note: stream autoclosed upon exiting this block).256 std::ifstream metadata_in(metadata_path);257 if (metadata_in.good()) {258 try {259 metadata_in >> metadata;260 LOG_INF("%s: previous metadata file found %s: %s\n", __func__, metadata_path.c_str(), metadata.dump().c_str());261 if (metadata.contains("url") && metadata.at("url").is_string()) {262 auto previous_url = metadata.at("url").get<std::string>();263 if (previous_url != url) {264 LOG_ERR("%s: Model URL mismatch: %s != %s\n", __func__, url.c_str(), previous_url.c_str());265 return false;266 }267 }268 if (metadata.contains("etag") && metadata.at("etag").is_string()) {269 etag = metadata.at("etag");270 }271 if (metadata.contains("lastModified") && metadata.at("lastModified").is_string()) {272 last_modified = metadata.at("lastModified");273 }274 } catch (const nlohmann::json::exception & e) {275 LOG_ERR("%s: error reading metadata file %s: %s\n", __func__, metadata_path.c_str(), e.what());276 return false;277 }278 }279 } else {280 LOG_INF("%s: no previous model file found %s\n", __func__, path.c_str());281 }282 283 // Send a HEAD request to retrieve the etag and last-modified headers284 struct common_load_model_from_url_headers {285 std::string etag;286 std::string last_modified;287 };288 289 common_load_model_from_url_headers headers;290 291 {292 typedef size_t(*CURLOPT_HEADERFUNCTION_PTR)(char *, size_t, size_t, void *);293 auto header_callback = [](char * buffer, size_t /*size*/, size_t n_items, void * userdata) -> size_t {294 common_load_model_from_url_headers * headers = (common_load_model_from_url_headers *) userdata;295 296 static std::regex header_regex("([^:]+): (.*)\r\n");297 static std::regex etag_regex("ETag", std::regex_constants::icase);298 static std::regex last_modified_regex("Last-Modified", std::regex_constants::icase);299 300 std::string header(buffer, n_items);301 std::smatch match;302 if (std::regex_match(header, match, header_regex)) {303 const std::string & key = match[1];304 const std::string & value = match[2];305 if (std::regex_match(key, match, etag_regex)) {306 headers->etag = value;307 } else if (std::regex_match(key, match, last_modified_regex)) {308 headers->last_modified = value;309 }310 }311 return n_items;312 };313 314 curl_easy_setopt(curl.get(), CURLOPT_NOBODY, 1L); // will trigger the HEAD verb315 curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 1L); // hide head request progress316 curl_easy_setopt(curl.get(), CURLOPT_HEADERFUNCTION, static_cast<CURLOPT_HEADERFUNCTION_PTR>(header_callback));317 curl_easy_setopt(curl.get(), CURLOPT_HEADERDATA, &headers);318 319 bool was_perform_successful = curl_perform_with_retry(url, curl.get(), CURL_MAX_RETRY, CURL_RETRY_DELAY_SECONDS);320 if (!was_perform_successful) {321 return false;322 }323 324 long http_code = 0;325 curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &http_code);326 if (http_code != 200) {327 // HEAD not supported, we don't know if the file has changed328 // force trigger downloading329 force_download = true;330 LOG_ERR("%s: HEAD invalid http status code received: %ld\n", __func__, http_code);331 }332 }333 334 bool should_download = !file_exists || force_download;335 if (!should_download) {336 if (!etag.empty() && etag != headers.etag) {337 LOG_WRN("%s: ETag header is different (%s != %s): triggering a new download\n", __func__, etag.c_str(), headers.etag.c_str());338 should_download = true;339 } else if (!last_modified.empty() && last_modified != headers.last_modified) {340 LOG_WRN("%s: Last-Modified header is different (%s != %s): triggering a new download\n", __func__, last_modified.c_str(), headers.last_modified.c_str());341 should_download = true;342 }343 }344 if (should_download) {345 std::string path_temporary = path + ".downloadInProgress";346 if (file_exists) {347 LOG_WRN("%s: deleting previous downloaded file: %s\n", __func__, path.c_str());348 if (remove(path.c_str()) != 0) {349 LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str());350 return false;351 }352 }353 354 // Set the output file355 356 struct FILE_deleter {357 void operator()(FILE * f) const {358 fclose(f);359 }360 };361 362 std::unique_ptr<FILE, FILE_deleter> outfile(fopen(path_temporary.c_str(), "wb"));363 if (!outfile) {364 LOG_ERR("%s: error opening local file for writing: %s\n", __func__, path.c_str());365 return false;366 }367 368 typedef size_t(*CURLOPT_WRITEFUNCTION_PTR)(void * data, size_t size, size_t nmemb, void * fd);369 auto write_callback = [](void * data, size_t size, size_t nmemb, void * fd) -> size_t {370 return fwrite(data, size, nmemb, (FILE *)fd);371 };372 curl_easy_setopt(curl.get(), CURLOPT_NOBODY, 0L);373 curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, static_cast<CURLOPT_WRITEFUNCTION_PTR>(write_callback));374 curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, outfile.get());375 376 // display download progress377 curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 0L);378 379 // helper function to hide password in URL380 auto llama_download_hide_password_in_url = [](const std::string & url) -> std::string {381 std::size_t protocol_pos = url.find("://");382 if (protocol_pos == std::string::npos) {383 return url; // Malformed URL384 }385 386 std::size_t at_pos = url.find('@', protocol_pos + 3);387 if (at_pos == std::string::npos) {388 return url; // No password in URL389 }390 391 return url.substr(0, protocol_pos + 3) + "********" + url.substr(at_pos);392 };393 394 // start the download395 LOG_INF("%s: trying to download model from %s to %s (server_etag:%s, server_last_modified:%s)...\n", __func__,396 llama_download_hide_password_in_url(url).c_str(), path.c_str(), headers.etag.c_str(), headers.last_modified.c_str());397 bool was_perform_successful = curl_perform_with_retry(url, curl.get(), CURL_MAX_RETRY, CURL_RETRY_DELAY_SECONDS);398 if (!was_perform_successful) {399 return false;400 }401 402 long http_code = 0;403 curl_easy_getinfo (curl.get(), CURLINFO_RESPONSE_CODE, &http_code);404 if (http_code < 200 || http_code >= 400) {405 LOG_ERR("%s: invalid http status code received: %ld\n", __func__, http_code);406 return false;407 }408 409 // Causes file to be closed explicitly here before we rename it.410 outfile.reset();411 412 // Write the updated JSON metadata file.413 metadata.update({414 {"url", url},415 {"etag", headers.etag},416 {"lastModified", headers.last_modified}417 });418 std::ofstream(metadata_path) << metadata.dump(4);419 LOG_INF("%s: file metadata saved: %s\n", __func__, metadata_path.c_str());420 421 if (rename(path_temporary.c_str(), path.c_str()) != 0) {422 LOG_ERR("%s: unable to rename file: %s to %s\n", __func__, path_temporary.c_str(), path.c_str());423 return false;424 }425 }426 427 return true;428}429 430// download multiple files from remote URLs to local paths431// the input is a vector of pairs <url, path>432static bool common_download_file_multiple(const std::vector<std::pair<std::string, std::string>> & urls, const std::string & bearer_token) {433 // Prepare download in parallel434 std::vector<std::future<bool>> futures_download;435 for (auto const & item : urls) {436 futures_download.push_back(std::async(std::launch::async, [bearer_token](const std::pair<std::string, std::string> & it) -> bool {437 return common_download_file_single(it.first, it.second, bearer_token);438 }, item));439 }440 441 // Wait for all downloads to complete442 for (auto & f : futures_download) {443 if (!f.get()) {444 return false;445 }446 }447 448 return true;449}450 451static bool common_download_model(452 const common_params_model & model,453 const std::string & bearer_token) {454 // Basic validation of the model.url455 if (model.url.empty()) {456 LOG_ERR("%s: invalid model url\n", __func__);457 return false;458 }459 460 if (!common_download_file_single(model.url, model.path, bearer_token)) {461 return false;462 }463 464 // check for additional GGUFs split to download465 int n_split = 0;466 {467 struct gguf_init_params gguf_params = {468 /*.no_alloc = */ true,469 /*.ctx = */ NULL,470 };471 auto * ctx_gguf = gguf_init_from_file(model.path.c_str(), gguf_params);472 if (!ctx_gguf) {473 LOG_ERR("\n%s: failed to load input GGUF from %s\n", __func__, model.path.c_str());474 return false;475 }476 477 auto key_n_split = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_COUNT);478 if (key_n_split >= 0) {479 n_split = gguf_get_val_u16(ctx_gguf, key_n_split);480 }481 482 gguf_free(ctx_gguf);483 }484 485 if (n_split > 1) {486 char split_prefix[PATH_MAX] = {0};487 char split_url_prefix[LLAMA_CURL_MAX_URL_LENGTH] = {0};488 489 // Verify the first split file format490 // and extract split URL and PATH prefixes491 {492 if (!llama_split_prefix(split_prefix, sizeof(split_prefix), model.path.c_str(), 0, n_split)) {493 LOG_ERR("\n%s: unexpected model file name: %s n_split=%d\n", __func__, model.path.c_str(), n_split);494 return false;495 }496 497 if (!llama_split_prefix(split_url_prefix, sizeof(split_url_prefix), model.url.c_str(), 0, n_split)) {498 LOG_ERR("\n%s: unexpected model url: %s n_split=%d\n", __func__, model.url.c_str(), n_split);499 return false;500 }501 }502 503 std::vector<std::pair<std::string, std::string>> urls;504 for (int idx = 1; idx < n_split; idx++) {505 char split_path[PATH_MAX] = {0};506 llama_split_path(split_path, sizeof(split_path), split_prefix, idx, n_split);507 508 char split_url[LLAMA_CURL_MAX_URL_LENGTH] = {0};509 llama_split_path(split_url, sizeof(split_url), split_url_prefix, idx, n_split);510 511 if (std::string(split_path) == model.path) {512 continue; // skip the already downloaded file513 }514 515 urls.push_back({split_url, split_path});516 }517 518 // Download in parallel519 common_download_file_multiple(urls, bearer_token);520 }521 522 return true;523}524 525/**526 * Allow getting the HF file from the HF repo with tag (like ollama), for example:527 * - bartowski/Llama-3.2-3B-Instruct-GGUF:q4528 * - bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M529 * - bartowski/Llama-3.2-3B-Instruct-GGUF:q5_k_s530 * Tag is optional, default to "latest" (meaning it checks for Q4_K_M first, then Q4, then if not found, return the first GGUF file in repo)531 *532 * Return pair of <repo, file> (with "repo" already having tag removed)533 *534 * Note: we use the Ollama-compatible HF API, but not using the blobId. Instead, we use the special "ggufFile" field which returns the value for "hf_file". This is done to be backward-compatible with existing cache files.535 */536static struct common_hf_file_res common_get_hf_file(const std::string & hf_repo_with_tag, const std::string & bearer_token) {537 auto parts = string_split<std::string>(hf_repo_with_tag, ':');538 std::string tag = parts.size() > 1 ? parts.back() : "latest";539 std::string hf_repo = parts[0];540 if (string_split<std::string>(hf_repo, '/').size() != 2) {541 throw std::invalid_argument("error: invalid HF repo format, expected <user>/<model>[:quant]\n");542 }543 544 // fetch model info from Hugging Face Hub API545 curl_ptr curl(curl_easy_init(), &curl_easy_cleanup);546 curl_slist_ptr http_headers;547 std::string res_str;548 549 std::string model_endpoint = get_model_endpoint();550 551 std::string url = model_endpoint + "v2/" + hf_repo + "/manifests/" + tag;552 curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());553 curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 1L);554 typedef size_t(*CURLOPT_WRITEFUNCTION_PTR)(void * ptr, size_t size, size_t nmemb, void * data);555 auto write_callback = [](void * ptr, size_t size, size_t nmemb, void * data) -> size_t {556 static_cast<std::string *>(data)->append((char * ) ptr, size * nmemb);557 return size * nmemb;558 };559 curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, static_cast<CURLOPT_WRITEFUNCTION_PTR>(write_callback));560 curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &res_str);561#if defined(_WIN32)562 curl_easy_setopt(curl.get(), CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA);563#endif564 if (!bearer_token.empty()) {565 std::string auth_header = "Authorization: Bearer " + bearer_token;566 http_headers.ptr = curl_slist_append(http_headers.ptr, auth_header.c_str());567 }568 // Important: the User-Agent must be "llama-cpp" to get the "ggufFile" field in the response569 http_headers.ptr = curl_slist_append(http_headers.ptr, "User-Agent: llama-cpp");570 http_headers.ptr = curl_slist_append(http_headers.ptr, "Accept: application/json");571 curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, http_headers.ptr);572 573 CURLcode res = curl_easy_perform(curl.get());574 575 if (res != CURLE_OK) {576 throw std::runtime_error("error: cannot make GET request to HF API");577 }578 579 long res_code;580 std::string ggufFile = "";581 std::string mmprojFile = "";582 curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &res_code);583 if (res_code == 200) {584 // extract ggufFile.rfilename in json, using regex585 {586 std::regex pattern("\"ggufFile\"[\\s\\S]*?\"rfilename\"\\s*:\\s*\"([^\"]+)\"");587 std::smatch match;588 if (std::regex_search(res_str, match, pattern)) {589 ggufFile = match[1].str();590 }591 }592 // extract mmprojFile.rfilename in json, using regex593 {594 std::regex pattern("\"mmprojFile\"[\\s\\S]*?\"rfilename\"\\s*:\\s*\"([^\"]+)\"");595 std::smatch match;596 if (std::regex_search(res_str, match, pattern)) {597 mmprojFile = match[1].str();598 }599 }600 } else if (res_code == 401) {601 throw std::runtime_error("error: model is private or does not exist; if you are accessing a gated model, please provide a valid HF token");602 } else {603 throw std::runtime_error(string_format("error from HF API, response code: %ld, data: %s", res_code, res_str.c_str()));604 }605 606 // check response607 if (ggufFile.empty()) {608 throw std::runtime_error("error: model does not have ggufFile");609 }610 611 return { hf_repo, ggufFile, mmprojFile };612}613 614#else615 616static bool common_download_file_single(const std::string &, const std::string &, const std::string &) {617 LOG_ERR("error: built without CURL, cannot download model from internet\n");618 return false;619}620 621static bool common_download_file_multiple(const std::vector<std::pair<std::string, std::string>> &, const std::string &) {622 LOG_ERR("error: built without CURL, cannot download model from the internet\n");623 return false;624}625 626static bool common_download_model(627 const common_params_model &,628 const std::string &) {629 LOG_ERR("error: built without CURL, cannot download model from the internet\n");630 return false;631}632 633static struct common_hf_file_res common_get_hf_file(const std::string &, const std::string &) {634 LOG_ERR("error: built without CURL, cannot download model from the internet\n");635 return {};636}637 638#endif // LLAMA_USE_CURL639 640//641// utils642//643 644static void common_params_handle_model(645 struct common_params_model & model,646 const std::string & bearer_token,647 const std::string & model_path_default,648 bool is_mmproj = false) { // TODO: move is_mmproj to an enum when we have more files?649 // handle pre-fill default model path and url based on hf_repo and hf_file650 {651 if (!model.hf_repo.empty()) {652 // short-hand to avoid specifying --hf-file -> default it to --model653 if (model.hf_file.empty()) {654 if (model.path.empty()) {655 auto auto_detected = common_get_hf_file(model.hf_repo, bearer_token);656 if (auto_detected.repo.empty() || auto_detected.ggufFile.empty()) {657 exit(1); // built without CURL, error message already printed658 }659 model.hf_repo = auto_detected.repo;660 model.hf_file = is_mmproj ? auto_detected.mmprojFile : auto_detected.ggufFile;661 } else {662 model.hf_file = model.path;663 }664 }665 666 std::string model_endpoint = get_model_endpoint();667 model.url = model_endpoint + model.hf_repo + "/resolve/main/" + model.hf_file;668 // make sure model path is present (for caching purposes)669 if (model.path.empty()) {670 // this is to avoid different repo having same file name, or same file name in different subdirs671 std::string filename = model.hf_repo + "_" + model.hf_file;672 // to make sure we don't have any slashes in the filename673 string_replace_all(filename, "/", "_");674 model.path = fs_get_cache_file(filename);675 }676 677 } else if (!model.url.empty()) {678 if (model.path.empty()) {679 auto f = string_split<std::string>(model.url, '#').front();680 f = string_split<std::string>(f, '?').front();681 model.path = fs_get_cache_file(string_split<std::string>(f, '/').back());682 }683 684 } else if (model.path.empty()) {685 model.path = model_path_default;686 }687 }688 689 // then, download it if needed690 if (!model.url.empty()) {691 bool ok = common_download_model(model, bearer_token);692 if (!ok) {693 LOG_ERR("error: failed to download model from %s\n", model.url.c_str());694 exit(1);695 }696 }697}698 699const std::vector<ggml_type> kv_cache_types = {700 GGML_TYPE_F32,701 GGML_TYPE_F16,702 GGML_TYPE_BF16,703 GGML_TYPE_Q8_0,704 GGML_TYPE_Q4_0,705 GGML_TYPE_Q4_1,706 GGML_TYPE_IQ4_NL,707 GGML_TYPE_Q5_0,708 GGML_TYPE_Q5_1,709};710 711static ggml_type kv_cache_type_from_str(const std::string & s) {712 for (const auto & type : kv_cache_types) {713 if (ggml_type_name(type) == s) {714 return type;715 }716 }717 throw std::runtime_error("Unsupported cache type: " + s);718}719 720static std::string get_all_kv_cache_types() {721 std::ostringstream msg;722 for (const auto & type : kv_cache_types) {723 msg << ggml_type_name(type) << (&type == &kv_cache_types.back() ? "" : ", ");724 }725 return msg.str();726}727 728//729// CLI argument parsing functions730//731 732static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {733 std::string arg;734 const std::string arg_prefix = "--";735 common_params & params = ctx_arg.params;736 737 std::unordered_map<std::string, common_arg *> arg_to_options;738 for (auto & opt : ctx_arg.options) {739 for (const auto & arg : opt.args) {740 arg_to_options[arg] = &opt;741 }742 }743 744 // handle environment variables745 for (auto & opt : ctx_arg.options) {746 std::string value;747 if (opt.get_value_from_env(value)) {748 try {749 if (opt.handler_void && (value == "1" || value == "true")) {750 opt.handler_void(params);751 }752 if (opt.handler_int) {753 opt.handler_int(params, std::stoi(value));754 }755 if (opt.handler_string) {756 opt.handler_string(params, value);757 continue;758 }759 } catch (std::exception & e) {760 throw std::invalid_argument(string_format(761 "error while handling environment variable \"%s\": %s\n\n", opt.env, e.what()));762 }763 }764 }765 766 // handle command line arguments767 auto check_arg = [&](int i) {768 if (i+1 >= argc) {769 throw std::invalid_argument("expected value for argument");770 }771 };772 773 for (int i = 1; i < argc; i++) {774 const std::string arg_prefix = "--";775 776 std::string arg = argv[i];777 if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {778 std::replace(arg.begin(), arg.end(), '_', '-');779 }780 if (arg_to_options.find(arg) == arg_to_options.end()) {781 throw std::invalid_argument(string_format("error: invalid argument: %s", arg.c_str()));782 }783 auto opt = *arg_to_options[arg];784 if (opt.has_value_from_env()) {785 fprintf(stderr, "warn: %s environment variable is set, but will be overwritten by command line argument %s\n", opt.env, arg.c_str());786 }787 try {788 if (opt.handler_void) {789 opt.handler_void(params);790 continue;791 }792 793 // arg with single value794 check_arg(i);795 std::string val = argv[++i];796 if (opt.handler_int) {797 opt.handler_int(params, std::stoi(val));798 continue;799 }800 if (opt.handler_string) {801 opt.handler_string(params, val);802 continue;803 }804 805 // arg with 2 values806 check_arg(i);807 std::string val2 = argv[++i];808 if (opt.handler_str_str) {809 opt.handler_str_str(params, val, val2);810 continue;811 }812 } catch (std::exception & e) {813 throw std::invalid_argument(string_format(814 "error while handling argument \"%s\": %s\n\n"815 "usage:\n%s\n\nto show complete usage, run with -h",816 arg.c_str(), e.what(), arg_to_options[arg]->to_string().c_str()));817 }818 }819 820 postprocess_cpu_params(params.cpuparams, nullptr);821 postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams);822 823 postprocess_cpu_params(params.speculative.cpuparams, ¶ms.cpuparams);824 postprocess_cpu_params(params.speculative.cpuparams_batch, ¶ms.cpuparams_batch);825 826 if (params.prompt_cache_all && (params.interactive || params.interactive_first)) {827 throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");828 }829 830 common_params_handle_model(params.model, params.hf_token, DEFAULT_MODEL_PATH);831 common_params_handle_model(params.speculative.model, params.hf_token, "");832 common_params_handle_model(params.vocoder.model, params.hf_token, "");833 834 // allow --mmproj to be set from -hf835 // assuming that mmproj is always in the same repo as text model836 if (!params.model.hf_repo.empty() && ctx_arg.ex == LLAMA_EXAMPLE_LLAVA) {837 params.mmproj.hf_repo = params.model.hf_repo;838 }839 common_params_handle_model(params.mmproj, params.hf_token, "", true);840 841 if (params.escape) {842 string_process_escapes(params.prompt);843 string_process_escapes(params.input_prefix);844 string_process_escapes(params.input_suffix);845 for (auto & antiprompt : params.antiprompt) {846 string_process_escapes(antiprompt);847 }848 for (auto & seq_breaker : params.sampling.dry_sequence_breakers) {849 string_process_escapes(seq_breaker);850 }851 }852 853 if (!params.kv_overrides.empty()) {854 params.kv_overrides.emplace_back();855 params.kv_overrides.back().key[0] = 0;856 }857 858 if (!params.tensor_buft_overrides.empty()) {859 params.tensor_buft_overrides.push_back({nullptr, nullptr});860 }861 862 if (params.reranking && params.embedding) {863 throw std::invalid_argument("error: either --embedding or --reranking can be specified, but not both");864 }865 866 if (!params.chat_template.empty() && !common_chat_verify_template(params.chat_template, params.use_jinja)) {867 throw std::runtime_error(string_format(868 "error: the supplied chat template is not supported: %s%s\n",869 params.chat_template.c_str(),870 params.use_jinja ? "" : "\nnote: llama.cpp was started without --jinja, we only support commonly used templates"871 ));872 }873 874 return true;875}876 877static void common_params_print_usage(common_params_context & ctx_arg) {878 auto print_options = [](std::vector<common_arg *> & options) {879 for (common_arg * opt : options) {880 printf("%s", opt->to_string().c_str());881 }882 };883 884 std::vector<common_arg *> common_options;885 std::vector<common_arg *> sparam_options;886 std::vector<common_arg *> specific_options;887 for (auto & opt : ctx_arg.options) {888 // in case multiple LLAMA_EXAMPLE_* are set, we prioritize the LLAMA_EXAMPLE_* matching current example889 if (opt.is_sparam) {890 sparam_options.push_back(&opt);891 } else if (opt.in_example(ctx_arg.ex)) {892 specific_options.push_back(&opt);893 } else {894 common_options.push_back(&opt);895 }896 }897 printf("----- common params -----\n\n");898 print_options(common_options);899 printf("\n\n----- sampling params -----\n\n");900 print_options(sparam_options);901 // TODO: maybe convert enum llama_example to string902 printf("\n\n----- example-specific params -----\n\n");903 print_options(specific_options);904}905 906static void common_params_print_completion(common_params_context & ctx_arg) {907 std::vector<common_arg *> common_options;908 std::vector<common_arg *> sparam_options;909 std::vector<common_arg *> specific_options;910 911 for (auto & opt : ctx_arg.options) {912 if (opt.is_sparam) {913 sparam_options.push_back(&opt);914 } else if (opt.in_example(ctx_arg.ex)) {915 specific_options.push_back(&opt);916 } else {917 common_options.push_back(&opt);918 }919 }920 921 printf("_llama_completions() {\n");922 printf(" local cur prev opts\n");923 printf(" COMPREPLY=()\n");924 printf(" cur=\"${COMP_WORDS[COMP_CWORD]}\"\n");925 printf(" prev=\"${COMP_WORDS[COMP_CWORD-1]}\"\n\n");926 927 printf(" opts=\"");928 auto print_options = [](const std::vector<common_arg *> & options) {929 for (const common_arg * opt : options) {930 for (const char * arg : opt->args) {931 printf("%s ", arg);932 }933 }934 };935 936 print_options(common_options);937 print_options(sparam_options);938 print_options(specific_options);939 printf("\"\n\n");940 941 printf(" case \"$prev\" in\n");942 printf(" --model)\n");943 printf(" COMPREPLY=( $(compgen -f -X '!*.gguf' -- \"$cur\") $(compgen -d -- \"$cur\") )\n");944 printf(" return 0\n");945 printf(" ;;\n");946 printf(" --grammar-file)\n");947 printf(" COMPREPLY=( $(compgen -f -X '!*.gbnf' -- \"$cur\") $(compgen -d -- \"$cur\") )\n");948 printf(" return 0\n");949 printf(" ;;\n");950 printf(" --chat-template-file)\n");951 printf(" COMPREPLY=( $(compgen -f -X '!*.jinja' -- \"$cur\") $(compgen -d -- \"$cur\") )\n");952 printf(" return 0\n");953 printf(" ;;\n");954 printf(" *)\n");955 printf(" COMPREPLY=( $(compgen -W \"${opts}\" -- \"$cur\") )\n");956 printf(" return 0\n");957 printf(" ;;\n");958 printf(" esac\n");959 printf("}\n\n");960 961 std::set<std::string> executables = {962 "llama-batched",963 "llama-batched-bench",964 "llama-bench",965 "llama-cli",966 "llama-convert-llama2c-to-ggml",967 "llama-cvector-generator",968 "llama-embedding",969 "llama-eval-callback",970 "llama-export-lora",971 "llama-gbnf-validator",972 "llama-gen-docs",973 "llama-gguf",974 "llama-gguf-hash",975 "llama-gguf-split",976 "llama-gritlm",977 "llama-imatrix",978 "llama-infill",979 "llama-llava-cli",980 "llama-llava-clip-quantize-cli",981 "llama-lookahead",982 "llama-lookup",983 "llama-lookup-create",984 "llama-lookup-merge",985 "llama-lookup-stats",986 "llama-minicpmv-cli",987 "llama-parallel",988 "llama-passkey",989 "llama-perplexity",990 "llama-q8dot",991 "llama-quantize",992 "llama-quantize-stats",993 "llama-qwen2vl-cli",994 "llama-retrieval",995 "llama-run",996 "llama-save-load-state",997 "llama-server",998 "llama-simple",999 "llama-simple-chat",1000 "llama-speculative",1001 "llama-speculative-simple",1002 "llama-tokenize",1003 "llama-tts",1004 "llama-vdot"1005 };1006 1007 for (const auto& exe : executables) {1008 printf("complete -F _llama_completions %s\n", exe.c_str());1009 }1010}1011 1012static std::vector<ggml_backend_dev_t> parse_device_list(const std::string & value) {1013 std::vector<ggml_backend_dev_t> devices;1014 auto dev_names = string_split<std::string>(value, ',');1015 if (dev_names.empty()) {1016 throw std::invalid_argument("no devices specified");1017 }1018 if (dev_names.size() == 1 && dev_names[0] == "none") {1019 devices.push_back(nullptr);1020 } else {1021 for (const auto & device : dev_names) {1022 auto * dev = ggml_backend_dev_by_name(device.c_str());1023 if (!dev || ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {1024 throw std::invalid_argument(string_format("invalid device: %s", device.c_str()));1025 }1026 devices.push_back(dev);1027 }1028 devices.push_back(nullptr);1029 }1030 return devices;1031}1032 1033static void add_rpc_devices(std::string servers) {1034 auto rpc_servers = string_split<std::string>(servers, ',');1035 if (rpc_servers.empty()) {1036 throw std::invalid_argument("no RPC servers specified");1037 }1038 ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC");1039 if (!rpc_reg) {1040 throw std::invalid_argument("failed to find RPC backend");1041 }1042 typedef ggml_backend_dev_t (*ggml_backend_rpc_add_device_t)(const char * endpoint);1043 ggml_backend_rpc_add_device_t ggml_backend_rpc_add_device_fn = (ggml_backend_rpc_add_device_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_device");1044 if (!ggml_backend_rpc_add_device_fn) {1045 throw std::invalid_argument("failed to find RPC device add function");1046 }1047 for (const auto & server : rpc_servers) {1048 ggml_backend_dev_t dev = ggml_backend_rpc_add_device_fn(server.c_str());1049 if (dev) {1050 ggml_backend_device_register(dev);1051 } else {1052 throw std::invalid_argument("failed to register RPC device");1053 }1054 }1055}1056 1057bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) {1058 auto ctx_arg = common_params_parser_init(params, ex, print_usage);1059 const common_params params_org = ctx_arg.params; // the example can modify the default params1060 1061 try {1062 if (!common_params_parse_ex(argc, argv, ctx_arg)) {1063 ctx_arg.params = params_org;1064 return false;1065 }1066 if (ctx_arg.params.usage) {1067 common_params_print_usage(ctx_arg);1068 if (ctx_arg.print_usage) {1069 ctx_arg.print_usage(argc, argv);1070 }1071 exit(0);1072 }1073 if (ctx_arg.params.completion) {1074 common_params_print_completion(ctx_arg);1075 exit(0);1076 }1077 } catch (const std::invalid_argument & ex) {1078 fprintf(stderr, "%s\n", ex.what());1079 ctx_arg.params = params_org;1080 return false;1081 }1082 1083 return true;1084}1085 1086static std::string list_builtin_chat_templates() {1087 std::vector<const char *> supported_tmpl;1088 int32_t res = llama_chat_builtin_templates(nullptr, 0);1089 supported_tmpl.resize(res);1090 res = llama_chat_builtin_templates(supported_tmpl.data(), supported_tmpl.size());1091 std::ostringstream msg;1092 for (auto & tmpl : supported_tmpl) {1093 msg << tmpl << (&tmpl == &supported_tmpl.back() ? "" : ", ");1094 }1095 return msg.str();1096}1097 1098common_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **)) {1099 // load dynamic backends1100 ggml_backend_load_all();1101 1102 common_params_context ctx_arg(params);1103 ctx_arg.print_usage = print_usage;1104 ctx_arg.ex = ex;1105 1106 std::string sampler_type_chars;1107 std::string sampler_type_names;1108 for (const auto & sampler : params.sampling.samplers) {1109 sampler_type_chars += common_sampler_type_to_chr(sampler);1110 sampler_type_names += common_sampler_type_to_str(sampler) + ";";1111 }1112 sampler_type_names.pop_back();1113 1114 1115 /**1116 * filter options by example1117 * rules:1118 * - all examples inherit options from LLAMA_EXAMPLE_COMMON1119 * - if LLAMA_EXAMPLE_* is set (other than COMMON), we only show the option in the corresponding example1120 * - if both {LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_*,} are set, we will prioritize the LLAMA_EXAMPLE_* matching current example1121 */1122 auto add_opt = [&](common_arg arg) {1123 if ((arg.in_example(ex) || arg.in_example(LLAMA_EXAMPLE_COMMON)) && !arg.is_exclude(ex)) {1124 ctx_arg.options.push_back(std::move(arg));1125 }1126 };1127 1128 1129 add_opt(common_arg(1130 {"-h", "--help", "--usage"},1131 "print usage and exit",1132 [](common_params & params) {1133 params.usage = true;1134 }1135 ));1136 add_opt(common_arg(1137 {"--version"},1138 "show version and build info",1139 [](common_params &) {1140 fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT);1141 fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET);1142 exit(0);1143 }1144 ));1145 add_opt(common_arg(1146 {"--completion-bash"},1147 "print source-able bash completion script for llama.cpp",1148 [](common_params & params) {1149 params.completion = true;1150 }1151 ));1152 add_opt(common_arg(1153 {"--verbose-prompt"},1154 string_format("print a verbose prompt before generation (default: %s)", params.verbose_prompt ? "true" : "false"),1155 [](common_params & params) {1156 params.verbose_prompt = true;1157 }1158 ));1159 add_opt(common_arg(1160 {"--no-display-prompt"},1161 string_format("don't print prompt at generation (default: %s)", !params.display_prompt ? "true" : "false"),1162 [](common_params & params) {1163 params.display_prompt = false;1164 }1165 ).set_examples({LLAMA_EXAMPLE_MAIN}));1166 add_opt(common_arg(1167 {"-co", "--color"},1168 string_format("colorise output to distinguish prompt and user input from generations (default: %s)", params.use_color ? "true" : "false"),1169 [](common_params & params) {1170 params.use_color = true;1171 }1172 ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_INFILL, LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP}));1173 add_opt(common_arg(1174 {"-t", "--threads"}, "N",1175 string_format("number of threads to use during generation (default: %d)", params.cpuparams.n_threads),1176 [](common_params & params, int value) {1177 params.cpuparams.n_threads = value;1178 if (params.cpuparams.n_threads <= 0) {1179 params.cpuparams.n_threads = std::thread::hardware_concurrency();1180 }1181 }1182 ).set_env("LLAMA_ARG_THREADS"));1183 add_opt(common_arg(1184 {"-tb", "--threads-batch"}, "N",1185 "number of threads to use during batch and prompt processing (default: same as --threads)",1186 [](common_params & params, int value) {1187 params.cpuparams_batch.n_threads = value;1188 if (params.cpuparams_batch.n_threads <= 0) {1189 params.cpuparams_batch.n_threads = std::thread::hardware_concurrency();1190 }1191 }1192 ));1193 add_opt(common_arg(1194 {"-C", "--cpu-mask"}, "M",1195 "CPU affinity mask: arbitrarily long hex. Complements cpu-range (default: \"\")",1196 [](common_params & params, const std::string & mask) {1197 params.cpuparams.mask_valid = true;1198 if (!parse_cpu_mask(mask, params.cpuparams.cpumask)) {1199 throw std::invalid_argument("invalid cpumask");1200 }