Codeprocastinator/optimized-tinyllama-covalent
0119
1#if defined(_MSC_VER)2#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING3#endif4 5#include "ggml.h"6#include "gguf.h"7 8#include "common.h"9#include "log.h"10#include "llama.h"11 12#include <algorithm>13#include <cinttypes>14#include <climits>15#include <cmath>16#include <codecvt>17#include <cstdarg>18#include <cstring>19#include <ctime>20#include <filesystem>21#include <fstream>22#include <iostream>23#include <iterator>24#include <regex>25#include <sstream>26#include <string>27#include <thread>28#include <unordered_map>29#include <unordered_set>30#include <vector>31 32#if defined(__APPLE__) && defined(__MACH__)33#include <sys/types.h>34#include <sys/sysctl.h>35#endif36 37#if defined(_WIN32)38#define WIN32_LEAN_AND_MEAN39#ifndef NOMINMAX40# define NOMINMAX41#endif42#include <locale>43#include <windows.h>44#include <fcntl.h>45#include <io.h>46#else47#include <sys/ioctl.h>48#include <sys/stat.h>49#include <unistd.h>50#endif51 52#if defined(_MSC_VER)53#pragma warning(disable: 4244 4267) // possible loss of data54#endif55 56//57// CPU utils58//59 60int32_t cpu_get_num_physical_cores() {61#ifdef __linux__62 // enumerate the set of thread siblings, num entries is num cores63 std::unordered_set<std::string> siblings;64 for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {65 std::ifstream thread_siblings("/sys/devices/system/cpu/cpu"66 + std::to_string(cpu) + "/topology/thread_siblings");67 if (!thread_siblings.is_open()) {68 break; // no more cpus69 }70 std::string line;71 if (std::getline(thread_siblings, line)) {72 siblings.insert(line);73 }74 }75 if (!siblings.empty()) {76 return static_cast<int32_t>(siblings.size());77 }78#elif defined(__APPLE__) && defined(__MACH__)79 int32_t num_physical_cores;80 size_t len = sizeof(num_physical_cores);81 int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);82 if (result == 0) {83 return num_physical_cores;84 }85 result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);86 if (result == 0) {87 return num_physical_cores;88 }89#elif defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later90 // TODO: windows + arm64 + mingw6491 unsigned int n_threads_win = std::thread::hardware_concurrency();92 unsigned int default_threads = n_threads_win > 0 ? (n_threads_win <= 4 ? n_threads_win : n_threads_win / 2) : 4;93 94 DWORD buffer_size = 0;95 if (!GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &buffer_size)) {96 if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {97 return default_threads;98 }99 }100 101 std::vector<char> buffer(buffer_size);102 if (!GetLogicalProcessorInformationEx(RelationProcessorCore, reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data()), &buffer_size)) {103 return default_threads;104 }105 106 int32_t num_physical_cores = 0;107 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data());108 while (buffer_size > 0) {109 if (info->Relationship == RelationProcessorCore) {110 num_physical_cores += info->Processor.GroupCount;111 }112 buffer_size -= info->Size;113 info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(reinterpret_cast<char*>(info) + info->Size);114 }115 116 return num_physical_cores > 0 ? num_physical_cores : default_threads;117#endif118 unsigned int n_threads = std::thread::hardware_concurrency();119 return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;120}121 122#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)123#include <pthread.h>124 125static void cpuid(unsigned leaf, unsigned subleaf,126 unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) {127 __asm__("movq\t%%rbx,%%rsi\n\t"128 "cpuid\n\t"129 "xchgq\t%%rbx,%%rsi"130 : "=a"(*eax), "=S"(*ebx), "=c"(*ecx), "=d"(*edx)131 : "0"(leaf), "2"(subleaf));132}133 134static int pin_cpu(int cpu) {135 cpu_set_t mask;136 CPU_ZERO(&mask);137 CPU_SET(cpu, &mask);138 return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);139}140 141static bool is_hybrid_cpu(void) {142 unsigned eax, ebx, ecx, edx;143 cpuid(7, 0, &eax, &ebx, &ecx, &edx);144 return !!(edx & (1u << 15));145}146 147static bool is_running_on_efficiency_core(void) {148 unsigned eax, ebx, ecx, edx;149 cpuid(0x1a, 0, &eax, &ebx, &ecx, &edx);150 int intel_atom = 0x20;151 int core_type = (eax & 0xff000000u) >> 24;152 return core_type == intel_atom;153}154 155static int cpu_count_math_cpus(int n_cpu) {156 int result = 0;157 for (int cpu = 0; cpu < n_cpu; ++cpu) {158 if (pin_cpu(cpu)) {159 return -1;160 }161 if (is_running_on_efficiency_core()) {162 continue; // efficiency cores harm lockstep threading163 }164 ++cpu; // hyperthreading isn't useful for linear algebra165 ++result;166 }167 return result;168}169 170#endif // __x86_64__ && __linux__171 172/**173 * Returns number of CPUs on system that are useful for math.174 */175int32_t cpu_get_num_math() {176#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)177 int n_cpu = sysconf(_SC_NPROCESSORS_ONLN);178 if (n_cpu < 1) {179 return cpu_get_num_physical_cores();180 }181 if (is_hybrid_cpu()) {182 cpu_set_t affinity;183 if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) {184 int result = cpu_count_math_cpus(n_cpu);185 pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity);186 if (result > 0) {187 return result;188 }189 }190 }191#endif192 return cpu_get_num_physical_cores();193}194 195// Helper for setting process priority196 197#if defined(_WIN32)198 199bool set_process_priority(enum ggml_sched_priority prio) {200 if (prio == GGML_SCHED_PRIO_NORMAL) {201 return true;202 }203 204 DWORD p = NORMAL_PRIORITY_CLASS;205 switch (prio) {206 case GGML_SCHED_PRIO_NORMAL: p = NORMAL_PRIORITY_CLASS; break;207 case GGML_SCHED_PRIO_MEDIUM: p = ABOVE_NORMAL_PRIORITY_CLASS; break;208 case GGML_SCHED_PRIO_HIGH: p = HIGH_PRIORITY_CLASS; break;209 case GGML_SCHED_PRIO_REALTIME: p = REALTIME_PRIORITY_CLASS; break;210 }211 212 if (!SetPriorityClass(GetCurrentProcess(), p)) {213 LOG_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError());214 return false;215 }216 217 return true;218}219 220#else // MacOS and POSIX221#include <sys/types.h>222#include <sys/resource.h>223 224bool set_process_priority(enum ggml_sched_priority prio) {225 if (prio == GGML_SCHED_PRIO_NORMAL) {226 return true;227 }228 229 int p = 0;230 switch (prio) {231 case GGML_SCHED_PRIO_NORMAL: p = 0; break;232 case GGML_SCHED_PRIO_MEDIUM: p = -5; break;233 case GGML_SCHED_PRIO_HIGH: p = -10; break;234 case GGML_SCHED_PRIO_REALTIME: p = -20; break;235 }236 237 if (!setpriority(PRIO_PROCESS, 0, p)) {238 LOG_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno);239 return false;240 }241 return true;242}243 244#endif245 246//247// CLI argument parsing248//249 250 251void postprocess_cpu_params(cpu_params& cpuparams, const cpu_params* role_model) {252 int32_t n_set = 0;253 254 if (cpuparams.n_threads < 0) {255 // Assuming everything about cpuparams is invalid256 if (role_model != nullptr) {257 cpuparams = *role_model;258 } else {259 cpuparams.n_threads = cpu_get_num_math();260 }261 }262 263 for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {264 if (cpuparams.cpumask[i]) {265 n_set++;266 }267 }268 269 if (n_set && n_set < cpuparams.n_threads) {270 // Not enough set bits, may experience performance issues.271 LOG_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads);272 }273}274 275bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THREADS]) {276 size_t dash_loc = range.find('-');277 if (dash_loc == std::string::npos) {278 LOG_ERR("Format of CPU range is invalid! Expected [<start>]-[<end>].\n");279 return false;280 }281 282 size_t start_i;283 size_t end_i;284 285 if (dash_loc == 0) {286 start_i = 0;287 } else {288 start_i = std::stoull(range.substr(0, dash_loc));289 if (start_i >= GGML_MAX_N_THREADS) {290 LOG_ERR("Start index out of bounds!\n");291 return false;292 }293 }294 295 if (dash_loc == range.length() - 1) {296 end_i = GGML_MAX_N_THREADS - 1;297 } else {298 end_i = std::stoull(range.substr(dash_loc + 1));299 if (end_i >= GGML_MAX_N_THREADS) {300 LOG_ERR("End index out of bounds!\n");301 return false;302 }303 }304 305 for (size_t i = start_i; i <= end_i; i++) {306 boolmask[i] = true;307 }308 309 return true;310}311 312bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREADS]) {313 // Discard potential 0x prefix314 size_t start_i = 0;315 if (mask.length() >= 2 && mask.substr(0, 2) == "0x") {316 start_i = 2;317 }318 319 size_t num_digits = mask.length() - start_i;320 if (num_digits > 128) num_digits = 128;321 322 size_t end_i = num_digits + start_i;323 324 for (size_t i = start_i, n = (num_digits*4 - 1); i < end_i; i++, n-=4) {325 char c = mask.at(i);326 int8_t id = c;327 328 if ((c >= '0' && c <= '9')) {329 id -= '0';330 } else if (c >= 'a' && c <= 'f') {331 id -= 'a' - 10;332 } else if (c >= 'A' && c <= 'F') {333 id -= 'A' - 10;334 } else {335 LOG_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i));336 return false;337 }338 339 boolmask[ n ] = boolmask[ n ] || ((id & 8) != 0);340 boolmask[n - 1] = boolmask[n - 1] || ((id & 4) != 0);341 boolmask[n - 2] = boolmask[n - 2] || ((id & 2) != 0);342 boolmask[n - 3] = boolmask[n - 3] || ((id & 1) != 0);343 }344 345 return true;346}347 348void common_init() {349 llama_log_set([](ggml_log_level level, const char * text, void * /*user_data*/) {350 if (LOG_DEFAULT_LLAMA <= common_log_verbosity_thold) {351 common_log_add(common_log_main(), level, "%s", text);352 }353 }, NULL);354 355#ifdef NDEBUG356 const char * build_type = "";357#else358 const char * build_type = " (debug)";359#endif360 361 LOG_INF("build: %d (%s) with %s for %s%s\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT, LLAMA_COMPILER, LLAMA_BUILD_TARGET, build_type);362}363 364std::string common_params_get_system_info(const common_params & params) {365 std::ostringstream os;366 367 os << "system_info: n_threads = " << params.cpuparams.n_threads;368 if (params.cpuparams_batch.n_threads != -1) {369 os << " (n_threads_batch = " << params.cpuparams_batch.n_threads << ")";370 }371#if defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later372 // TODO: windows + arm64 + mingw64373 DWORD logicalProcessorCount = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);374 os << " / " << logicalProcessorCount << " | " << llama_print_system_info();375#else376 os << " / " << std::thread::hardware_concurrency() << " | " << llama_print_system_info();377#endif378 379 return os.str();380}381 382//383// String utils384//385 386std::string string_format(const char * fmt, ...) {387 va_list ap;388 va_list ap2;389 va_start(ap, fmt);390 va_copy(ap2, ap);391 int size = vsnprintf(NULL, 0, fmt, ap);392 GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT393 std::vector<char> buf(size + 1);394 int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);395 GGML_ASSERT(size2 == size);396 va_end(ap2);397 va_end(ap);398 return std::string(buf.data(), size);399}400 401std::string string_strip(const std::string & str) {402 size_t start = 0;403 size_t end = str.size();404 while (start < end && std::isspace(str[start])) {405 start++;406 }407 while (end > start && std::isspace(str[end - 1])) {408 end--;409 }410 return str.substr(start, end - start);411}412 413std::string string_get_sortable_timestamp() {414 using clock = std::chrono::system_clock;415 416 const clock::time_point current_time = clock::now();417 const time_t as_time_t = clock::to_time_t(current_time);418 char timestamp_no_ns[100];419 std::strftime(timestamp_no_ns, 100, "%Y_%m_%d-%H_%M_%S", std::localtime(&as_time_t));420 421 const int64_t ns = std::chrono::duration_cast<std::chrono::nanoseconds>(422 current_time.time_since_epoch() % 1000000000).count();423 char timestamp_ns[11];424 snprintf(timestamp_ns, 11, "%09" PRId64, ns);425 426 return std::string(timestamp_no_ns) + "." + std::string(timestamp_ns);427}428 429void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {430 if (search.empty()) {431 return;432 }433 std::string builder;434 builder.reserve(s.length());435 size_t pos = 0;436 size_t last_pos = 0;437 while ((pos = s.find(search, last_pos)) != std::string::npos) {438 builder.append(s, last_pos, pos - last_pos);439 builder.append(replace);440 last_pos = pos + search.length();441 }442 builder.append(s, last_pos, std::string::npos);443 s = std::move(builder);444}445 446std::string regex_escape(const std::string & s) {447 static const std::regex special_chars("[.^$|()*+?\\[\\]{}\\\\]");448 return std::regex_replace(s, special_chars, "\\$0");449}450 451std::string string_join(const std::vector<std::string> & values, const std::string & separator) {452 std::ostringstream result;453 for (size_t i = 0; i < values.size(); ++i) {454 if (i > 0) {455 result << separator;456 }457 result << values[i];458 }459 return result.str();460}461 462std::vector<std::string> string_split(const std::string & str, const std::string & delimiter) {463 std::vector<std::string> parts;464 size_t start = 0;465 size_t end = str.find(delimiter);466 467 while (end != std::string::npos) {468 parts.push_back(str.substr(start, end - start));469 start = end + delimiter.length();470 end = str.find(delimiter, start);471 }472 473 parts.push_back(str.substr(start));474 475 return parts;476}477 478std::string string_repeat(const std::string & str, size_t n) {479 if (n == 0) {480 return "";481 }482 483 std::string result;484 result.reserve(str.length() * n);485 486 for (size_t i = 0; i < n; ++i) {487 result += str;488 }489 490 return result;491}492 493std::string string_from(bool value) {494 return value ? "true" : "false";495}496 497std::string string_from(const std::vector<int> & values) {498 std::stringstream buf;499 500 buf << "[ ";501 bool first = true;502 for (auto e : values) {503 if (first) {504 first = false;505 } else {506 buf << ", ";507 }508 buf << std::to_string(e);509 }510 buf << " ]";511 512 return buf.str();513}514 515std::string string_from(const struct llama_context * ctx, const std::vector<llama_token> & tokens) {516 std::stringstream buf;517 518 buf << "[ ";519 520 bool first = true;521 for (const auto & token : tokens) {522 if (!first) {523 buf << ", ";524 } else {525 first = false;526 }527 528 auto detokenized = common_token_to_piece(ctx, token);529 530 detokenized.erase(531 std::remove_if(532 detokenized.begin(),533 detokenized.end(),534 [](const unsigned char c) { return !std::isprint(c); }),535 detokenized.end());536 537 buf << "'" << detokenized << "'"538 << ":" << std::to_string(token);539 }540 541 buf << " ]";542 543 return buf.str();544}545 546std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch) {547 std::stringstream buf;548 549 buf << "[ ";550 551 bool first = true;552 for (int i = 0; i < batch.n_tokens; ++i) {553 if (!first) {554 buf << ", ";555 } else {556 first = false;557 }558 559 auto detokenized = common_token_to_piece(ctx, batch.token[i]);560 561 detokenized.erase(562 std::remove_if(563 detokenized.begin(),564 detokenized.end(),565 [](const unsigned char c) { return !std::isprint(c); }),566 detokenized.end());567 568 buf << "\n" << std::to_string(i)569 << ", token '" << detokenized << "'"570 << ", pos " << std::to_string(batch.pos[i])571 << ", n_seq_id " << std::to_string(batch.n_seq_id[i])572 << ", seq_id " << std::to_string(batch.seq_id[i][0])573 << ", logits " << std::to_string(batch.logits[i]);574 }575 576 buf << " ]";577 578 return buf.str();579}580 581void string_process_escapes(std::string & input) {582 std::size_t input_len = input.length();583 std::size_t output_idx = 0;584 585 for (std::size_t input_idx = 0; input_idx < input_len; ++input_idx) {586 if (input[input_idx] == '\\' && input_idx + 1 < input_len) {587 switch (input[++input_idx]) {588 case 'n': input[output_idx++] = '\n'; break;589 case 'r': input[output_idx++] = '\r'; break;590 case 't': input[output_idx++] = '\t'; break;591 case '\'': input[output_idx++] = '\''; break;592 case '\"': input[output_idx++] = '\"'; break;593 case '\\': input[output_idx++] = '\\'; break;594 case 'x':595 // Handle \x12, etc596 if (input_idx + 2 < input_len) {597 const char x[3] = { input[input_idx + 1], input[input_idx + 2], 0 };598 char *err_p = nullptr;599 const long val = std::strtol(x, &err_p, 16);600 if (err_p == x + 2) {601 input_idx += 2;602 input[output_idx++] = char(val);603 break;604 }605 }606 // fall through607 default: input[output_idx++] = '\\';608 input[output_idx++] = input[input_idx]; break;609 }610 } else {611 input[output_idx++] = input[input_idx];612 }613 }614 615 input.resize(output_idx);616}617 618bool string_parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides) {619 const char * sep = strchr(data, '=');620 if (sep == nullptr || sep - data >= 128) {621 LOG_ERR("%s: malformed KV override '%s'\n", __func__, data);622 return false;623 }624 llama_model_kv_override kvo;625 std::strncpy(kvo.key, data, sep - data);626 kvo.key[sep - data] = 0;627 sep++;628 if (strncmp(sep, "int:", 4) == 0) {629 sep += 4;630 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;631 kvo.val_i64 = std::atol(sep);632 } else if (strncmp(sep, "float:", 6) == 0) {633 sep += 6;634 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;635 kvo.val_f64 = std::atof(sep);636 } else if (strncmp(sep, "bool:", 5) == 0) {637 sep += 5;638 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;639 if (std::strcmp(sep, "true") == 0) {640 kvo.val_bool = true;641 } else if (std::strcmp(sep, "false") == 0) {642 kvo.val_bool = false;643 } else {644 LOG_ERR("%s: invalid boolean value for KV override '%s'\n", __func__, data);645 return false;646 }647 } else if (strncmp(sep, "str:", 4) == 0) {648 sep += 4;649 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;650 if (strlen(sep) > 127) {651 LOG_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data);652 return false;653 }654 strncpy(kvo.val_str, sep, 127);655 kvo.val_str[127] = '\0';656 } else {657 LOG_ERR("%s: invalid type for KV override '%s'\n", __func__, data);658 return false;659 }660 overrides.emplace_back(std::move(kvo));661 return true;662}663 664//665// Filesystem utils666//667 668// Validate if a filename is safe to use669// To validate a full path, split the path by the OS-specific path separator, and validate each part with this function670bool fs_validate_filename(const std::string & filename) {671 if (!filename.length()) {672 // Empty filename invalid673 return false;674 }675 if (filename.length() > 255) {676 // Limit at common largest possible filename on Linux filesystems677 // to avoid unnecessary further validation678 // (On systems with smaller limits it will be caught by the OS)679 return false;680 }681 682 std::u32string filename_utf32;683 try {684#if defined(__clang__)685 // disable C++17 deprecation warning for std::codecvt_utf8686# pragma clang diagnostic push687# pragma clang diagnostic ignored "-Wdeprecated-declarations"688#endif689 std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> converter;690 691#if defined(__clang__)692# pragma clang diagnostic pop693#endif694 695 filename_utf32 = converter.from_bytes(filename);696 697 // If the reverse conversion mismatches, it means overlong UTF-8 sequences were used,698 // or invalid encodings were encountered. Reject such attempts699 std::string filename_reencoded = converter.to_bytes(filename_utf32);700 if (filename_reencoded != filename) {701 return false;702 }703 } catch (const std::exception &) {704 return false;705 }706 707 // Check for forbidden codepoints:708 // - Control characters709 // - Unicode equivalents of illegal characters710 // - UTF-16 surrogate pairs711 // - UTF-8 replacement character712 // - Byte order mark (BOM)713 // - Illegal characters: / \ : * ? " < > |714 for (char32_t c : filename_utf32) {715 if (c <= 0x1F // Control characters (C0)716 || c == 0x7F // Control characters (DEL)717 || (c >= 0x80 && c <= 0x9F) // Control characters (C1)718 || c == 0xFF0E // Fullwidth Full Stop (period equivalent)719 || c == 0x2215 // Division Slash (forward slash equivalent)720 || c == 0x2216 // Set Minus (backslash equivalent)721 || (c >= 0xD800 && c <= 0xDFFF) // UTF-16 surrogate pairs722 || c == 0xFFFD // Replacement Character (UTF-8)723 || c == 0xFEFF // Byte Order Mark (BOM)724 || c == '/' || c == '\\' || c == ':' || c == '*' // Illegal characters725 || c == '?' || c == '"' || c == '<' || c == '>' || c == '|') {726 return false;727 }728 }729 730 // Reject any leading or trailing ' ', or any trailing '.', these are stripped on Windows and will cause a different filename731 // Unicode and other whitespace is not affected, only 0x20 space732 if (filename.front() == ' ' || filename.back() == ' ' || filename.back() == '.') {733 return false;734 }735 736 // Reject any ".." (currently stricter than necessary, it should be fine to just check for == ".." instead)737 if (filename.find("..") != std::string::npos) {738 return false;739 }740 741 // Reject "."742 if (filename == ".") {743 return false;744 }745 746 return true;747}748 749// returns true if successful, false otherwise750bool fs_create_directory_with_parents(const std::string & path) {751#ifdef _WIN32752 std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;753 std::wstring wpath = converter.from_bytes(path);754 755 // if the path already exists, check whether it's a directory756 const DWORD attributes = GetFileAttributesW(wpath.c_str());757 if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {758 return true;759 }760 761 size_t pos_slash = 0;762 763 // process path from front to back, procedurally creating directories764 while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {765 const std::wstring subpath = wpath.substr(0, pos_slash);766 const wchar_t * test = subpath.c_str();767 768 const bool success = CreateDirectoryW(test, NULL);769 if (!success) {770 const DWORD error = GetLastError();771 772 // if the path already exists, ensure that it's a directory773 if (error == ERROR_ALREADY_EXISTS) {774 const DWORD attributes = GetFileAttributesW(subpath.c_str());775 if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {776 return false;777 }778 } else {779 return false;780 }781 }782 783 pos_slash += 1;784 }785 786 return true;787#else788 // if the path already exists, check whether it's a directory789 struct stat info;790 if (stat(path.c_str(), &info) == 0) {791 return S_ISDIR(info.st_mode);792 }793 794 size_t pos_slash = 1; // skip leading slashes for directory creation795 796 // process path from front to back, procedurally creating directories797 while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {798 const std::string subpath = path.substr(0, pos_slash);799 struct stat info;800 801 // if the path already exists, ensure that it's a directory802 if (stat(subpath.c_str(), &info) == 0) {803 if (!S_ISDIR(info.st_mode)) {804 return false;805 }806 } else {807 // create parent directories808 const int ret = mkdir(subpath.c_str(), 0755);809 if (ret != 0) {810 return false;811 }812 }813 814 pos_slash += 1;815 }816 817 return true;818#endif // _WIN32819}820 821std::string fs_get_cache_directory() {822 std::string cache_directory = "";823 auto ensure_trailing_slash = [](std::string p) {824 // Make sure to add trailing slash825 if (p.back() != DIRECTORY_SEPARATOR) {826 p += DIRECTORY_SEPARATOR;827 }828 return p;829 };830 if (getenv("LLAMA_CACHE")) {831 cache_directory = std::getenv("LLAMA_CACHE");832 } else {833#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX)834 if (std::getenv("XDG_CACHE_HOME")) {835 cache_directory = std::getenv("XDG_CACHE_HOME");836 } else {837 cache_directory = std::getenv("HOME") + std::string("/.cache/");838 }839#elif defined(__APPLE__)840 cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");841#elif defined(_WIN32)842 cache_directory = std::getenv("LOCALAPPDATA");843#else844# error Unknown architecture845#endif846 cache_directory = ensure_trailing_slash(cache_directory);847 cache_directory += "llama.cpp";848 }849 return ensure_trailing_slash(cache_directory);850}851 852std::string fs_get_cache_file(const std::string & filename) {853 GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);854 std::string cache_directory = fs_get_cache_directory();855 const bool success = fs_create_directory_with_parents(cache_directory);856 if (!success) {857 throw std::runtime_error("failed to create cache directory: " + cache_directory);858 }859 return cache_directory + filename;860}861 862 863//864// Model utils865//866 867struct common_init_result common_init_from_params(common_params & params) {868 common_init_result iparams;869 auto mparams = common_model_params_to_llama(params);870 871 llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);872 if (model == NULL) {873 LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());874 return iparams;875 }876 877 const llama_vocab * vocab = llama_model_get_vocab(model);878 879 if (params.reranking) {880 bool ok = true;881 882 if (llama_vocab_bos(vocab) == LLAMA_TOKEN_NULL) {883 LOG_WRN("%s: warning: vocab does not have a BOS token, reranking will not work\n", __func__);884 ok = false;885 }886 887 if (llama_vocab_eos(vocab) == LLAMA_TOKEN_NULL) {888 LOG_WRN("%s: warning: vocab does not have an EOS token, reranking will not work\n", __func__);889 ok = false;890 }891 892 if (llama_vocab_sep(vocab) == LLAMA_TOKEN_NULL) {893 LOG_WRN("%s: warning: vocab does not have a SEP token, reranking will not work\n", __func__);894 ok = false;895 }896 897 if (!ok) {898 llama_model_free(model);899 900 return iparams;901 }902 }903 904 auto cparams = common_context_params_to_llama(params);905 906 llama_context * lctx = llama_init_from_model(model, cparams);907 if (lctx == NULL) {908 LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str());909 llama_model_free(model);910 return iparams;911 }912 913 if (params.ctx_shift && !llama_kv_self_can_shift(lctx)) {914 LOG_WRN("%s: KV cache shifting is not supported for this context, disabling KV cache shifting\n", __func__);915 params.ctx_shift = false;916 }917 918 if (!params.control_vectors.empty()) {919 if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;920 if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_model_n_layer(model);921 922 const auto cvec = common_control_vector_load(params.control_vectors);923 if (cvec.n_embd == -1) {924 llama_free(lctx);925 llama_model_free(model);926 927 return iparams;928 }929 930 int err = llama_apply_adapter_cvec(931 lctx,932 cvec.data.data(),933 cvec.data.size(),934 cvec.n_embd,935 params.control_vector_layer_start,936 params.control_vector_layer_end);937 if (err) {938 llama_free(lctx);939 llama_model_free(model);940 941 return iparams;942 }943 }944 945 // load and optionally apply lora adapters946 for (auto & la : params.lora_adapters) {947 llama_adapter_lora_ptr lora;948 lora.reset(llama_adapter_lora_init(model, la.path.c_str()));949 if (lora == nullptr) {950 LOG_ERR("%s: failed to apply lora adapter '%s'\n", __func__, la.path.c_str());951 llama_free(lctx);952 llama_model_free(model);953 return iparams;954 }955 956 la.ptr = lora.get();957 iparams.lora.emplace_back(std::move(lora)); // copy to list of loaded adapters958 }959 960 if (!params.lora_init_without_apply) {961 common_set_adapter_lora(lctx, params.lora_adapters);962 }963 964 if (params.sampling.ignore_eos && llama_vocab_eos(vocab) == LLAMA_TOKEN_NULL) {965 LOG_WRN("%s: warning: vocab does not have an EOS token, ignoring --ignore-eos\n", __func__);966 params.sampling.ignore_eos = false;967 }968 969 if (params.sampling.ignore_eos) {970 for (llama_token i = 0; i < llama_vocab_n_tokens(vocab); i++) {971 if (llama_vocab_is_eog(vocab, i)) {972 LOG_INF("%s: added %s logit bias = %f\n", __func__, common_token_to_piece(lctx, i).c_str(), -INFINITY);973 params.sampling.logit_bias.push_back({i, -INFINITY});974 }975 }976 }977 978 if (params.sampling.penalty_last_n == -1) {979 LOG_INF("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));980 params.sampling.penalty_last_n = llama_n_ctx(lctx);981 }982 983 if (params.sampling.dry_penalty_last_n == -1) {984 LOG_INF("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));985 params.sampling.dry_penalty_last_n = llama_n_ctx(lctx);986 }987 988 if (params.warmup) {989 LOG_WRN("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__);990 991 llama_set_warmup(lctx, true);992 993 std::vector<llama_token> tmp;994 llama_token bos = llama_vocab_bos(vocab);995 llama_token eos = llama_vocab_eos(vocab);996 997 // some models (e.g. T5) don't have a BOS token998 if (bos != LLAMA_TOKEN_NULL) {999 tmp.push_back(bos);1000 }1001 if (eos != LLAMA_TOKEN_NULL) {1002 tmp.push_back(eos);1003 }1004 if (tmp.empty()) {1005 tmp.push_back(0);1006 }1007 1008 if (llama_model_has_encoder(model)) {1009 llama_encode(lctx, llama_batch_get_one(tmp.data(), tmp.size()));1010 llama_token decoder_start_token_id = llama_model_decoder_start_token(model);1011 if (decoder_start_token_id == LLAMA_TOKEN_NULL) {1012 decoder_start_token_id = bos;1013 }1014 tmp.clear();1015 tmp.push_back(decoder_start_token_id);1016 }1017 if (llama_model_has_decoder(model)) {1018 llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch)));1019 }1020 llama_kv_self_clear(lctx);1021 llama_synchronize(lctx);1022 llama_perf_context_reset(lctx);1023 llama_set_warmup(lctx, false);1024 }1025 1026 iparams.model.reset(model);1027 iparams.context.reset(lctx);1028 1029 return iparams;1030}1031 1032std::string get_model_endpoint() {1033 const char * model_endpoint_env = getenv("MODEL_ENDPOINT");1034 // We still respect the use of environment-variable "HF_ENDPOINT" for backward-compatibility.1035 const char * hf_endpoint_env = getenv("HF_ENDPOINT");1036 const char * endpoint_env = model_endpoint_env ? model_endpoint_env : hf_endpoint_env;1037 std::string model_endpoint = "https://huggingface.co/";1038 if (endpoint_env) {1039 model_endpoint = endpoint_env;1040 if (model_endpoint.back() != '/') model_endpoint += '/';1041 }1042 return model_endpoint;1043}1044 1045void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora) {1046 llama_clear_adapter_lora(ctx);1047 for (auto & la : lora) {1048 if (la.scale != 0.0f) {1049 llama_set_adapter_lora(ctx, la.ptr, la.scale);1050 }1051 }1052}1053 1054struct llama_model_params common_model_params_to_llama(common_params & params) {1055 auto mparams = llama_model_default_params();1056 1057 if (!params.devices.empty()) {1058 mparams.devices = params.devices.data();1059 }1060 1061 if (params.n_gpu_layers != -1) {1062 mparams.n_gpu_layers = params.n_gpu_layers;1063 }1064 1065 mparams.main_gpu = params.main_gpu;1066 mparams.split_mode = params.split_mode;1067 mparams.tensor_split = params.tensor_split;1068 mparams.use_mmap = params.use_mmap;1069 mparams.use_mlock = params.use_mlock;1070 mparams.check_tensors = params.check_tensors;1071 1072 if (params.kv_overrides.empty()) {1073 mparams.kv_overrides = NULL;1074 } else {1075 GGML_ASSERT(params.kv_overrides.back().key[0] == 0 && "KV overrides not terminated with empty key");1076 mparams.kv_overrides = params.kv_overrides.data();1077 }1078 1079 if (params.tensor_buft_overrides.empty()) {1080 mparams.tensor_buft_overrides = NULL;1081 } else {1082 GGML_ASSERT(params.tensor_buft_overrides.back().pattern == nullptr && "Tensor buffer overrides not terminated with empty pattern");1083 mparams.tensor_buft_overrides = params.tensor_buft_overrides.data();1084 }1085 1086 return mparams;1087}1088 1089struct llama_context_params common_context_params_to_llama(const common_params & params) {1090 auto cparams = llama_context_default_params();1091 1092 cparams.n_ctx = params.n_ctx;1093 cparams.n_seq_max = params.n_parallel;1094 cparams.n_batch = params.n_batch;1095 cparams.n_ubatch = params.n_ubatch;1096 cparams.n_threads = params.cpuparams.n_threads;1097 cparams.n_threads_batch = params.cpuparams_batch.n_threads == -1 ?1098 params.cpuparams.n_threads : params.cpuparams_batch.n_threads;1099 cparams.logits_all = params.logits_all;1100 cparams.embeddings = params.embedding;1101 cparams.rope_scaling_type = params.rope_scaling_type;1102 cparams.rope_freq_base = params.rope_freq_base;1103 cparams.rope_freq_scale = params.rope_freq_scale;1104 cparams.yarn_ext_factor = params.yarn_ext_factor;1105 cparams.yarn_attn_factor = params.yarn_attn_factor;1106 cparams.yarn_beta_fast = params.yarn_beta_fast;1107 cparams.yarn_beta_slow = params.yarn_beta_slow;1108 cparams.yarn_orig_ctx = params.yarn_orig_ctx;1109 cparams.pooling_type = params.pooling_type;1110 cparams.attention_type = params.attention_type;1111 cparams.defrag_thold = params.defrag_thold;1112 cparams.cb_eval = params.cb_eval;1113 cparams.cb_eval_user_data = params.cb_eval_user_data;1114 cparams.offload_kqv = !params.no_kv_offload;1115 cparams.flash_attn = params.flash_attn;1116 cparams.no_perf = params.no_perf;1117 1118 if (params.reranking) {1119 cparams.embeddings = true;1120 cparams.pooling_type = LLAMA_POOLING_TYPE_RANK;1121 }1122 1123 cparams.type_k = params.cache_type_k;1124 cparams.type_v = params.cache_type_v;1125 1126 return cparams;1127}1128 1129struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const cpu_params & params) {1130 struct ggml_threadpool_params tpp;1131 1132 ggml_threadpool_params_init(&tpp, params.n_threads); // setup the defaults1133 1134 if (params.mask_valid) {1135 std::memcpy(&tpp.cpumask, ¶ms.cpumask, GGML_MAX_N_THREADS);1136 }1137 1138 tpp.prio = params.priority;1139 tpp.poll = params.poll;1140 tpp.strict_cpu = params.strict_cpu;1141 1142 return tpp;1143}1144 1145//1146// Batch utils1147//1148 1149void common_batch_clear(struct llama_batch & batch) {1150 batch.n_tokens = 0;1151}1152 1153void common_batch_add(1154 struct llama_batch & batch,1155 llama_token id,1156 llama_pos pos,1157 const std::vector<llama_seq_id> & seq_ids,1158 bool logits) {1159 GGML_ASSERT(batch.seq_id[batch.n_tokens] && "llama_batch size exceeded");1160 1161 batch.token [batch.n_tokens] = id;1162 batch.pos [batch.n_tokens] = pos;1163 batch.n_seq_id[batch.n_tokens] = seq_ids.size();1164 for (size_t i = 0; i < seq_ids.size(); ++i) {1165 batch.seq_id[batch.n_tokens][i] = seq_ids[i];1166 }1167 batch.logits [batch.n_tokens] = logits;1168 1169 batch.n_tokens++;1170}1171 1172//1173// Token utils1174//1175 1176size_t common_lcp(const llama_tokens & a, const llama_tokens & b) {1177 size_t i;1178 for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) {}1179 1180 return i;1181}1182 1183size_t common_lcs(const llama_tokens & a, const llama_tokens & b) {1184 // check for empty sequences1185 if (a.empty() || b.empty()) {1186 return 0;1187 }1188 1189 // get the lengths of the input sequences1190 size_t a_len = a.size();1191 size_t b_len = b.size();1192 1193 // initialize the maximum length of the longest common subsequence (LCS)1194 size_t max_length = 0;1195 1196 // use two rows instead of a 2D matrix to optimize space1197 std::vector<size_t> prev_row(b_len + 1, 0);1198 std::vector<size_t> curr_row(b_len + 1, 0);1199 1200 // iterate through the elements of a