Felipe97/llama-cpp-compiled
01.1k
1#include "ggml.h"2#include "gguf.h"3 4#include "build-info.h"5#include "common.h"6#include "fit.h"7#include "log.h"8#include "llama.h"9#include "sampling.h"10#include "speculative.h"11#include "unicode.h"12 13#include <algorithm>14#include <cinttypes>15#include <climits>16#include <cmath>17#include <chrono>18#include <cstdarg>19#include <cstring>20#include <ctime>21#include <filesystem>22#include <fstream>23#include <iostream>24#include <iterator>25#include <regex>26#include <sstream>27#include <string>28#include <thread>29#include <unordered_set>30#include <vector>31 32#if defined(__APPLE__) && defined(__MACH__)33#include <sys/types.h>34#include <sys/sysctl.h>35#endif36 37#if defined(_WIN32)38#define WIN32_LEAN_AND_MEAN39#ifndef NOMINMAX40# define NOMINMAX41#endif42#include <locale>43#include <windows.h>44#include <string.h>45#include <fcntl.h>46#include <io.h>47#else48#include <sys/ioctl.h>49#include <sys/stat.h>50#include <unistd.h>51#endif52 53#if defined(__linux__)54#include <sys/types.h>55#include <pwd.h>56#endif57 58#if defined(_AIX)59#include <sys/systemcfg.h>60#endif61 62#if defined(_MSC_VER)63#pragma warning(disable: 4244 4267) // possible loss of data64#endif65 66common_time_meas::common_time_meas(int64_t & t_acc, bool disable) : t_start_us(disable ? -1 : ggml_time_us()), t_acc(t_acc) {}67 68common_time_meas::~common_time_meas() {69 if (t_start_us >= 0) {70 t_acc += ggml_time_us() - t_start_us;71 }72}73 74//75// CPU utils76//77 78int32_t common_cpu_get_num_physical_cores() {79#if defined(_AIX)80 int32_t logical_cpus = _system_configuration.ncpus;81 int32_t smt_threads = _system_configuration.smt_threads;82 if (smt_threads > 0) {83 return static_cast<int32_t>(logical_cpus / smt_threads);84 }85 if (logical_cpus > 0) {86 return static_cast<int32_t>(logical_cpus);87 }88#elif defined(__linux__)89 // enumerate the set of thread siblings, num entries is num cores90 std::unordered_set<std::string> siblings;91 for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {92 std::ifstream thread_siblings("/sys/devices/system/cpu/cpu"93 + std::to_string(cpu) + "/topology/thread_siblings");94 if (!thread_siblings.is_open()) {95 break; // no more cpus96 }97 std::string line;98 if (std::getline(thread_siblings, line)) {99 siblings.insert(line);100 }101 }102 if (!siblings.empty()) {103 return static_cast<int32_t>(siblings.size());104 }105#elif defined(__APPLE__) && defined(__MACH__)106 int32_t num_physical_cores;107 size_t len = sizeof(num_physical_cores);108 int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);109 if (result == 0) {110 return num_physical_cores;111 }112 result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);113 if (result == 0) {114 return num_physical_cores;115 }116#elif defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later117 // TODO: windows + arm64 + mingw64118 unsigned int n_threads_win = std::thread::hardware_concurrency();119 unsigned int default_threads = n_threads_win > 0 ? (n_threads_win <= 4 ? n_threads_win : n_threads_win / 2) : 4;120 121 DWORD buffer_size = 0;122 if (!GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &buffer_size)) {123 if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {124 return default_threads;125 }126 }127 128 std::vector<char> buffer(buffer_size);129 if (!GetLogicalProcessorInformationEx(RelationProcessorCore, reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data()), &buffer_size)) {130 return default_threads;131 }132 133 int32_t num_physical_cores = 0;134 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data());135 while (buffer_size > 0) {136 if (info->Relationship == RelationProcessorCore) {137 num_physical_cores += info->Processor.GroupCount;138 }139 buffer_size -= info->Size;140 info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(reinterpret_cast<char*>(info) + info->Size);141 }142 143 return num_physical_cores > 0 ? num_physical_cores : default_threads;144#endif145 unsigned int n_threads = std::thread::hardware_concurrency();146 return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;147}148 149#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)150#include <pthread.h>151 152static void cpuid(unsigned leaf, unsigned subleaf,153 unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) {154 __asm__("movq\t%%rbx,%%rsi\n\t"155 "cpuid\n\t"156 "xchgq\t%%rbx,%%rsi"157 : "=a"(*eax), "=S"(*ebx), "=c"(*ecx), "=d"(*edx)158 : "0"(leaf), "2"(subleaf));159}160 161static int pin_cpu(int cpu) {162 cpu_set_t mask;163 CPU_ZERO(&mask);164 CPU_SET(cpu, &mask);165 return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);166}167 168static bool is_hybrid_cpu(void) {169 unsigned eax, ebx, ecx, edx;170 cpuid(7, 0, &eax, &ebx, &ecx, &edx);171 return !!(edx & (1u << 15));172}173 174static bool is_running_on_efficiency_core(void) {175 unsigned eax, ebx, ecx, edx;176 cpuid(0x1a, 0, &eax, &ebx, &ecx, &edx);177 int intel_atom = 0x20;178 int core_type = (eax & 0xff000000u) >> 24;179 return core_type == intel_atom;180}181 182static int cpu_count_math_cpus(int n_cpu) {183 int result = 0;184 for (int cpu = 0; cpu < n_cpu; ++cpu) {185 if (pin_cpu(cpu)) {186 return -1;187 }188 if (is_running_on_efficiency_core()) {189 continue; // efficiency cores harm lockstep threading190 }191 ++cpu; // hyperthreading isn't useful for linear algebra192 ++result;193 }194 return result;195}196 197#endif // __x86_64__ && __linux__198 199/**200 * Returns number of CPUs on system that are useful for math.201 */202int32_t common_cpu_get_num_math() {203#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)204 int n_cpu = sysconf(_SC_NPROCESSORS_ONLN);205 if (n_cpu < 1) {206 return common_cpu_get_num_physical_cores();207 }208 if (is_hybrid_cpu()) {209 cpu_set_t affinity;210 if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) {211 int result = cpu_count_math_cpus(n_cpu);212 pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity);213 if (result > 0) {214 return result;215 }216 }217 }218#elif defined(__powerpc64__) || defined(__powerpc__)219 int32_t smt_factor = 1;220 int phy_cpus = common_cpu_get_num_physical_cores();221 int logical_cpus = sysconf(_SC_NPROCESSORS_ONLN);222 if (phy_cpus > 0 && logical_cpus > phy_cpus) {223 smt_factor = logical_cpus / phy_cpus;224 }225 return phy_cpus * std::min(smt_factor, 2);226#endif227 return common_cpu_get_num_physical_cores();228}229 230// Helper for setting process priority231 232#if defined(_WIN32)233 234bool set_process_priority(enum ggml_sched_priority prio) {235 if (prio == GGML_SCHED_PRIO_NORMAL) {236 return true;237 }238 239 DWORD p = NORMAL_PRIORITY_CLASS;240 switch (prio) {241 case GGML_SCHED_PRIO_LOW: p = BELOW_NORMAL_PRIORITY_CLASS; break;242 case GGML_SCHED_PRIO_NORMAL: p = NORMAL_PRIORITY_CLASS; break;243 case GGML_SCHED_PRIO_MEDIUM: p = ABOVE_NORMAL_PRIORITY_CLASS; break;244 case GGML_SCHED_PRIO_HIGH: p = HIGH_PRIORITY_CLASS; break;245 case GGML_SCHED_PRIO_REALTIME: p = REALTIME_PRIORITY_CLASS; break;246 }247 248 if (!SetPriorityClass(GetCurrentProcess(), p)) {249 COM_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError());250 return false;251 }252 253 return true;254}255 256#else // MacOS and POSIX257#include <sys/types.h>258#include <sys/resource.h>259 260bool set_process_priority(enum ggml_sched_priority prio) {261 if (prio == GGML_SCHED_PRIO_NORMAL) {262 return true;263 }264 265 int p = 0;266 switch (prio) {267 case GGML_SCHED_PRIO_LOW: p = 5; break;268 case GGML_SCHED_PRIO_NORMAL: p = 0; break;269 case GGML_SCHED_PRIO_MEDIUM: p = -5; break;270 case GGML_SCHED_PRIO_HIGH: p = -10; break;271 case GGML_SCHED_PRIO_REALTIME: p = -20; break;272 }273 274 if (setpriority(PRIO_PROCESS, 0, p) != 0) {275 COM_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno);276 return false;277 }278 return true;279}280 281#endif282 283//284// CLI argument parsing285//286 287 288void postprocess_cpu_params(common_cpu_params & cpuparams, const common_cpu_params * role_model) {289 int32_t n_set = 0;290 291 if (cpuparams.n_threads < 0) {292 // Assuming everything about cpuparams is invalid293 if (role_model != nullptr) {294 cpuparams = *role_model;295 } else {296 cpuparams.n_threads = common_cpu_get_num_math();297 }298 }299 300 for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {301 if (cpuparams.cpumask[i]) {302 n_set++;303 }304 }305 306 if (n_set && n_set < cpuparams.n_threads) {307 // Not enough set bits, may experience performance issues.308 COM_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads);309 }310}311 312bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THREADS]) {313 size_t dash_loc = range.find('-');314 if (dash_loc == std::string::npos) {315 COM_ERR("%s", "Format of CPU range is invalid! Expected [<start>]-[<end>].\n");316 return false;317 }318 319 size_t start_i;320 size_t end_i;321 322 if (dash_loc == 0) {323 start_i = 0;324 } else {325 start_i = std::stoull(range.substr(0, dash_loc));326 if (start_i >= GGML_MAX_N_THREADS) {327 COM_ERR("%s", "Start index out of bounds!\n");328 return false;329 }330 }331 332 if (dash_loc == range.length() - 1) {333 end_i = GGML_MAX_N_THREADS - 1;334 } else {335 end_i = std::stoull(range.substr(dash_loc + 1));336 if (end_i >= GGML_MAX_N_THREADS) {337 COM_ERR("%s", "End index out of bounds!\n");338 return false;339 }340 }341 342 for (size_t i = start_i; i <= end_i; i++) {343 boolmask[i] = true;344 }345 346 return true;347}348 349bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREADS]) {350 // Discard potential 0x prefix351 size_t start_i = 0;352 if (mask.length() >= 2 && mask.substr(0, 2) == "0x") {353 start_i = 2;354 }355 356 size_t num_digits = mask.length() - start_i;357 num_digits = std::min<size_t>(num_digits, 128);358 359 size_t end_i = num_digits + start_i;360 361 for (size_t i = start_i, n = (num_digits*4 - 1); i < end_i; i++, n-=4) {362 char c = mask.at(i);363 int8_t id = c;364 365 if ((c >= '0' && c <= '9')) {366 id -= '0';367 } else if (c >= 'a' && c <= 'f') {368 id -= 'a' - 10;369 } else if (c >= 'A' && c <= 'F') {370 id -= 'A' - 10;371 } else {372 COM_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i));373 return false;374 }375 376 boolmask[ n ] = boolmask[ n ] || ((id & 8) != 0);377 boolmask[n - 1] = boolmask[n - 1] || ((id & 4) != 0);378 boolmask[n - 2] = boolmask[n - 2] || ((id & 2) != 0);379 boolmask[n - 3] = boolmask[n - 3] || ((id & 1) != 0);380 }381 382 return true;383}384 385void common_init() {386#if defined(_WIN32)387 SetConsoleOutputCP(CP_UTF8);388 SetConsoleCP(CP_UTF8);389#endif390 391 common_log_set_prefix(common_log_main(), true);392 common_log_set_timestamps(common_log_main(), true);393 394 llama_log_set(common_log_default_callback, NULL);395}396 397void common_params_print_info(const common_params & params, bool print_devices) {398#ifdef NDEBUG399 const char * build_type = "";400#else401 const char * build_type = " (debug)";402#endif403 COM_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type);404 405 const int verbosity = common_log_get_verbosity_thold();406 COM_INF("%s: verbosity = %d (adjust with the `-lv N` CLI arg)\n", __func__, verbosity);407 408 // device enumeration creates a primary context on CUDA backends, skip it when the caller does not own any device409 if (print_devices && verbosity >= LOG_LEVEL_TRACE) {410 COM_TRC("%s", "device_info:\n");411 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {412 auto * dev = ggml_backend_dev_get(i);413 size_t free, total;414 ggml_backend_dev_memory(dev, &free, &total);415 COM_TRC(" - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);416 }417 }418 COM_TRC("%s\n", common_params_get_system_info(params).c_str());419}420 421std::string common_params_get_system_info(const common_params & params) {422 std::ostringstream os;423 424 os << "system_info: n_threads = " << params.cpuparams.n_threads;425 if (params.cpuparams_batch.n_threads != -1) {426 os << " (n_threads_batch = " << params.cpuparams_batch.n_threads << ")";427 }428#if defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later429 // TODO: windows + arm64 + mingw64430 DWORD logicalProcessorCount = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);431 os << " / " << logicalProcessorCount << " | " << llama_print_system_info();432#else433 os << " / " << std::thread::hardware_concurrency() << " | " << llama_print_system_info();434#endif435 436 return os.str();437}438 439//440// String utils441//442 443std::string string_format(const char * fmt, ...) {444 va_list ap;445 va_list ap2;446 va_start(ap, fmt);447 va_copy(ap2, ap);448 int size = vsnprintf(NULL, 0, fmt, ap);449 GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT450 std::vector<char> buf(size + 1);451 int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);452 GGML_ASSERT(size2 == size);453 va_end(ap2);454 va_end(ap);455 return std::string(buf.data(), size);456}457 458std::string string_strip(const std::string & str) {459 size_t start = 0;460 size_t end = str.size();461 while (start < end && std::isspace(str[start])) {462 start++;463 }464 while (end > start && std::isspace(str[end - 1])) {465 end--;466 }467 return str.substr(start, end - start);468}469 470std::string string_lcs(std::string_view a, std::string_view b) {471 if (a.empty() || b.empty()) return {};472 473 std::vector<std::vector<size_t>> dp(a.size() + 1, std::vector<size_t>(b.size() + 1, 0));474 size_t best_len = 0;475 size_t best_end_a = 0;476 477 for (size_t i = 1; i <= a.size(); ++i) {478 for (size_t j = 1; j <= b.size(); ++j) {479 if (a[i - 1] == b[j - 1]) {480 dp[i][j] = dp[i - 1][j - 1] + 1;481 if (dp[i][j] > best_len) {482 best_len = dp[i][j];483 best_end_a = i;484 }485 }486 }487 }488 return std::string(a.substr(best_end_a - best_len, best_len));489}490 491std::string string_get_sortable_timestamp() {492 using clock = std::chrono::system_clock;493 494 const clock::time_point current_time = clock::now();495 const time_t as_time_t = clock::to_time_t(current_time);496 char timestamp_no_ns[100];497 std::strftime(timestamp_no_ns, 100, "%Y_%m_%d-%H_%M_%S", std::localtime(&as_time_t));498 499 const int64_t ns = std::chrono::duration_cast<std::chrono::nanoseconds>(500 current_time.time_since_epoch() % 1000000000).count();501 char timestamp_ns[11];502 snprintf(timestamp_ns, 11, "%09" PRId64, ns);503 504 return std::string(timestamp_no_ns) + "." + std::string(timestamp_ns);505}506 507void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {508 if (search.empty()) {509 return;510 }511 std::string builder;512 builder.reserve(s.length());513 size_t pos = 0;514 size_t last_pos = 0;515 while ((pos = s.find(search, last_pos)) != std::string::npos) {516 builder.append(s, last_pos, pos - last_pos);517 builder.append(replace);518 last_pos = pos + search.length();519 }520 builder.append(s, last_pos, std::string::npos);521 s = std::move(builder);522}523 524std::string regex_escape(const std::string & s) {525 static const std::regex special_chars("[.^$|()*+?\\[\\]{}\\\\]");526 return std::regex_replace(s, special_chars, "\\$&");527}528 529std::string string_join(const std::vector<std::string> & values, const std::string & separator) {530 std::ostringstream result;531 for (size_t i = 0; i < values.size(); ++i) {532 if (i > 0) {533 result << separator;534 }535 result << values[i];536 }537 return result.str();538}539 540std::vector<std::string> string_split(const std::string & str, const std::string & delimiter) {541 std::vector<std::string> parts;542 size_t start = 0;543 size_t end = str.find(delimiter);544 545 while (end != std::string::npos) {546 parts.push_back(str.substr(start, end - start));547 start = end + delimiter.length();548 end = str.find(delimiter, start);549 }550 551 parts.push_back(str.substr(start));552 553 return parts;554}555 556std::string string_repeat(const std::string & str, size_t n) {557 if (n == 0) {558 return "";559 }560 561 std::string result;562 result.reserve(str.length() * n);563 564 for (size_t i = 0; i < n; ++i) {565 result += str;566 }567 568 return result;569}570 571std::string string_from(bool value) {572 return value ? "true" : "false";573}574 575std::string string_from(const std::vector<int> & values) {576 std::stringstream buf;577 578 buf << "[ ";579 bool first = true;580 for (auto e : values) {581 if (first) {582 first = false;583 } else {584 buf << ", ";585 }586 buf << std::to_string(e);587 }588 buf << " ]";589 590 return buf.str();591}592 593std::string string_from(const struct llama_context * ctx, const std::vector<llama_token> & tokens) {594 std::stringstream buf;595 596 buf << "[ ";597 598 bool first = true;599 for (const auto & token : tokens) {600 if (!first) {601 buf << ", ";602 } else {603 first = false;604 }605 606 auto detokenized = common_token_to_piece(ctx, token);607 608 buf << "'" << detokenized << "'"609 << ":" << std::to_string(token);610 }611 612 buf << " ]";613 614 return buf.str();615}616 617std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch) {618 std::stringstream buf;619 620 buf << "[ ";621 622 bool first = true;623 for (int i = 0; i < batch.n_tokens; ++i) {624 if (!first) {625 buf << ", ";626 } else {627 first = false;628 }629 630 auto detokenized = common_token_to_piece(ctx, batch.token[i]);631 632 buf << "\n" << std::to_string(i)633 << ", token '" << detokenized << "'"634 << ", pos " << std::to_string(batch.pos[i])635 << ", n_seq_id " << std::to_string(batch.n_seq_id[i])636 << ", seq_id " << std::to_string(batch.seq_id[i][0])637 << ", logits " << std::to_string(batch.logits[i]);638 }639 640 buf << " ]";641 642 return buf.str();643}644 645void string_process_escapes(std::string & input) {646 std::size_t input_len = input.length();647 std::size_t output_idx = 0;648 649 for (std::size_t input_idx = 0; input_idx < input_len; ++input_idx) {650 if (input[input_idx] == '\\' && input_idx + 1 < input_len) {651 switch (input[++input_idx]) {652 case 'n': input[output_idx++] = '\n'; break;653 case 'r': input[output_idx++] = '\r'; break;654 case 't': input[output_idx++] = '\t'; break;655 case '\'': input[output_idx++] = '\''; break;656 case '\"': input[output_idx++] = '\"'; break;657 case '\\': input[output_idx++] = '\\'; break;658 case 'x':659 // Handle \x12, etc660 if (input_idx + 2 < input_len) {661 const char x[3] = { input[input_idx + 1], input[input_idx + 2], 0 };662 char *err_p = nullptr;663 const long val = std::strtol(x, &err_p, 16);664 if (err_p == x + 2) {665 input_idx += 2;666 input[output_idx++] = char(val);667 break;668 }669 }670 // fall through671 default: input[output_idx++] = '\\';672 input[output_idx++] = input[input_idx]; break;673 }674 } else {675 input[output_idx++] = input[input_idx];676 }677 }678 679 input.resize(output_idx);680}681 682bool string_parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides) {683 const char * sep = strchr(data, '=');684 if (sep == nullptr || sep - data >= 128) {685 COM_ERR("%s: malformed KV override '%s'\n", __func__, data);686 return false;687 }688 llama_model_kv_override kvo;689 std::strncpy(kvo.key, data, sep - data);690 kvo.key[sep - data] = 0;691 sep++;692 if (strncmp(sep, "int:", 4) == 0) {693 sep += 4;694 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;695 kvo.val_i64 = std::atol(sep);696 } else if (strncmp(sep, "float:", 6) == 0) {697 sep += 6;698 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;699 kvo.val_f64 = std::atof(sep);700 } else if (strncmp(sep, "bool:", 5) == 0) {701 sep += 5;702 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;703 if (std::strcmp(sep, "true") == 0) {704 kvo.val_bool = true;705 } else if (std::strcmp(sep, "false") == 0) {706 kvo.val_bool = false;707 } else {708 COM_ERR("%s: invalid boolean value for KV override '%s'\n", __func__, data);709 return false;710 }711 } else if (strncmp(sep, "str:", 4) == 0) {712 sep += 4;713 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;714 if (strlen(sep) > 127) {715 COM_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data);716 return false;717 }718 strncpy(kvo.val_str, sep, 127);719 kvo.val_str[127] = '\0';720 } else {721 COM_ERR("%s: invalid type for KV override '%s'\n", __func__, data);722 return false;723 }724 overrides.emplace_back(std::move(kvo));725 return true;726}727 728static inline bool glob_class_match(const char c, const char * pattern, const char * class_end) {729 const char * class_start = pattern;730 bool negated = false;731 732 if (*class_start == '!') {733 negated = true;734 class_start++;735 }736 737 // If first character after negation is ']' or '-', treat it as literal738 if (*class_start == ']' || *class_start == '-') {739 if (class_start < class_end && *class_start == c) {740 return !negated;741 }742 class_start++;743 }744 745 bool matched = false;746 747 while (class_start < class_end) {748 if (class_start + 2 < class_end && class_start[1] == '-' && class_start[2] != ']') {749 char start_char = *class_start;750 char end_char = class_start[2];751 if (c >= start_char && c <= end_char) {752 matched = true;753 break;754 }755 class_start += 3;756 } else {757 if (*class_start == c) {758 matched = true;759 break;760 }761 class_start++;762 }763 }764 765 return negated ? !matched : matched;766}767 768// simple glob: * matches non-/ chars, ** matches anything including /, [] matches character class769static inline bool glob_match(const char * pattern, const char * str) {770 if (*pattern == '\0') {771 return *str == '\0';772 }773 if (pattern[0] == '*' && pattern[1] == '*') {774 const char * p = pattern + 2;775 if (glob_match(p, str)) return true;776 if (*str != '\0') return glob_match(pattern, str + 1);777 return false;778 }779 if (*pattern == '*') {780 const char * p = pattern + 1;781 for (; *str != '\0' && *str != '/'; str++) {782 if (glob_match(p, str)) return true;783 }784 return glob_match(p, str);785 }786 if (*pattern == '?' && *str != '\0' && *str != '/') {787 return glob_match(pattern + 1, str + 1);788 }789 if (*pattern == '[') {790 const char * class_end = pattern + 1;791 // If first character after '[' is ']' or '-', treat it as literal792 if (*class_end == ']' || *class_end == '-') {793 class_end++;794 }795 while (*class_end != '\0' && *class_end != ']') {796 class_end++;797 }798 if (*class_end == ']') {799 if (*str == '\0') return false;800 bool matched = glob_class_match(*str, pattern + 1, class_end);801 return matched && glob_match(class_end + 1, str + 1);802 } else {803 if (*str == '[') {804 return glob_match(pattern + 1, str + 1);805 }806 return false;807 }808 }809 if (*pattern == *str) {810 return glob_match(pattern + 1, str + 1);811 }812 return false;813}814 815bool glob_match(const std::string & pattern, const std::string & str) {816 return glob_match(pattern.c_str(), str.c_str());817}818 819//820// Filesystem utils821//822 823// Validate if a filename is safe to use824// To validate a full path, split the path by the OS-specific path separator, and validate each part with this function825bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {826 if (!filename.length()) {827 // Empty filename invalid828 return false;829 }830 if (filename.length() > 255) {831 // Limit at common largest possible filename on Linux filesystems832 // to avoid unnecessary further validation833 // (On systems with smaller limits it will be caught by the OS)834 return false;835 }836 837 size_t offset = 0;838 while (offset < filename.size()) {839 utf8_parse_result result = common_parse_utf8_codepoint(filename, offset);840 841 if (result.status != utf8_parse_result::SUCCESS) {842 return false;843 }844 uint32_t c = result.codepoint;845 846 if ((result.bytes_consumed == 2 && c < 0x80) ||847 (result.bytes_consumed == 3 && c < 0x800) ||848 (result.bytes_consumed == 4 && c < 0x10000)) {849 return false;850 }851 852 // Check for forbidden codepoints:853 // - Control characters854 // - Unicode equivalents of illegal characters855 // - UTF-16 surrogate pairs856 // - UTF-8 replacement character857 // - Byte order mark (BOM)858 // - Illegal characters: / \ : * ? " < > |859 if (c <= 0x1F // Control characters (C0)860 || c == 0x7F // Control characters (DEL)861 || (c >= 0x80 && c <= 0x9F) // Control characters (C1)862 || c == 0xFF0E // Fullwidth Full Stop (period equivalent)863 || c == 0x2215 // Division Slash (forward slash equivalent)864 || c == 0x2216 // Set Minus (backslash equivalent)865 || (c >= 0xD800 && c <= 0xDFFF) // UTF-16 surrogate pairs866 || c > 0x10FFFF // Max Unicode limit867 || c == 0xFFFD // Replacement Character (UTF-8)868 || c == 0xFEFF // Byte Order Mark (BOM)869 || c == ':' || c == '*' // Illegal characters870 || c == '?' || c == '"' || c == '<' || c == '>' || c == '|') {871 return false;872 }873 if (!allow_subdirs && (c == '/' || c == '\\')) {874 // Subdirectories not allowed, reject path separators875 return false;876 }877 offset += result.bytes_consumed;878 }879 880 // Reject any leading or trailing ' ', or any trailing '.', these are stripped on Windows and will cause a different filename881 // Unicode and other whitespace is not affected, only 0x20 space882 if (filename.front() == ' ' || filename.back() == ' ' || filename.back() == '.') {883 return false;884 }885 886 // Reject any ".." (currently stricter than necessary, it should be fine to just check for == ".." instead)887 if (filename.find("..") != std::string::npos) {888 return false;889 }890 891 // Reject "."892 if (filename == ".") {893 return false;894 }895 896 return true;897}898 899#include <iostream>900 901 902#ifdef _WIN32903static std::wstring utf8_to_wstring(const std::string & str) {904 if (str.empty()) {905 return std::wstring();906 }907 908 int size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0);909 910 if (size <= 0) {911 return std::wstring();912 }913 914 std::wstring wstr(size, 0);915 MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), &wstr[0], size);916 917 return wstr;918}919#endif920 921// returns true if successful, false otherwise922bool fs_create_directory_with_parents(const std::string & path) {923#ifdef _WIN32924 std::wstring wpath = utf8_to_wstring(path);925 926 // if the path already exists, check whether it's a directory927 const DWORD attributes = GetFileAttributesW(wpath.c_str());928 if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {929 return true;930 }931 932 size_t pos_slash = 0;933 934 // process path from front to back, procedurally creating directories935 while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {936 const std::wstring subpath = wpath.substr(0, pos_slash);937 938 pos_slash += 1;939 940 // skip the drive letter, in some systems it can return an access denied error941 if (subpath.length() == 2 && subpath[1] == ':') {942 continue;943 }944 945 const bool success = CreateDirectoryW(subpath.c_str(), NULL);946 947 if (!success) {948 const DWORD error = GetLastError();949 950 // if the path already exists, ensure that it's a directory951 if (error == ERROR_ALREADY_EXISTS) {952 const DWORD attributes = GetFileAttributesW(subpath.c_str());953 if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {954 return false;955 }956 } else {957 return false;958 }959 }960 }961 962 return true;963#else964 // if the path already exists, check whether it's a directory965 struct stat info;966 if (stat(path.c_str(), &info) == 0) {967 return S_ISDIR(info.st_mode);968 }969 970 size_t pos_slash = 1; // skip leading slashes for directory creation971 972 // process path from front to back, procedurally creating directories973 while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {974 const std::string subpath = path.substr(0, pos_slash);975 struct stat info;976 977 // if the path already exists, ensure that it's a directory978 if (stat(subpath.c_str(), &info) == 0) {979 if (!S_ISDIR(info.st_mode)) {980 return false;981 }982 } else {983 // create parent directories984 const int ret = mkdir(subpath.c_str(), 0755);985 if (ret != 0) {986 return false;987 }988 }989 990 pos_slash += 1;991 }992 993 return true;994#endif // _WIN32995}996 997bool fs_is_directory(const std::string & path) {998 std::filesystem::path dir(path);999 return std::filesystem::exists(dir) && std::filesystem::is_directory(dir);1000}1001 1002std::string common_get_env(const std::string & name) {1003 const char * value = std::getenv(name.c_str());1004 return value == nullptr ? "" : value;1005}1006 1007void common_set_env(const std::string & name, const std::string & value) {1008#if defined(_WIN32)1009 _putenv_s(name.c_str(), value.c_str());1010#else1011 if (value.empty()) {1012 unsetenv(name.c_str());1013 } else {1014 setenv(name.c_str(), value.c_str(), 1);1015 }1016#endif1017}1018 1019std::string fs_get_cache_directory() {1020 std::string cache_directory = "";1021 auto ensure_trailing_slash = [](std::string p) {1022 // Make sure to add trailing slash1023 if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {1024 p += DIRECTORY_SEPARATOR;1025 }1026 return p;1027 };1028 cache_directory = common_get_env("LLAMA_CACHE");1029 if (cache_directory.empty()) {1030#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \1031 defined(__OpenBSD__) || defined(__NetBSD__)1032 const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");1033 const std::string home = common_get_env("HOME");1034 if (!xdg_cache_home.empty()) {1035 cache_directory = xdg_cache_home;1036 } else if (!home.empty()) {1037 cache_directory = home + "/.cache/";1038 } else {1039#if defined(__linux__)1040 /* no $HOME is defined, fallback to getpwuid */1041 struct passwd *pw = getpwuid(getuid());1042 if ((!pw) || (!pw->pw_dir)) {1043 throw std::runtime_error("Failed to find $HOME directory");1044 }1045 1046 cache_directory = std::string(pw->pw_dir) + std::string("/.cache/");1047#else /* defined(__linux__) */1048 throw std::runtime_error("Failed to find $HOME directory");1049#endif /* defined(__linux__) */1050 }1051#elif defined(__APPLE__)1052 cache_directory = common_get_env("HOME");1053 if (cache_directory.empty()) {1054 throw std::runtime_error("Failed to find $HOME directory");1055 }1056 cache_directory += "/Library/Caches/";1057#elif defined(_WIN32)1058 cache_directory = common_get_env("LOCALAPPDATA");1059 if (cache_directory.empty()) {1060 throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");1061 }1062#elif defined(__EMSCRIPTEN__)1063 GGML_ABORT("not implemented on this platform");1064#else1065# error Unknown architecture1066#endif1067 cache_directory = ensure_trailing_slash(cache_directory);1068 cache_directory += "llama.cpp";1069 }1070 return ensure_trailing_slash(cache_directory);1071}1072 1073std::string fs_get_config_directory() {1074 std::string config_directory = "";1075 auto ensure_trailing_slash = [](std::string p) {1076 if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {1077 p += DIRECTORY_SEPARATOR;1078 }1079 return p;1080 };1081#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \1082 defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)1083 const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");1084 const std::string home = common_get_env("HOME");1085 if (!xdg_config_home.empty()) {1086 config_directory = xdg_config_home;1087 } else if (!home.empty()) {1088 config_directory = home + "/.config/";1089 } else {1090#if defined(__linux__)1091 /* no $HOME is defined, fallback to getpwuid */1092 struct passwd *pw = getpwuid(getuid());1093 if ((!pw) || (!pw->pw_dir)) {1094 throw std::runtime_error("Failed to find $HOME directory");1095 }1096 1097 config_directory = std::string(pw->pw_dir) + std::string("/.config/");1098#else1099 throw std::runtime_error("Failed to find $HOME directory");1100#endif1101 }1102#elif defined(_WIN32)1103 config_directory = common_get_env("APPDATA");1104 if (config_directory.empty()) {1105 throw std::runtime_error("Failed to find %APPDATA% directory");1106 }1107#elif defined(__EMSCRIPTEN__)1108 // caller decides what to do when there is no config directory1109 throw std::runtime_error("not implemented on this platform");1110#else1111# error Unknown architecture1112#endif1113 config_directory = ensure_trailing_slash(config_directory);1114 config_directory += "llama.cpp";1115 return ensure_trailing_slash(config_directory);1116}1117 1118std::string fs_get_cache_file(const std::string & filename) {1119 GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);1120 std::string cache_directory = fs_get_cache_directory();1121 const bool success = fs_create_directory_with_parents(cache_directory);1122 if (!success) {1123 throw std::runtime_error("failed to create cache directory: " + cache_directory);1124 }1125 return cache_directory + filename;1126}1127 1128std::vector<common_file_info> fs_list(const std::string & path, bool include_directories) {1129 std::vector<common_file_info> files;1130 if (path.empty()) return files;1131 1132 std::filesystem::path dir(path);1133 if (!std::filesystem::exists(dir) || !std::filesystem::is_directory(dir)) {1134 return files;1135 }1136 1137 for (const auto & entry : std::filesystem::directory_iterator(dir)) {1138 try {1139 // Only include regular files (skip directories)1140 const auto & p = entry.path();1141 if (std::filesystem::is_regular_file(p)) {1142 common_file_info info;1143 info.path = p.string();1144 info.name = p.filename().string();1145 info.is_dir = false;1146 try {1147 info.size = static_cast<size_t>(std::filesystem::file_size(p));1148 } catch (const std::filesystem::filesystem_error &) {1149 info.size = 0;1150 }1151 files.push_back(std::move(info));1152 } else if (include_directories && std::filesystem::is_directory(p)) {1153 common_file_info info;1154 info.path = p.string();1155 info.name = p.filename().string();1156 info.size = 0; // Directories have no size1157 info.is_dir = true;1158 files.push_back(std::move(info));1159 }1160 } catch (const std::filesystem::filesystem_error &) {1161 // skip entries we cannot inspect1162 continue;1163 }1164 }1165 1166 return files;1167}1168 1169std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode) {1170#ifdef _WIN321171 int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0);1172 if (!wlen) { return std::ifstream(); }1173 std::vector<wchar_t> wfname(wlen);1174 (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen);1175 return std::ifstream(wfname.data(), mode);1176#else1177 return std::ifstream(fname, mode);1178#endif1179}1180 1181//1182// TTY utils1183//1184 1185bool tty_can_use_colors() {1186 // Check NO_COLOR environment variable (https://no-color.org/)1187 if (const char * no_color = std::getenv("NO_COLOR")) {1188 if (no_color[0] != '\0') {1189 return false;1190 }1191 }1192 1193 // Check TERM environment variable1194 if (const char * term = std::getenv("TERM")) {1195 if (std::strcmp(term, "dumb") == 0) {1196 return false;1197 }1198 }1199 1200 // Check if stdout and stderr are connected to a terminal