Felipe97/llama-cpp-compiled

1#include "llama-mmap.h"2 3#include "llama-impl.h"4 5#include "ggml.h"6 7#include <cstring>8#include <climits>9#include <cstdlib>10#include <stdexcept>11#include <cerrno>12#include <algorithm>13 14#ifdef __has_include15    #if __has_include(<unistd.h>)16        #include <unistd.h>17        #include <fcntl.h>18        #include <sys/stat.h>19        #if defined(_POSIX_MAPPED_FILES)20            #include <sys/mman.h>21        #endif22        #if defined(_POSIX_MEMLOCK_RANGE)23            #include <sys/resource.h>24        #endif25    #endif26#endif27 28#if defined(_WIN32)29    #define WIN32_LEAN_AND_MEAN30    #ifndef NOMINMAX31        #define NOMINMAX32    #endif33    #include <windows.h>34    #ifndef PATH_MAX35        #define PATH_MAX MAX_PATH36    #endif37    #include <io.h>38#endif39 40#if defined(__APPLE__)41#include <TargetConditionals.h>42#endif43 44#ifdef _WIN3245#    define llama_mmap_ftell _ftelli6446#    define llama_mmap_fseek _fseeki6447#else48#    define llama_mmap_ftell ftello49#    define llama_mmap_fseek fseeko50#endif51 52// TODO: consider moving to llama-impl.h if needed in more places53#if defined(_WIN32)54static std::string llama_format_win_err(DWORD err) {55    LPSTR buf;56    size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,57                                 NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);58    if (!size) {59        return "FormatMessageA failed";60    }61    std::string ret(buf, size);62    LocalFree(buf);63    return ret;64}65#endif66 67// llama_file68 69struct llama_file::impl {70#if defined(_WIN32)71    HANDLE fp_win32;72    std::string GetErrorMessageWin32(DWORD error_code) const {73        std::string ret;74        LPSTR lpMsgBuf = NULL;75        DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,76                                    NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);77        if (!bufLen) {78            ret = format("Win32 error code: %lx", error_code);79        } else {80            ret = lpMsgBuf;81            LocalFree(lpMsgBuf);82        }83 84        return ret;85    }86 87    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {88        fp = ggml_fopen(fname, mode);89        if (fp == NULL) {90            throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));91        }92        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));93        seek(0, SEEK_END);94        size = tell();95        seek(0, SEEK_SET);96    }97 98    impl(FILE * file) : owns_fp(false) {99        fp = file;100        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));101        seek(0, SEEK_END);102        size = tell();103        seek(0, SEEK_SET);104    }105 106    size_t tell() const {107        LARGE_INTEGER li;108        li.QuadPart = 0;109        BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);110        if (!ret) {111            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));112        }113 114        return li.QuadPart;115    }116 117    void seek(size_t offset, int whence) const {118        static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");119        static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");120        static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");121 122        LARGE_INTEGER li;123        li.QuadPart = offset;124        BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);125        if (!ret) {126            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));127        }128    }129 130    void read_raw(void * ptr, size_t len) {131        size_t bytes_read = 0;132        while (bytes_read < len) {133            size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);134            DWORD chunk_read = 0;135            BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);136            if (!result) {137                throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));138            }139            if (chunk_read < chunk_size || chunk_read == 0) {140                throw std::runtime_error("unexpectedly reached end of file");141            }142 143            bytes_read += chunk_read;144        }145    }146 147    uint32_t read_u32() {148        uint32_t val;149        read_raw(&val, sizeof(val));150        return val;151    }152 153    void write_raw(const void * ptr, size_t len) const {154        size_t bytes_written = 0;155        while (bytes_written < len) {156            size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);157            DWORD chunk_written = 0;158            BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);159            if (!result) {160                throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));161            }162            if (chunk_written < chunk_size || chunk_written == 0) {163                throw std::runtime_error("unexpectedly failed to write bytes");164            }165 166            bytes_written += chunk_written;167        }168    }169 170    void write_u32(uint32_t val) const {171        write_raw(&val, sizeof(val));172    }173 174    bool has_direct_io() const {175        return true;176    }177 178    ~impl() {179        if (fp && owns_fp) {180            std::fclose(fp);181        }182    }183#else184    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {185#ifdef __linux__186        // Try unbuffered I/O for read only187        if (use_direct_io && std::strcmp(mode, "rb") == 0) {188            if (init_fd()) {189                return;190            }191            LLAMA_LOG_WARN("Failed to open file '%s' with error: %s. Falling back to buffered I/O",192                           fname, strerror(errno));193        }194#endif195        init_fp(mode);196    }197 198#ifdef __linux__199    bool init_fd() {200        fd = open(fname.c_str(), O_RDONLY | O_DIRECT);201 202        if (fd != -1) {203            struct stat file_stats{};204            fstat(fd, &file_stats);205 206            size = file_stats.st_size;207            alignment = file_stats.st_blksize;208 209            off_t ret = lseek(fd, 0, SEEK_SET);210            if (ret == -1) {211                throw std::runtime_error(format("seek error: %s", strerror(errno)));212            }213            return true;214        }215        return false;216    }217#endif218 219    void init_fp(const char * mode) {220        fp = ggml_fopen(fname.c_str(), mode);221        if (fp == NULL) {222            throw std::runtime_error(format("failed to open %s: %s", fname.c_str(), strerror(errno)));223        }224        seek(0, SEEK_END);225        size = tell();226        seek(0, SEEK_SET);227    }228 229    impl(FILE * file) : fname("(file*)"), owns_fp(false) {230        fp = file;231        seek(0, SEEK_END);232        size = tell();233        seek(0, SEEK_SET);234    }235 236    size_t tell() const {237        if (fd == -1) {238            off_t ret = llama_mmap_ftell(fp);239            if (ret == -1) {240                throw std::runtime_error(format("ftell error: %s", strerror(errno)));241            }242 243            return (size_t) ret;244        }245 246        off_t pos = lseek(fd, 0, SEEK_CUR);247        if (pos == -1) {248            throw std::runtime_error(format("lseek error: %s", strerror(errno)));249        }250        return (size_t) pos;251    }252 253    void seek(size_t offset, int whence) const {254        off_t ret = 0;255        if (fd == -1) {256            ret = llama_mmap_fseek(fp, offset, whence);257        } else {258            ret = lseek(fd, offset, whence);259        }260        if (ret == -1) {261            throw std::runtime_error(format("seek error: %s", strerror(errno)));262        }263    }264 265    void read_raw_unsafe(void * ptr, size_t len) {266        if (len == 0) {267            return;268        }269        errno = 0;270        if (fd == -1) {271            const size_t curr_off = tell();272            const size_t to_read = std::min(len, size - curr_off);273 274            std::size_t ret = std::fread(ptr, to_read, 1, fp);275            if (ferror(fp)) {276                throw std::runtime_error(format("read error: %s", strerror(errno)));277            }278            if (to_read > 0 && ret != 1) {279                throw std::runtime_error("unexpectedly reached end of file");280            }281        } else {282            size_t bytes_read = 0;283            while (bytes_read < len) {284                const size_t to_read = len - bytes_read;285                ssize_t ret = ::read(fd, reinterpret_cast<char *>(ptr) + bytes_read, to_read);286 287                if (ret == -1) {288                    if (errno == EINTR) {289                        continue;  // Interrupted by signal, retry290                    }291                    // Fallback to std::fread in case the DMA controller cannot access the buffer292                    if (errno == EFAULT || errno == EINVAL) {293                        LLAMA_LOG_WARN("%s: Falling back to buffered IO due to %s\n", __func__, strerror(errno));294                        auto curr_off = tell();295                        close(fd);296                        fd = -1;297                        alignment = 1;298                        init_fp("rb");299                        seek(curr_off, SEEK_SET);300                        read_raw_unsafe(ptr, len);301                        return;302                    }303                    throw std::runtime_error(format("read error: %s", strerror(errno)));304                }305                if (ret == 0) {306                    // EOF: allow if this read was only pulling alignment padding past file end307                    off_t pos = lseek(fd, 0, SEEK_CUR);308                    if (pos != -1 && (size_t) pos == size) {309                        std::memset(reinterpret_cast<char *>(ptr) + bytes_read, 0, len - bytes_read);310                        return;311                    }312                    throw std::runtime_error("unexpectedly reached end of file");313                }314 315                bytes_read += (size_t) ret;316            }317        }318    }319 320    void read_aligned_chunk(void * dest, size_t size) {321        size_t offset = tell();322        off_t aligned_offset = offset & ~(alignment - 1);323        off_t offset_from_alignment = offset - aligned_offset;324        size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);325 326        void * raw_buffer = nullptr;327        int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read);328        if (ret != 0) {329            throw std::runtime_error(format("posix_memalign failed with error %d", ret));330        }331 332        struct aligned_buffer_deleter {333            void operator()(void * p) const { free(p); }334        };335        std::unique_ptr<void, aligned_buffer_deleter> buffer(raw_buffer);336 337        seek(aligned_offset, SEEK_SET);338        read_raw_unsafe(buffer.get(), bytes_to_read);339 340        uintptr_t actual_data = reinterpret_cast<uintptr_t>(buffer.get()) + offset_from_alignment;341        memcpy(dest, reinterpret_cast<void *>(actual_data), size);342    }343 344    void read_raw(void * ptr, size_t len) {345        if (has_direct_io()) {346            read_aligned_chunk(ptr, len);347        } else {348            read_raw_unsafe(ptr, len);349        }350    }351 352    uint32_t read_u32() {353        uint32_t ret;354        read_raw(&ret, sizeof(ret));355        return ret;356    }357 358    void write_raw(const void * ptr, size_t len) const {359        if (len == 0) {360            return;361        }362        errno = 0;363        size_t ret = std::fwrite(ptr, len, 1, fp);364        if (ret != 1) {365            throw std::runtime_error(format("write error: %s", strerror(errno)));366        }367    }368 369    void write_u32(uint32_t val) const {370        write_raw(&val, sizeof(val));371    }372 373    bool has_direct_io() const {374        return fd != -1 && alignment > 1;375    }376 377    ~impl() {378        if (fd != -1) {379            close(fd);380        } else if (owns_fp) {381            std::fclose(fp);382        }383    }384    int fd = -1;385    std::string fname;386#endif387 388    size_t read_alignment() const {389        return alignment;390    }391 392    size_t alignment = 1;393 394    FILE * fp{};395    size_t size{};396    bool owns_fp = true;397};398 399llama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) :400    pimpl(std::make_unique<impl>(fname, mode, use_direct_io)) {}401 402llama_file::llama_file(FILE * file) : pimpl(std::make_unique<impl>(file)) {}403 404llama_file::~llama_file() = default;405 406size_t llama_file::tell() const { return pimpl->tell(); }407size_t llama_file::size() const { return pimpl->size; }408 409size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }410bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }411 412int llama_file::file_id() const {413#ifdef _WIN32414    return _fileno(pimpl->fp);415#else416    if (pimpl->fd != -1) {417        return pimpl->fd;418    }419#if defined(fileno)420    return fileno(pimpl->fp);421#else422    return ::fileno(pimpl->fp);423#endif424#endif425}426 427void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }428void llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }429#ifdef _WIN32430void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }431#else432void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); }433#endif434 435uint32_t llama_file::read_u32() { return pimpl->read_u32(); }436 437void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }438void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }439 440// llama_mmap441 442#if defined(_POSIX_MAPPED_FILES) || defined(_WIN32)443// merge `ranges` and return their complement within [0, limit)444static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t limit) {445    llama_mmap::ranges res;446    std::sort(ranges.begin(), ranges.end());447 448    size_t pos = 0;449    for (const auto & range : ranges) {450        const size_t beg = std::min(range.first,  limit);451        const size_t end = std::min(range.second, limit);452        if (beg > pos) {453            res.emplace_back(pos, beg);454        }455        pos = std::max(pos, end);456    }457    if (pos < limit) {458        res.emplace_back(pos, limit);459    }460 461    return res;462}463#endif464 465struct llama_mmap::impl {466#ifdef _POSIX_MAPPED_FILES467    std::vector<std::pair<size_t, size_t>> mapped_fragments;468 469    impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {470        size = file->size();471        int fd = file->file_id();472        int flags = MAP_SHARED;473        if (numa) { prefetch = 0; }474#ifdef __linux__475        if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {476            LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",477                    strerror(errno));478        }479        // MAP_POPULATE would fault in the lazy ranges too480        if (prefetch && lazy_ranges.empty()) { flags |= MAP_POPULATE; }481#endif482        addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);483        if (addr == MAP_FAILED) {484            throw std::runtime_error(format("mmap failed: %s", strerror(errno)));485        }486 487        // page-aligned madvise over [beg, end), clamped to the file488        auto advise = [&](size_t beg, size_t end, int advice, const char * name) {489            const size_t page_size = sysconf(_SC_PAGESIZE);490            beg = beg & ~(page_size - 1);491            end = std::min((end + page_size - 1) & ~(page_size - 1), file->size());492            if (beg >= end) {493                return;494            }495            if (posix_madvise((char *) addr + beg, end - beg, advice)) {496                LLAMA_LOG_WARN("warning: posix_madvise(.., %s) failed: %s\n", name, strerror(errno));497            }498        };499 500        if (prefetch > 0) {501            for (const auto & range : ranges_complement(lazy_ranges, std::min(file->size(), prefetch))) {502                advise(range.first, range.second, POSIX_MADV_WILLNEED, "POSIX_MADV_WILLNEED");503            }504        }505        for (const auto & range : lazy_ranges) {506            advise(range.first, range.second, POSIX_MADV_RANDOM, "POSIX_MADV_RANDOM");507        }508        if (numa) {509            if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {510                LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",511                        strerror(errno));512            }513        }514 515        mapped_fragments.emplace_back(0, file->size());516    }517 518    static void align_range(size_t * first, size_t * last, size_t page_size) {519        size_t offset_in_page = *first & (page_size - 1);520        size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;521        *first += offset_to_page;522 523        *last = *last & ~(page_size - 1);524 525        if (*last <= *first) {526            *last = *first;527        }528    }529 530    void unmap_fragment(size_t first, size_t last) {531        int page_size = sysconf(_SC_PAGESIZE);532        align_range(&first, &last, page_size);533        size_t len = last - first;534 535        if (len == 0) {536            return;537        }538 539        GGML_ASSERT(first % page_size == 0);540        GGML_ASSERT(last % page_size == 0);541        GGML_ASSERT(last > first);542 543        void * next_page_start = (uint8_t *) addr + first;544 545        if (munmap(next_page_start, len)) {546            LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));547        }548 549        std::vector<std::pair<size_t, size_t>> new_mapped_fragments;550        for (const auto & frag : mapped_fragments) {551            if (frag.first < first && frag.second > last) {552                new_mapped_fragments.emplace_back(frag.first, first);553                new_mapped_fragments.emplace_back(last, frag.second);554            } else if (frag.first < first && frag.second > first) {555                new_mapped_fragments.emplace_back(frag.first, first);556            } else if (frag.first < last && frag.second > last) {557                new_mapped_fragments.emplace_back(last, frag.second);558            } else if (frag.first >= first && frag.second <= last) {559            } else {560                new_mapped_fragments.push_back(frag);561            }562        }563        mapped_fragments = std::move(new_mapped_fragments);564    }565 566    ~impl() {567        for (const auto & frag : mapped_fragments) {568            if (munmap((char *) addr + frag.first, frag.second - frag.first)) {569                LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));570            }571        }572    }573#elif defined(_WIN32)574    HANDLE hMapping = nullptr;575 576    impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {577        GGML_UNUSED(numa);578 579        size = file->size();580 581        HANDLE hFile = (HANDLE) _get_osfhandle(file->file_id());582 583        hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);584 585        if (hMapping == NULL) {586            DWORD error = GetLastError();587            throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));588        }589 590        addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);591        DWORD error = GetLastError();592 593        if (addr == NULL) {594            CloseHandle(hMapping);595            throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));596        }597 598        if (prefetch > 0) {599#if _WIN32_WINNT >= 0x602600            BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);601            HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");602 603            pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");604 605            if (pPrefetchVirtualMemory) {606                std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;607                for (const auto & range : ranges_complement(lazy_ranges, std::min(size, prefetch))) {608                    WIN32_MEMORY_RANGE_ENTRY entry;609                    entry.VirtualAddress = (char *) addr + range.first;610                    entry.NumberOfBytes  = (SIZE_T) (range.second - range.first);611                    entries.push_back(entry);612                }613                if (!entries.empty() &&614                        !pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0)) {615                    LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",616                            llama_format_win_err(GetLastError()).c_str());617                }618            }619#else620            LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n");621#endif622        }623    }624 625    void unmap_fragment(size_t first, size_t last) {626        GGML_UNUSED(first);627        GGML_UNUSED(last);628    }629 630    ~impl() {631        if (hMapping) {632            if (addr) {633                if (!UnmapViewOfFile(addr)) {634                    LLAMA_LOG_WARN("warning: UnmapViewOfFile failed: %s\n",635                            llama_format_win_err(GetLastError()).c_str());636                }637            }638            if (!CloseHandle(hMapping)) {639                LLAMA_LOG_WARN("warning: CloseHandle failed: %s\n",640                        llama_format_win_err(GetLastError()).c_str());641            }642        }643    }644#else645    impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {646        GGML_UNUSED(file);647        GGML_UNUSED(prefetch);648        GGML_UNUSED(numa);649        GGML_UNUSED(lazy_ranges);650 651        throw std::runtime_error("mmap not supported");652    }653 654    void unmap_fragment(size_t first, size_t last) {655        GGML_UNUSED(first);656        GGML_UNUSED(last);657 658        throw std::runtime_error("mmap not supported");659    }660#endif661 662    void * addr;663    size_t size;664};665 666llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa,667        const ranges & lazy_ranges) : pimpl(std::make_unique<impl>(file, prefetch, numa, lazy_ranges)) {}668llama_mmap::~llama_mmap() = default;669 670size_t llama_mmap::size() const { return pimpl->size; }671void * llama_mmap::addr() const { return pimpl->addr; }672 673void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }674 675#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)676const bool llama_mmap::SUPPORTED  = true;677#else678const bool llama_mmap::SUPPORTED  = false;679#endif680 681// llama_mlock682 683struct llama_mlock::impl {684#ifdef _POSIX_MEMLOCK_RANGE685    static size_t lock_granularity() {686        return (size_t) sysconf(_SC_PAGESIZE);687    }688 689    bool raw_lock(const void * addr, size_t size) const {690        if (!mlock(addr, size)) {691            return true;692        }693 694#ifdef __APPLE__695#define MLOCK_SUGGESTION \696        "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \697        "decreasing 'vm.global_no_user_wire_amount'.  Also try increasing RLIMIT_MEMLOCK (ulimit -l).\n"698#else699#define MLOCK_SUGGESTION \700        "Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\n"701#endif702 703        char* errmsg = std::strerror(errno);704        bool suggest = (errno == ENOMEM);705#if defined(TARGET_OS_VISION) || defined(TARGET_OS_TV) || defined(_AIX) || defined(__HAIKU__)706        // visionOS/tvOS/Haiku don't support RLIMIT_MEMLOCK707        // Skip resource limit checks on these platforms708        suggest = false;709#else710        struct rlimit lock_limit;711        if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {712            suggest = false;713        }714        if (suggest && ((uint64_t)lock_limit.rlim_max > (uint64_t)lock_limit.rlim_cur + size)) {715            suggest = false;716        }717#endif718 719        LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",720                size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");721        return false;722    }723 724    static void raw_unlock(void * addr, size_t size) {725        if (munlock(addr, size)) {726            LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));727        }728    }729#elif defined(_WIN32)730    static size_t lock_granularity() {731        SYSTEM_INFO si;732        GetSystemInfo(&si);733        return (size_t) si.dwPageSize;734    }735 736    bool raw_lock(void * ptr, size_t len) const {737        for (int tries = 1; ; tries++) {738            if (VirtualLock(ptr, len)) {739                return true;740            }741            if (tries == 2) {742                LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",743                    len, size, llama_format_win_err(GetLastError()).c_str());744                return false;745            }746 747            SIZE_T min_ws_size, max_ws_size;748            if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {749                LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n",750                        llama_format_win_err(GetLastError()).c_str());751                return false;752            }753            size_t increment = len + 1048576;754            min_ws_size += increment;755            max_ws_size += increment;756            if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {757                LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n",758                        llama_format_win_err(GetLastError()).c_str());759                return false;760            }761        }762    }763 764    static void raw_unlock(void * ptr, size_t len) {765        if (!VirtualUnlock(ptr, len)) {766            LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n",767                    llama_format_win_err(GetLastError()).c_str());768        }769    }770#else771    static size_t lock_granularity() {772        return (size_t) 65536;773    }774 775    bool raw_lock(const void * addr, size_t len) const {776        LLAMA_LOG_WARN("warning: mlock not supported on this system\n");777        return false;778    }779 780    static void raw_unlock(const void * addr, size_t len) {}781#endif782 783    impl() : addr(NULL), size(0), failed_already(false) {}784 785    void init(void * ptr) {786        GGML_ASSERT(addr == NULL && size == 0);787        addr = ptr;788    }789 790    void grow_to(size_t target_size) {791        GGML_ASSERT(addr);792        if (failed_already) {793            return;794        }795        size_t granularity = lock_granularity();796        target_size = (target_size + granularity - 1) & ~(granularity - 1);797        if (target_size > size) {798            if (raw_lock((uint8_t *) addr + size, target_size - size)) {799                size = target_size;800            } else {801                failed_already = true;802            }803        }804    }805 806    void * addr;807    size_t size;808 809    bool failed_already;810};811 812llama_mlock::llama_mlock() : pimpl(std::make_unique<impl>()) {}813llama_mlock::~llama_mlock() = default;814 815void llama_mlock::init(void * ptr) { pimpl->init(ptr); }816void llama_mlock::grow_to(size_t target_size) { pimpl->grow_to(target_size); }817 818#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)819const bool llama_mlock::SUPPORTED = true;820#else821const bool llama_mlock::SUPPORTED = false;822#endif823 824size_t llama_path_max() {825    return PATH_MAX;826}827