Felipe97/llama-cpp-compiled

1// GGUF binary parser adapted from the huggingface/gguf package.2// Reference: https://github.com/huggingface/huggingface.js3 4#include "gguf-model-data.h"5 6#include "common.h"7#include "ggml-cpp.h"8#include "gguf.h"9 10#include <algorithm>11#include <cstdio>12#include <cstring>13#include <filesystem>14#include <fstream>15 16#include "http.h"17#define JSON_ASSERT GGML_ASSERT18#include <nlohmann/json.hpp>19 20// Equivalent of RangeView21struct gguf_buf_reader {22    const char * data;23    size_t       size;24    size_t       pos;25 26    gguf_buf_reader(const std::vector<char> & buf) : data(buf.data()), size(buf.size()), pos(0) {}27 28    bool has_n_bytes(size_t n) const {29        return pos + n <= size;30    }31 32    template <typename T>33    bool read_val(T & out) {34        if (!has_n_bytes(sizeof(T))) {35            return false;36        }37        memcpy(&out, data + pos, sizeof(T));38        pos += sizeof(T);39        return true;40    }41 42    bool read_str(std::string & out) {43        uint64_t len;44        if (!read_val(len)) {45            return false;46        }47        if (!has_n_bytes((size_t)len)) {48            return false;49        }50        out.assign(data + pos, (size_t)len);51        pos += (size_t)len;52        return true;53    }54 55    bool skip(size_t n) {56        if (!has_n_bytes(n)) {57            return false;58        }59        pos += n;60        return true;61    }62};63 64static size_t gguf_val_type_size(int32_t vtype) {65    switch (vtype) {66        case GGUF_TYPE_UINT8:   return 1;67        case GGUF_TYPE_INT8:    return 1;68        case GGUF_TYPE_UINT16:  return 2;69        case GGUF_TYPE_INT16:   return 2;70        case GGUF_TYPE_UINT32:  return 4;71        case GGUF_TYPE_INT32:   return 4;72        case GGUF_TYPE_FLOAT32: return 4;73        case GGUF_TYPE_BOOL:    return 1;74        case GGUF_TYPE_UINT64:  return 8;75        case GGUF_TYPE_INT64:   return 8;76        case GGUF_TYPE_FLOAT64: return 8;77        default:                return 0; // string/array handled separately78    }79}80 81// Equivalent of readMetadataValue(), skips unused values rather than storing82static bool gguf_skip_value(gguf_buf_reader & r, int32_t vtype) {83    if (vtype == GGUF_TYPE_STRING) {84        std::string tmp;85        return r.read_str(tmp);86    }87    if (vtype == GGUF_TYPE_ARRAY) {88        int32_t elem_type;89        uint64_t count;90        if (!r.read_val(elem_type)) {91            return false;92        }93        if (!r.read_val(count)) {94            return false;95        }96        if (elem_type == GGUF_TYPE_STRING) {97            for (uint64_t i = 0; i < count; i++) {98                std::string tmp;99                if (!r.read_str(tmp)) {100                    return false;101                }102            }103            return true;104        }105        if (elem_type == GGUF_TYPE_ARRAY) {106            // nested arrays - recurse107            for (uint64_t i = 0; i < count; i++) {108                if (!gguf_skip_value(r, GGUF_TYPE_ARRAY)) {109                    return false;110                }111            }112            return true;113        }114        size_t elem_sz = gguf_val_type_size(elem_type);115        if (elem_sz == 0) {116            return false;117        }118        return r.skip((size_t)count * elem_sz);119    }120    size_t sz = gguf_val_type_size(vtype);121    if (sz == 0) {122        return false;123    }124    return r.skip(sz);125}126 127static bool gguf_read_uint32_val(gguf_buf_reader & r, int32_t vtype, uint32_t & out) {128    // Handle array-valued fields (e.g. per-layer head counts in hybrid models)129    // by reading the first element as a representative value.130    if (vtype == GGUF_TYPE_ARRAY) {131        int32_t elem_type;132        uint64_t count;133        if (!r.read_val(elem_type)) {134            return false;135        }136        if (!r.read_val(count)) {137            return false;138        }139        if (count == 0) {140            return false;141        }142        // Read first element, skip the rest143        if (!gguf_read_uint32_val(r, elem_type, out)) {144            return false;145        }146        for (uint64_t i = 1; i < count; i++) {147            size_t sz = gguf_val_type_size(elem_type);148            if (sz == 0) {149                return false;150            }151            if (!r.skip(sz)) {152                return false;153            }154        }155        return true;156    }157    if (vtype == GGUF_TYPE_UINT8) {158        uint8_t v;159        if (!r.read_val(v)) {160            return false;161        }162        out = v;163        return true;164    }165    if (vtype == GGUF_TYPE_INT8) {166        int8_t v;167        if (!r.read_val(v)) {168            return false;169        }170        out = (uint32_t)v;171        return true;172    }173    if (vtype == GGUF_TYPE_UINT16) {174        uint16_t v;175        if (!r.read_val(v)) {176            return false;177        }178        out = v;179        return true;180    }181    if (vtype == GGUF_TYPE_INT16) {182        int16_t v;183        if (!r.read_val(v)) {184            return false;185        }186        out = (uint32_t)v;187        return true;188    }189    if (vtype == GGUF_TYPE_UINT32) {190        uint32_t v;191        if (!r.read_val(v)) {192            return false;193        }194        out = v;195        return true;196    }197    if (vtype == GGUF_TYPE_INT32) {198        int32_t v;199        if (!r.read_val(v)) {200            return false;201        }202        out = (uint32_t)v;203        return true;204    }205    if (vtype == GGUF_TYPE_UINT64) {206        uint64_t v;207        if (!r.read_val(v)) {208            return false;209        }210        out = (uint32_t)v;211        return true;212    }213    if (vtype == GGUF_TYPE_INT64) {214        int64_t v;215        if (!r.read_val(v)) {216            return false;217        }218        out = (uint32_t)v;219        return true;220    }221    return false;222}223 224// Follows the same header -> KV -> tensor parsing sequence as gguf() huggingface/gguf225static std::optional<gguf_remote_model> gguf_parse_meta(const std::vector<char> & buf) {226    gguf_buf_reader r(buf);227 228    // Header: magic(4) + version(4) + tensor_count(8) + kv_count(8) = 24 bytes minimum229    uint32_t magic_raw;230    if (!r.read_val(magic_raw)) {231        return std::nullopt;232    }233    if (memcmp(&magic_raw, "GGUF", 4) != 0) {234        fprintf(stderr, "gguf_parse_meta: invalid magic\n");235        return std::nullopt;236    }237 238    uint32_t version;239    if (!r.read_val(version)) {240        return std::nullopt;241    }242    if (version < 2 || version > 3) {243        fprintf(stderr, "gguf_parse_meta: unsupported version %u\n", version);244        return std::nullopt;245    }246 247    int64_t tensor_count_raw;248    int64_t kv_count_raw;249    if (!r.read_val(tensor_count_raw)) {250        return std::nullopt;251    }252    if (!r.read_val(kv_count_raw)) {253        return std::nullopt;254    }255 256    uint64_t tensor_count = (uint64_t)tensor_count_raw;257    uint64_t kv_count     = (uint64_t)kv_count_raw;258 259    gguf_remote_model model;260 261    std::string arch_prefix;262 263    // Parse KV pairs264    for (uint64_t i = 0; i < kv_count; i++) {265        std::string key;266        if (!r.read_str(key)) {267            return std::nullopt;268        }269 270        int32_t vtype;271        if (!r.read_val(vtype)) {272            return std::nullopt;273        }274 275        if (key == "general.architecture" && vtype == GGUF_TYPE_STRING) {276            if (!r.read_str(model.architecture)) {277                return std::nullopt;278            }279            arch_prefix = model.architecture + ".";280            continue;281        }282 283        // Extract split.count for proper handling of split files284        if (key == "split.count") {285            uint32_t v;286            if (!gguf_read_uint32_val(r, vtype, v)) {287                return std::nullopt;288            }289            model.n_split = (uint16_t)v;290            continue;291        }292 293        // Extract split.tensors.count so we can verify we have all tensors294        if (key == "split.tensors.count") {295            uint32_t v;296            if (!gguf_read_uint32_val(r, vtype, v)) {297                return std::nullopt;298            }299            model.n_split_tensors = v;300            continue;301        }302 303        if (!arch_prefix.empty()) {304            uint32_t * target = nullptr;305 306            if      (key == arch_prefix + "embedding_length")         { target = &model.n_embd; }307            else if (key == arch_prefix + "feed_forward_length")      { target = &model.n_ff; }308            else if (key == arch_prefix + "block_count")              { target = &model.n_layer; }309            else if (key == arch_prefix + "attention.head_count")     { target = &model.n_head; }310            else if (key == arch_prefix + "attention.head_count_kv")  { target = &model.n_head_kv; }311            else if (key == arch_prefix + "expert_count")             { target = &model.n_expert; }312            else if (key == arch_prefix + "attention.key_length")     { target = &model.n_embd_head_k; }313            else if (key == arch_prefix + "attention.value_length")   { target = &model.n_embd_head_v; }314 315            if (target) {316                if (!gguf_read_uint32_val(r, vtype, *target)) {317                    return std::nullopt;318                }319                continue;320            }321        }322 323        if (!gguf_skip_value(r, vtype)) {324            return std::nullopt;325        }326    }327 328    // Parse tensor info entries329    model.tensors.reserve((size_t)tensor_count);330    for (uint64_t i = 0; i < tensor_count; i++) {331        gguf_remote_tensor t;332 333        if (!r.read_str(t.name)) {334            return std::nullopt;335        }336        if (!r.read_val(t.n_dims)) {337            return std::nullopt;338        }339 340        if (t.n_dims > 4) {341            fprintf(stderr, "gguf_parse_meta: tensor '%s' has %u dims (max 4)\n", t.name.c_str(), t.n_dims);342            return std::nullopt;343        }344 345        for (uint32_t d = 0; d < t.n_dims; d++) {346            if (!r.read_val(t.ne[d])) {347                return std::nullopt;348            }349        }350 351        int32_t type_raw;352        if (!r.read_val(type_raw)) {353            return std::nullopt;354        }355        t.type = (ggml_type)type_raw;356 357        uint64_t offset;358        if (!r.read_val(offset)) {359            return std::nullopt;360        }361 362        // Infer n_vocab from token_embd.weight363        if (t.name == "token_embd.weight") {364            model.n_vocab = (uint32_t)t.ne[1];365        }366 367        model.tensors.push_back(std::move(t));368    }369 370    return model;371}372 373// cache handling for local download374static std::string get_default_cache_dir() {375    return fs_get_cache_directory() + "gguf-headers/";376}377 378static std::string sanitize_for_path(const std::string & s) {379    std::string out = s;380    for (char & c : out) {381        if (c == '/' || c == '\\' || c == ':') {382            c = '_';383        }384    }385    return out;386}387 388static bool read_file(const std::string & path, std::vector<char> & out) {389    std::ifstream f(path, std::ios::binary | std::ios::ate);390    if (!f.good()) {391        return false;392    }393    auto sz = f.tellg();394    if (sz <= 0) {395        return false;396    }397    out.resize((size_t)sz);398    f.seekg(0);399    f.read(out.data(), sz);400    return f.good();401}402 403static bool write_file(const std::string & path, const std::vector<char> & data) {404    std::ofstream f(path, std::ios::binary | std::ios::trunc);405    if (!f.good()) {406        return false;407    }408    f.write(data.data(), (std::streamsize)data.size());409    return f.good();410}411 412// HuggingFace file auto-detection and HTTP download413static std::pair<long, std::vector<char>> gguf_http_get(414        const std::string & url,415        const httplib::Headers & headers = {},416        int timeout_sec = 60) {417    try {418        auto [cli, parts] = common_http_client(url);419 420        if (timeout_sec > 0) {421            cli.set_read_timeout(timeout_sec, 0);422            cli.set_write_timeout(timeout_sec, 0);423        }424        cli.set_connection_timeout(30, 0);425 426        std::vector<char> body;427        auto res = cli.Get(parts.path, headers,428            [&](const char * data, size_t len) {429                body.insert(body.end(), data, data + len);430                return true;431            }, nullptr);432 433        if (!res) {434            fprintf(stderr, "gguf_fetch: HTTP request failed for %s (error %d)\n",435                    url.c_str(), (int)res.error());436            return {-1, {}};437        }438        return {res->status, std::move(body)};439    } catch (const std::exception & e) {440        fprintf(stderr, "gguf_fetch: HTTP error: %s\n", e.what());441        return {-1, {}};442    }443}444 445// Find the filename for given repo/quant.446// For split models, returns the first shard (the one containing "00001-of-")447// split_prefix is set to the portion before "-00001-of-XXXXX.gguf" when a split file is found448static std::string detect_gguf_filename(const std::string & repo, const std::string & quant,449                                        std::string & split_prefix) {450    split_prefix.clear();451    std::string api_url = "https://huggingface.co/api/models/" + repo;452 453    auto [code, body] = gguf_http_get(api_url, {}, 30);454    if (code != 200 || body.empty()) {455        fprintf(stderr, "gguf_fetch: failed to query HF API for %s (HTTP %ld)\n", repo.c_str(), code);456        return "";457    }458 459    nlohmann::json j;460    try {461        j = nlohmann::json::parse(body.begin(), body.end());462    } catch (...) {463        fprintf(stderr, "gguf_fetch: failed to parse HF API response\n");464        return "";465    }466 467    if (!j.contains("siblings") || !j["siblings"].is_array()) {468        fprintf(stderr, "gguf_fetch: unexpected HF API response format\n");469        return "";470    }471 472    std::vector<std::string> matches;473    std::string quant_upper = quant;474    for (char & c : quant_upper) { c = (char)toupper(c); }475 476    for (const auto & sibling : j["siblings"]) {477        if (!sibling.contains("rfilename")) { continue; }478        std::string fname = sibling["rfilename"].get<std::string>();479        if (fname.size() < 5 || fname.substr(fname.size() - 5) != ".gguf") {480            continue;481        }482 483        std::string fname_upper = fname;484        for (char & c : fname_upper) { c = (char)toupper(c); }485        if (fname_upper.find(quant_upper) != std::string::npos) {486            matches.push_back(fname);487        }488    }489 490    if (matches.empty()) {491        fprintf(stderr, "gguf_fetch: no .gguf files matching '%s' in %s\n", quant.c_str(), repo.c_str());492        return "";493    }494 495    std::sort(matches.begin(), matches.end());496 497    // Prefer non-split, non-supplementary file498    for (const auto & m : matches) {499        if (m.find("-of-") == std::string::npos && m.find("mmproj") == std::string::npos) {500            return m;501        }502    }503 504    // Return the first shard (00001-of-) and extract the prefix505    for (const auto & m : matches) {506        auto pos = m.find("-00001-of-");507        if (pos != std::string::npos) {508            split_prefix = m.substr(0, pos);509            return m;510        }511    }512 513    return matches[0];514}515 516static std::optional<gguf_remote_model> fetch_and_parse(517        const std::string & repo,518        const std::string & filename,519        const std::string & cache_path,520        bool verbose) {521    std::string url = "https://huggingface.co/" + repo + "/resolve/main/" + filename;522 523    // Progressive download inspired by RangeView.fetchChunk()524    // Start at 2MB, double each time, cap at 64MB525    size_t chunk_size = 2 * 1024 * 1024;526    const size_t max_chunk = 64 * 1024 * 1024;527 528    while (chunk_size <= max_chunk) {529        if (verbose) {530            fprintf(stderr, "gguf_fetch: downloading %zu bytes from %s\n", chunk_size, filename.c_str());531        }532 533        char range_buf[64];534        snprintf(range_buf, sizeof(range_buf), "bytes=0-%zu", chunk_size - 1);535        httplib::Headers headers = {{"Range", range_buf}};536 537        auto [code, body] = gguf_http_get(url, headers, 120);538        if (code != 200 && code != 206) {539            fprintf(stderr, "gguf_fetch: HTTP %ld fetching %s\n", code, url.c_str());540            return std::nullopt;541        }542 543        if (body.empty()) {544            fprintf(stderr, "gguf_fetch: empty response\n");545            return std::nullopt;546        }547 548        auto result = gguf_parse_meta(body);549        if (result.has_value()) {550            write_file(cache_path, body);551            return result;552        }553 554        if (code == 200) {555            fprintf(stderr, "gguf_fetch: server returned full response but metadata parse failed\n");556            return std::nullopt;557        }558 559        // Parse failed, try larger chunk560        chunk_size *= 2;561    }562 563    fprintf(stderr, "gguf_fetch: metadata exceeds 64MB, giving up\n");564    return std::nullopt;565}566 567static std::string get_cache_file_path(const std::string& cdir, const std::string& repo_part, const std::string& filename) {568    std::string fname_part = sanitize_for_path(filename);569    return cdir + "/" + repo_part + "--" + fname_part + ".partial";570}571 572// Try cache first, then fetch and parse a single GGUF shard.573static std::optional<gguf_remote_model> fetch_or_cached(574        const std::string & repo,575        const std::string & filename,576        const std::string & cdir,577        const std::string & repo_part,578        bool verbose) {579    std::string cache_path = get_cache_file_path(cdir, repo_part, filename);580 581    {582        std::vector<char> cached;583        if (std::filesystem::exists(cache_path) && read_file(cache_path, cached)) {584            auto result = gguf_parse_meta(cached);585            if (result.has_value()) {586                if (verbose) {587                    fprintf(stderr, "gguf_fetch: loaded from cache: %s\n", cache_path.c_str());588                }589                return result;590            }591        }592    }593 594    fs_create_directory_with_parents(cdir);595    return fetch_and_parse(repo, filename, cache_path, verbose);596}597 598std::optional<gguf_remote_model> gguf_fetch_model_meta(599        const std::string & repo,600        const std::string & quant,601        const std::string & cache_dir,602        bool verbose) {603    std::string cdir = cache_dir.empty() ? get_default_cache_dir() : cache_dir;604    std::string repo_part = sanitize_for_path(repo);605 606    std::string split_prefix;607    std::string filename = detect_gguf_filename(repo, quant, split_prefix);608    if (filename.empty()) {609        return std::nullopt;610    }611 612    auto model_opt = fetch_or_cached(repo, filename, cdir, repo_part, verbose);613    if (!model_opt.has_value()) {614        fprintf(stderr, "gguf_fetch: failed to fetch %s\n", filename.c_str());615        return std::nullopt;616    }617 618    auto & model = model_opt.value();619 620    // If the model is split across multiple files we need to fetch the remaining shards metadata621    if (model.n_split > 1) {622        if (split_prefix.empty()) {623            fprintf(stderr, "gguf_fetch: model reports %u splits but filename has no split pattern\n", model.n_split);624            return std::nullopt;625        }626 627        if (verbose) {628            fprintf(stderr, "gguf_fetch: split model with %u shards, fetching remaining %u...\n",629                    model.n_split, model.n_split - 1);630        }631 632        for (int i = 2; i <= model.n_split; i++) {633            char buf_num[32];634            char buf_tot[32];635            snprintf(buf_num, sizeof(buf_num), "%05d", i);636            snprintf(buf_tot, sizeof(buf_tot), "%05d", (int)model.n_split);637            std::string shard_name = split_prefix + "-" + buf_num + "-of-" + buf_tot + ".gguf";638 639            auto shard = fetch_or_cached(repo, shard_name, cdir, repo_part, verbose);640            if (!shard.has_value()) {641                fprintf(stderr, "gguf_fetch: failed to fetch shard %d: %s\n", i, shard_name.c_str());642                return std::nullopt;643            }644 645            model.tensors.insert(model.tensors.end(),646                std::make_move_iterator(shard->tensors.begin()),647                std::make_move_iterator(shard->tensors.end()));648        }649 650        if (model.n_split_tensors > 0 && model.tensors.size() != model.n_split_tensors) {651            fprintf(stderr, "gguf_fetch: WARNING: expected %u tensors from split.tensors.count, got %zu\n",652                    model.n_split_tensors, model.tensors.size());653        }654    }655 656    return model_opt;657}658 659gguf_context_ptr gguf_fetch_gguf_ctx(660        const std::string & repo,661        const std::string & quant,662        const std::string & cache_dir,663        bool verbose) {664    std::string cdir = cache_dir.empty() ? get_default_cache_dir() : cache_dir;665    std::string repo_part = sanitize_for_path(repo);666 667    std::string split_prefix;668    std::string filename = detect_gguf_filename(repo, quant, split_prefix);669 670    if (filename.empty()) {671        return nullptr;672    }673 674    auto model_opt = fetch_or_cached(repo, filename, cdir, repo_part, verbose);675    if (!model_opt.has_value()) {676        fprintf(stderr, "gguf_fetch: failed to fetch %s\n", filename.c_str());677        return nullptr;678    }679 680    auto & model = model_opt.value();681 682    const std::string cache_path = get_cache_file_path(cdir, repo_part, filename);683 684    ggml_context_ptr ggml_ctx_ptr;685    ggml_context * ggml_ctx{};686    gguf_init_params params{true, &ggml_ctx};687    gguf_context_ptr ctx{gguf_init_from_file(cache_path.c_str(), params)};688    ggml_ctx_ptr.reset(ggml_ctx);689 690    if (ctx == nullptr) {691        fprintf(stderr, "gguf_fetch: gguf_init_from_file failed\n");692        return nullptr;693    }694 695    // If the model is split across multiple files we need to fetch the remaining shards metadata696    if (model.n_split > 1) {697        if (split_prefix.empty()) {698            fprintf(stderr, "gguf_fetch: model reports %u splits but filename has no split pattern\n", model.n_split);699            return nullptr;700        }701 702        if (verbose) {703            fprintf(stderr, "gguf_fetch: split model with %u shards, fetching remaining %u...\n",704                    model.n_split, model.n_split - 1);705        }706 707        for (int i = 2; i <= model.n_split; i++) {708            char buf_num[32];709            char buf_tot[32];710            snprintf(buf_num, sizeof(buf_num), "%05d", i);711            snprintf(buf_tot, sizeof(buf_tot), "%05d", (int)model.n_split);712            std::string shard_name = split_prefix + "-" + buf_num + "-of-" + buf_tot + ".gguf";713 714            auto shard = fetch_or_cached(repo, shard_name, cdir, repo_part, verbose);715            if (!shard.has_value()) {716                fprintf(stderr, "gguf_fetch: failed to fetch shard %d: %s\n", i, shard_name.c_str());717                return nullptr;718            }719 720            // Load tensors from shard and add to main gguf_context721            const std::string shard_path = get_cache_file_path(cdir, repo_part, shard_name);722            ggml_context_ptr shard_ggml_ctx_ptr;723            ggml_context * shard_ggml_ctx{};724            gguf_init_params shard_params{true, &shard_ggml_ctx};725            gguf_context_ptr shard_ctx{gguf_init_from_file(shard_path.c_str(), shard_params)};726            shard_ggml_ctx_ptr.reset(shard_ggml_ctx);727 728            if (shard_ctx == nullptr) {729                fprintf(stderr, "gguf_fetch: shard gguf_init_from_file failed\n");730                return nullptr;731            }732 733            for (ggml_tensor * t = ggml_get_first_tensor(shard_ggml_ctx); t; t = ggml_get_next_tensor(shard_ggml_ctx, t)) {734                gguf_add_tensor(ctx.get(), t);735            }736        }737 738        gguf_set_val_u16(ctx.get(), "split.count", 1);739    }740 741    return ctx;742}743